diff --git a/normalization/languages/__init__.py b/normalization/languages/__init__.py index 1e01e2f..b265568 100644 --- a/normalization/languages/__init__.py +++ b/normalization/languages/__init__.py @@ -7,6 +7,7 @@ german, italian, norwegian, + portuguese, spanish, swedish, ) @@ -24,6 +25,7 @@ "german", "italian", "norwegian", + "portuguese", "spanish", "swedish", "get_language_registry", diff --git a/normalization/languages/portuguese/__init__.py b/normalization/languages/portuguese/__init__.py new file mode 100644 index 0000000..e59f3cc --- /dev/null +++ b/normalization/languages/portuguese/__init__.py @@ -0,0 +1,7 @@ +from .operators import PortugueseOperators +from .replacements import PORTUGUESE_REPLACEMENTS + +__all__ = [ + "PortugueseOperators", + "PORTUGUESE_REPLACEMENTS", +] diff --git a/normalization/languages/portuguese/clitics.py b/normalization/languages/portuguese/clitics.py new file mode 100644 index 0000000..33f70d8 --- /dev/null +++ b/normalization/languages/portuguese/clitics.py @@ -0,0 +1,38 @@ +"""Merge hyphenated Portuguese clitic pronouns into single tokens for WER scoring. + +Oral Portuguese often attaches clitics to imperatives (``envie-me`` / ``enviame``). +ASR may hyphenate or space them; this step canonicalizes to the glued form. +""" + +import re + +_CLITIC_SUFFIXES = ( + "me", + "te", + "se", + "lo", + "la", + "lhe", + "lhes", + "nos", + "vos", + "los", + "las", + "no", + "na", + "lho", + "lha", + "lhes", +) + +_RE_HYPHEN_CLITIC = re.compile( + rf"\b(\w+)-({'|'.join(_CLITIC_SUFFIXES)})\b", + re.IGNORECASE, +) + + +def merge_hyphenated_clitics(text: str) -> str: + """``digame-me`` / ``diga-me`` → ``digame``.""" + return _RE_HYPHEN_CLITIC.sub( + lambda m: m.group(1).lower() + m.group(2).lower(), text + ) diff --git a/normalization/languages/portuguese/number_normalizer.py b/normalization/languages/portuguese/number_normalizer.py new file mode 100644 index 0000000..e837f13 --- /dev/null +++ b/normalization/languages/portuguese/number_normalizer.py @@ -0,0 +1,66 @@ +"""Portuguese number normalizer using text2num's alpha2digit. + +Converts spelled-out numbers to digits (e.g. vinte e tres → 23) and handles +mixed digit+word forms (e.g. 3 milhoes → tres milhoes) before conversion. +""" + +import re + +from text_to_num import alpha2digit + +_RE_MIXED_NUMBER = re.compile( + r"\b(\d+)\s+(mil|milhao|milhoes|milhão|milhões|bilhao|bilhoes|bilhão|bilhões)\b", + re.IGNORECASE, +) + +_RE_UM = re.compile(r"\bum\b", re.IGNORECASE) +_RE_DOIS = re.compile(r"\bdois\b", re.IGNORECASE) + +# text2num expects accented Portuguese forms; STT often drops accents. +_UNACCENTED_NUMBER_FORMS: dict[str, str] = { + "milhoes": "milhões", + "milhao": "milhão", + "bilhoes": "bilhões", + "bilhao": "bilhão", + "tres": "três", + "seis": "seis", +} + + +def _restore_number_accents(text: str) -> str: + for unaccented, canonical in _UNACCENTED_NUMBER_FORMS.items(): + text = re.sub(rf"\b{unaccented}\b", canonical, text, flags=re.IGNORECASE) + return text + + +def _fix_remaining_words(text: str) -> str: + """Replace number words alpha2digit did not convert.""" + text = _RE_UM.sub("1", text) + text = _RE_DOIS.sub("2", text) + return text + + +class PortugueseNumberNormalizer: + """Convert Portuguese spelled-out numbers to digits via text2num.alpha2digit.""" + + def __init__(self, digit_words: dict[str, str]) -> None: + self._digit_to_word = {v: k for k, v in digit_words.items()} + + def _normalize_mixed_numbers(self, text: str) -> str: + """Convert ``3 milhoes`` → ``tres milhoes`` so alpha2digit yields 3000000.""" + + def replace(match: re.Match[str]) -> str: + number = match.group(1) + multiplier = match.group(2) + if len(number) == 1 and number in self._digit_to_word: + return f"{self._digit_to_word[number]} {multiplier}" + return match.group(0) + + return _RE_MIXED_NUMBER.sub(replace, text) + + def __call__(self, text: str) -> str: + text = _restore_number_accents(text) + text = self._normalize_mixed_numbers(text) + text = alpha2digit(text, "pt") + text = _fix_remaining_words(text) + return text diff --git a/normalization/languages/portuguese/operators.py b/normalization/languages/portuguese/operators.py new file mode 100644 index 0000000..7b8e0ee --- /dev/null +++ b/normalization/languages/portuguese/operators.py @@ -0,0 +1,133 @@ +import re + +from normalization.languages.base import LanguageConfig, LanguageOperators +from normalization.languages.portuguese.clitics import merge_hyphenated_clitics +from normalization.languages.portuguese.number_normalizer import ( + PortugueseNumberNormalizer, +) +from normalization.languages.portuguese.replacements import PORTUGUESE_REPLACEMENTS +from normalization.languages.portuguese.sentence_replacements import ( + PORTUGUESE_SENTENCE_REPLACEMENTS, +) +from normalization.languages.registry import register_language + +_PORTUGUESE_DIGIT_WORDS: dict[str, str] = { + "zero": "0", + "um": "1", + "dois": "2", + "tres": "3", + "quatro": "4", + "cinco": "5", + "seis": "6", + "sete": "7", + "oito": "8", + "nove": "9", +} + +PORTUGUESE_CONFIG = LanguageConfig( + code="pt", + decimal_separator=",", + decimal_word="virgula", + thousand_separator=" ", + symbols_to_words={ + "@": "arroba", + ".": "ponto", + "+": "mais", + "=": "igual a", + ">": "maior que", + "<": "menor que", + "°": "grau", + "°C": "graus celsius", + "°F": "graus fahrenheit", + "%": "por cento", + }, + currency_symbol_to_word={ + "€": "euros", + "$": "dolares", + "£": "libras", + "¢": "centavos", + "¥": "ienes", + }, + filler_words=[ + # Hesitations / backchannels only — avoid stripping meaningful words + # (e.g. "claro", "bem" in "tudo bem", "pronto" in "e pronto, foi assim"). + "ah", + "eh", + "ehm", + "entao", + "portanto", + "mm", + "mmm", + "mhm", + "hmm", + "hm", + "hum", + "humm", + ], + sentence_replacements=PORTUGUESE_SENTENCE_REPLACEMENTS, + digit_words=_PORTUGUESE_DIGIT_WORDS, + number_words=[ + *_PORTUGUESE_DIGIT_WORDS, + "dez", + "onze", + "doze", + "treze", + "catorze", + "quinze", + "dezasseis", + "dezesseis", + "dezassete", + "dezessete", + "dezoito", + "dezenove", + "vinte", + "trinta", + "quarenta", + "cinquenta", + "sessenta", + "setenta", + "oitenta", + "noventa", + "cem", + "cento", + "mil", + "milhao", + "milhoes", + "milhão", + "milhões", + "bilhao", + "bilhoes", + "bilhão", + "bilhões", + ], + plus_word="mais", +) + + +@register_language +class PortugueseOperators(LanguageOperators): + """Portuguese language operators: clitics, written numbers, word replacements.""" + + def __init__(self) -> None: + super().__init__(PORTUGUESE_CONFIG) + self._number_normalizer = PortugueseNumberNormalizer( + PORTUGUESE_CONFIG.digit_words or {} + ) + + def expand_contractions(self, text: str) -> str: + """Merge hyphenated clitic pronouns (envie-me → enviame).""" + return merge_hyphenated_clitics(text) + + def expand_written_numbers(self, text: str) -> str: + """Convert Portuguese spelled-out numbers to digits (vinte e tres → 23).""" + return self._number_normalizer(text) + + def fix_one_word_in_numeric_contexts(self, text: str) -> str: + text = re.sub(r"(\d+)\s+um\b", r"\1 1", text, flags=re.IGNORECASE) + text = re.sub(r"\bum\s+(\d)", r"1 \1", text, flags=re.IGNORECASE) + text = re.sub(r"(\d+)um\b", r"\1 1", text, flags=re.IGNORECASE) + text = re.sub(r"\bum(\d)", r"1 \1", text, flags=re.IGNORECASE) + return text + + def get_word_replacements(self) -> dict[str, str]: + return PORTUGUESE_REPLACEMENTS diff --git a/normalization/languages/portuguese/replacements.py b/normalization/languages/portuguese/replacements.py new file mode 100644 index 0000000..9d54ba1 --- /dev/null +++ b/normalization/languages/portuguese/replacements.py @@ -0,0 +1,42 @@ +"""Single-token Portuguese colloquial / ASR variants → canonical form for WER.""" + +PORTUGUESE_REPLACEMENTS: dict[str, str] = { + # Oral preposition / contraction variants + "pra": "para", + # Gender agreement in short confirmations (telephony slips) + "propria": "proprio", + "proprias": "proprios", + # Adjective / adverb spelling variants + "maioritariamente": "maioritario", + "media": "medio", + # Email / compound spelling + "emails": "email", + "e-mail": "email", + "e-mails": "email", + # Morphological number (singular/plural oral agreement) + "rotativos": "rotativo", + "turnos": "turno", + "pesos": "peso", + "quantos": "quanto", + # PT-PT / PT-BR orthography (canonical: PT-PT) + "controles": "controlos", + "equipe": "equipa", + "confirmar-se": "confirmares", + # Clitic / verb form variants (ASR subjunctive vs imperative) + "envieme": "enviame", + "adeco": "adecco", + "adec": "adecco", + # Abbreviations / titles + "dr": "doutor", + "dra": "doutora", + "sr": "senhor", + "sra": "senhora", + "exmo": "excelentissimo", + "exma": "excelentissima", + "etc": "etcetera", + "tel": "telefone", + "vs": "versus", + "versus": "versus", + # Acknowledgments / spelling variants + "okay": "ok", +} diff --git a/normalization/languages/portuguese/sentence_replacements.py b/normalization/languages/portuguese/sentence_replacements.py new file mode 100644 index 0000000..354d56a --- /dev/null +++ b/normalization/languages/portuguese/sentence_replacements.py @@ -0,0 +1,17 @@ +"""Multi-word Portuguese phrase normalization (oral variants, compounds, ASR repairs).""" + +PORTUGUESE_SENTENCE_REPLACEMENTS: dict[str, str] = { + # Email compounds (hyphenated forms run in text_pre; spaced forms after cleanup) + "e-mails": "email", + "e-mail": "email", + "e mail": "email", + "e mails": "email", + # Morphological number / construction variants + "turnos rotativos": "turno rotativo", + "em quantos pesos": "de quanto peso", + # PT-PT / PT-BR orthography + "confirmar se": "confirmares", + # Clitic spacing (ASR splits glued imperatives) + "envie me": "enviame", + "diga me": "digame", +} diff --git a/tests/e2e/files/gladia-3/pt.csv b/tests/e2e/files/gladia-3/pt.csv new file mode 100644 index 0000000..3044d83 --- /dev/null +++ b/tests/e2e/files/gladia-3/pt.csv @@ -0,0 +1,30 @@ +input,expected +eh entao sim,sim +ah mm sim,sim +claro estou,claro estou +esta tudo bem,esta tudo bem +e pronto foi assim,e pronto foi assim +pra trabalhar,para trabalhar +Se vou ser chamada pra trabalhar,se vou ser chamada para trabalhar +Sim e propria,sim e proprio +maioritariamente,maioritario +media,medio +emails,email +e-mails,email +turnos rotativos,turno rotativo +em quantos pesos,de quanto peso +envie-me,enviame +enviame,enviame +diga-me,digame +diga me,digame +controles,controlos +equipe,equipa +adeco,adecco +adec,adecco +vinte e tres,23 +cem euros,100 euros +Maria,maria +María,maria +test@example.com,test arroba example ponto com +"12,5 euros",12 virgula 5 euros +posso ajudar,posso ajudar diff --git a/tests/unit/languages/portuguese_number_normalizer_test.py b/tests/unit/languages/portuguese_number_normalizer_test.py new file mode 100644 index 0000000..5eae6e4 --- /dev/null +++ b/tests/unit/languages/portuguese_number_normalizer_test.py @@ -0,0 +1,37 @@ +import pytest + +from normalization.languages.portuguese.number_normalizer import ( + PortugueseNumberNormalizer, +) + +_DIGIT_WORDS = { + "zero": "0", + "um": "1", + "dois": "2", + "tres": "3", + "quatro": "4", + "cinco": "5", + "seis": "6", + "sete": "7", + "oito": "8", + "nove": "9", +} + + +@pytest.fixture +def normalizer(): + return PortugueseNumberNormalizer(_DIGIT_WORDS) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("vinte e tres", "23"), + ("cem", "100"), + ("duzentos", "200"), + ("3 milhoes", "3000000"), + ("um", "1"), + ], +) +def test_portuguese_number_normalizer(normalizer, text: str, expected: str): + assert normalizer(text) == expected diff --git a/tests/unit/languages/portuguese_operators_test.py b/tests/unit/languages/portuguese_operators_test.py new file mode 100644 index 0000000..4da4f50 --- /dev/null +++ b/tests/unit/languages/portuguese_operators_test.py @@ -0,0 +1,87 @@ +import pytest + +from normalization.languages.portuguese.clitics import merge_hyphenated_clitics +from normalization.languages.portuguese.operators import PortugueseOperators +from normalization.pipeline.loader import load_pipeline + + +@pytest.fixture +def operators(): + return PortugueseOperators() + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("envie-me", "envieme"), + ("Envie-me", "envieme"), + ("diga-me", "digame"), + ("mande-me o documento", "mandeme o documento"), + ("fale-me", "faleme"), + ], +) +def test_merge_hyphenated_clitics(text: str, expected: str): + assert merge_hyphenated_clitics(text) == expected + + +def test_portuguese_clitic_pipeline_variants(): + pipeline = load_pipeline("gladia-3", "pt") + for variant in ("envie-me", "enviame", "envie me"): + assert pipeline.normalize(variant) == pipeline.normalize("enviame") + + for variant in ("diga-me", "digame", "diga me"): + assert pipeline.normalize(variant) == pipeline.normalize("digame") + + +def test_portuguese_filler_removal(): + pipeline = load_pipeline("gladia-3", "pt") + assert pipeline.normalize("eh entao sim") == pipeline.normalize("sim") + assert pipeline.normalize("ah mm sim") == pipeline.normalize("sim") + # Meaningful affirmatives / idioms must survive normalization + assert pipeline.normalize("claro estou") == pipeline.normalize("claro estou") + assert pipeline.normalize("esta tudo bem") == pipeline.normalize("esta tudo bem") + assert pipeline.normalize("e pronto foi assim") == pipeline.normalize( + "e pronto foi assim" + ) + + +def test_portuguese_pra_para(): + pipeline = load_pipeline("gladia-3", "pt") + assert pipeline.normalize("pra trabalhar") == pipeline.normalize("para trabalhar") + + +def test_portuguese_gender_and_spelling_variants(): + pipeline = load_pipeline("gladia-3", "pt") + assert pipeline.normalize("propria") == pipeline.normalize("proprio") + assert pipeline.normalize("maioritariamente") == pipeline.normalize("maioritario") + assert pipeline.normalize("media") == pipeline.normalize("medio") + + +def test_portuguese_email_and_number_variants(): + pipeline = load_pipeline("gladia-3", "pt") + assert pipeline.normalize("emails") == pipeline.normalize("email") + assert pipeline.normalize("e-mails") == pipeline.normalize("email") + assert pipeline.normalize("turnos rotativos") == pipeline.normalize( + "turno rotativo" + ) + assert pipeline.normalize("em quantos pesos") == pipeline.normalize( + "de quanto peso" + ) + + +def test_portuguese_pt_pt_br_variants(): + pipeline = load_pipeline("gladia-3", "pt") + assert pipeline.normalize("controles") == pipeline.normalize("controlos") + assert pipeline.normalize("equipe") == pipeline.normalize("equipa") + + +def test_portuguese_brand_variants(): + pipeline = load_pipeline("gladia-3", "pt") + assert pipeline.normalize("adeco") == pipeline.normalize("adecco") + assert pipeline.normalize("adec") == pipeline.normalize("adecco") + + +def test_portuguese_written_numbers(): + pipeline = load_pipeline("gladia-3", "pt") + assert pipeline.normalize("vinte e tres") == "23" + assert pipeline.normalize("cem euros") == "100 euros"