From 9b8a6889fd71c1b6e0b31f40fbf2820a15a26a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9nich=20Bon=20=C4=86iri=C4=87?= Date: Tue, 20 Oct 2026 19:25:13 -0600 Subject: [PATCH 1/2] fix(validator): support regional locale fallback and multi-byte UTF-8 capitalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Match base language prefix in GetValidatorByLang when a regional locale (such as pt_BR or es_MX) is not pre-registered in GlobalValidatorMapping, preserving the active locale for field name translation via translator.Tr instead of falling back to en_US. - Slice field.ErrorMsg by rune index instead of byte index when capitalizing the initial Latin character in MyValidator.Check to prevent UTF-8 corruption on multi-byte accented characters. - Add unit tests covering regional locale fallback and multi-byte Latin capitalization. Co-authored-by: Antigravity Signed-off-by: Rénich Bon Ćirić --- internal/base/validator/validator.go | 53 ++++++++++++++-- internal/base/validator/validator_test.go | 73 +++++++++++++++++++++++ 2 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 internal/base/validator/validator_test.go diff --git a/internal/base/validator/validator.go b/internal/base/validator/validator.go index 9c7f6ec41..2b7374b19 100644 --- a/internal/base/validator/validator.go +++ b/internal/base/validator/validator.go @@ -24,6 +24,7 @@ import ( "fmt" "reflect" "strings" + "sync" "unicode" "github.com/apache/answer/internal/base/reason" @@ -96,7 +97,10 @@ type FormErrorField struct { } // GlobalValidatorMapping is a mapping from validator to translator used -var GlobalValidatorMapping = make(map[i18n.Language]*MyValidator, 0) +var ( + GlobalValidatorMapping = make(map[i18n.Language]*MyValidator, 0) + validatorMappingMu sync.RWMutex +) func init() { for _, t := range allLanguageTranslators { @@ -180,9 +184,44 @@ func createDefaultValidator(la i18n.Language) *validator.Validate { } func GetValidatorByLang(lang i18n.Language) *MyValidator { - if GlobalValidatorMapping[lang] != nil { - return GlobalValidatorMapping[lang] + validatorMappingMu.RLock() + if v := GlobalValidatorMapping[lang]; v != nil { + validatorMappingMu.RUnlock() + return v + } + validatorMappingMu.RUnlock() + + baseLang := string(lang) + if idx := strings.IndexAny(baseLang, "_-"); idx > 0 { + baseLang = baseLang[:idx] } + + for _, t := range allLanguageTranslators { + tBase := string(t.La) + if idx := strings.IndexAny(tBase, "_-"); idx > 0 { + tBase = tBase[:idx] + } + if strings.EqualFold(baseLang, tBase) { + validatorMappingMu.Lock() + defer validatorMappingMu.Unlock() + if v := GlobalValidatorMapping[lang]; v != nil { + return v + } + tran, val := getTran(t.Lo), createDefaultValidator(lang) + if t.RegisterFunc != nil { + if err := t.RegisterFunc(val, tran); err != nil { + log.Error(err) + break + } + } + v := &MyValidator{Validate: val, Tran: tran, Lang: lang} + GlobalValidatorMapping[lang] = v + return v + } + } + + validatorMappingMu.RLock() + defer validatorMappingMu.RUnlock() return GlobalValidatorMapping[i18n.DefaultLanguage] } @@ -196,12 +235,16 @@ func (m *MyValidator) Check(value any) (errFields []*FormErrorField, err error) if len(field.ErrorField) == 0 { continue } - firstRune := []rune(field.ErrorMsg)[0] + runes := []rune(field.ErrorMsg) + if len(runes) == 0 { + continue + } + firstRune := runes[0] if !unicode.IsLetter(firstRune) || !unicode.Is(unicode.Latin, firstRune) { continue } upperFirstRune := unicode.ToUpper(firstRune) - field.ErrorMsg = string(upperFirstRune) + field.ErrorMsg[1:] + field.ErrorMsg = string(upperFirstRune) + string(runes[1:]) if !strings.HasSuffix(field.ErrorMsg, ".") { field.ErrorMsg += "." } diff --git a/internal/base/validator/validator_test.go b/internal/base/validator/validator_test.go new file mode 100644 index 000000000..583e2bf2e --- /dev/null +++ b/internal/base/validator/validator_test.go @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package validator + +import ( + "strings" + "testing" + "unicode/utf8" + + "github.com/segmentfault/pacman/i18n" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type sampleCommentReq struct { + OriginalText string `validate:"required,notblank,gte=2,lte=5" json:"original_text"` +} + +type sampleAccentedFieldReq struct { + Email string `validate:"required,email" json:"émail"` +} + +func TestGetValidatorByLang_RegionalLocaleFallback(t *testing.T) { + req := &sampleCommentReq{OriginalText: "123456"} + + esMXValidator := GetValidatorByLang(i18n.Language("es_MX")) + require.NotNil(t, esMXValidator) + assert.Equal(t, i18n.Language("es_MX"), esMXValidator.Lang) + + errFields, err := esMXValidator.Check(req) + require.Error(t, err) + require.Len(t, errFields, 1) + assert.Contains(t, errFields[0].ErrorMsg, "debe tener un máximo de 5") + + ptBRValidator := GetValidatorByLang(i18n.Language("pt_BR")) + require.NotNil(t, ptBRValidator) + assert.Equal(t, i18n.Language("pt_BR"), ptBRValidator.Lang) + + errFieldsPT, errPT := ptBRValidator.Check(req) + require.Error(t, errPT) + require.Len(t, errFieldsPT, 1) + assert.Contains(t, errFieldsPT[0].ErrorMsg, "máximo") +} + +func TestCheck_MultiByteLatinCapitalization(t *testing.T) { + req := &sampleAccentedFieldReq{Email: "not-an-email"} + + esValidator := GetValidatorByLang(i18n.LanguageSpanish) + errFields, err := esValidator.Check(req) + require.Error(t, err) + require.Len(t, errFields, 1) + + msg := errFields[0].ErrorMsg + assert.True(t, utf8.ValidString(msg), "error message must be valid UTF-8, got: %q", msg) + assert.True(t, strings.HasPrefix(msg, "Émail "), "expected prefix 'Émail ', got: %q", msg) +} From 39d4b8a6058ebdaccce1ada0a4cf3c16ae961f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9nich=20Bon=20=C4=86iri=C4=87?= Date: Fri, 6 Nov 2026 09:40:59 -0600 Subject: [PATCH 2/2] fix(validator): release lock on fallback and resolve regional Accept-Language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move regional validator registration into registerRegionalValidator so the write lock is released on every path. A failing RegisterFunc previously left the lock held and the default-language fallback then blocked on RLock. - Resolve Accept-Language through a language.Matcher built from the configured language options, so regional variants such as es-MX, pt-AO or zh-HK reach the closest configured locale instead of falling back to en_US. Unmatched languages still default to English. - Add tests for the deadlock regression and the middleware resolution table. Co-authored-by: Antigravity Signed-off-by: Rénich Bon Ćirić --- internal/base/middleware/accept_language.go | 6 ++ .../base/middleware/accept_language_test.go | 92 +++++++++++++++++++ internal/base/translator/provider.go | 53 ++++++++++- internal/base/validator/validator.go | 59 +++++++----- internal/base/validator/validator_test.go | 36 ++++++++ 5 files changed, 221 insertions(+), 25 deletions(-) create mode 100644 internal/base/middleware/accept_language_test.go diff --git a/internal/base/middleware/accept_language.go b/internal/base/middleware/accept_language.go index a4e1b2f53..abc0d8ae1 100644 --- a/internal/base/middleware/accept_language.go +++ b/internal/base/middleware/accept_language.go @@ -54,6 +54,12 @@ func ExtractAndSetAcceptLanguage(ctx *gin.Context) { } } + // A regional variant such as es-MX or zh-HK resolves to the closest configured language. + if matched, ok := translator.MatchLanguage(tag[0]); ok { + ctx.Set(constant.AcceptLanguageFlag, i18n.Language(matched)) + return + } + // default language ctx.Set(constant.AcceptLanguageFlag, i18n.LanguageEnglish) } diff --git a/internal/base/middleware/accept_language_test.go b/internal/base/middleware/accept_language_test.go new file mode 100644 index 000000000..9e4093836 --- /dev/null +++ b/internal/base/middleware/accept_language_test.go @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package middleware + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/apache/answer/internal/base/constant" + "github.com/apache/answer/internal/base/translator" + "github.com/gin-gonic/gin" + "github.com/segmentfault/pacman/i18n" + "github.com/stretchr/testify/assert" +) + +func resolveAcceptLanguage(t *testing.T, header string) i18n.Language { + t.Helper() + gin.SetMode(gin.TestMode) + + var resolved i18n.Language + router := gin.New() + router.Use(ExtractAndSetAcceptLanguage) + router.GET("/", func(ctx *gin.Context) { + value, _ := ctx.Get(constant.AcceptLanguageFlag) + resolved, _ = value.(i18n.Language) + ctx.Status(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodGet, "/", nil) + if header != "" { + req.Header.Set(constant.AcceptLanguageFlag, header) + } + router.ServeHTTP(httptest.NewRecorder(), req) + return resolved +} + +func TestExtractAndSetAcceptLanguage(t *testing.T) { + original := translator.LanguageOptions + translator.SetLanguageOptions([]*translator.LangOption{ + {Label: "Default", Value: translator.DefaultLangOption}, + {Label: "English", Value: "en_US"}, + {Label: "Español", Value: "es_ES"}, + {Label: "Português do Brasil", Value: "pt_BR"}, + {Label: "Português", Value: "pt_PT"}, + {Label: "简体中文", Value: "zh_CN"}, + {Label: "繁體中文", Value: "zh_TW"}, + }) + t.Cleanup(func() { translator.SetLanguageOptions(original) }) + + tests := []struct { + name string + header string + want i18n.Language + }{ + {"exact configured locale", "es-ES", "es_ES"}, + {"exact locale with quality", "pt-BR,pt;q=0.8,en;q=0.5", "pt_BR"}, + {"regional Spanish falls back to configured Spanish", "es-MX", "es_ES"}, + {"Latin American Spanish", "es-419", "es_ES"}, + {"regional Portuguese prefers European Portuguese", "pt-AO", "pt_PT"}, + {"Hong Kong Chinese resolves to Traditional", "zh-HK", "zh_TW"}, + {"Singapore Chinese resolves to Simplified", "zh-SG", "zh_CN"}, + {"regional English", "en-GB", "en_US"}, + {"unsupported language uses English", "tlh", "en_US"}, + {"empty header uses English", "", "en_US"}, + {"malformed header uses English", "!!!", "en_US"}, + {"oversized header uses English", strings.Repeat("es-MX,", 100), "en_US"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, resolveAcceptLanguage(t, tt.header)) + }) + } +} diff --git a/internal/base/translator/provider.go b/internal/base/translator/provider.go index 4f0059e36..459506b7e 100644 --- a/internal/base/translator/provider.go +++ b/internal/base/translator/provider.go @@ -25,11 +25,13 @@ import ( "path/filepath" "sort" "strings" + "sync/atomic" "github.com/google/wire" myTran "github.com/segmentfault/pacman/contrib/i18n" "github.com/segmentfault/pacman/i18n" "github.com/segmentfault/pacman/log" + "golang.org/x/text/language" "gopkg.in/yaml.v3" ) @@ -51,8 +53,55 @@ const DefaultLangOption = "Default" var ( // LanguageOptions language LanguageOptions []*LangOption + + // languageMatcher resolves regional language tags to a configured language option. + languageMatcher atomic.Pointer[languageMatch] ) +// languageMatch pairs a language matcher with the option values it was built from, +// so the matcher index maps back to the configured value such as es_ES. +type languageMatch struct { + matcher language.Matcher + values []string +} + +// SetLanguageOptions sets the configured language options and rebuilds the matcher +// used by MatchLanguage. +func SetLanguageOptions(options []*LangOption) { + LanguageOptions = options + + values := make([]string, 0, len(options)) + tags := make([]language.Tag, 0, len(options)) + for _, option := range options { + if option.Value == DefaultLangOption { + continue + } + tag, err := language.Parse(strings.ReplaceAll(option.Value, "_", "-")) + if err != nil { + log.Warnf("language option %q is not a valid language tag: %s", option.Value, err) + continue + } + values = append(values, option.Value) + tags = append(tags, tag) + } + languageMatcher.Store(&languageMatch{matcher: language.NewMatcher(tags), values: values}) +} + +// MatchLanguage returns the configured language option that best matches the given +// tag, for example es_ES for es-MX, pt_PT for pt-AO or zh_TW for zh-HK. It reports +// false when no configured language is a plausible match for the tag. +func MatchLanguage(tag language.Tag) (string, bool) { + match := languageMatcher.Load() + if match == nil || len(match.values) == 0 { + return "", false + } + _, idx, confidence := match.matcher.Match(tag) + if confidence == language.No || idx < 0 || idx >= len(match.values) { + return "", false + } + return match.values[idx], true +} + // NewTranslator new a translator func NewTranslator(c *I18n) (tr i18n.Translator, err error) { entries, err := os.ReadDir(c.BundleDir) @@ -120,10 +169,10 @@ func NewTranslator(c *I18n) (tr i18n.Translator, err error) { if err != nil { return nil, fmt.Errorf("i18n file parsing failed: %s", err) } - LanguageOptions = s.LangOption - for _, option := range LanguageOptions { + for _, option := range s.LangOption { option.Label = fmt.Sprintf("%s (%d%%)", option.Label, option.Progress) } + SetLanguageOptions(s.LangOption) return GlobalTrans, err } diff --git a/internal/base/validator/validator.go b/internal/base/validator/validator.go index 2b7374b19..aced954d7 100644 --- a/internal/base/validator/validator.go +++ b/internal/base/validator/validator.go @@ -183,6 +183,37 @@ func createDefaultValidator(la i18n.Language) *validator.Validate { return validate } +// baseLanguage returns the language subtag of a locale such as pt_BR or es-MX. +func baseLanguage(lang string) string { + if idx := strings.IndexAny(lang, "_-"); idx > 0 { + return lang[:idx] + } + return lang +} + +// registerRegionalValidator builds and caches a validator for a regional locale +// (such as es_MX) on top of the translator of its base language. The write lock +// is scoped to this call so callers never hold it while falling back. It returns +// nil if the translations cannot be registered. +func registerRegionalValidator(lang i18n.Language, t *TranslatorLocal) *MyValidator { + validatorMappingMu.Lock() + defer validatorMappingMu.Unlock() + + if v := GlobalValidatorMapping[lang]; v != nil { + return v + } + tran, val := getTran(t.Lo), createDefaultValidator(lang) + if t.RegisterFunc != nil { + if err := t.RegisterFunc(val, tran); err != nil { + log.Error(err) + return nil + } + } + v := &MyValidator{Validate: val, Tran: tran, Lang: lang} + GlobalValidatorMapping[lang] = v + return v +} + func GetValidatorByLang(lang i18n.Language) *MyValidator { validatorMappingMu.RLock() if v := GlobalValidatorMapping[lang]; v != nil { @@ -191,33 +222,15 @@ func GetValidatorByLang(lang i18n.Language) *MyValidator { } validatorMappingMu.RUnlock() - baseLang := string(lang) - if idx := strings.IndexAny(baseLang, "_-"); idx > 0 { - baseLang = baseLang[:idx] - } - + baseLang := baseLanguage(string(lang)) for _, t := range allLanguageTranslators { - tBase := string(t.La) - if idx := strings.IndexAny(tBase, "_-"); idx > 0 { - tBase = tBase[:idx] + if !strings.EqualFold(baseLang, baseLanguage(string(t.La))) { + continue } - if strings.EqualFold(baseLang, tBase) { - validatorMappingMu.Lock() - defer validatorMappingMu.Unlock() - if v := GlobalValidatorMapping[lang]; v != nil { - return v - } - tran, val := getTran(t.Lo), createDefaultValidator(lang) - if t.RegisterFunc != nil { - if err := t.RegisterFunc(val, tran); err != nil { - log.Error(err) - break - } - } - v := &MyValidator{Validate: val, Tran: tran, Lang: lang} - GlobalValidatorMapping[lang] = v + if v := registerRegionalValidator(lang, t); v != nil { return v } + break } validatorMappingMu.RLock() diff --git a/internal/base/validator/validator_test.go b/internal/base/validator/validator_test.go index 583e2bf2e..380092f91 100644 --- a/internal/base/validator/validator_test.go +++ b/internal/base/validator/validator_test.go @@ -20,10 +20,15 @@ package validator import ( + "errors" "strings" "testing" + "time" "unicode/utf8" + "github.com/go-playground/locales/en" + ut "github.com/go-playground/universal-translator" + "github.com/go-playground/validator/v10" "github.com/segmentfault/pacman/i18n" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -59,6 +64,37 @@ func TestGetValidatorByLang_RegionalLocaleFallback(t *testing.T) { assert.Contains(t, errFieldsPT[0].ErrorMsg, "máximo") } +// A failing RegisterFunc must release the write lock before the fallback takes +// the read lock, otherwise the lookup deadlocks. +func TestGetValidatorByLang_RegisterFuncFailureDoesNotDeadlock(t *testing.T) { + failing := &TranslatorLocal{ + La: i18n.Language("xx_XX"), + Lo: en.New(), + RegisterFunc: func(*validator.Validate, ut.Translator) error { + return errors.New("register failed") + }, + } + original := allLanguageTranslators + allLanguageTranslators = append(append([]*TranslatorLocal{}, original...), failing) + t.Cleanup(func() { allLanguageTranslators = original }) + + result := make(chan *MyValidator, 1) + go func() { result <- GetValidatorByLang(i18n.Language("xx_YY")) }() + + select { + case v := <-result: + require.NotNil(t, v) + assert.Equal(t, i18n.DefaultLanguage, v.Lang) + case <-time.After(5 * time.Second): + t.Fatal("GetValidatorByLang deadlocked after a RegisterFunc failure") + } + + validatorMappingMu.RLock() + _, cached := GlobalValidatorMapping[i18n.Language("xx_YY")] + validatorMappingMu.RUnlock() + assert.False(t, cached, "a failed registration must not be cached") +} + func TestCheck_MultiByteLatinCapitalization(t *testing.T) { req := &sampleAccentedFieldReq{Email: "not-an-email"}