fix(validator): support regional locale fallback and multi-byte UTF-8 capitalization

internal/base/middleware/accept_language.go

@@ -54,6 +54,12 @@ func ExtractAndSetAcceptLanguage(ctx *gin.Context) {
 		}
 	}
 
+	// A regional variant such as es-MX or zh-HK resolves to the closest configured language.
+	if matched, ok := translator.MatchLanguage(tag[0]); ok {
+		ctx.Set(constant.AcceptLanguageFlag, i18n.Language(matched))
+		return
+	}
+
 	// default language
 	ctx.Set(constant.AcceptLanguageFlag, i18n.LanguageEnglish)
 }

internal/base/middleware/accept_language_test.go

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package middleware
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/apache/answer/internal/base/constant"
+	"github.com/apache/answer/internal/base/translator"
+	"github.com/gin-gonic/gin"
+	"github.com/segmentfault/pacman/i18n"
+	"github.com/stretchr/testify/assert"
+)
+
+func resolveAcceptLanguage(t *testing.T, header string) i18n.Language {
+	t.Helper()
+	gin.SetMode(gin.TestMode)
+
+	var resolved i18n.Language
+	router := gin.New()
+	router.Use(ExtractAndSetAcceptLanguage)
+	router.GET("/", func(ctx *gin.Context) {
+		value, _ := ctx.Get(constant.AcceptLanguageFlag)
+		resolved, _ = value.(i18n.Language)
+		ctx.Status(http.StatusNoContent)
+	})
+
+	req := httptest.NewRequest(http.MethodGet, "/", nil)
+	if header != "" {
+		req.Header.Set(constant.AcceptLanguageFlag, header)
+	}
+	router.ServeHTTP(httptest.NewRecorder(), req)
+	return resolved
+}
+
+func TestExtractAndSetAcceptLanguage(t *testing.T) {
+	original := translator.LanguageOptions
+	translator.SetLanguageOptions([]*translator.LangOption{
+		{Label: "Default", Value: translator.DefaultLangOption},
+		{Label: "English", Value: "en_US"},
+		{Label: "Español", Value: "es_ES"},
+		{Label: "Português do Brasil", Value: "pt_BR"},
+		{Label: "Português", Value: "pt_PT"},
+		{Label: "简体中文", Value: "zh_CN"},
+		{Label: "繁體中文", Value: "zh_TW"},
+	})
+	t.Cleanup(func() { translator.SetLanguageOptions(original) })
+
+	tests := []struct {
+		name   string
+		header string
+		want   i18n.Language
+	}{
+		{"exact configured locale", "es-ES", "es_ES"},
+		{"exact locale with quality", "pt-BR,pt;q=0.8,en;q=0.5", "pt_BR"},
+		{"regional Spanish falls back to configured Spanish", "es-MX", "es_ES"},
+		{"Latin American Spanish", "es-419", "es_ES"},
+		{"regional Portuguese prefers European Portuguese", "pt-AO", "pt_PT"},
+		{"Hong Kong Chinese resolves to Traditional", "zh-HK", "zh_TW"},
+		{"Singapore Chinese resolves to Simplified", "zh-SG", "zh_CN"},
+		{"regional English", "en-GB", "en_US"},
+		{"unsupported language uses English", "tlh", "en_US"},
+		{"empty header uses English", "", "en_US"},
+		{"malformed header uses English", "!!!", "en_US"},
+		{"oversized header uses English", strings.Repeat("es-MX,", 100), "en_US"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, resolveAcceptLanguage(t, tt.header))
+		})
+	}
+}

internal/base/translator/provider.go

@@ -25,11 +25,13 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync/atomic"
 
 	"github.com/google/wire"
 	myTran "github.com/segmentfault/pacman/contrib/i18n"
 	"github.com/segmentfault/pacman/i18n"
 	"github.com/segmentfault/pacman/log"
+	"golang.org/x/text/language"
 	"gopkg.in/yaml.v3"
 )
 
@@ -51,8 +53,55 @@ const DefaultLangOption = "Default"
 var (
 	// LanguageOptions language
 	LanguageOptions []*LangOption
+
+	// languageMatcher resolves regional language tags to a configured language option.
+	languageMatcher atomic.Pointer[languageMatch]
 )
 
+// languageMatch pairs a language matcher with the option values it was built from,
+// so the matcher index maps back to the configured value such as es_ES.
+type languageMatch struct {
+	matcher language.Matcher
+	values  []string
+}
+
+// SetLanguageOptions sets the configured language options and rebuilds the matcher
+// used by MatchLanguage.
+func SetLanguageOptions(options []*LangOption) {
+	LanguageOptions = options
+
+	values := make([]string, 0, len(options))
+	tags := make([]language.Tag, 0, len(options))
+	for _, option := range options {
+		if option.Value == DefaultLangOption {
+			continue
+		}
+		tag, err := language.Parse(strings.ReplaceAll(option.Value, "_", "-"))
+		if err != nil {
+			log.Warnf("language option %q is not a valid language tag: %s", option.Value, err)
+			continue
+		}
+		values = append(values, option.Value)
+		tags = append(tags, tag)
+	}
+	languageMatcher.Store(&languageMatch{matcher: language.NewMatcher(tags), values: values})
+}
+
+// MatchLanguage returns the configured language option that best matches the given
+// tag, for example es_ES for es-MX, pt_PT for pt-AO or zh_TW for zh-HK. It reports
+// false when no configured language is a plausible match for the tag.
+func MatchLanguage(tag language.Tag) (string, bool) {
+	match := languageMatcher.Load()
+	if match == nil || len(match.values) == 0 {
+		return "", false
+	}
+	_, idx, confidence := match.matcher.Match(tag)
+	if confidence == language.No || idx < 0 || idx >= len(match.values) {
+		return "", false
+	}
+	return match.values[idx], true
+}
+
 // NewTranslator new a translator
 func NewTranslator(c *I18n) (tr i18n.Translator, err error) {
 	entries, err := os.ReadDir(c.BundleDir)
@@ -120,10 +169,10 @@ func NewTranslator(c *I18n) (tr i18n.Translator, err error) {
 	if err != nil {
 		return nil, fmt.Errorf("i18n file parsing failed: %s", err)
 	}
-	LanguageOptions = s.LangOption
-	for _, option := range LanguageOptions {
+	for _, option := range s.LangOption {
 		option.Label = fmt.Sprintf("%s (%d%%)", option.Label, option.Progress)
 	}
+	SetLanguageOptions(s.LangOption)
 	return GlobalTrans, err
 }
 

internal/base/validator/validator.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"sync"
 	"unicode"
 
 	"github.com/apache/answer/internal/base/reason"
@@ -96,7 +97,10 @@ type FormErrorField struct {
 }
 
 // GlobalValidatorMapping is a mapping from validator to translator used
-var GlobalValidatorMapping = make(map[i18n.Language]*MyValidator, 0)
+var (
+	GlobalValidatorMapping = make(map[i18n.Language]*MyValidator, 0)
+	validatorMappingMu     sync.RWMutex
+)
 
 func init() {
 	for _, t := range allLanguageTranslators {
@@ -179,10 +183,58 @@ func createDefaultValidator(la i18n.Language) *validator.Validate {
 	return validate
 }
 
+// baseLanguage returns the language subtag of a locale such as pt_BR or es-MX.
+func baseLanguage(lang string) string {
+	if idx := strings.IndexAny(lang, "_-"); idx > 0 {
+		return lang[:idx]
+	}
+	return lang
+}
+
+// registerRegionalValidator builds and caches a validator for a regional locale
+// (such as es_MX) on top of the translator of its base language. The write lock
+// is scoped to this call so callers never hold it while falling back. It returns
+// nil if the translations cannot be registered.
+func registerRegionalValidator(lang i18n.Language, t *TranslatorLocal) *MyValidator {
+	validatorMappingMu.Lock()
+	defer validatorMappingMu.Unlock()
+
+	if v := GlobalValidatorMapping[lang]; v != nil {
+		return v
+	}
+	tran, val := getTran(t.Lo), createDefaultValidator(lang)
+	if t.RegisterFunc != nil {
+		if err := t.RegisterFunc(val, tran); err != nil {
+			log.Error(err)
+			return nil
+		}
+	}
+	v := &MyValidator{Validate: val, Tran: tran, Lang: lang}
+	GlobalValidatorMapping[lang] = v
+	return v
+}
+
 func GetValidatorByLang(lang i18n.Language) *MyValidator {
-	if GlobalValidatorMapping[lang] != nil {
-		return GlobalValidatorMapping[lang]
+	validatorMappingMu.RLock()
+	if v := GlobalValidatorMapping[lang]; v != nil {
+		validatorMappingMu.RUnlock()
+		return v
+	}
+	validatorMappingMu.RUnlock()
+
+	baseLang := baseLanguage(string(lang))
+	for _, t := range allLanguageTranslators {
+		if !strings.EqualFold(baseLang, baseLanguage(string(t.La))) {
+			continue
+		}
+		if v := registerRegionalValidator(lang, t); v != nil {
+			return v
+		}
+		break
 	}
+
+	validatorMappingMu.RLock()
+	defer validatorMappingMu.RUnlock()
 	return GlobalValidatorMapping[i18n.DefaultLanguage]
 }
 
@@ -196,12 +248,16 @@ func (m *MyValidator) Check(value any) (errFields []*FormErrorField, err error)
 			if len(field.ErrorField) == 0 {
 				continue
 			}
-			firstRune := []rune(field.ErrorMsg)[0]
+			runes := []rune(field.ErrorMsg)
+			if len(runes) == 0 {
+				continue
+			}
+			firstRune := runes[0]
 			if !unicode.IsLetter(firstRune) || !unicode.Is(unicode.Latin, firstRune) {
 				continue
 			}
 			upperFirstRune := unicode.ToUpper(firstRune)
-			field.ErrorMsg = string(upperFirstRune) + field.ErrorMsg[1:]
+			field.ErrorMsg = string(upperFirstRune) + string(runes[1:])
 			if !strings.HasSuffix(field.ErrorMsg, ".") {
 				field.ErrorMsg += "."
 			}

internal/base/validator/validator_test.go

@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package validator
+
+import (
+	"errors"
+	"strings"
+	"testing"
+	"time"
+	"unicode/utf8"
+
+	"github.com/go-playground/locales/en"
+	ut "github.com/go-playground/universal-translator"
+	"github.com/go-playground/validator/v10"
+	"github.com/segmentfault/pacman/i18n"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type sampleCommentReq struct {
+	OriginalText string `validate:"required,notblank,gte=2,lte=5" json:"original_text"`
+}
+
+type sampleAccentedFieldReq struct {
+	Email string `validate:"required,email" json:"émail"`
+}
+
+func TestGetValidatorByLang_RegionalLocaleFallback(t *testing.T) {
+	req := &sampleCommentReq{OriginalText: "123456"}
+
+	esMXValidator := GetValidatorByLang(i18n.Language("es_MX"))
+	require.NotNil(t, esMXValidator)
+	assert.Equal(t, i18n.Language("es_MX"), esMXValidator.Lang)
+
+	errFields, err := esMXValidator.Check(req)
+	require.Error(t, err)
+	require.Len(t, errFields, 1)
+	assert.Contains(t, errFields[0].ErrorMsg, "debe tener un máximo de 5")
+
+	ptBRValidator := GetValidatorByLang(i18n.Language("pt_BR"))
+	require.NotNil(t, ptBRValidator)
+	assert.Equal(t, i18n.Language("pt_BR"), ptBRValidator.Lang)
+
+	errFieldsPT, errPT := ptBRValidator.Check(req)
+	require.Error(t, errPT)
+	require.Len(t, errFieldsPT, 1)
+	assert.Contains(t, errFieldsPT[0].ErrorMsg, "máximo")
+}
+
+// A failing RegisterFunc must release the write lock before the fallback takes
+// the read lock, otherwise the lookup deadlocks.
+func TestGetValidatorByLang_RegisterFuncFailureDoesNotDeadlock(t *testing.T) {
+	failing := &TranslatorLocal{
+		La: i18n.Language("xx_XX"),
+		Lo: en.New(),
+		RegisterFunc: func(*validator.Validate, ut.Translator) error {
+			return errors.New("register failed")
+		},
+	}
+	original := allLanguageTranslators
+	allLanguageTranslators = append(append([]*TranslatorLocal{}, original...), failing)
+	t.Cleanup(func() { allLanguageTranslators = original })
+
+	result := make(chan *MyValidator, 1)
+	go func() { result <- GetValidatorByLang(i18n.Language("xx_YY")) }()
+
+	select {
+	case v := <-result:
+		require.NotNil(t, v)
+		assert.Equal(t, i18n.DefaultLanguage, v.Lang)
+	case <-time.After(5 * time.Second):
+		t.Fatal("GetValidatorByLang deadlocked after a RegisterFunc failure")
+	}
+
+	validatorMappingMu.RLock()
+	_, cached := GlobalValidatorMapping[i18n.Language("xx_YY")]
+	validatorMappingMu.RUnlock()
+	assert.False(t, cached, "a failed registration must not be cached")
+}
+
+func TestCheck_MultiByteLatinCapitalization(t *testing.T) {
+	req := &sampleAccentedFieldReq{Email: "not-an-email"}
+
+	esValidator := GetValidatorByLang(i18n.LanguageSpanish)
+	errFields, err := esValidator.Check(req)
+	require.Error(t, err)
+	require.Len(t, errFields, 1)
+
+	msg := errFields[0].ErrorMsg
+	assert.True(t, utf8.ValidString(msg), "error message must be valid UTF-8, got: %q", msg)
+	assert.True(t, strings.HasPrefix(msg, "Émail "), "expected prefix 'Émail ', got: %q", msg)
+}