diff --git a/README.md b/README.md index a3826d445d20..4cc97b4e491e 100644 --- a/README.md +++ b/README.md @@ -41,16 +41,17 @@ The database is updated automatically on a schedule via the [update-db](.github/ The generated JSON files are published to the `gh-pages` branch and served at `https://app.lizardbyte.dev/GameDB/`. -| Endpoint | Description | URL | -|-------------|-------------------------------------------------------|-----------------------------------------------------------| -| Buckets | Game name search index, split by first two characters | `https://app.lizardbyte.dev/GameDB/buckets/.json` | -| Characters | Individual character details and all characters | `https://app.lizardbyte.dev/GameDB/characters/.json` | -| Collections | Individual collection details and all collections | `https://app.lizardbyte.dev/GameDB/collections/.json` | -| Franchises | Individual franchise details and all franchises | `https://app.lizardbyte.dev/GameDB/franchises/.json` | -| Games | Individual game details (no aggregate `all.json`) | `https://app.lizardbyte.dev/GameDB/games/.json` | -| Platforms | Individual platform details and all platforms | `https://app.lizardbyte.dev/GameDB/platforms/.json` | -| Videos | Individual YouTube video metadata | `https://app.lizardbyte.dev/GameDB/videos/.json` | -| Stats | Total item counts per category | `https://app.lizardbyte.dev/GameDB/stats.json` | +| Endpoint | Description | URL | +|-------------------|-------------------------------------------------------|------------------------------------------------------------------------------| +| Buckets | Game name search index, split by first two characters | `https://app.lizardbyte.dev/GameDB/buckets/.json` | +| Localized buckets | Optional regional game title search index | `https://app.lizardbyte.dev/GameDB/buckets/localized//.json` | +| Characters | Individual character details and all characters | `https://app.lizardbyte.dev/GameDB/characters/.json` | +| Collections | Individual collection details and all collections | `https://app.lizardbyte.dev/GameDB/collections/.json` | +| Franchises | Individual franchise details and all franchises | `https://app.lizardbyte.dev/GameDB/franchises/.json` | +| Games | Individual game details (no aggregate `all.json`) | `https://app.lizardbyte.dev/GameDB/games/.json` | +| Platforms | Individual platform details and all platforms | `https://app.lizardbyte.dev/GameDB/platforms/.json` | +| Videos | Individual YouTube video metadata | `https://app.lizardbyte.dev/GameDB/videos/.json` | +| Stats | Total item counts per category | `https://app.lizardbyte.dev/GameDB/stats.json` | `all.json` files (e.g. `characters/all.json`) contain a summary of every item in that category as a single dictionary keyed by ID. @@ -61,6 +62,23 @@ alphanumeric characters of the game name (lowercased), e.g. `ha.json` for games character. Games whose names do not start with two alphanumeric characters are grouped into `@.json`. Each bucket contains a dictionary of `{ id: { name } }` entries, keeping individual files small for fast lookups. +### Optional game title and cover localization + +Individual game records retain their original `name` and may also contain IGDB's `game_localizations` array. +Each localization has a `name`, an optional `cover.url`, and a `region` with an `identifier`, `name`, and `category`. +Use the localized cover URL when present, falling back to the game's original `cover.url` otherwise. IGDB regions can +represent a locale or a continent, so the identifier should be used as supplied by IGDB rather than assumed to be +a language code. If a game has no localization for the desired region, use its original `name`. + +Localized titles are indexed separately under `buckets/localized//.json`. The region path component +is the lowercased `region.identifier`. A localized bucket uses the first two Unicode alphanumeric characters of the +localized title, lowercased, ignoring punctuation and spaces; titles with no alphanumeric characters use `@`. +Bucket entries keep the familiar `{ id: { name } }` shape, with `name` set to the localized title. Only games with +a title for that region appear in its buckets. To search both translated and original titles, search the selected +region's bucket and the original bucket, then deduplicate results by game ID. + +The existing game `name`, bucket URLs, and original bucket contents remain available without selecting a region. + ## Development Code contributors can use the [Developer Setup](docs/developerSetup.md) guide. diff --git a/src/update_db.py b/src/update_db.py index 31dd7fc147e3..bcdfee8edb5a 100644 --- a/src/update_db.py +++ b/src/update_db.py @@ -363,6 +363,34 @@ def _build_buckets_and_collect_videos(full_dict: dict) -> tuple: return buckets, all_videos +def _build_localized_buckets(full_dict: dict) -> dict: + """Index localized game titles by IGDB region and Unicode name prefix. + + The original name and its buckets are left alone so existing consumers keep + receiving the same data. IGDB localizations are regional rather than purely + language based, and regions can represent locales or continents. + """ + buckets = {} + + for game_id, game_data in full_dict['games'].items(): + for localization in game_data.get('game_localizations', []): + name = localization.get('name') + region = localization.get('region') + identifier = region.get('identifier') if isinstance(region, dict) else None + if not isinstance(name, str) or not name.strip() or not isinstance(identifier, str): + continue + + # Region identifiers become path components in the static API. + identifier = identifier.lower() + if not re.fullmatch(r'[a-z0-9_-]+', identifier): + continue + + prefix = ''.join(char.lower() for char in name if char.isalnum())[:2] or '@' + buckets.setdefault(identifier, {}).setdefault(prefix, {})[game_id] = {'name': name} + + return buckets + + def _resolve_video_groups(all_videos: list, cache_file: str, group_size: int) -> list: """ Resolve the list of video groups, using and updating the cache file. @@ -518,6 +546,11 @@ def get_data(): 'external_games.url', 'franchise.name', 'franchises.name', + 'game_localizations.cover.url', + 'game_localizations.name', + 'game_localizations.region.category', + 'game_localizations.region.identifier', + 'game_localizations.region.name', 'game_modes.name', 'genres.name', 'involved_companies.company.name', @@ -617,6 +650,12 @@ def get_data(): file_path = os.path.join(args.out_dir, 'buckets', str(bucket)) write_json_files(file_path=file_path, data=bucket_data) + localized_buckets = _build_localized_buckets(full_dict=full_dict) + for region, region_buckets in localized_buckets.items(): + for bucket, bucket_data in region_buckets.items(): + file_path = os.path.join(args.out_dir, 'buckets', 'localized', region, bucket) + write_json_files(file_path=file_path, data=bucket_data) + all_videos.sort() all_video_groups = _resolve_video_groups( all_videos=all_videos, diff --git a/tests/unit/test_update_db.py b/tests/unit/test_update_db.py index 49eca47cc266..2fdd8d8d4cee 100644 --- a/tests/unit/test_update_db.py +++ b/tests/unit/test_update_db.py @@ -317,6 +317,71 @@ def test_build_buckets_no_videos(): assert all_videos == [] +def test_build_localized_buckets_by_region_and_unicode_prefix(): + full_dict = {'games': { + 1: {'name': 'The Legend', 'game_localizations': [ + {'name': 'ゼルダの伝説', 'region': {'identifier': 'JP'}}, + {'name': 'La Légende', 'region': {'identifier': 'fr-FR'}}, + ]}, + 2: {'name': 'Another Game', 'game_localizations': [ + {'name': 'ゼルダ II', 'region': {'identifier': 'jp'}}, + ]}, + 3: {'name': 'No Translation'}, + }} + + assert udb._build_localized_buckets(full_dict) == { + 'jp': {'ゼル': {1: {'name': 'ゼルダの伝説'}, 2: {'name': 'ゼルダ II'}}}, + 'fr-fr': {'la': {1: {'name': 'La Légende'}}}, + } + + +def test_build_localized_buckets_skips_incomplete_or_unsafe_entries(): + full_dict = {'games': {1: {'name': 'Original', 'game_localizations': [ + {'name': '', 'region': {'identifier': 'jp'}}, + {'name': 'Title', 'region': {'identifier': '../outside'}}, + {'name': 'Title', 'region': 7}, + {'name': 'Title', 'region': {}}, + {'name': '!!!', 'region': {'identifier': 'jp'}}, + ]}}} + + assert udb._build_localized_buckets(full_dict) == {'jp': {'@': {1: {'name': '!!!'}}}} + + +def test_get_data_writes_localized_buckets_without_changing_legacy_files(tmp_path, monkeypatch): + monkeypatch.setattr(udb, 'args', _make_args(tmp_path)) + game = { + 'id': 1, + 'name': 'The Legend', + 'cover': {'url': '//images.igdb.com/igdb/image/upload/t_thumb/original.jpg'}, + 'game_localizations': [{ + 'name': 'ゼルダの伝説', + 'cover': {'url': '//images.igdb.com/igdb/image/upload/t_thumb/localized.jpg'}, + 'region': {'identifier': 'jp'}, + }], + } + full_dict = {'games': {1: game}, 'platforms': {}} + + with patch('src.update_db._fetch_all_endpoints', return_value=full_dict) as fetch, \ + patch('src.update_db._append_related_items'), \ + patch('src.update_db._resolve_video_groups', return_value=[]): + udb.get_data() + + fields = fetch.call_args.kwargs['request_dict']['games']['fields'] + assert 'game_localizations.cover.url' in fields + assert 'game_localizations.name' in fields + assert 'game_localizations.region.identifier' in fields + + output = tmp_path / 'out' + assert json.loads((output / 'buckets' / 'th.json').read_text()) == {'1': {'name': 'The Legend'}} + assert json.loads((output / 'buckets' / 'localized' / 'jp' / 'ゼル.json').read_text()) == { + '1': {'name': 'ゼルダの伝説'}, + } + written_game = json.loads((output / 'games' / '1.json').read_text()) + assert written_game['name'] == 'The Legend' + assert written_game['cover'] == game['cover'] + assert written_game['game_localizations'] == game['game_localizations'] + + def test_resolve_video_groups_no_cache(tmp_path): cache = str(tmp_path / 'cache' / 'vg.json') all_videos = [f'v{i}' for i in range(5)]