NTSC-U and NTSC-J support

.gitattributes

@@ -4,3 +4,10 @@
 # Patch files must stay LF: git apply matches context bytes against LF upstream sources
 *.patch -text
 translator/tests/Translator.Tests/TestAssets/**/*.bin binary
+
+# Region tables reproduced by tools/region/
+projects/mkwii-ntsc-*/MAP.txt linguist-generated=true
+projects/mkwii-ntsc-*/region_port.json linguist-generated=true
+projects/mkwii-ntsc-j/data_addresses.txt linguist-generated=true
+runtime/include/guest_abi/guest_address_table.inc linguist-generated=true
+runtime/include/guest_function_imports.inc linguist-generated=true

.gitignore

@@ -66,7 +66,9 @@ project.lock.json
 /.vs/
 /.vscode/
 /.idea/
-/tools/
+/tools/*
+!/tools/region/
+__pycache__/
 /TEMP/
 /tmp/
 /test_output/

Launcher/Build-Installer.ps1

@@ -23,12 +23,23 @@ $publish = Join-Path $workRoot 'publish'
 $payloadRoot = Join-Path $workRoot 'payload'
 $setupProject = Join-Path $PSScriptRoot 'WiiCompiled.Setup.Windows\WiiCompiled.Setup.Windows.csproj'
 $translatorProject = Join-Path $repoRoot 'translator\src\Translator.Cli\Translator.Cli.csproj'
-$projectFile = Join-Path $repoRoot 'projects\mkwii\recomp.yml'
 
-# Everything Mario-Kart-specific in the manifest below is read from the project file rather than
-# restated here, and Test-PinnedFacts.ps1 checks the copies that cannot read it (the C++ entry
-# address, the C# endpoint constant, the lists this script and the installed host both carry).
-$pins = Get-MkwProjectPins $projectFile
+# Every supported release is one projects/<name>/recomp.yml. Everything Mario-Kart-specific in the
+# manifest below is read from those files rather than restated here, and Test-PinnedFacts.ps1
+# checks the copies that cannot read them (the C++ entry address, the C# Retro-WFC table, the
+# lists this script and the installed host both carry).
+$releaseProjects = @(Get-ChildItem -LiteralPath (Join-Path $repoRoot 'projects') -Directory |
+    Where-Object { Test-Path -LiteralPath (Join-Path $_.FullName 'recomp.yml') -PathType Leaf } |
+    Sort-Object Name)
+if ($releaseProjects.Count -eq 0) { throw 'No translation project defines a supported release.' }
+$supportedReleases = @(foreach ($releaseProject in $releaseProjects) {
+    $pins = Get-MkwProjectPins (Join-Path $releaseProject.FullName 'recomp.yml')
+    [ordered]@{ GameId = $pins.GameId; DolSha256 = $pins.DolSha256; RelSha256 = $pins.RelSha256 }
+})
+foreach ($key in @('GameId', 'DolSha256')) {
+    $duplicates = @($supportedReleases | Group-Object { $_[$key] } | Where-Object Count -gt 1)
+    if ($duplicates.Count -gt 0) { throw "Two translation projects pin the same $($key): $($duplicates[0].Name)." }
+}
 # The audit reports by throwing, which aborts this run; nothing native has been invoked yet, so
 # there is no exit code to inspect.
 & (Join-Path $PSScriptRoot 'Test-PinnedFacts.ps1') -RepositoryRoot $repoRoot
@@ -243,7 +254,9 @@ if (Test-Path -LiteralPath $runtimeDeveloperBuild) {
 }
 $auroraExtern = Join-Path $workspace 'aurora-main\extern'
 Get-ChildItem -LiteralPath $auroraExtern -Directory -Force | Remove-Item -Recurse -Force
-Copy-Directory (Join-Path $repoRoot 'projects\mkwii') (Join-Path $workspace 'projects\mkwii')
+foreach ($releaseProject in $releaseProjects) {
+    Copy-Directory $releaseProject.FullName (Join-Path $workspace "projects\$($releaseProject.Name)")
+}
 Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'LocalBuild.ps1') -Destination (Join-Path $workspace 'LocalBuild.ps1')
 # LocalBuild.ps1 dot-sources the canonical configure flags from this sibling.
 Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'NativeBuildFlags.ps1') -Destination (Join-Path $workspace 'NativeBuildFlags.ps1')
@@ -326,22 +339,20 @@ foreach ($required in @('ToolkitFingerprint','TranslationFingerprint','NativeToo
 $productVersion = ((& $setupHost --version) -join '').Trim()
 if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($productVersion)) { throw 'The setup host did not report its version.' }
 
+# Schema 3 replaced schema 2's single Expected* release fields with SupportedReleases.
 $manifest = [ordered]@{
-    SchemaVersion = 2
+    SchemaVersion = 3
     ProductVersion = $productVersion
-    ExpectedGameId = $pins.GameId
-    ExpectedDolSha256 = $pins.DolSha256
-    ExpectedRelSha256 = $pins.RelSha256
+    SupportedReleases = $supportedReleases
     ToolkitReleaseTag = if ($ToolkitReleaseTag) { $ToolkitReleaseTag } else { '' }
     BuildModel = 'local-translation-and-compilation'
-    RetroWfcPayloadUri = $pins.RetroWfcPayloadUri
     ToolkitFingerprint = $identities.ToolkitFingerprint
     ToolkitPackageFingerprint = $identities.ToolkitPackageFingerprint
     RuntimeAssetsFingerprint = $identities.RuntimeAssetsFingerprint
     TranslationFingerprint = $identities.TranslationFingerprint
     NativeToolchainFingerprint = $identities.NativeToolchainFingerprint
 }
-$manifest | ConvertTo-Json | Set-Content (Join-Path $payloadRoot 'payload-manifest.json') -Encoding UTF8
+$manifest | ConvertTo-Json -Depth 4 | Set-Content (Join-Path $payloadRoot 'payload-manifest.json') -Encoding UTF8
 
 Write-Host '[4/6] Enforcing the copyright and generated-code boundary...'
 # Both audits are PowerShell scripts that throw directly on failure (Test-PayloadBoundary.ps1

Launcher/LocalBuild.ps1

@@ -166,7 +166,6 @@ $toolchainBin = Join-Path $toolchain 'llvm-mingw\bin'
 $cc = Join-Path $toolchainBin 'x86_64-w64-mingw32-clang.exe'
 $cxx = Join-Path $toolchainBin 'x86_64-w64-mingw32-clang++.exe'
 $windres = Join-Path $toolchainBin 'x86_64-w64-mingw32-windres.exe'
-$project = Join-Path $Workspace 'projects\mkwii\recomp.yml'
 $assets = Join-Path $Workspace 'Assets'
 $generated = Join-Path $Workspace 'generated'
 $functions = Join-Path $generated 'functions'
@@ -183,14 +182,18 @@ $retroRoot = if ([string]::IsNullOrWhiteSpace($RetroRewindPackageDirectory)) {
 
 foreach ($required in @(
     @($translator, 'Bundled translator'), @($cmake, 'Bundled CMake'), @($ninja, 'Bundled Ninja'),
-    @($cc, 'Bundled C compiler'), @($cxx, 'Bundled C++ compiler'), @($windres, 'Bundled resource compiler'), @($project, 'Translation project'),
+    @($cc, 'Bundled C compiler'), @($cxx, 'Bundled C++ compiler'), @($windres, 'Bundled resource compiler'),
     @((Join-Path $assets 'main.dol'), 'Extracted main.dol'), @((Join-Path $assets 'StaticR.rel'), 'Extracted StaticR.rel')
 )) { Assert-File $required[0] $required[1] }
 
 # Everything Mario-Kart-specific this script needs is read from the project file rather than
 # restated here: the translation entry point below, and (through the retro-rewind profile the
-# translator loads itself) the module guest/link bases the mod translation uses.
-$pins = Get-MkwProjectPins $project
+# translator loads itself) the module guest/link bases the mod translation uses. The project is
+# the one pinned to the extracted main.dol, which decides the region.
+$selectedProject = Resolve-MkwProjectForAssets $Workspace $assets
+$project = $selectedProject.Path
+$pins = $selectedProject.Pins
+$retroWfcPayloadFile = "binary\payload.$(($pins.RetroWfcPayloadUri -split 'g=')[-1]).bin"
 
 # Three independent knobs: $translatorThreads (the translator's own worker threads; peak memory is
 # one heap that barely moves with thread count, so clamping it only slows translation), $translatedJobs
@@ -243,7 +246,8 @@ try {
         # outputs plus --prune-stale mean Ninja only recompiles shards whose bytes actually moved.
         $reuseBase = $false
         if (-not $ForceCleanBuild -and -not [string]::IsNullOrWhiteSpace($TranslationFingerprint) -and
-            (Get-RecordedFingerprint $translationProvenance 'TranslationFingerprint') -eq $TranslationFingerprint) {
+            (Get-RecordedFingerprint $translationProvenance 'TranslationFingerprint') -eq $TranslationFingerprint -and
+            (Get-RecordedFingerprint $translationProvenance 'DolSha256') -eq $pins.DolSha256) {
             $artifacts = @($baseMetadata, $baseManifest, (Join-Path $generated 'base_translation_sources.bin'),
                 (Join-Path $generated 'base_translation_mod_awareness.json'))
             $reuseBase = -not ($artifacts | Where-Object { -not (Test-Path -LiteralPath $_ -PathType Leaf) })
@@ -312,11 +316,11 @@ try {
 
             Invoke-Checked $translator @(
                 'emit-base-manifest', '--project', $project, '--out', $baseManifestDir,
-                '--functions-dir', $functions, '--translation-output-metadata', $baseMetadata, '--region', 'P'
+                '--functions-dir', $functions, '--translation-output-metadata', $baseMetadata, '--region', $pins.Region
             ) 'Creating the local base translation manifest' -StepId 'emit-base-manifest'
 
             if (-not [string]::IsNullOrWhiteSpace($TranslationFingerprint)) {
-                [ordered]@{ SchemaVersion = 1; TranslationFingerprint = $TranslationFingerprint } |
+                [ordered]@{ SchemaVersion = 1; TranslationFingerprint = $TranslationFingerprint; DolSha256 = $pins.DolSha256 } |
                     ConvertTo-Json | Set-Content -LiteralPath $translationProvenance -Encoding UTF8
             }
         }
@@ -329,14 +333,14 @@ try {
                 'translate-mod', '--project', $project, '--profile', 'retro-rewind',
                 '--base-manifest', $baseManifest, '--base-translation-output-metadata', $baseMetadata,
                 '--code-pul', $codePul, '--mod-root', $retroRoot, '--mod-name', 'Retro Rewind',
-                '--region', 'P', '--out', $retroOut, '--prefer-cached-inputs', '--emit-cpp',
+                '--region', $pins.Region, '--out', $retroOut, '--prefer-cached-inputs', '--emit-cpp',
                 '--threads', $translatorThreads
             )
             if ($SkipRetroWfcPayload) {
                 $translateModArguments += '--skip-retro-wfc'
             } elseif (-not [string]::IsNullOrWhiteSpace($RetroWfcOfflineDirectory)) {
                 $offlineRoot = [IO.Path]::GetFullPath($RetroWfcOfflineDirectory)
-                $offlinePayload = Join-Path $offlineRoot 'binary\payload.RMCPD00.bin'
+                $offlinePayload = Join-Path $offlineRoot $retroWfcPayloadFile
                 Assert-File $offlinePayload 'Offline Retro-WFC shared payload'
                 $translateModArguments += @('--retro-wfc-payload', $offlinePayload)
             } else {
@@ -447,8 +451,8 @@ try {
                 RelSha256 = $relSha
                 CodePulSha256 = if ($isRetro) { Get-MkwFileSha256 (Join-Path $retroRoot 'Binaries\Code.pul') } else { $null }
                 RetroWfcPayloadMode = if (-not $isRetro) { $null } elseif ($SkipRetroWfcPayload) { 'skipped' } else { $RetroWfcPayloadOrigin }
-                RetroWfcPayloadSha256 = if ($isRetro -and -not $SkipRetroWfcPayload) { Get-MkwFileSha256 (Join-Path $RetroWfcOfflineDirectory 'binary\payload.RMCPD00.bin') } else { $null }
-                RetroWfcPayloadLength = if ($isRetro -and -not $SkipRetroWfcPayload) { (Get-Item -LiteralPath (Join-Path $RetroWfcOfflineDirectory 'binary\payload.RMCPD00.bin')).Length } else { $null }
+                RetroWfcPayloadSha256 = if ($isRetro -and -not $SkipRetroWfcPayload) { Get-MkwFileSha256 (Join-Path $RetroWfcOfflineDirectory $retroWfcPayloadFile) } else { $null }
+                RetroWfcPayloadLength = if ($isRetro -and -not $SkipRetroWfcPayload) { (Get-Item -LiteralPath (Join-Path $RetroWfcOfflineDirectory $retroWfcPayloadFile)).Length } else { $null }
                 Compiler = "llvm-mingw clang-22 sha256:$compilerSha"
             }
             $provenance | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $Destination 'local-build.json') -Encoding UTF8

Launcher/NativeBuildFlags.ps1

@@ -93,15 +93,15 @@ function Get-MkwBuildSafePath([string]$Path, [string]$Kind, [string]$MarkerFile)
 
 function Get-MkwProjectPins([string]$ProjectFile) {
     <#
-    The Mario Kart Wii facts pinned by projects/mkwii/recomp.yml (game identity, clean input
+    The Mario Kart Wii facts pinned by a projects/*/recomp.yml (game identity, clean input
     digests, translation entry point, Retro-WFC endpoint), read from one place instead of being
     restated; Test-PinnedFacts.ps1 checks the copies that can't read YAML (C++ header, C#
     constants) against these values. Parsing is literal, not a YAML dependency, since the file is
     machine-written with a fixed shape.
     #>
     Assert-File $ProjectFile 'Translation project file'
     $pins = [ordered]@{
-        GameId = ''; DolSha256 = ''; RelSha256 = ''; EntryPoint = ''; RetroWfcPayloadUri = ''
+        GameId = ''; Region = ''; DolSha256 = ''; RelSha256 = ''; EntryPoint = ''; RetroWfcPayloadUri = ''
     }
     $section = ''
     $inputKey = ''
@@ -112,6 +112,8 @@ function Get-MkwProjectPins([string]$ProjectFile) {
         if ($section -eq 'inputs' -and $line -match '^\s{2}([A-Za-z0-9_]+):\s*$') { $inputKey = $Matches[1]; continue }
         if ($section -eq 'project' -and $line -match '^\s*game_id:\s*(\S+)\s*$') {
             $pins.GameId = $Matches[1]
+        } elseif ($section -eq 'project' -and $line -match '^\s*region:\s*(\S+)\s*$') {
+            $pins.Region = $Matches[1]
         } elseif ($section -eq 'inputs' -and $line -match '^\s*sha256:\s*([0-9a-fA-F]{64})\s*$') {
             if ($inputKey -eq 'dol') { $pins.DolSha256 = $Matches[1].ToLowerInvariant() }
             elseif ($inputKey -eq 'rel') { $pins.RelSha256 = $Matches[1].ToLowerInvariant() }
@@ -130,6 +132,30 @@ function Get-MkwProjectPins([string]$ProjectFile) {
     return $pins
 }
 
+function Resolve-MkwProjectForAssets([string]$Workspace, [string]$AssetsDirectory) {
+    <#
+    The translation project pinned to the main.dol in AssetsDirectory, after checking that
+    StaticR.rel and any extracted disc data belong to the same release.
+    #>
+    $dolSha = Get-MkwFileSha256 (Join-Path $AssetsDirectory 'main.dol')
+    foreach ($file in Get-ChildItem -LiteralPath (Join-Path $Workspace 'projects') -Filter 'recomp.yml' -Recurse -File) {
+        $pins = Get-MkwProjectPins $file.FullName
+        if ($pins.DolSha256 -ne $dolSha) { continue }
+        if ((Get-MkwFileSha256 (Join-Path $AssetsDirectory 'StaticR.rel')) -ne $pins.RelSha256) {
+            throw "StaticR.rel does not match the clean $($pins.GameId) release of main.dol."
+        }
+        $bootBin = Join-Path $AssetsDirectory 'DATA\sys\boot.bin'
+        if (Test-Path -LiteralPath $bootBin -PathType Leaf) {
+            $discId = [Text.Encoding]::ASCII.GetString([IO.File]::ReadAllBytes($bootBin), 0, 6)
+            if ($discId -ne $pins.GameId) {
+                throw "The extracted game data is $discId but main.dol is $($pins.GameId); extract the game again."
+            }
+        }
+        return [pscustomobject]@{ Path = $file.FullName; Pins = $pins }
+    }
+    throw "main.dol (sha256 $dolSha) is not a supported clean Mario Kart Wii release."
+}
+
 function Invoke-Checked([string]$FilePath, [string[]]$Arguments, [string]$Description,
     [string]$LogPrefix = 'MKWCBUILD', [string]$StepId = '', [bool]$WaitForProcessTree = $true) {
     <#