Add NTSC-U, NTSC-J and NTSC-K support (supersedes #104)

.gitattributes

@@ -4,3 +4,12 @@
 # Patch files must stay LF: git apply matches context bytes against LF upstream sources
 *.patch -text
 translator/tests/Translator.Tests/TestAssets/**/*.bin binary
+
+# Generated region tables and headers: reproduced byte-for-byte by the tools in
+# tools/region/ (see its README). The NTSC-U data_addresses.txt is hand-built and not marked.
+projects/mkwii-ntsc-*/MAP.txt linguist-generated=true
+projects/mkwii-ntsc-*/MAP_REPORT.md linguist-generated=true
+projects/mkwii-ntsc-*/region_port.json linguist-generated=true
+projects/mkwii-ntsc-j/data_addresses.txt linguist-generated=true
+projects/mkwii-ntsc-k/data_addresses.txt linguist-generated=true
+runtime/include/region/rmc*.h linguist-generated=true

.gitignore

@@ -65,7 +65,9 @@ project.lock.json
 /.vs/
 /.vscode/
 /.idea/
-/tools/
+/tools/*
+!/tools/region/
+__pycache__/
 /TEMP/
 /tmp/
 /test_output/

Launcher/LocalBuild.ps1

@@ -166,8 +166,52 @@ $toolchainBin = Join-Path $toolchain 'llvm-mingw\bin'
 $cc = Join-Path $toolchainBin 'x86_64-w64-mingw32-clang.exe'
 $cxx = Join-Path $toolchainBin 'x86_64-w64-mingw32-clang++.exe'
 $windres = Join-Path $toolchainBin 'x86_64-w64-mingw32-windres.exe'
-$project = Join-Path $Workspace 'projects\mkwii\recomp.yml'
 $assets = Join-Path $Workspace 'Assets'
+$mainDol = Join-Path $assets 'main.dol'
+$staticRel = Join-Path $assets 'StaticR.rel'
+$bootBin = Join-Path $assets 'DATA\sys\boot.bin'
+
+$project = Join-Path $Workspace 'projects\mkwii\recomp.yml'
+$expectedRelSha = '16d9d146112541fefea701ecb5bc1a496f9d50e4a752fbb5b6778e7c6399f67d'
+$expectedLetter = 'P'
+
+if (Test-Path -LiteralPath $mainDol) {
+    $dolSha = Get-MkwFileSha256 $mainDol
+    if ($dolSha -eq 'd2beec1b1645fcd134efe9e7e63774b546667764ed8d431029daccd725995694') {
+        $project = Join-Path $Workspace 'projects\mkwii-ntsc-u\recomp.yml'
+        $expectedRelSha = '1168107f8fdef27a356df76036db55afe4fbf7752606dbac991726701133a617'
+        $expectedLetter = 'E'
+    } elseif ($dolSha -eq '1b9621ef7c5d97dada103e50e5389730e67f3c2545dda592edd4b5843655af91') {
+        $project = Join-Path $Workspace 'projects\mkwii-ntsc-j\recomp.yml'
+        $expectedRelSha = '88539012d357a1420724e51dc7e351192ce696da4b0045994895518a3fad6fae'
+        $expectedLetter = 'J'
+    } elseif ($dolSha -eq '3098a1e9259b4915e32a4ccb5a1f124823f2a0914db29cd2476e10bdb01a77da') {
+        $project = Join-Path $Workspace 'projects\mkwii-ntsc-k\recomp.yml'
+        $expectedRelSha = 'f441b08e4ccc2d64aadcac8429973f10ea6b1cc8f175500fc084983775f7b3e5'
+        $expectedLetter = 'K'
+    } elseif ($dolSha -ne '80d18895b39c63bd80f457398bfcbb91b7d16ac116a41a88967e954080155b05') {
+        throw "main.dol sha256 ($dolSha) does not match any supported clean Mario Kart Wii release."
+    }
+}
+
+if ($buildsRetro -and $expectedLetter -eq 'K') {
+    throw 'Retro Rewind is not supported for NTSC-K; use -Profile base for the Korean game.'
+}
+
+if (Test-Path -LiteralPath $staticRel) {
+    $relSha = Get-MkwFileSha256 $staticRel
+    if ($relSha -cne $expectedRelSha) {
+        throw "StaticR.rel sha256 does not match the clean revision for region $expectedLetter."
+    }
+}
+
+if (Test-Path -LiteralPath $bootBin) {
+    $bytes = [IO.File]::ReadAllBytes($bootBin)
+    $gameId = [Text.Encoding]::ASCII.GetString($bytes, 0, [Math]::Min(6, $bytes.Length))
+    if ($gameId.Length -ge 4 -and $gameId[3] -cne $expectedLetter) {
+        throw "DATA\sys\boot.bin is for disc $gameId, but main.dol is region $expectedLetter. Stale disc assets detected; re-extract your game."
+    }
+}
 $generated = Join-Path $Workspace 'generated'
 $functions = Join-Path $generated 'functions'
 $baseMetadata = Join-Path $generated 'base_translation_output.json'
@@ -190,7 +234,7 @@ foreach ($required in @(
 # Everything Mario-Kart-specific this script needs is read from the project file rather than
 # restated here: the translation entry point below, and (through the retro-rewind profile the
 # translator loads itself) the module guest/link bases the mod translation uses.
-$pins = Get-MkwProjectPins $project
+$pins = Get-MkwProjectPins $project -BaseOnly:(-not $buildsRetro)
 
 # Three independent knobs: $translatorThreads (the translator's own worker threads; peak memory is
 # one heap that barely moves with thread count, so clamping it only slows translation), $translatedJobs
@@ -312,7 +356,7 @@ try {
 
             Invoke-Checked $translator @(
                 'emit-base-manifest', '--project', $project, '--out', $baseManifestDir,
-                '--functions-dir', $functions, '--translation-output-metadata', $baseMetadata, '--region', 'P'
+                '--functions-dir', $functions, '--translation-output-metadata', $baseMetadata, '--region', $expectedLetter
             ) 'Creating the local base translation manifest' -StepId 'emit-base-manifest'
 
             if (-not [string]::IsNullOrWhiteSpace($TranslationFingerprint)) {
@@ -329,14 +373,14 @@ try {
                 'translate-mod', '--project', $project, '--profile', 'retro-rewind',
                 '--base-manifest', $baseManifest, '--base-translation-output-metadata', $baseMetadata,
                 '--code-pul', $codePul, '--mod-root', $retroRoot, '--mod-name', 'Retro Rewind',
-                '--region', 'P', '--out', $retroOut, '--prefer-cached-inputs', '--emit-cpp',
+                '--region', $expectedLetter, '--out', $retroOut, '--prefer-cached-inputs', '--emit-cpp',
                 '--threads', $translatorThreads
             )
             if ($SkipRetroWfcPayload) {
                 $translateModArguments += '--skip-retro-wfc'
             } elseif (-not [string]::IsNullOrWhiteSpace($RetroWfcOfflineDirectory)) {
                 $offlineRoot = [IO.Path]::GetFullPath($RetroWfcOfflineDirectory)
-                $offlinePayload = Join-Path $offlineRoot 'binary\payload.RMCPD00.bin'
+                $offlinePayload = Join-Path $offlineRoot "binary\payload.RMC${expectedLetter}D00.bin"
                 Assert-File $offlinePayload 'Offline Retro-WFC shared payload'
                 $translateModArguments += @('--retro-wfc-payload', $offlinePayload)
             } else {
@@ -446,8 +490,8 @@ try {
                 RelSha256 = $relSha
                 CodePulSha256 = if ($isRetro) { Get-MkwFileSha256 (Join-Path $retroRoot 'Binaries\Code.pul') } else { $null }
                 RetroWfcPayloadMode = if (-not $isRetro) { $null } elseif ($SkipRetroWfcPayload) { 'skipped' } else { $RetroWfcPayloadOrigin }
-                RetroWfcPayloadSha256 = if ($isRetro -and -not $SkipRetroWfcPayload) { Get-MkwFileSha256 (Join-Path $RetroWfcOfflineDirectory 'binary\payload.RMCPD00.bin') } else { $null }
-                RetroWfcPayloadLength = if ($isRetro -and -not $SkipRetroWfcPayload) { (Get-Item -LiteralPath (Join-Path $RetroWfcOfflineDirectory 'binary\payload.RMCPD00.bin')).Length } else { $null }
+                RetroWfcPayloadSha256 = if ($isRetro -and -not $SkipRetroWfcPayload) { Get-MkwFileSha256 (Join-Path $RetroWfcOfflineDirectory "binary\payload.RMC${expectedLetter}D00.bin") } else { $null }
+                RetroWfcPayloadLength = if ($isRetro -and -not $SkipRetroWfcPayload) { (Get-Item -LiteralPath (Join-Path $RetroWfcOfflineDirectory "binary\payload.RMC${expectedLetter}D00.bin")).Length } else { $null }
                 Compiler = "llvm-mingw clang-22 sha256:$compilerSha"
             }
             $provenance | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $Destination 'local-build.json') -Encoding UTF8

Launcher/NativeBuildFlags.ps1

@@ -91,7 +91,7 @@ function Get-MkwBuildSafePath([string]$Path, [string]$Kind, [string]$MarkerFile)
         'letters, digits and spaces, or make a safe junction location available.')
 }
 
-function Get-MkwProjectPins([string]$ProjectFile) {
+function Get-MkwProjectPins([string]$ProjectFile, [switch]$BaseOnly) {
     <#
     The Mario Kart Wii facts pinned by projects/mkwii/recomp.yml (game identity, clean input
     digests, translation entry point, Retro-WFC endpoint), read from one place instead of being
@@ -122,7 +122,10 @@ function Get-MkwProjectPins([string]$ProjectFile) {
             $pins.RetroWfcPayloadUri = $Matches[1]
         }
     }
+    # Base translation has no dependency on a mod profile or its download endpoint.
+    # Installer/release callers still require the endpoint by default.
     foreach ($name in @($pins.Keys)) {
+        if ($BaseOnly -and $name -eq 'RetroWfcPayloadUri') { continue }
         if ([string]::IsNullOrWhiteSpace($pins[$name])) {
             throw "$ProjectFile does not pin $name; the project file is not the shape this build expects."
         }

Launcher/WiiCompiled.Setup.Common/RetroWfcPayload.cs

@@ -12,21 +12,33 @@ public sealed record RetroWfcPayloadSnapshot(string Directory, string Sha256, lo
 
 /// <summary>
 /// Downloads and verifies the Retro-WFC payload (a small, RSA-signed blob served from a single
-/// fixed endpoint). Shared by both installers - moved here from WiiCompiled.Setup.Windows's
+/// regional endpoint). Shared by both installers - moved here from WiiCompiled.Setup.Windows's
 /// InputValidation.cs, which keeps every one of these method names as thin forwarding wrappers so
 /// its many existing call sites (ProductRepairService.cs, LocalBuildService.cs, Installation.cs,
 /// SelfTests.cs) needed no changes.
 /// </summary>
 public static class RetroWfcPayload
 {
     private const long MaximumRetroWfcPayloadBytes = 16L * 1024 * 1024;
-    // The payload is tens of kilobytes from a single fixed endpoint 30s is good.
+    // The payload is tens of kilobytes from a fixed regional endpoint; allow 30 seconds.
     private static readonly TimeSpan RetroWfcDownloadTimeout = TimeSpan.FromSeconds(30);
     private static readonly TimeSpan RetroWfcRetryDelay = TimeSpan.FromSeconds(1);
 
     public const string CurrentRetroWfcPayloadUri = "https://rwfc.net/api/wfc/payload?g=RMCPD00";
-    private static readonly string RetroWfcOfflinePayloadFile =
-        Path.Combine("binary", "payload.RMCPD00.bin");
+    public static string PayloadGame(string gameId) => gameId switch
+    {
+        "RMCP01" => "RMCPD00",
+        "RMCE01" => "RMCED00",
+        "RMCJ01" => "RMCJD00",
+        "RMCK01" => "RMCKD00",
+        _ => throw new InvalidDataException($"Unsupported Retro-WFC base game '{gameId}'.")
+    };
+
+    public static string PayloadRelativePath(string gameId) =>
+        Path.Combine("binary", $"payload.{PayloadGame(gameId)}.bin");
+
+    public static string PayloadUri(string gameId) =>
+        $"https://rwfc.net/api/wfc/payload?g={PayloadGame(gameId)}";
 
     /// <summary>Retro-WFC production payload signing key (PROD PayloadPublicKey). Verifies
     /// the payload past the 0x110 header; on-console stage1 pins the same key, so
@@ -46,46 +58,48 @@ public static class RetroWfcPayload
     private const int RetroWfcPayloadMinimumBytes = 0x130;
 
     public static string ValidateStagedRetroWfcPayloadDirectory(string stagedDirectory,
-        RSAParameters? signingKey = null)
+        RSAParameters? signingKey = null, string gameId = "RMCP01")
     {
         if (string.IsNullOrWhiteSpace(stagedDirectory))
             throw new InvalidDataException("The staged Retro-WFC payload directory is missing.");
 
         var root = Path.GetFullPath(stagedDirectory);
-        var payload = Path.Combine(root, RetroWfcOfflinePayloadFile);
+        var relativePath = PayloadRelativePath(gameId);
+        var payload = Path.Combine(root, relativePath);
         try
         {
             if ((File.GetAttributes(payload) & FileAttributes.Directory) != 0)
                 throw new InvalidDataException(
-                    "The staged Retro-WFC payload directory does not contain binary\\payload.RMCPD00.bin.");
+                    $"The staged Retro-WFC payload directory does not contain {relativePath}.");
         }
         catch (FileNotFoundException)
         {
             throw new InvalidDataException(
-                "The staged Retro-WFC payload directory does not contain binary\\payload.RMCPD00.bin.");
+                $"The staged Retro-WFC payload directory does not contain {relativePath}.");
         }
         catch (DirectoryNotFoundException)
         {
             throw new InvalidDataException(
-                "The staged Retro-WFC payload directory does not contain binary\\payload.RMCPD00.bin.");
+                $"The staged Retro-WFC payload directory does not contain {relativePath}.");
         }
-        ValidateRetroWfcPayloadFile(payload, signingKey);
+        ValidateRetroWfcPayloadFile(payload, signingKey, PayloadGame(gameId));
         return root;
     }
 
     public static string ResolveRetroWfcPayloadFile(string stagedDirectory,
-        RSAParameters? signingKey = null) =>
-        Path.Combine(ValidateStagedRetroWfcPayloadDirectory(stagedDirectory, signingKey),
-            RetroWfcOfflinePayloadFile);
+        RSAParameters? signingKey = null, string gameId = "RMCP01") =>
+        Path.Combine(ValidateStagedRetroWfcPayloadDirectory(stagedDirectory, signingKey, gameId),
+            PayloadRelativePath(gameId));
 
     public static string ComputeRetroWfcPayloadSha256(string stagedDirectory,
-        RSAParameters? signingKey = null) =>
-        Sha256File(ResolveRetroWfcPayloadFile(stagedDirectory, signingKey));
+        RSAParameters? signingKey = null, string gameId = "RMCP01") =>
+        Sha256File(ResolveRetroWfcPayloadFile(stagedDirectory, signingKey, gameId));
 
     public static void ValidateRetroWfcPayloadUri(string uriText)
     {
-        if (!string.Equals(uriText, CurrentRetroWfcPayloadUri, StringComparison.Ordinal))
-            throw new InvalidDataException("The installer does not define the fixed Retro-WFC payload endpoint.");
+        if (!new[] { "RMCP01", "RMCE01", "RMCJ01", "RMCK01" }
+            .Any(gameId => string.Equals(uriText, PayloadUri(gameId), StringComparison.Ordinal)))
+            throw new InvalidDataException("The installer does not define this fixed Retro-WFC payload endpoint.");
     }
 
     public static async Task<RetroWfcPayloadSnapshot> DownloadRetroWfcPayloadAsync(string uriText,
@@ -96,7 +110,8 @@ public static async Task<RetroWfcPayloadSnapshot> DownloadRetroWfcPayloadAsync(s
         var uri = new Uri(uriText, UriKind.Absolute);
 
         var root = Path.GetFullPath(destinationDirectory);
-        var destination = Path.Combine(root, RetroWfcOfflinePayloadFile);
+        var payloadGame = uri.Query[3..]; // ValidateRetroWfcPayloadUri accepts only the exact ?g= form.
+        var destination = Path.Combine(root, "binary", $"payload.{payloadGame}.bin");
         Directory.CreateDirectory(Path.GetDirectoryName(destination)!);
         using var handler = new HttpClientHandler { AllowAutoRedirect = false };
         using var client = new HttpClient(handler) { Timeout = Timeout.InfiniteTimeSpan };
@@ -154,7 +169,7 @@ private static async Task<RetroWfcPayloadSnapshot> DownloadRetroWfcPayloadAttemp
                 }
                 actualSha256 = Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant();
             }
-            ValidateRetroWfcPayloadFile(temporary);
+            ValidateRetroWfcPayloadFile(temporary, expectedGame: uri.Query[3..]);
             File.Move(temporary, destination, overwrite: true);
             return new RetroWfcPayloadSnapshot(root, actualSha256, total);
         }
@@ -181,7 +196,8 @@ public static bool IsTransientRetroWfcDownloadFailure(Exception exception,
                request.StatusCode is { } status && (int)status >= 500;
     }
 
-    private static void ValidateRetroWfcPayloadFile(string payload, RSAParameters? signingKey = null)
+    private static void ValidateRetroWfcPayloadFile(string payload, RSAParameters? signingKey = null,
+        string expectedGame = "RMCPD00")
     {
         byte[] image;
         using (var stream = File.OpenRead(payload))
@@ -201,6 +217,10 @@ private static void ValidateRetroWfcPayloadFile(string payload, RSAParameters? s
             throw new InvalidDataException(
                 $"The Retro-WFC payload declares {declaredSize} bytes but contains {image.Length}.");
 
+        if (image.Length < 0x144 ||
+            System.Text.Encoding.ASCII.GetString(image, 0x138, 12).TrimEnd('\0') != expectedGame)
+            throw new InvalidDataException($"The Retro-WFC payload is not for {expectedGame}.");
+
         using var rsa = RSA.Create();
         rsa.ImportParameters(signingKey ?? new RSAParameters
         {