Faster updates mac and linux

.gitignore

@@ -1,4 +1,5 @@
 /generated/
+/runtime/native-index.json
 /Assets/
 /assets/
 /PulsarPacks/

Launcher/Build-Installer.ps1

@@ -112,6 +112,20 @@ Assert-Directory $dependencySources 'Pinned offline dependency sources'
 # back to compiling aurora from source, which is what this package exists to
 # avoid - and a mismatched one would be an ABI hazard.
 $portableClangHash = Get-MkwFileSha256 (Join-Path $portableTools 'llvm-mingw\bin\clang-22.exe')
+# The identity the runtime build compares against the package's runtime archive.
+function Get-RuntimeSourceFingerprint([string]$RuntimeDirectory, [string]$DependenciesDirectory) {
+    $arguments = @("-DMKW_RUNTIME_DIR=$(ConvertTo-MkwCMakePath $RuntimeDirectory)")
+    $cppWinRt = Join-Path $DependenciesDirectory 'cppwinrt'
+    if (Test-Path -LiteralPath (Join-Path $cppWinRt 'winrt\base.h') -PathType Leaf) {
+        $arguments += "-DMKW_CPPWINRT_INCLUDE_DIR=$(ConvertTo-MkwCMakePath $cppWinRt)"
+    }
+    $arguments += @('-P', (ConvertTo-MkwCMakePath (Join-Path $RuntimeDirectory 'cmake\RuntimeFingerprint.cmake')))
+    $output = @(& (Join-Path $portableTools 'CMake\bin\cmake.exe') @arguments)
+    if ($LASTEXITCODE -ne 0) { throw "Runtime fingerprinting failed with exit code $LASTEXITCODE." }
+    $line = $output | Where-Object { "$_" -match '^runtime_fingerprint=[0-9a-f]{64}$' } | Select-Object -First 1
+    if (-not $line) { throw "RuntimeFingerprint.cmake printed no fingerprint: $($output -join ' ')" }
+    return "$line".Substring('runtime_fingerprint='.Length)
+}
 function Test-NativePrebuiltCurrent([string]$PackageDirectory) {
     $provenancePath = Join-Path $PackageDirectory 'provenance.json'
     if (-not (Test-Path -LiteralPath $provenancePath -PathType Leaf)) { return $false }
@@ -129,18 +143,23 @@ function Test-NativePrebuiltCurrent([string]$PackageDirectory) {
     # release ships runtime/third_party sources that first-party TUs compile
     # against, so archives harvested before a vendored change are stale.
     $recordedThirdParty = $provenance.PSObject.Properties['ThirdPartySourceFingerprint']
+    # The precompiled runtime archive must be from these runtime sources; a package without one
+    # would make every installation compile the runtime itself.
+    $recordedRuntime = $provenance.PSObject.Properties['RuntimeSourceFingerprint']
     return ($provenance.CompilerSha256 -eq $portableClangHash) -and
            ($provenance.FlagFingerprint -eq (Get-MkwNativeFlagFingerprint)) -and
            ($null -ne $recordedDawn) -and
            ($recordedDawn.Value -eq (Get-MkwDawnRuntimeSha256 $dependencySources)) -and
            ($null -ne $recordedAurora) -and
            ($recordedAurora.Value -eq (Get-MkwAuroraSourceFingerprint (Join-Path $repoRoot 'aurora-main'))) -and
            ($null -ne $recordedThirdParty) -and
-           ($recordedThirdParty.Value -eq (Get-MkwThirdPartySourceFingerprint (Join-Path $repoRoot 'runtime\third_party')))
+           ($recordedThirdParty.Value -eq (Get-MkwThirdPartySourceFingerprint (Join-Path $repoRoot 'runtime\third_party'))) -and
+           ($null -ne $recordedRuntime) -and
+           ($recordedRuntime.Value -eq (Get-RuntimeSourceFingerprint (Join-Path $repoRoot 'runtime') $dependencySources))
 }
 $nativePrebuiltPackage = Join-Path $dependencySources 'native_prebuilt'
 if (-not (Test-NativePrebuiltCurrent $nativePrebuiltPackage)) {
-    Write-Host '[0/6] Building the precompiled aurora and third-party package...'
+    Write-Host '[0/6] Building the precompiled aurora, third-party and runtime package...'
     & (Join-Path $PSScriptRoot 'Prepare-NativePrebuilt.ps1') `
         -PortableToolsDirectory $portableTools -DependencySourceDirectory $dependencySources
     if ($LASTEXITCODE -ne 0) { throw 'Precompiled aurora package preparation failed.' }
@@ -222,6 +241,21 @@ Copy-Item -LiteralPath (Join-Path $PSScriptRoot 'NativeBuildFlags.ps1') -Destina
 [IO.Directory]::CreateDirectory((Join-Path $workspace 'Dependencies')) | Out-Null
 foreach ($name in $requiredDependencies) { Copy-Directory (Join-Path $dependencySources $name) (Join-Path $workspace "Dependencies\$name") }
 
+# The translation identity covers the runtime's native registrations through this index (see
+# ToolkitFingerprint), so it is extracted from exactly the staged runtime sources.
+$nativeIndex = Join-Path $workspace 'runtime\native-index.json'
+& $translator emit-native-index --project (Join-Path $workspace 'projects\mkwii\recomp.yml') --out $nativeIndex
+if ($LASTEXITCODE -ne 0) { throw "Native index extraction failed with exit code $LASTEXITCODE." }
+Assert-File $nativeIndex 'Runtime native index'
+# A staged runtime that fingerprints differently from the shipped archive would make every
+# installation silently compile the runtime from source.
+$stagedRuntimeFingerprint = Get-RuntimeSourceFingerprint (Join-Path $workspace 'runtime') (Join-Path $workspace 'Dependencies')
+$packagedRuntimeFingerprint = (Get-Content -LiteralPath (Join-Path $workspace 'Dependencies\native_prebuilt\provenance.json') -Raw |
+    ConvertFrom-Json).RuntimeSourceFingerprint
+if ($stagedRuntimeFingerprint -ne $packagedRuntimeFingerprint) {
+    throw "The staged runtime ($stagedRuntimeFingerprint) does not match the precompiled runtime archive ($packagedRuntimeFingerprint)."
+}
+
 [IO.Directory]::CreateDirectory((Join-Path $payloadRoot 'host')) | Out-Null
 Copy-Item $setupHost (Join-Path $payloadRoot 'host\WiiCompiled-Setup.exe')
 

Launcher/Prepare-NativePrebuilt.ps1

@@ -1,7 +1,8 @@
 # Builds the redistributable precompiled aurora + third-party package: aurora (~43% of local build
 # CPU time) and vendored Crypto++ are identical for every user, so this configures runtime/ like
-# LocalBuild.ps1 (both dot-source NativeBuildFlags.ps1), builds just that closure, and harvests the
-# archives plus a generated CMake description into launcher/artifacts/dependencies/native_prebuilt.
+# LocalBuild.ps1 (both dot-source NativeBuildFlags.ps1), builds just that closure plus the
+# game-independent runtime archive, and harvests the archives and a generated CMake description
+# into launcher/artifacts/dependencies/native_prebuilt.
 [CmdletBinding(PositionalBinding = $false)]
 param(
     [string]$PortableToolsDirectory = 'launcher/artifacts/portable-tools',
@@ -87,24 +88,29 @@ try {
         -AdditionalArguments @("-DMKW_NATIVE_PREBUILT_EXPORT_DIR=$(ConvertTo-MkwCMakePath $exportDirectory)")
     Invoke-Checked $cmake $configure 'Configuring the aurora/third-party staging build' -LogPrefix 'MKWNP'
 
+    $metaPath = Join-Path $exportDirectory 'meta.txt'
+    Assert-File $metaPath 'Native prebuilt export metadata'
+    $meta = @{}
+    foreach ($line in Get-Content -LiteralPath $metaPath) {
+        if ($line -match '^([a-z0-9_]+)=(.*)$') { $meta[$Matches[1]] = $Matches[2] }
+    }
+    # The game-independent runtime ships precompiled too; it is outside the probe closure.
+    if (-not $meta.ContainsKey('runtime_fingerprint')) {
+        throw 'The staging configure exported no runtime_fingerprint; the x86_64 package must carry the runtime archive.'
+    }
+
     # mkw_np_probe links exactly what mkw_runtime_common and the public products
     # link, so building it compiles the whole redistributable closure and nothing
     # else - and a successful link proves the harvested archives are complete.
-    Invoke-Checked $cmake @('--build', $stage, '--target', 'mkw_np_probe', '--parallel', "$Parallel") `
-        'Building the aurora/third-party closure' -LogPrefix 'MKWNP'
+    Invoke-Checked $cmake @('--build', $stage, '--target', 'mkw_np_probe', '--target', 'mkw_runtime',
+        '--parallel', "$Parallel") 'Building the aurora/third-party closure and the runtime' -LogPrefix 'MKWNP'
 } finally {
     $env:PATH = $oldPath
 }
 
 # ---------------------------------------------------------------------------
-# Read the description CMake wrote, plus the resolved command lines Ninja holds.
+# Read the resolved command lines Ninja holds.
 # ---------------------------------------------------------------------------
-$metaPath = Join-Path $exportDirectory 'meta.txt'
-Assert-File $metaPath 'Native prebuilt export metadata'
-$meta = @{}
-foreach ($line in Get-Content -LiteralPath $metaPath) {
-    if ($line -match '^([a-z0-9_]+)=(.*)$') { $meta[$Matches[1]] = $Matches[2] }
-}
 
 $buildNinja = Join-Path $stage 'build.ninja'
 Assert-File $buildNinja 'Generated build.ninja'
@@ -234,6 +240,12 @@ foreach ($name in ($targets.Keys | Sort-Object)) {
 }
 if ($sharedImports.Count -eq 0) { throw 'No shared libraries were harvested; the SDL/zlib/libpng DLLs would be missing.' }
 if ($dawnLinkerFile) { $linkerFileToReference[$dawnLinkerFile] = 'dawn::webgpu_dawn' }
+$runtimeTarget = $targets['mkw_runtime']
+if ($null -eq $runtimeTarget -or $runtimeTarget.Type -ne 'STATIC_LIBRARY' -or
+    -not $linkerFileToReference.ContainsKey($runtimeTarget.LinkerFile)) {
+    throw 'The runtime archive (mkw_runtime) was not built and harvested.'
+}
+$runtimeArchiveReference = $linkerFileToReference[$runtimeTarget.LinkerFile]
 
 # ---------------------------------------------------------------------------
 # Rewrite the compile/link interface into workspace-relative tokens.
@@ -297,6 +309,9 @@ $packageLinkItems = @($linkItems | ForEach-Object {
     if (StartsWithPath $absolute $dependencyRoot) { return "`@DEPS`@/$(RelativeTo $absolute $dependencyRoot)" }
     throw "Link item is neither a system library nor a harvested archive: $item"
 })
+if ($packageLinkItems -contains $runtimeArchiveReference) {
+    throw 'The runtime archive leaked into the aurora link interface.'
+}
 
 # fmt's `-include cstdlib` (and anything like it) is recorded from a C++ compile
 # line, so re-scope it to C++. The runtime also assembles generated .S blobs
@@ -333,6 +348,11 @@ $generated.Add('')
 $generated.Add("set(MKW_NP_DAWN_CONFIG_DIR `"$(ConvertTo-Token $meta['dawn_config_dir'] 'dawn')`")")
 $generated.Add("set(MKW_NP_DAWN_VERSION `"$($meta['aurora_dawn_version'])`")")
 $generated.Add('')
+$generated.Add('# The precompiled game-independent runtime; RuntimeLibrary.cmake decides whether it applies.')
+$generated.Add("set(MKW_NP_RUNTIME_ARCHIVE `"$runtimeArchiveReference`")")
+$generated.Add("set(MKW_NP_RUNTIME_FINGERPRINT `"$($meta['runtime_fingerprint'])`")")
+$generated.Add("set(MKW_NP_RUNTIME_CPU_PROFILE `"$($meta['runtime_cpu_profile'])`")")
+$generated.Add('')
 $generated.Add('# Shared third-party libraries keep imported SHARED targets so that')
 $generated.Add('# $<TARGET_RUNTIME_DLLS> still copies them next to the game executable.')
 foreach ($import in $sharedImports) {
@@ -393,6 +413,9 @@ $provenance = [ordered]@{
     # release build and the local build can refuse a package aurora has outgrown.
     AuroraSourceFingerprint = $auroraSourceFingerprint
     ThirdPartySourceFingerprint = $thirdPartySourceFingerprint
+    # runtime/cmake/RuntimeFingerprint.cmake over the runtime sources the archive was built from.
+    RuntimeSourceFingerprint = $meta['runtime_fingerprint']
+    RuntimeCpuProfile = $meta['runtime_cpu_profile']
     Sdl3Target = $meta['aurora_sdl3_target']
     HarvestedLibraryCount = @($linkerFileToReference.Keys).Count
     Contents = $contents

Launcher/Prepare-NativePrebuilt.sh

@@ -2,8 +2,9 @@
 # Builds the redistributable precompiled aurora + third-party package for native Linux: aurora
 # (~43% of local build CPU time per Prepare-NativePrebuilt.ps1), Crypto++ and mbed TLS are identical
 # for every user under the pinned toolchain prepare-portable-tools.sh bundles, so this configures
-# runtime/ against that toolchain, builds just that closure, and harvests the archives plus a
-# generated CMake description into an output package - the Linux counterpart to
+# runtime/ against that toolchain, builds just that closure (plus, on x86_64, the game-independent
+# runtime archive), and harvests the archives plus a generated CMake description into an output
+# package - the Linux counterpart to
 # Launcher/Prepare-NativePrebuilt.ps1, consumed by the same platform-agnostic
 # runtime/cmake/NativePrebuilt.cmake either script's package works with unmodified.
 #
@@ -191,12 +192,23 @@ if [[ "${cxx_sha256}" != "${compiler_sha256}" ]]; then
     compiler_sha256="${compiler_sha256}:${cxx_sha256}"
 fi
 
+# The game-independent runtime ships precompiled on x86_64 only; aarch64 compiles it locally.
+runtime_fingerprint=""
+if [[ "$arch" == x86_64 ]]; then
+    runtime_fingerprint=$("$cmake_bin" -DMKW_RUNTIME_DIR="$runtime_source" \
+        -P "$runtime_source/cmake/RuntimeFingerprint.cmake" | sed -n 's/^runtime_fingerprint=\([0-9a-f]\{64\}\)$/\1/p')
+    [[ -n "$runtime_fingerprint" ]] || fail "RuntimeFingerprint.cmake printed no runtime fingerprint"
+fi
+
 if [[ "$print_fingerprint_only" -eq 1 ]]; then
     printf 'compiler_sha256=%s\n' "$compiler_sha256"
     printf 'flag_fingerprint=%s\n' "$flag_fingerprint"
     printf 'aurora_fingerprint=%s\n' "$aurora_fingerprint"
     printf 'third_party_fingerprint=%s\n' "$third_party_fingerprint"
     printf 'mbedtls_fingerprint=%s\n' "$mbedtls_fingerprint"
+    if [[ -n "$runtime_fingerprint" ]]; then
+        printf 'runtime_fingerprint=%s\n' "$runtime_fingerprint"
+    fi
     exit 0
 fi
 
@@ -220,20 +232,30 @@ echo "Prepare-NativePrebuilt.sh: configuring the aurora/third-party staging buil
     -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \
     -DMKW_NATIVE_PREBUILT_EXPORT_DIR="$export_dir"
 
+meta_path="$export_dir/meta.txt"
+assert_file "$meta_path" "Native prebuilt export metadata"
+get_meta() { awk -F= -v k="$1" '$0 ~ "^" k "=" { sub(/^[^=]*=/, ""); print; exit }' "$meta_path"; }
+
 # mkw_np_probe links exactly what mkw_runtime_common and the public products link, so building it
 # compiles the whole redistributable closure and nothing else - and a successful link proves the
-# harvested archives are complete.
+# harvested archives are complete. The runtime archive is outside that closure and built alongside.
+build_targets=(--target mkw_np_probe)
+meta_runtime_fingerprint=$(get_meta runtime_fingerprint)
+if [[ -n "$meta_runtime_fingerprint" ]]; then
+    if [[ -n "$runtime_fingerprint" && "$meta_runtime_fingerprint" != "$runtime_fingerprint" ]]; then
+        fail "The configure's runtime fingerprint ($meta_runtime_fingerprint) differs from RuntimeFingerprint.cmake's ($runtime_fingerprint)"
+    fi
+    build_targets+=(--target mkw_runtime)
+elif [[ -n "$runtime_fingerprint" ]]; then
+    fail "The staging configure exported no runtime_fingerprint; the $arch package must carry the runtime archive"
+fi
 echo "Prepare-NativePrebuilt.sh: building the aurora/third-party closure..."
-"$cmake_bin" --build "$stage_dir" --target mkw_np_probe --parallel "$parallel"
+"$cmake_bin" --build "$stage_dir" "${build_targets[@]}" --parallel "$parallel"
 
 # ---------------------------------------------------------------------------
-# Read the description CMake wrote, plus the resolved command lines Ninja holds.
+# Read the resolved command lines Ninja holds.
 # ---------------------------------------------------------------------------
 
-meta_path="$export_dir/meta.txt"
-assert_file "$meta_path" "Native prebuilt export metadata"
-get_meta() { awk -F= -v k="$1" '$0 ~ "^" k "=" { sub(/^[^=]*=/, ""); print; exit }' "$meta_path"; }
-
 build_ninja="$stage_dir/build.ninja"
 assert_file "$build_ninja" "Generated build.ninja"
 
@@ -397,6 +419,13 @@ while IFS='|' read -r name type file linkerfile; do
         linker_file_to_reference["$linkerfile"]="@PKG@/$relative_linker"
     fi
 done < "$targets_txt"
+runtime_archive_ref=""
+if [[ -n "$meta_runtime_fingerprint" ]]; then
+    runtime_linker=$(awk -F'|' '$1 == "mkw_runtime" && $2 == "STATIC_LIBRARY" { print $4; exit }' "$targets_txt")
+    [[ -n "$runtime_linker" && -f "$runtime_linker" ]] || fail "The runtime archive (mkw_runtime) was not built"
+    runtime_archive_ref=${linker_file_to_reference["$(normalize "$runtime_linker")"]:-}
+    [[ "$runtime_archive_ref" == "@PKG@/lib/"* ]] || fail "The runtime archive (mkw_runtime) was not harvested"
+fi
 mbedtls_refs=()
 mbedtls_txt="$export_dir/mbedtls.txt"
 assert_file "$mbedtls_txt" "Mbed TLS export targets list"
@@ -498,6 +527,11 @@ for item in "${link_items[@]}"; do
     esac
     fail "Link item is neither a system library nor a harvested archive: $item"
 done
+if [[ -n "$runtime_archive_ref" ]]; then
+    for item in "${package_link_items[@]}"; do
+        [[ "$item" != "$runtime_archive_ref" ]] || fail "The runtime archive leaked into the aurora link interface"
+    done
+fi
 
 # ---------------------------------------------------------------------------
 # Emit the consumer-facing CMake description.
@@ -562,6 +596,12 @@ generated_cmake="$output_dir/native_prebuilt.cmake"
     format_cmake_block MKW_NP_MBEDTLS_LIBRARIES "${mbedtls_refs[@]}"
     echo 'set(MKW_NP_MBEDTLS_INCLUDE_DIR "@PKG@/include/mbedtls")'
     printf 'set(MKW_NP_MBEDTLS_FINGERPRINT "%s")\n' "$mbedtls_fingerprint"
+    if [[ -n "$runtime_archive_ref" ]]; then
+        # The precompiled game-independent runtime; RuntimeLibrary.cmake decides whether it applies.
+        printf 'set(MKW_NP_RUNTIME_ARCHIVE "%s")\n' "$runtime_archive_ref"
+        printf 'set(MKW_NP_RUNTIME_FINGERPRINT "%s")\n' "$meta_runtime_fingerprint"
+        printf 'set(MKW_NP_RUNTIME_CPU_PROFILE "%s")\n' "$(get_meta runtime_cpu_profile)"
+    fi
     format_cmake_block MKW_NP_AURORA_TARGETS "aurora::gx" "aurora::pad" "aurora::si" "aurora::vi" "aurora::mtx"
     echo ""
     printf 'set(MKW_NP_DAWN_CONFIG_DIR "%s")\n' "$dawn_config_token"
@@ -597,12 +637,13 @@ harvested_count=${#linker_file_to_reference[@]}
 
 python3 - "$output_dir" "$compiler_sha256" "$compiler_version" "$flag_fingerprint" \
     "$dawn_version" "$dawn_runtime_sha256" "$aurora_fingerprint" "$third_party_fingerprint" \
-    "$sdl3_target" "$harvested_count" "$mbedtls_fingerprint" <<'PY'
+    "$sdl3_target" "$harvested_count" "$mbedtls_fingerprint" \
+    "${runtime_archive_ref:+$meta_runtime_fingerprint}" "${runtime_archive_ref:+$(get_meta runtime_cpu_profile)}" <<'PY'
 import hashlib, json, os, sys, datetime
 
 (output_dir, compiler_sha256, compiler_version, flag_fingerprint, dawn_version,
  dawn_runtime_sha256, aurora_fingerprint, third_party_fingerprint, sdl3_target,
- harvested_count, mbedtls_fingerprint) = sys.argv[1:]
+ harvested_count, mbedtls_fingerprint, runtime_fingerprint, runtime_cpu_profile) = sys.argv[1:]
 
 contents = []
 for root, dirs, files in os.walk(output_dir):
@@ -644,6 +685,9 @@ provenance = {
     "HarvestedLibraryCount": int(harvested_count),
     "Contents": contents,
 }
+if runtime_fingerprint:
+    provenance["RuntimeSourceFingerprint"] = runtime_fingerprint
+    provenance["RuntimeCpuProfile"] = runtime_cpu_profile
 with open(os.path.join(output_dir, "provenance.json"), "w", encoding="utf-8") as fh:
     json.dump(provenance, fh, indent=2)
     fh.write("\n")

Launcher/Test-PinnedFacts.ps1

@@ -127,7 +127,7 @@ foreach ($mandatory in ([regex]::Matches($parameterBlock,
 }
 
 # --- The build steps the installer's progress bar acts on.
-$installProgress = Read-SourceFile (Join-Path $setup 'InstallProgress.cs') 'InstallProgress.cs'
+$installProgress = Read-SourceFile (Join-Path $common 'InstallProgress.cs') 'InstallProgress.cs'
 $knownSteps = Get-QuotedList $installProgress '(?s)class BuildStepIds\s*\{(.*?)\n\}' `
     'const string \w+ = "([^"]+)";' 'BuildStepIds'
 $emittedSteps = @([regex]::Matches($localBuild, "(?:-StepId|Write-MkwBuildStep)\s+'([^']+)'") |