diff --git a/eng/pipelines/common/evaluate-default-paths.yml b/eng/pipelines/common/evaluate-default-paths.yml index dba1aaad202b60..cdaa0b71445755 100644 --- a/eng/pipelines/common/evaluate-default-paths.yml +++ b/eng/pipelines/common/evaluate-default-paths.yml @@ -291,6 +291,22 @@ jobs: - ${{ parameters._const_paths._always_exclude }} - ${{ parameters._const_paths._perf_pipeline_specific_only }} + # The checked-in CoreCLR wasm call helpers (src/coreclr/vm/wasm/{browser,wasi}/callhelpers-*.cpp) + # are generated from the framework's [UnmanagedCallersOnly] methods, P/Invokes and QCalls, so any + # change to that managed code can make them stale. Triggers the wasm CoreCLR build legs that run + # wasm-coreclr-callhelpers-check.yml. + - subset: wasm_coreclr_callhelpers + combined: true + include: + - src/libraries/*/src/* + - src/coreclr/System.Private.CoreLib/* + - src/coreclr/tools/aot/ILCompiler.ReadyToRun/* + - src/coreclr/tools/Common/* + - src/coreclr/vm/wasm/* + - eng/wasm/WasmPInvokeModules.props + exclude: + - ${{ parameters._const_paths._always_exclude }} + # wasm/runtimetests coreCLR need to be run - subset: wasm_coreclr_runtimetests combined: true diff --git a/eng/pipelines/common/templates/wasi-wasm-coreclr-build-only.yml b/eng/pipelines/common/templates/wasi-wasm-coreclr-build-only.yml index dfee081a0c2d93..1ab7055386983f 100644 --- a/eng/pipelines/common/templates/wasi-wasm-coreclr-build-only.yml +++ b/eng/pipelines/common/templates/wasi-wasm-coreclr-build-only.yml @@ -32,6 +32,7 @@ jobs: timeoutInMinutes: 120 condition: ${{ parameters.condition }} postBuildSteps: + - template: /eng/pipelines/common/templates/wasm-coreclr-callhelpers-check.yml - template: /eng/pipelines/common/wasm-post-build-steps.yml parameters: publishArtifactsForWorkload: ${{ parameters.publishArtifactsForWorkload }} diff --git a/eng/pipelines/common/templates/wasm-coreclr-callhelpers-check.yml b/eng/pipelines/common/templates/wasm-coreclr-callhelpers-check.yml new file mode 100644 index 00000000000000..02f11f7132e055 --- /dev/null +++ b/eng/pipelines/common/templates/wasm-coreclr-callhelpers-check.yml @@ -0,0 +1,53 @@ +# Regenerates the checked-in CoreCLR WebAssembly call helpers for this leg's target OS +# (src/coreclr/vm/wasm//callhelpers-*.cpp) from the testhost and crossgen2 the leg just built, +# and fails if they differ from what is committed. A stale table builds fine but crashes the +# corerun that links it, for example with "GetUnmanagedCallersOnlyThunk: unknown thunk" when a +# new [UnmanagedCallersOnly] method is missing. See src/coreclr/vm/wasm/generate-coreclr-helpers.md. +# +# Must run after a clr+libs build of osGroup (browser or wasi) in buildConfig on a Linux host. +parameters: + osGroup: '' + osSubgroup: '' + archType: '' + buildConfig: '' + runtimeFlavor: '' + runtimeVariant: '' + helixQueues: '' + targetRid: '' + nameSuffix: '' + platform: '' + shouldContinueOnError: false + +steps: + - bash: | + set -u + target_os='${{ parameters.osGroup }}' + configuration='$(_BuildConfig)' + table_dir="src/coreclr/vm/wasm/$target_os/" + + src/coreclr/vm/wasm/generate-coreclr-helpers.sh -c "$configuration" --target-os "$target_os" + generator_exit=$? + if [[ $generator_exit -ne 0 ]]; then + echo "##vso[task.logissue type=error]Regenerating the $target_os CoreCLR WebAssembly call helpers failed (exit $generator_exit)." + exit $generator_exit + fi + + if [[ -z "$(git status --porcelain -- "$table_dir")" ]]; then + echo "The checked-in $target_os call helpers are up to date." + exit 0 + fi + + echo "##vso[task.logissue type=error]The checked-in CoreCLR WebAssembly call helpers in $table_dir are out of date. Regenerate them and commit the result; see the step log for the diff and the commands." + git --no-pager status --short -- "$table_dir" + git --no-pager diff -- "$table_dir" + echo + echo "The managed code that feeds these tables ([UnmanagedCallersOnly] methods, P/Invokes, QCalls)" + echo "changed without the tables being regenerated. To fix, run from the repository root:" + echo + echo " ./build.sh clr+libs -os $target_os -c $configuration" + echo " src/coreclr/vm/wasm/generate-coreclr-helpers.sh -c $configuration --target-os $target_os" + echo + echo "and commit the changes under $table_dir." + exit 1 + displayName: Check ${{ parameters.osGroup }} CoreCLR call helpers are up to date + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 1dd60140236382..69055cf0bf41a7 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -137,6 +137,8 @@ extends: buildArgs: -s clr+libs+libs.tests+packs -c Release -rc $(_BuildConfig) /p:TestAssemblies=false /p:TestWasmBuildTests=true /p:ArchiveTests=true /p:InstallWorkloadForTesting=false /p:BuildCrossgen2HostPackForWorkloadTesting=true timeoutInMinutes: 120 postBuildSteps: + # Runs on the Linux-hosted browser_wasm job only; the step skips itself on Windows. + - template: /eng/pipelines/common/templates/wasm-coreclr-callhelpers-check.yml - template: /eng/pipelines/common/wasm-post-build-steps.yml parameters: publishArtifactsForWorkload: true @@ -151,6 +153,7 @@ extends: eq(variables['wasmDarcDependenciesChanged'], true), eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_tools_illink.containsChange'], true), eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_wasm_coreclr_runtimetests.containsChange'], true), + eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_wasm_coreclr_callhelpers.containsChange'], true), eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_wasmbuildtests.containsChange'], true), eq(variables['isRollingBuild'], true)) @@ -167,6 +170,7 @@ extends: eq(variables['wasmDarcDependenciesChanged'], true), eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_tools_illink.containsChange'], true), eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_wasm_coreclr_runtimetests.containsChange'], true), + eq(stageDependencies.EvaluatePaths.evaluate_paths.outputs['SetPathVars_wasm_coreclr_callhelpers.containsChange'], true), eq(variables['isRollingBuild'], true)) # CoreCLR src/tests subset run on wasmtime via Helix diff --git a/src/coreclr/vm/wasm/generate-coreclr-helpers.cmd b/src/coreclr/vm/wasm/generate-coreclr-helpers.cmd index 020edfca8f131c..d75fe752394e9a 100644 --- a/src/coreclr/vm/wasm/generate-coreclr-helpers.cmd +++ b/src/coreclr/vm/wasm/generate-coreclr-helpers.cmd @@ -2,9 +2,10 @@ setlocal enabledelayedexpansion :: Default configuration -set "configuration=Debug" +set "configuration=Release" set "browser_scan_path_override=" set "wasi_scan_path_override=" +set "target_os=" :: Get the repo root (script is in src/coreclr/vm/wasm). :: This must be computed before argument parsing, because SHIFT also shifts %0. @@ -18,12 +19,14 @@ set "usage=!usage!^ Options:^ - -c, --configuration ^ Build configuration (default: Debug)^ + -c, --configuration ^ Build configuration (default: Release)^ -s, --scan-path ^ Override the default browser scan path^ -w, --wasi-scan-path ^ Override the default wasi scan path^ + -t, --target-os ^ Regenerate only this flavor (default: both)^ + -h, --help Show this help message" :parse_args @@ -34,6 +37,8 @@ if /i "%~1"=="-s" goto :set_scan_path if /i "%~1"=="--scan-path" goto :set_scan_path if /i "%~1"=="-w" goto :set_wasi_scan_path if /i "%~1"=="--wasi-scan-path" goto :set_wasi_scan_path +if /i "%~1"=="-t" goto :set_target_os +if /i "%~1"=="--target-os" goto :set_target_os if /i "%~1"=="-h" goto :show_help if /i "%~1"=="--help" goto :show_help @@ -59,6 +64,12 @@ shift shift goto :parse_args +:set_target_os +set target_os=%~2 +shift +shift +goto :parse_args + :show_help echo !usage! exit /b 0 @@ -71,6 +82,15 @@ if /i not "%configuration%"=="Debug" if /i not "%configuration%"=="Release" if / exit /b 1 ) +:: Validate target OS to prevent injection +if not "%target_os%"=="" if /i not "%target_os%"=="browser" if /i not "%target_os%"=="wasi" ( + echo Error: Invalid target OS "%target_os%". Must be browser or wasi. + exit /b 1 +) + +set "target_os_prop=" +if not "%target_os%"=="" set "target_os_prop=-p:CallHelperTargetOS=%target_os%" + echo Configuration: %configuration% echo Repo root: %repo_root% @@ -95,7 +115,7 @@ if not "%wasi_scan_path_override%"=="" ( set wasi_prop="-p:WasiScanPath=!wasi_scan_path_override!" ) -call .\dotnet.cmd build "%generator_proj%" -t:GenerateCallHelpers "-p:Configuration=%configuration%" %browser_prop% %wasi_prop% +call .\dotnet.cmd build "%generator_proj%" -t:GenerateCallHelpers "-p:Configuration=%configuration%" %browser_prop% %wasi_prop% %target_os_prop% if errorlevel 1 exit /b 1 echo Done! diff --git a/src/coreclr/vm/wasm/generate-coreclr-helpers.md b/src/coreclr/vm/wasm/generate-coreclr-helpers.md index 48e9439cfc8838..361f46c14df52f 100644 --- a/src/coreclr/vm/wasm/generate-coreclr-helpers.md +++ b/src/coreclr/vm/wasm/generate-coreclr-helpers.md @@ -19,7 +19,8 @@ to look at when a regenerated table is not what was expected. The relink targets for browser and wasi apps run the same crossgen2 mode over the app's own assembly closure, so these checked-in files and a relinked app are produced by one code path. -The scripts generate **both** WebAssembly variations: +By default the scripts generate **both** WebAssembly variations; pass `--target-os browser` or +`--target-os wasi` to regenerate only one: | Target OS | Output directory | Default scan path (testhost) | |-----------|---------------------------------|------------------------------| @@ -52,22 +53,23 @@ regenerated files in the same change. A module missing from the list surfaces as The generator scans the **managed framework assemblies** in the `testhost` folder produced by a `clr+libs` build, and runs the self-contained crossgen2 that the same build produces. Because the -scripts generate both the `browser` and `wasi` variations, you must build **both** WebAssembly -flavors before running them. The first build of either flavor also downloads and +scripts generate both the `browser` and `wasi` variations by default, you must build **both** +WebAssembly flavors before running them, or pass `--target-os` to regenerate only the flavor you +built. The first build of either flavor also downloads and provisions the Emscripten SDK (emsdk) automatically. From the repository root: **Windows:** ```cmd -.\build.cmd clr+libs -os browser -c Debug -.\build.cmd clr+libs -os wasi -c Debug +.\build.cmd clr+libs -os browser -c Release +.\build.cmd clr+libs -os wasi -c Release ``` **Linux/macOS:** ```bash -./build.sh clr+libs -os browser -c Debug -./build.sh clr+libs -os wasi -c Debug +./build.sh clr+libs -os browser -c Release +./build.sh clr+libs -os wasi -c Release ``` Notes: @@ -77,28 +79,59 @@ Notes: - If a required `testhost` scan path or crossgen2 is missing, the script stops and prints the exact `build` command needed to produce it. +## Which configuration to generate from + +Generate the checked-in tables from a **Release** build. The CI check regenerates them from +Release and compares exactly. + +The tables are linked into every configuration's corerun, so their entries must not depend on the +configuration. Debug and Release libraries, a Checked or Release CoreLib, and a Checked or Release +crossgen2 all produce the same P/Invoke, reverse-thunk and interpreter-to-managed entries. The +trailing `// , ...` comments on the P/Invoke table are the exception. They list the +assemblies that still reference each import after the library build trims them, and that set +differs by configuration. For example, Debug `System.Net.NameResolution` keeps a reference to +`SystemNative_GetErrNo` that Release trims. Tables regenerated from Debug therefore show +comment-only diffs that the CI check rejects. + ## Running the generator Once both flavors are built, run the script from anywhere (it resolves the repo root itself): **Windows:** ```cmd -src\coreclr\vm\wasm\generate-coreclr-helpers.cmd -c Debug +src\coreclr\vm\wasm\generate-coreclr-helpers.cmd -c Release ``` **Linux/macOS:** ```bash -src/coreclr/vm/wasm/generate-coreclr-helpers.sh -c Debug +src/coreclr/vm/wasm/generate-coreclr-helpers.sh -c Release ``` ### Options | Option | Description | |--------|-------------| -| `-c`, `--configuration ` | Build configuration (default: `Debug`). Determines the default scan paths. | +| `-c`, `--configuration ` | Build configuration (default: `Release`). Determines the default scan paths. | | `-s`, `--scan-path ` | Override the default **browser** scan path. | | `-w`, `--wasi-scan-path ` | Override the default **wasi** scan path. | +| `-t`, `--target-os ` | Regenerate only this flavor (default: both). | | `-h`, `--help` | Show usage. | After running, review and commit any changes to the generated files under `src/coreclr/vm/wasm/browser/` and `src/coreclr/vm/wasm/wasi/`. + +## CI check + +The Release `browser_wasm` CoreCLR build leg and the `wasi_wasm` CoreCLR build-only leg in +`eng/pipelines/runtime.yml` run +[`wasm-coreclr-callhelpers-check.yml`](../../../../eng/pipelines/common/templates/wasm-coreclr-callhelpers-check.yml) +after building. It regenerates the leg's own flavor with `--target-os` and fails if +`src/coreclr/vm/wasm//` differs from what is committed, printing the diff and the commands to +fix it. A stale table still builds, but the corerun that links it crashes at run time; a missing +`[UnmanagedCallersOnly]` thunk, for example, fails startup with +`GetUnmanagedCallersOnlyThunk: unknown thunk`. Library tests do not catch this because the app +relink generates its own tables. + +The legs are triggered by the `wasm_coreclr_callhelpers` path subset in +`eng/pipelines/common/evaluate-default-paths.yml`, which covers the library sources, CoreLib and +the generator, so a change that only touches managed interop still runs the check. diff --git a/src/coreclr/vm/wasm/generate-coreclr-helpers.proj b/src/coreclr/vm/wasm/generate-coreclr-helpers.proj index 6a4b06404a9d72..627b095bb5f19f 100644 --- a/src/coreclr/vm/wasm/generate-coreclr-helpers.proj +++ b/src/coreclr/vm/wasm/generate-coreclr-helpers.proj @@ -6,10 +6,14 @@ of once per shell language. See generate-coreclr-helpers.md. Properties: - Configuration Debug (default), Release or Checked. Has to match the configuration the + Configuration Release (the scripts' default), Debug or Checked. Has to match the configuration the scanned assemblies were built in. BrowserScanPath Overrides the browser scan path. WasiScanPath Overrides the wasi scan path. + CallHelperTargetOS + browser or wasi to regenerate only that flavor's tables (default: both). + Lets a build that produced only one flavor, such as a CI leg, regenerate + its own tables. --> @@ -31,11 +35,11 @@ - <_WasmCallHelperTarget Include="browser"> + <_WasmCallHelperTarget Include="browser" Condition="'$(CallHelperTargetOS)' == '' or '$(CallHelperTargetOS)' == 'browser'"> $([MSBuild]::EnsureTrailingSlash('$(BrowserScanPath)')) $(_TesthostDir)$(NetCoreAppCurrent)-browser-$(Configuration)-wasm/shared/Microsoft.NETCore.App/$(_FrameworkVersion)/ - <_WasmCallHelperTarget Include="wasi"> + <_WasmCallHelperTarget Include="wasi" Condition="'$(CallHelperTargetOS)' == '' or '$(CallHelperTargetOS)' == 'wasi'"> $([MSBuild]::EnsureTrailingSlash('$(WasiScanPath)')) $(_TesthostDir)$(NetCoreAppCurrent)-wasi-$(Configuration)-wasm/shared/Microsoft.NETCore.App/$(_FrameworkVersion)/ @@ -81,8 +85,13 @@ + + + + - + <_TargetOS>%(_WasmCallHelperTarget.Identity) <_ScanPath>%(_WasmCallHelperTarget.ScanPath) diff --git a/src/coreclr/vm/wasm/generate-coreclr-helpers.sh b/src/coreclr/vm/wasm/generate-coreclr-helpers.sh index c45bf9f729eda6..0fcefa39e21bfe 100755 --- a/src/coreclr/vm/wasm/generate-coreclr-helpers.sh +++ b/src/coreclr/vm/wasm/generate-coreclr-helpers.sh @@ -3,16 +3,18 @@ set -euo pipefail # Default configuration -configuration="Debug" +configuration="Release" browser_scan_path_override="" wasi_scan_path_override="" +target_os="" usage="Usage: $0 [options] Options: - -c, --configuration Build configuration (default: Debug) + -c, --configuration Build configuration (default: Release) -s, --scan-path Override the default browser scan path -w, --wasi-scan-path Override the default wasi scan path + -t, --target-os Regenerate only this flavor (default: both) -h, --help Show this help message" # Parse arguments @@ -30,6 +32,10 @@ while [[ $# -gt 0 ]]; do wasi_scan_path_override="$2" shift 2 ;; + -t|--target-os) + target_os="$2" + shift 2 + ;; -h|--help) echo "$usage" exit 0 @@ -53,6 +59,16 @@ case "$config_lower" in ;; esac +target_os="$(echo "$target_os" | tr '[:upper:]' '[:lower:]')" +case "$target_os" in + ""|browser|wasi) + ;; + *) + echo "Error: Invalid target OS \"$target_os\". Must be browser or wasi." + exit 1 + ;; +esac + # Get the repo root (script is in src/coreclr/vm/wasm) script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd "$script_dir/../../../.." && pwd)" @@ -78,6 +94,10 @@ if [[ -n "$wasi_scan_path_override" ]]; then args+=("-p:WasiScanPath=$wasi_scan_path_override") fi +if [[ -n "$target_os" ]]; then + args+=("-p:CallHelperTargetOS=$target_os") +fi + ./dotnet.sh "${args[@]}" echo "Done!" diff --git a/src/coreclr/vm/wasm/wasi/callhelpers-pinvoke.cpp b/src/coreclr/vm/wasm/wasi/callhelpers-pinvoke.cpp index 0f0f9ca8e353ab..b90d906e3ccdee 100644 --- a/src/coreclr/vm/wasm/wasi/callhelpers-pinvoke.cpp +++ b/src/coreclr/vm/wasm/wasi/callhelpers-pinvoke.cpp @@ -344,7 +344,7 @@ static const Entry s_libSystem_Native [] = { DllImportEntry(SystemNative_GetCryptographicallySecureRandomBytes) // System.IO.Compression DllImportEntry(SystemNative_GetCwd) // System.Private.CoreLib DllImportEntry(SystemNative_GetDefaultSearchOrderPseudoHandle) // System.Private.CoreLib - DllImportEntry(SystemNative_GetErrNo) // System.Net.NameResolution, System.Private.CoreLib + DllImportEntry(SystemNative_GetErrNo) // System.Private.CoreLib DllImportEntry(SystemNative_GetHostEntryForName) // System.Net.NameResolution DllImportEntry(SystemNative_GetHostName) // System.Net.NameResolution DllImportEntry(SystemNative_GetIPv4Address) // System.Net.Primitives, System.Net.Sockets @@ -369,7 +369,7 @@ static const Entry s_libSystem_Native [] = { DllImportEntry(SystemNative_GetSystemTimeAsTicks) // System.Private.CoreLib DllImportEntry(SystemNative_GetTimeZoneData) // System.Private.CoreLib DllImportEntry(SystemNative_GetTimestamp) // System.Private.CoreLib - DllImportEntry(SystemNative_GetWasiSocketDescriptor) // System.Net.Sockets + DllImportEntry(SystemNative_GetWasiSocketDescriptor) // System.Private.CoreLib DllImportEntry(SystemNative_IsMemfdSupported) // System.IO.MemoryMappedFiles DllImportEntry(SystemNative_LChflags) // System.Private.CoreLib DllImportEntry(SystemNative_LChflagsCanSetHiddenFlag) // System.Private.CoreLib @@ -446,7 +446,7 @@ static const Entry s_libSystem_Native [] = { DllImportEntry(SystemNative_TryGetIPPacketInformation) // System.Net.Sockets DllImportEntry(SystemNative_UTimensat) // System.Private.CoreLib DllImportEntry(SystemNative_Unlink) // System.IO.MemoryMappedFiles, System.Private.CoreLib - DllImportEntry(SystemNative_WasiSubscribeSocketPollable) // System.Net.Sockets + DllImportEntry(SystemNative_WasiSubscribeSocketPollable) // System.Private.CoreLib DllImportEntry(SystemNative_Write) // System.Console, System.Net.Sockets, System.Private.CoreLib DllImportEntry(SystemNative_WriteToNonblocking) // System.Private.CoreLib DllImportEntry(SystemNative_WriteV) // System.Private.CoreLib