Fix aspnetcore perf-build CI failures found in first test runs

eng/pipelines/aspnetcore-perf-build-jobs.yml

@@ -14,11 +14,16 @@
 # the moved eng/pipelines/tools/stage-bcs-nupkg-aspnetcore.ps1, and publish one
 # pipeline artifact per config.
 #
-# Job names (Windows_x64_build / Windows_x86_build / Windows_arm64_build /
-# Linux_x64_build / Linux_arm64_build) are load-bearing:
+# Job names (Windows_build / Linux_x64_build / Linux_arm64_build) are load-bearing:
 # they must match the `dependencyJobName` values in
 # eng/pipelines/upload-build-artifacts-jobs.yml's aspnetcore_* branches.
 #
+# SUBMODULES: aspnetcore's build compiles vendored submodules (MessagePack-CSharp,
+# googletest) that eng/build.{cmd,sh} do NOT init. The reference perf-build.yml got
+# these for free via default-build.yml@self; here we init them explicitly with
+# `submodules: true` on the aspnetcore checkout (both are public GitHub repos, so
+# the mirror checkout clones them anonymously).
+#
 # PUBLIC-FEED rationale: aspnetcore's own ci-public.yml builds every public PR with
 # `_InternalRuntimeDownloadArgs` empty, proving public feeds suffice for a from-
 # source pack build. We therefore drop enable-internal-runtimes / get-delegation-sas
@@ -41,27 +46,25 @@ parameters:
 jobs:
 
 # ===========================================================================
-# Windows build jobs (one self-sufficient job per arch, mirroring the Linux
-# jobs below). Each job is independently gated by its config boolean, checks out
-# the aspnetcore mirror + this repo, runs aspnetcore's eng/build.cmd standalone
-# for its arch, packs the per-RID archive, and publishes one artifact.
+# Windows build job (x64 / x86 / arm64 in ONE agent, mirroring aspnetcore's
+# ci-public.yml Windows_build). build.cmd is invoked up to three times on the
+# same machine: x64 FIRST with -nativeToolsOnMachine (builds the native ANCM
+# bits for all Windows arches), then x86 and arm64 with -noBuildNative because
+# they cross-pack against the x64 native output. Splitting these into separate
+# agents (v1's first attempt) is not viable: x86/arm64 have no native bits of
+# their own and fail packing (NU5026 aspnetcorev2.dll not found).
 #
-# Unlike aspnetcore ci-public.yml -- which builds x64/x86/arm64 sequentially in
-# ONE job so x86/arm64 reuse the x64 step's managed build and on-machine native
-# toolchain -- these jobs run on separate agents, so each must be self-sufficient:
-#   * every arch carries -all (full managed build) and -nativeToolsOnMachine
-#     (set up the native toolchain the shared x64 step used to provide);
-#   * x86/arm64 keep -noBuildNative (faithful to ci-public's per-arch args; their
-#     native runtime bits restore from packages rather than cross-building on the
-#     x64 host). The first real queued run validates standalone native restore.
+# The whole job is gated on ANY windows config being enabled; the x64 build step
+# always runs (it is the shared native prerequisite) while each arch's staging +
+# artifact publish is gated on its own config boolean.
 #
-# Job names (Windows_x64_build / Windows_x86_build / Windows_arm64_build) are
-# load-bearing: they must match the `dependencyJobName` values in
-# eng/pipelines/upload-build-artifacts-jobs.yml's aspnetcore_*_windows branches.
+# Job name (Windows_build) is load-bearing: it must match the
+# `dependencyJobName` for all three aspnetcore_*_windows branches in
+# eng/pipelines/upload-build-artifacts-jobs.yml.
 # ===========================================================================
-- ${{ if eq(parameters.aspnetcore_x64_windows, true) }}:
-  - job: Windows_x64_build
-    displayName: 'Build: Windows x64 (perf-build)'
+- ${{ if or(eq(parameters.aspnetcore_x64_windows, true), eq(parameters.aspnetcore_x86_windows, true), eq(parameters.aspnetcore_arm64_windows, true)) }}:
+  - job: Windows_build
+    displayName: 'Build: Windows x64/x86/arm64 (perf-build)'
     timeoutInMinutes: 180
     pool:
       name: $(DncEngInternalBuildPool)
@@ -71,10 +74,10 @@ jobs:
       _StageNupkgScript: $(Agent.BuildDirectory)\s\performance\eng\pipelines\tools\stage-bcs-nupkg-aspnetcore.ps1
       _ShippingDir: $(Agent.BuildDirectory)\s\aspnetcore\artifacts\packages\Release\Shipping
       _StagingRoot: $(Build.ArtifactStagingDirectory)\bcs
-      MSBUILDUSESERVER: "1"
     steps:
     - checkout: ${{ parameters.aspnetcoreRepoAlias }}
       path: s/aspnetcore
+      submodules: true
       fetchDepth: 1
       fetchTags: false
       clean: true
@@ -84,6 +87,8 @@ jobs:
       fetchTags: false
       clean: true
 
+    # x64 first: builds native ANCM for all Windows arches (no -noBuildNative).
+    # Always runs when the job runs -- it is the shared prerequisite for x86/arm64.
     - script: >-
         eng\build.cmd
         -ci
@@ -102,150 +107,80 @@ jobs:
       env:
         MSBUILDUSESERVER: "1"
 
-    - pwsh: >-
-        & "$(_StageNupkgScript)" -Rids win-x64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)"
-      displayName: 'Stage BCS nupkg (win-x64)'
-    - task: PublishPipelineArtifact@1
-      displayName: 'Publish BuildArtifacts_windows_x64_Release_aspnetcore'
-      inputs:
-        targetPath: $(_StagingRoot)\BuildArtifacts_windows_x64_Release_aspnetcore
-        artifactName: BuildArtifacts_windows_x64_Release_aspnetcore
+    - ${{ if eq(parameters.aspnetcore_x86_windows, true) }}:
+      - script: >-
+          eng\build.cmd
+          -ci
+          -prepareMachine
+          -Configuration Release
+          -arch x86
+          -pack
+          -all
+          -noBuildJava
+          -noBuildNative
+          /p:OnlyPackPlatformSpecificPackages=true
+          $(_BuildArgs)
+          /bl:artifacts/log/Release/Build.x86.binlog
+        workingDirectory: $(_AspNetCoreRoot)
+        displayName: Build x86
+        env:
+          MSBUILDUSESERVER: "1"
 
-    - task: PublishPipelineArtifact@1
-      displayName: 'Publish Windows x64 build logs'
-      condition: always()
-      continueOnError: true
-      inputs:
-        targetPath: $(_AspNetCoreRoot)\artifacts\log
-        artifactName: Windows_x64_perf_build_Logs_Attempt_$(System.JobAttempt)
+    - ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}:
+      - script: >-
+          eng\build.cmd
+          -ci
+          -prepareMachine
+          -Configuration Release
+          -arch arm64
+          -pack
+          -noBuildJava
+          -noBuildNative
+          /p:OnlyPackPlatformSpecificPackages=true
+          $(_BuildArgs)
+          /bl:artifacts/log/Release/Build.arm64.binlog
+        workingDirectory: $(_AspNetCoreRoot)
+        displayName: Build ARM64
+        env:
+          MSBUILDUSESERVER: "1"
 
-# ===========================================================================
-# Windows x86 build job
-# ===========================================================================
-- ${{ if eq(parameters.aspnetcore_x86_windows, true) }}:
-  - job: Windows_x86_build
-    displayName: 'Build: Windows x86 (perf-build)'
-    timeoutInMinutes: 180
-    pool:
-      name: $(DncEngInternalBuildPool)
-      demands: ImageOverride -equals windows.vs2026preview.scout.amd64
-    variables:
-      _AspNetCoreRoot: $(Agent.BuildDirectory)\s\aspnetcore
-      _StageNupkgScript: $(Agent.BuildDirectory)\s\performance\eng\pipelines\tools\stage-bcs-nupkg-aspnetcore.ps1
-      _ShippingDir: $(Agent.BuildDirectory)\s\aspnetcore\artifacts\packages\Release\Shipping
-      _StagingRoot: $(Build.ArtifactStagingDirectory)\bcs
-      MSBUILDUSESERVER: "1"
-    steps:
-    - checkout: ${{ parameters.aspnetcoreRepoAlias }}
-      path: s/aspnetcore
-      fetchDepth: 1
-      fetchTags: false
-      clean: true
-    - checkout: ${{ parameters.performanceRepoAlias }}
-      path: s/performance
-      fetchDepth: 1
-      fetchTags: false
-      clean: true
+    - ${{ if eq(parameters.aspnetcore_x64_windows, true) }}:
+      - pwsh: >-
+          & "$(_StageNupkgScript)" -Rids win-x64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)"
+        displayName: 'Stage BCS nupkg (win-x64)'
+      - task: PublishPipelineArtifact@1
+        displayName: 'Publish BuildArtifacts_windows_x64_Release_aspnetcore'
+        inputs:
+          targetPath: $(_StagingRoot)\BuildArtifacts_windows_x64_Release_aspnetcore
+          artifactName: BuildArtifacts_windows_x64_Release_aspnetcore
 
-    - script: >-
-        eng\build.cmd
-        -ci
-        -prepareMachine
-        -nativeToolsOnMachine
-        -Configuration Release
-        -arch x86
-        -pack
-        -all
-        -noBuildJava
-        -noBuildNative
-        /p:OnlyPackPlatformSpecificPackages=true
-        $(_BuildArgs)
-        /bl:artifacts/log/Release/Build.x86.binlog
-      workingDirectory: $(_AspNetCoreRoot)
-      displayName: Build x86
-      env:
-        MSBUILDUSESERVER: "1"
+    - ${{ if eq(parameters.aspnetcore_x86_windows, true) }}:
+      - pwsh: >-
+          & "$(_StageNupkgScript)" -Rids win-x86 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)"
+        displayName: 'Stage BCS nupkg (win-x86)'
+      - task: PublishPipelineArtifact@1
+        displayName: 'Publish BuildArtifacts_windows_x86_Release_aspnetcore'
+        inputs:
+          targetPath: $(_StagingRoot)\BuildArtifacts_windows_x86_Release_aspnetcore
+          artifactName: BuildArtifacts_windows_x86_Release_aspnetcore
 
-    - pwsh: >-
-        & "$(_StageNupkgScript)" -Rids win-x86 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)"
-      displayName: 'Stage BCS nupkg (win-x86)'
-    - task: PublishPipelineArtifact@1
-      displayName: 'Publish BuildArtifacts_windows_x86_Release_aspnetcore'
-      inputs:
-        targetPath: $(_StagingRoot)\BuildArtifacts_windows_x86_Release_aspnetcore
-        artifactName: BuildArtifacts_windows_x86_Release_aspnetcore
+    - ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}:
+      - pwsh: >-
+          & "$(_StageNupkgScript)" -Rids win-arm64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)"
+        displayName: 'Stage BCS nupkg (win-arm64)'
+      - task: PublishPipelineArtifact@1
+        displayName: 'Publish BuildArtifacts_windows_arm64_Release_aspnetcore'
+        inputs:
+          targetPath: $(_StagingRoot)\BuildArtifacts_windows_arm64_Release_aspnetcore
+          artifactName: BuildArtifacts_windows_arm64_Release_aspnetcore
 
     - task: PublishPipelineArtifact@1
-      displayName: 'Publish Windows x86 build logs'
+      displayName: 'Publish Windows build logs'
       condition: always()
       continueOnError: true
       inputs:
         targetPath: $(_AspNetCoreRoot)\artifacts\log
-        artifactName: Windows_x86_perf_build_Logs_Attempt_$(System.JobAttempt)
-
-# ===========================================================================
-# Windows arm64 build job
-# ===========================================================================
-- ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}:
-  - job: Windows_arm64_build
-    displayName: 'Build: Windows ARM64 (perf-build)'
-    timeoutInMinutes: 180
-    pool:
-      name: $(DncEngInternalBuildPool)
-      demands: ImageOverride -equals windows.vs2026preview.scout.amd64
-    variables:
-      _AspNetCoreRoot: $(Agent.BuildDirectory)\s\aspnetcore
-      _StageNupkgScript: $(Agent.BuildDirectory)\s\performance\eng\pipelines\tools\stage-bcs-nupkg-aspnetcore.ps1
-      _ShippingDir: $(Agent.BuildDirectory)\s\aspnetcore\artifacts\packages\Release\Shipping
-      _StagingRoot: $(Build.ArtifactStagingDirectory)\bcs
-      MSBUILDUSESERVER: "1"
-    steps:
-    - checkout: ${{ parameters.aspnetcoreRepoAlias }}
-      path: s/aspnetcore
-      fetchDepth: 1
-      fetchTags: false
-      clean: true
-    - checkout: ${{ parameters.performanceRepoAlias }}
-      path: s/performance
-      fetchDepth: 1
-      fetchTags: false
-      clean: true
-
-    - script: >-
-        eng\build.cmd
-        -ci
-        -prepareMachine
-        -nativeToolsOnMachine
-        -Configuration Release
-        -arch arm64
-        -pack
-        -all
-        -noBuildJava
-        -noBuildNative
-        /p:OnlyPackPlatformSpecificPackages=true
-        $(_BuildArgs)
-        /bl:artifacts/log/Release/Build.arm64.binlog
-      workingDirectory: $(_AspNetCoreRoot)
-      displayName: Build ARM64
-      env:
-        MSBUILDUSESERVER: "1"
-
-    - pwsh: >-
-        & "$(_StageNupkgScript)" -Rids win-arm64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)"
-      displayName: 'Stage BCS nupkg (win-arm64)'
-    - task: PublishPipelineArtifact@1
-      displayName: 'Publish BuildArtifacts_windows_arm64_Release_aspnetcore'
-      inputs:
-        targetPath: $(_StagingRoot)\BuildArtifacts_windows_arm64_Release_aspnetcore
-        artifactName: BuildArtifacts_windows_arm64_Release_aspnetcore
-
-    - task: PublishPipelineArtifact@1
-      displayName: 'Publish Windows arm64 build logs'
-      condition: always()
-      continueOnError: true
-      inputs:
-        targetPath: $(_AspNetCoreRoot)\artifacts\log
-        artifactName: Windows_arm64_perf_build_Logs_Attempt_$(System.JobAttempt)
+        artifactName: Windows_perf_build_Logs_Attempt_$(System.JobAttempt)
 
 # ===========================================================================
 # Linux x64 build job
@@ -265,6 +200,7 @@ jobs:
     steps:
     - checkout: ${{ parameters.aspnetcoreRepoAlias }}
       path: s/aspnetcore
+      submodules: true
       fetchDepth: 1
       fetchTags: false
       clean: true
@@ -323,6 +259,7 @@ jobs:
     steps:
     - checkout: ${{ parameters.aspnetcoreRepoAlias }}
       path: s/aspnetcore
+      submodules: true
       fetchDepth: 1
       fetchTags: false
       clean: true
@@ -340,11 +277,22 @@ jobs:
         --pack
         --all
         --no-build-java
+        -p:CrossBuild=true
         -p:OnlyPackPlatformSpecificPackages=true
         $(_BuildArgs)
         /bl:artifacts/log/Release/Build.linux-arm64.binlog
       workingDirectory: $(_AspNetCoreRoot)
       displayName: Build linux-arm64
+      # arm64 is a cross-build from the x64 host. The NativeAOT aspnetcoretools
+      # tool links native code targeting aarch64-linux-gnu, which needs the arm64
+      # sysroot (crt objects + libc/libdl/libm/libgcc). The plain 1es-ubuntu-2204
+      # image has no /crossrootfs/arm64, so this step runs in the dotnet cross
+      # prereq container (target: crossArm64) that bundles clang + the sysroot,
+      # with ROOTFS_DIR + -p:CrossBuild=true. Mirrors aspnetcore ci-public.yml.
+      # Only this step runs in the container; pack/publish stay on the host.
+      target: crossArm64
+      env:
+        ROOTFS_DIR: /crossrootfs/arm64
 
     - pwsh: >-
         & "$(_StageNupkgScript)" -Rids linux-arm64 -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)"