From 4ad6d5143a08790c1c64e54907a4ff72585e7939 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 11 May 2026 23:42:15 +0000 Subject: [PATCH 01/12] Initial plan From 41de9f8114569e4303b695c7b821ec0ca40b9263 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 11 May 2026 23:45:30 +0000 Subject: [PATCH 02/12] Split package publishing into ordered buckets with feed availability polling Agent-Logs-Url: https://github.com/dotnet/workload-versions/sessions/d59ee713-02c8-46e2-9df1-537b310335df Co-authored-by: marcpopMSFT <12663534+marcpopMSFT@users.noreply.github.com> --- eng/pipelines/official.yml | 68 +++++++++++++++-- eng/split-package-publish-buckets.ps1 | 87 +++++++++++++++++++++ eng/wait-for-package-availability.ps1 | 105 ++++++++++++++++++++++++++ 3 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 eng/split-package-publish-buckets.ps1 create mode 100644 eng/wait-for-package-availability.ps1 diff --git a/eng/pipelines/official.yml b/eng/pipelines/official.yml index 42d0de864..c9aa06241 100644 --- a/eng/pipelines/official.yml +++ b/eng/pipelines/official.yml @@ -254,24 +254,82 @@ extends: inputs: artifactName: Artifacts targetPath: $(System.DefaultWorkingDirectory)/artifacts + - powershell: >- + eng/split-package-publish-buckets.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages' + displayName: 🟣 Split packages into publish buckets # 1ES docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs/nuget-packages # DotNetCoreCLI@2 docs: https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/dotnet-core-cli-v2 - ${{ if eq(parameters.publishToAzDO, true) }}: - task: 1ES.PublishNuget@1 - displayName: 🟣 Publish packages to AzDO + displayName: 🟣 Publish packs to AzDO inputs: useDotNetTask: true - packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/**/*.nupkg + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs/**/*.nupkg + packageParentPath: $(System.DefaultWorkingDirectory)/artifacts + publishVstsFeed: ${{ parameters.azDOPublishFeed }} + - powershell: >- + eng/wait-for-package-availability.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs' + -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ parameters.azDOPublishFeed }}/nuget/v3/index.json' + displayName: 🟣 Wait for packs in AzDO + - task: 1ES.PublishNuget@1 + displayName: 🟣 Publish manifests to AzDO + inputs: + useDotNetTask: true + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests/**/*.nupkg + packageParentPath: $(System.DefaultWorkingDirectory)/artifacts + publishVstsFeed: ${{ parameters.azDOPublishFeed }} + - powershell: >- + eng/wait-for-package-availability.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests' + -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ parameters.azDOPublishFeed }}/nuget/v3/index.json' + displayName: 🟣 Wait for manifests in AzDO + - task: 1ES.PublishNuget@1 + displayName: 🟣 Publish workload set to AzDO + inputs: + useDotNetTask: true + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/workloadSet/**/*.nupkg packageParentPath: $(System.DefaultWorkingDirectory)/artifacts publishVstsFeed: ${{ parameters.azDOPublishFeed }} - ${{ if eq(parameters.publishToNuGet, true) }}: - task: 1ES.PublishNuget@1 - displayName: 🟣 Publish packages to NuGet.org + displayName: 🟣 Publish packs to NuGet.org + inputs: + useDotNetTask: false + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs/**/*.nupkg + packageParentPath: $(System.DefaultWorkingDirectory)/artifacts + nuGetFeedType: external + publishVstsFeed: https://api.nuget.org/v3/index.json + # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a + publishFeedCredentials: nuget.org (dotnetframework) + - powershell: >- + eng/wait-for-package-availability.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs' + -feedIndexUrl 'https://api.nuget.org/v3/index.json' + displayName: 🟣 Wait for packs in NuGet.org + - task: 1ES.PublishNuget@1 + displayName: 🟣 Publish manifests to NuGet.org + inputs: + useDotNetTask: false + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests/**/*.nupkg + packageParentPath: $(System.DefaultWorkingDirectory)/artifacts + nuGetFeedType: external + publishVstsFeed: https://api.nuget.org/v3/index.json + # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a + publishFeedCredentials: nuget.org (dotnetframework) + - powershell: >- + eng/wait-for-package-availability.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests' + -feedIndexUrl 'https://api.nuget.org/v3/index.json' + displayName: 🟣 Wait for manifests in NuGet.org + - task: 1ES.PublishNuget@1 + displayName: 🟣 Publish workload set to NuGet.org inputs: useDotNetTask: false - packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/**/*.nupkg + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/workloadSet/**/*.nupkg packageParentPath: $(System.DefaultWorkingDirectory)/artifacts nuGetFeedType: external publishVstsFeed: https://api.nuget.org/v3/index.json # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a - publishFeedCredentials: nuget.org (dotnetframework) \ No newline at end of file + publishFeedCredentials: nuget.org (dotnetframework) diff --git a/eng/split-package-publish-buckets.ps1 b/eng/split-package-publish-buckets.ps1 new file mode 100644 index 000000000..804eab136 --- /dev/null +++ b/eng/split-package-publish-buckets.ps1 @@ -0,0 +1,87 @@ +# This script splits package files into publish buckets. +# The buckets are: +# 1) packs: all non-manifest packages except the workload set package +# 2) manifests: package IDs containing 'manifest' +# 3) workloadSet: package IDs matching Microsoft.NET.Workloads. + +param ( + [Parameter(Mandatory = $true)] [string] $packagesPath +) + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function Get-PackageMetadata { + param ( + [Parameter(Mandatory = $true)] [string] $packageFilePath + ) + + $archive = [System.IO.Compression.ZipFile]::OpenRead($packageFilePath) + try { + $nuspecEntry = $archive.Entries | Where-Object { $_.Name.EndsWith('.nuspec', [System.StringComparison]::OrdinalIgnoreCase) } | Select-Object -First 1 + if (-not $nuspecEntry) { + throw "Could not find .nuspec in package '$packageFilePath'." + } + + $stream = $nuspecEntry.Open() + try { + $reader = New-Object System.IO.StreamReader($stream) + try { + $nuspecXml = [xml]$reader.ReadToEnd() + } finally { + $reader.Dispose() + } + } finally { + $stream.Dispose() + } + + $id = $nuspecXml.package.metadata.id + if ([string]::IsNullOrWhiteSpace($id)) { + throw "Package ID is missing in '$packageFilePath'." + } + + return $id + } finally { + $archive.Dispose() + } +} + +$packagesPath = (Resolve-Path -Path $packagesPath).Path +$bucketRoot = Join-Path -Path $packagesPath -ChildPath 'publishBuckets' + +if (Test-Path -Path $bucketRoot) { + Remove-Item -Path $bucketRoot -Recurse -Force +} + +$bucketDirectories = @{ + packs = (Join-Path -Path $bucketRoot -ChildPath 'packs') + manifests = (Join-Path -Path $bucketRoot -ChildPath 'manifests') + workloadSet = (Join-Path -Path $bucketRoot -ChildPath 'workloadSet') +} + +$bucketDirectories.Values | ForEach-Object { + $null = New-Item -Path $_ -ItemType Directory -Force +} + +$allPackages = Get-ChildItem -Path $packagesPath -Recurse -Filter *.nupkg -File | Where-Object { $_.FullName -notlike "$bucketRoot*" } +if (-not $allPackages) { + throw "No .nupkg files found under '$packagesPath'." +} + +foreach ($package in $allPackages) { + $packageId = Get-PackageMetadata -packageFilePath $package.FullName + + $bucketName = 'packs' + if ($packageId -match '^microsoft\.net\.workloads\.\d+\.\d+\.\d+$') { + $bucketName = 'workloadSet' + } elseif ($packageId -match 'manifest') { + $bucketName = 'manifests' + } + + Copy-Item -Path $package.FullName -Destination $bucketDirectories[$bucketName] -Force +} + +Write-Host "Split packages under '$packagesPath' into publish buckets:" +$bucketDirectories.GetEnumerator() | ForEach-Object { + $count = (Get-ChildItem -Path $_.Value -Filter *.nupkg -File).Count + Write-Host " $($_.Key): $count" +} diff --git a/eng/wait-for-package-availability.ps1 b/eng/wait-for-package-availability.ps1 new file mode 100644 index 000000000..ff19dcb48 --- /dev/null +++ b/eng/wait-for-package-availability.ps1 @@ -0,0 +1,105 @@ +# This script waits for package versions to appear on a NuGet V3 feed. +# It polls package versions sequentially every 5 minutes. + +param ( + [Parameter(Mandatory = $true)] [string] $packagesPath, + [Parameter(Mandatory = $true)] [string] $feedIndexUrl +) + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function Get-PackageMetadata { + param ( + [Parameter(Mandatory = $true)] [string] $packageFilePath + ) + + $archive = [System.IO.Compression.ZipFile]::OpenRead($packageFilePath) + try { + $nuspecEntry = $archive.Entries | Where-Object { $_.Name.EndsWith('.nuspec', [System.StringComparison]::OrdinalIgnoreCase) } | Select-Object -First 1 + if (-not $nuspecEntry) { + throw "Could not find .nuspec in package '$packageFilePath'." + } + + $stream = $nuspecEntry.Open() + try { + $reader = New-Object System.IO.StreamReader($stream) + try { + $nuspecXml = [xml]$reader.ReadToEnd() + } finally { + $reader.Dispose() + } + } finally { + $stream.Dispose() + } + + $id = $nuspecXml.package.metadata.id + $version = $nuspecXml.package.metadata.version + if ([string]::IsNullOrWhiteSpace($id) -or [string]::IsNullOrWhiteSpace($version)) { + throw "Package ID/version is missing in '$packageFilePath'." + } + + return @{ + Id = $id + Version = $version + } + } finally { + $archive.Dispose() + } +} + +function Get-PackageBaseAddress { + param ( + [Parameter(Mandatory = $true)] [string] $indexUrl + ) + + $index = Invoke-RestMethod -Method Get -Uri $indexUrl + $resource = $index.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress/3.0.0*' } | Select-Object -First 1 + if (-not $resource) { + throw "Could not find PackageBaseAddress in '$indexUrl'." + } + + return $resource.'@id'.TrimEnd('/') +} + +$packagesPath = (Resolve-Path -Path $packagesPath).Path +$packageFiles = Get-ChildItem -Path $packagesPath -Filter *.nupkg -File +if (-not $packageFiles) { + throw "No .nupkg files found under '$packagesPath'." +} + +$packageBaseAddress = Get-PackageBaseAddress -indexUrl $feedIndexUrl +$pollingIntervalSeconds = 300 +$maxAttempts = 60 + +foreach ($packageFile in $packageFiles) { + $packageMetadata = Get-PackageMetadata -packageFilePath $packageFile.FullName + $idLower = $packageMetadata.Id.ToLowerInvariant() + $versionLower = $packageMetadata.Version.ToLowerInvariant() + $registrationUrl = "$packageBaseAddress/$idLower/index.json" + + Write-Host "Waiting for $($packageMetadata.Id) $($packageMetadata.Version) on $feedIndexUrl" + + $isAvailable = $false + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + try { + $registrationIndex = Invoke-RestMethod -Method Get -Uri $registrationUrl + if ($registrationIndex.versions -contains $versionLower) { + Write-Host " Available after attempt $attempt." + $isAvailable = $true + break + } + + Write-Host " Not available yet (attempt $attempt of $maxAttempts)." + } catch { + Write-Host " Failed to query package index on attempt $attempt of $maxAttempts. Error: $($_.Exception.Message)" + } + + if ($attempt -lt $maxAttempts) { + Start-Sleep -Seconds $pollingIntervalSeconds + } + } + + if (-not $isAvailable) { + throw "Timed out waiting for $($packageMetadata.Id) $($packageMetadata.Version) in '$feedIndexUrl'." + } +} From 82d340daa0cf31af88b7f2b7b2a6e4caab715153 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 11 May 2026 23:46:00 +0000 Subject: [PATCH 03/12] Make publish bucket package-id matching explicitly case-insensitive Agent-Logs-Url: https://github.com/dotnet/workload-versions/sessions/d59ee713-02c8-46e2-9df1-537b310335df Co-authored-by: marcpopMSFT <12663534+marcpopMSFT@users.noreply.github.com> --- eng/split-package-publish-buckets.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/split-package-publish-buckets.ps1 b/eng/split-package-publish-buckets.ps1 index 804eab136..f6ff16f8c 100644 --- a/eng/split-package-publish-buckets.ps1 +++ b/eng/split-package-publish-buckets.ps1 @@ -71,9 +71,9 @@ foreach ($package in $allPackages) { $packageId = Get-PackageMetadata -packageFilePath $package.FullName $bucketName = 'packs' - if ($packageId -match '^microsoft\.net\.workloads\.\d+\.\d+\.\d+$') { + if ($packageId -imatch '^microsoft\.net\.workloads\.\d+\.\d+\.\d+$') { $bucketName = 'workloadSet' - } elseif ($packageId -match 'manifest') { + } elseif ($packageId -imatch 'manifest') { $bucketName = 'manifests' } From 8dbc126b09473b6dae1acd11ea1b7fc8d80670b1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 11 May 2026 23:46:40 +0000 Subject: [PATCH 04/12] Handle workload set package-id variants when splitting publish buckets Agent-Logs-Url: https://github.com/dotnet/workload-versions/sessions/d59ee713-02c8-46e2-9df1-537b310335df Co-authored-by: marcpopMSFT <12663534+marcpopMSFT@users.noreply.github.com> --- eng/split-package-publish-buckets.ps1 | 4 +++- eng/wait-for-package-availability.ps1 | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/split-package-publish-buckets.ps1 b/eng/split-package-publish-buckets.ps1 index f6ff16f8c..d2b9759ce 100644 --- a/eng/split-package-publish-buckets.ps1 +++ b/eng/split-package-publish-buckets.ps1 @@ -70,8 +70,10 @@ if (-not $allPackages) { foreach ($package in $allPackages) { $packageId = Get-PackageMetadata -packageFilePath $package.FullName + $packageIdLower = $packageId.ToLowerInvariant() + $bucketName = 'packs' - if ($packageId -imatch '^microsoft\.net\.workloads\.\d+\.\d+\.\d+$') { + if ($packageIdLower.StartsWith('microsoft.net.workloads.') -and $packageIdLower -notmatch '\.msi\.') { $bucketName = 'workloadSet' } elseif ($packageId -imatch 'manifest') { $bucketName = 'manifests' diff --git a/eng/wait-for-package-availability.ps1 b/eng/wait-for-package-availability.ps1 index ff19dcb48..b0a76e700 100644 --- a/eng/wait-for-package-availability.ps1 +++ b/eng/wait-for-package-availability.ps1 @@ -1,5 +1,5 @@ # This script waits for package versions to appear on a NuGet V3 feed. -# It polls package versions sequentially every 5 minutes. +# It polls package versions sequentially using the interval configured below. param ( [Parameter(Mandatory = $true)] [string] $packagesPath, From 7d2efd5521ab3cd40305b5d2eef450b49f1f4917 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 11 May 2026 23:47:37 +0000 Subject: [PATCH 05/12] Fix AzDO feed polling URL and make polling settings configurable Agent-Logs-Url: https://github.com/dotnet/workload-versions/sessions/d59ee713-02c8-46e2-9df1-537b310335df Co-authored-by: marcpopMSFT <12663534+marcpopMSFT@users.noreply.github.com> --- eng/pipelines/official.yml | 4 ++-- eng/split-package-publish-buckets.ps1 | 8 ++++++-- eng/wait-for-package-availability.ps1 | 7 +++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/official.yml b/eng/pipelines/official.yml index c9aa06241..c90c78e57 100644 --- a/eng/pipelines/official.yml +++ b/eng/pipelines/official.yml @@ -271,7 +271,7 @@ extends: - powershell: >- eng/wait-for-package-availability.ps1 -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs' - -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ parameters.azDOPublishFeed }}/nuget/v3/index.json' + -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ replace(parameters.azDOPublishFeed, '/', '/_packaging/') }}/nuget/v3/index.json' displayName: 🟣 Wait for packs in AzDO - task: 1ES.PublishNuget@1 displayName: 🟣 Publish manifests to AzDO @@ -283,7 +283,7 @@ extends: - powershell: >- eng/wait-for-package-availability.ps1 -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests' - -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ parameters.azDOPublishFeed }}/nuget/v3/index.json' + -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ replace(parameters.azDOPublishFeed, '/', '/_packaging/') }}/nuget/v3/index.json' displayName: 🟣 Wait for manifests in AzDO - task: 1ES.PublishNuget@1 displayName: 🟣 Publish workload set to AzDO diff --git a/eng/split-package-publish-buckets.ps1 b/eng/split-package-publish-buckets.ps1 index d2b9759ce..942b6d84e 100644 --- a/eng/split-package-publish-buckets.ps1 +++ b/eng/split-package-publish-buckets.ps1 @@ -10,6 +10,10 @@ param ( Add-Type -AssemblyName System.IO.Compression.FileSystem +$workloadSetPrefix = 'microsoft.net.workloads.' +$msiExclusionPattern = '\.msi\.' +$manifestPattern = 'manifest' + function Get-PackageMetadata { param ( [Parameter(Mandatory = $true)] [string] $packageFilePath @@ -73,9 +77,9 @@ foreach ($package in $allPackages) { $packageIdLower = $packageId.ToLowerInvariant() $bucketName = 'packs' - if ($packageIdLower.StartsWith('microsoft.net.workloads.') -and $packageIdLower -notmatch '\.msi\.') { + if ($packageIdLower.StartsWith($workloadSetPrefix) -and $packageIdLower -notmatch $msiExclusionPattern) { $bucketName = 'workloadSet' - } elseif ($packageId -imatch 'manifest') { + } elseif ($packageId -imatch $manifestPattern) { $bucketName = 'manifests' } diff --git a/eng/wait-for-package-availability.ps1 b/eng/wait-for-package-availability.ps1 index b0a76e700..4351afc5e 100644 --- a/eng/wait-for-package-availability.ps1 +++ b/eng/wait-for-package-availability.ps1 @@ -3,7 +3,9 @@ param ( [Parameter(Mandatory = $true)] [string] $packagesPath, - [Parameter(Mandatory = $true)] [string] $feedIndexUrl + [Parameter(Mandatory = $true)] [string] $feedIndexUrl, + [int] $pollingIntervalSeconds = 300, + [int] $maxAttempts = 60 ) Add-Type -AssemblyName System.IO.Compression.FileSystem @@ -68,9 +70,6 @@ if (-not $packageFiles) { } $packageBaseAddress = Get-PackageBaseAddress -indexUrl $feedIndexUrl -$pollingIntervalSeconds = 300 -$maxAttempts = 60 - foreach ($packageFile in $packageFiles) { $packageMetadata = Get-PackageMetadata -packageFilePath $packageFile.FullName $idLower = $packageMetadata.Id.ToLowerInvariant() From 829a437ab34de7cb7a5c0b2f0195ffe9fb10f417 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 11 May 2026 23:48:09 +0000 Subject: [PATCH 06/12] Align manifest bucket match logic with lowercase package-id comparisons Agent-Logs-Url: https://github.com/dotnet/workload-versions/sessions/d59ee713-02c8-46e2-9df1-537b310335df Co-authored-by: marcpopMSFT <12663534+marcpopMSFT@users.noreply.github.com> --- eng/split-package-publish-buckets.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/split-package-publish-buckets.ps1 b/eng/split-package-publish-buckets.ps1 index 942b6d84e..08c2b3c34 100644 --- a/eng/split-package-publish-buckets.ps1 +++ b/eng/split-package-publish-buckets.ps1 @@ -79,7 +79,7 @@ foreach ($package in $allPackages) { $bucketName = 'packs' if ($packageIdLower.StartsWith($workloadSetPrefix) -and $packageIdLower -notmatch $msiExclusionPattern) { $bucketName = 'workloadSet' - } elseif ($packageId -imatch $manifestPattern) { + } elseif ($packageIdLower -match $manifestPattern) { $bucketName = 'manifests' } From cd5dbd7fccb0c747fa7cca3a4767ec5ce4cc6c28 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Tue, 12 May 2026 11:12:22 -0700 Subject: [PATCH 07/12] Fail fast on non-retryable feed polling HTTP errors --- eng/wait-for-package-availability.ps1 | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/eng/wait-for-package-availability.ps1 b/eng/wait-for-package-availability.ps1 index 4351afc5e..fe542a062 100644 --- a/eng/wait-for-package-availability.ps1 +++ b/eng/wait-for-package-availability.ps1 @@ -63,6 +63,22 @@ function Get-PackageBaseAddress { return $resource.'@id'.TrimEnd('/') } +function Get-HttpStatusCodeFromException { + param ( + [Parameter(Mandatory = $true)] $exception + ) + + if ($exception -and $exception.Response) { + try { + return [int]$exception.Response.StatusCode.value__ + } catch { + return $null + } + } + + return $null +} + $packagesPath = (Resolve-Path -Path $packagesPath).Path $packageFiles = Get-ChildItem -Path $packagesPath -Filter *.nupkg -File if (-not $packageFiles) { @@ -90,7 +106,16 @@ foreach ($packageFile in $packageFiles) { Write-Host " Not available yet (attempt $attempt of $maxAttempts)." } catch { - Write-Host " Failed to query package index on attempt $attempt of $maxAttempts. Error: $($_.Exception.Message)" + $statusCode = Get-HttpStatusCodeFromException -exception $_.Exception + if ($statusCode -in @(400, 401, 403, 409, 410, 422)) { + throw "Non-retryable HTTP status code $statusCode querying '$registrationUrl'. Error: $($_.Exception.Message)" + } + + if ($statusCode) { + Write-Host " Failed to query package index on attempt $attempt of $maxAttempts with HTTP $statusCode. Error: $($_.Exception.Message)" + } else { + Write-Host " Failed to query package index on attempt $attempt of $maxAttempts. Error: $($_.Exception.Message)" + } } if ($attempt -lt $maxAttempts) { From f3220f2ea083d2a830d37f685cca2621a0b46cc2 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Wed, 13 May 2026 16:23:14 -0700 Subject: [PATCH 08/12] Add checkout step to publish deployment job Deployment jobs don't automatically checkout source code, so the eng/split-package-publish-buckets.ps1 and eng/wait-for-package-availability.ps1 scripts were not available. Add checkout: self with fetchDepth: 1 to make the scripts accessible during the publish stage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/official.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/official.yml b/eng/pipelines/official.yml index c90c78e57..e7484462f 100644 --- a/eng/pipelines/official.yml +++ b/eng/pipelines/official.yml @@ -248,6 +248,8 @@ extends: runOnce: deploy: steps: + - checkout: self + fetchDepth: 1 # 1ES docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/inputs - task: 1ES.DownloadPipelineArtifact@1 displayName: 🟣 Download build artifacts From b7c8dac8e0614a4f185088978bef48441d67f157 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Wed, 13 May 2026 16:29:33 -0700 Subject: [PATCH 09/12] Revert "Add checkout step to publish deployment job" This reverts commit f3220f2ea083d2a830d37f685cca2621a0b46cc2. --- eng/pipelines/official.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/official.yml b/eng/pipelines/official.yml index e7484462f..c90c78e57 100644 --- a/eng/pipelines/official.yml +++ b/eng/pipelines/official.yml @@ -248,8 +248,6 @@ extends: runOnce: deploy: steps: - - checkout: self - fetchDepth: 1 # 1ES docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/inputs - task: 1ES.DownloadPipelineArtifact@1 displayName: 🟣 Download build artifacts From 21f877b71fba7d4d165f83eb52de58728e46002b Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Wed, 13 May 2026 16:57:14 -0700 Subject: [PATCH 10/12] Ship publish scripts as build artifacts for release job 1ES release jobs block checkout, so eng/ scripts aren't available on disk in the Publish stage. Copy the two publish scripts into the artifacts/publishScripts directory during the Build stage so they're downloaded alongside the packages in the Publish stage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/official.yml | 10 +++++----- eng/pipelines/templates/jobs/workload-build.yml | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/official.yml b/eng/pipelines/official.yml index c90c78e57..8bb2fd6df 100644 --- a/eng/pipelines/official.yml +++ b/eng/pipelines/official.yml @@ -255,7 +255,7 @@ extends: artifactName: Artifacts targetPath: $(System.DefaultWorkingDirectory)/artifacts - powershell: >- - eng/split-package-publish-buckets.ps1 + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/split-package-publish-buckets.ps1 -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages' displayName: 🟣 Split packages into publish buckets # 1ES docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs/nuget-packages @@ -269,7 +269,7 @@ extends: packageParentPath: $(System.DefaultWorkingDirectory)/artifacts publishVstsFeed: ${{ parameters.azDOPublishFeed }} - powershell: >- - eng/wait-for-package-availability.ps1 + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1 -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs' -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ replace(parameters.azDOPublishFeed, '/', '/_packaging/') }}/nuget/v3/index.json' displayName: 🟣 Wait for packs in AzDO @@ -281,7 +281,7 @@ extends: packageParentPath: $(System.DefaultWorkingDirectory)/artifacts publishVstsFeed: ${{ parameters.azDOPublishFeed }} - powershell: >- - eng/wait-for-package-availability.ps1 + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1 -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests' -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ replace(parameters.azDOPublishFeed, '/', '/_packaging/') }}/nuget/v3/index.json' displayName: 🟣 Wait for manifests in AzDO @@ -304,7 +304,7 @@ extends: # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a publishFeedCredentials: nuget.org (dotnetframework) - powershell: >- - eng/wait-for-package-availability.ps1 + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1 -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs' -feedIndexUrl 'https://api.nuget.org/v3/index.json' displayName: 🟣 Wait for packs in NuGet.org @@ -319,7 +319,7 @@ extends: # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a publishFeedCredentials: nuget.org (dotnetframework) - powershell: >- - eng/wait-for-package-availability.ps1 + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1 -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests' -feedIndexUrl 'https://api.nuget.org/v3/index.json' displayName: 🟣 Wait for manifests in NuGet.org diff --git a/eng/pipelines/templates/jobs/workload-build.yml b/eng/pipelines/templates/jobs/workload-build.yml index 0104ae9a7..496db2ac5 100644 --- a/eng/pipelines/templates/jobs/workload-build.yml +++ b/eng/pipelines/templates/jobs/workload-build.yml @@ -115,6 +115,12 @@ jobs: displayName: 🟣 Build solution # Name is required to reference the variables created within this build step in other stages. name: BuildSolution + - powershell: | + $destDir = "$(System.DefaultWorkingDirectory)/artifacts/publishScripts" + New-Item -ItemType Directory -Path $destDir -Force | Out-Null + Copy-Item "eng/split-package-publish-buckets.ps1" $destDir + Copy-Item "eng/wait-for-package-availability.ps1" $destDir + displayName: 🟣 Copy publish scripts to artifacts - ${{ if eq(parameters.createVSInsertion, true) }}: - task: 1ES.PublishPipelineArtifact@1 displayName: 🟣 Publish workload artifacts From af6112433e8effcbe2fdc00f3142b05cef230b0d Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 14 May 2026 14:38:50 -0700 Subject: [PATCH 11/12] Fix publish script artifact path to use staging directory The Arcade template publishes the Artifacts pipeline artifact from Build.ArtifactStagingDirectory, not System.DefaultWorkingDirectory. The source repo is checked out to 'source-branch' subdirectory, and build outputs are gathered to the staging directory before publishing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/templates/jobs/workload-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/workload-build.yml b/eng/pipelines/templates/jobs/workload-build.yml index 496db2ac5..2b1a1b2b9 100644 --- a/eng/pipelines/templates/jobs/workload-build.yml +++ b/eng/pipelines/templates/jobs/workload-build.yml @@ -116,7 +116,7 @@ jobs: # Name is required to reference the variables created within this build step in other stages. name: BuildSolution - powershell: | - $destDir = "$(System.DefaultWorkingDirectory)/artifacts/publishScripts" + $destDir = "$(Build.ArtifactStagingDirectory)/artifacts/publishScripts" New-Item -ItemType Directory -Path $destDir -Force | Out-Null Copy-Item "eng/split-package-publish-buckets.ps1" $destDir Copy-Item "eng/wait-for-package-availability.ps1" $destDir From b15710126e754f6a144ad31032c938dc1ab62dcb Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 2 Jul 2026 14:02:17 -0700 Subject: [PATCH 12/12] Classify architecture-specific workload set MSI packages as workload sets Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/split-package-publish-buckets.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/split-package-publish-buckets.ps1 b/eng/split-package-publish-buckets.ps1 index 08c2b3c34..514b12579 100644 --- a/eng/split-package-publish-buckets.ps1 +++ b/eng/split-package-publish-buckets.ps1 @@ -2,7 +2,7 @@ # The buckets are: # 1) packs: all non-manifest packages except the workload set package # 2) manifests: package IDs containing 'manifest' -# 3) workloadSet: package IDs matching Microsoft.NET.Workloads. +# 3) workloadSet: package IDs starting with Microsoft.NET.Workloads. (including architecture-specific MSI variants) param ( [Parameter(Mandatory = $true)] [string] $packagesPath @@ -11,7 +11,6 @@ param ( Add-Type -AssemblyName System.IO.Compression.FileSystem $workloadSetPrefix = 'microsoft.net.workloads.' -$msiExclusionPattern = '\.msi\.' $manifestPattern = 'manifest' function Get-PackageMetadata { @@ -77,7 +76,7 @@ foreach ($package in $allPackages) { $packageIdLower = $packageId.ToLowerInvariant() $bucketName = 'packs' - if ($packageIdLower.StartsWith($workloadSetPrefix) -and $packageIdLower -notmatch $msiExclusionPattern) { + if ($packageIdLower.StartsWith($workloadSetPrefix)) { $bucketName = 'workloadSet' } elseif ($packageIdLower -match $manifestPattern) { $bucketName = 'manifests'