Split NuGet publishing into ordered buckets with feed-availability gates

eng/pipelines/official.yml

@@ -254,24 +254,82 @@ extends:
                   inputs:
                     artifactName: Artifacts
                     targetPath: $(System.DefaultWorkingDirectory)/artifacts
+                - powershell: >-
+                    $(System.DefaultWorkingDirectory)/artifacts/publishScripts/split-package-publish-buckets.ps1
+                    -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages'
+                  displayName: 🟣 Split packages into publish buckets
                 # 1ES docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs/nuget-packages
                 # DotNetCoreCLI@2 docs: https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/dotnet-core-cli-v2
                 - ${{ if eq(parameters.publishToAzDO, true) }}:
                   - task: 1ES.PublishNuget@1
-                    displayName: 🟣 Publish packages to AzDO
+                    displayName: 🟣 Publish packs to AzDO
                     inputs:
                       useDotNetTask: true
-                      packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/**/*.nupkg
+                      packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs/**/*.nupkg
+                      packageParentPath: $(System.DefaultWorkingDirectory)/artifacts
+                      publishVstsFeed: ${{ parameters.azDOPublishFeed }}
+                  - powershell: >-
+                      $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1
+                      -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs'
+                      -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ replace(parameters.azDOPublishFeed, '/', '/_packaging/') }}/nuget/v3/index.json'
+                    displayName: 🟣 Wait for packs in AzDO
+                  - task: 1ES.PublishNuget@1
+                    displayName: 🟣 Publish manifests to AzDO
+                    inputs:
+                      useDotNetTask: true
+                      packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests/**/*.nupkg
+                      packageParentPath: $(System.DefaultWorkingDirectory)/artifacts
+                      publishVstsFeed: ${{ parameters.azDOPublishFeed }}
+                  - powershell: >-
+                      $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1
+                      -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests'
+                      -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ replace(parameters.azDOPublishFeed, '/', '/_packaging/') }}/nuget/v3/index.json'
+                    displayName: 🟣 Wait for manifests in AzDO
+                  - task: 1ES.PublishNuget@1
+                    displayName: 🟣 Publish workload set to AzDO
+                    inputs:
+                      useDotNetTask: true
+                      packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/workloadSet/**/*.nupkg
                       packageParentPath: $(System.DefaultWorkingDirectory)/artifacts
                       publishVstsFeed: ${{ parameters.azDOPublishFeed }}
                 - ${{ if eq(parameters.publishToNuGet, true) }}:
                   - task: 1ES.PublishNuget@1
-                    displayName: 🟣 Publish packages to NuGet.org
+                    displayName: 🟣 Publish packs to NuGet.org
+                    inputs:
+                      useDotNetTask: false
+                      packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs/**/*.nupkg
+                      packageParentPath: $(System.DefaultWorkingDirectory)/artifacts
+                      nuGetFeedType: external
+                      publishVstsFeed: https://api.nuget.org/v3/index.json
+                      # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a
+                      publishFeedCredentials: nuget.org (dotnetframework)
+                  - powershell: >-
+                      $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1
+                      -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs'
+                      -feedIndexUrl 'https://api.nuget.org/v3/index.json'
+                    displayName: 🟣 Wait for packs in NuGet.org
+                  - task: 1ES.PublishNuget@1
+                    displayName: 🟣 Publish manifests to NuGet.org
+                    inputs:
+                      useDotNetTask: false
+                      packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests/**/*.nupkg
+                      packageParentPath: $(System.DefaultWorkingDirectory)/artifacts
+                      nuGetFeedType: external
+                      publishVstsFeed: https://api.nuget.org/v3/index.json
+                      # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a
+                      publishFeedCredentials: nuget.org (dotnetframework)
+                  - powershell: >-
+                      $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1
+                      -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests'
+                      -feedIndexUrl 'https://api.nuget.org/v3/index.json'
+                    displayName: 🟣 Wait for manifests in NuGet.org
+                  - task: 1ES.PublishNuget@1
+                    displayName: 🟣 Publish workload set to NuGet.org
                     inputs:
                       useDotNetTask: false
-                      packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/**/*.nupkg
+                      packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/workloadSet/**/*.nupkg
                       packageParentPath: $(System.DefaultWorkingDirectory)/artifacts
                       nuGetFeedType: external
                       publishVstsFeed: https://api.nuget.org/v3/index.json
                       # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a
-                      publishFeedCredentials: nuget.org (dotnetframework)
+                      publishFeedCredentials: nuget.org (dotnetframework)

eng/pipelines/templates/jobs/workload-build.yml

@@ -115,6 +115,12 @@ jobs:
         displayName: 🟣 Build solution
         # Name is required to reference the variables created within this build step in other stages.
         name: BuildSolution
+      - powershell: |
+          $destDir = "$(Build.ArtifactStagingDirectory)/artifacts/publishScripts"
+          New-Item -ItemType Directory -Path $destDir -Force | Out-Null
+          Copy-Item "eng/split-package-publish-buckets.ps1" $destDir
+          Copy-Item "eng/wait-for-package-availability.ps1" $destDir
+        displayName: 🟣 Copy publish scripts to artifacts
       - ${{ if eq(parameters.createVSInsertion, true) }}:
         - task: 1ES.PublishPipelineArtifact@1
           displayName: 🟣 Publish workload artifacts

eng/split-package-publish-buckets.ps1

@@ -0,0 +1,92 @@
+# This script splits package files into publish buckets.
+# The buckets are:
+# 1) packs: all non-manifest packages except the workload set package
+# 2) manifests: package IDs containing 'manifest'
+# 3) workloadSet: package IDs starting with Microsoft.NET.Workloads. (including architecture-specific MSI variants)
+
+param (
+  [Parameter(Mandatory = $true)] [string] $packagesPath
+)
+
+Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+$workloadSetPrefix = 'microsoft.net.workloads.'
+$manifestPattern = 'manifest'
+
+function Get-PackageMetadata {
+  param (
+    [Parameter(Mandatory = $true)] [string] $packageFilePath
+  )
+
+  $archive = [System.IO.Compression.ZipFile]::OpenRead($packageFilePath)
+  try {
+    $nuspecEntry = $archive.Entries | Where-Object { $_.Name.EndsWith('.nuspec', [System.StringComparison]::OrdinalIgnoreCase) } | Select-Object -First 1
+    if (-not $nuspecEntry) {
+      throw "Could not find .nuspec in package '$packageFilePath'."
+    }
+
+    $stream = $nuspecEntry.Open()
+    try {
+      $reader = New-Object System.IO.StreamReader($stream)
+      try {
+        $nuspecXml = [xml]$reader.ReadToEnd()
+      } finally {
+        $reader.Dispose()
+      }
+    } finally {
+      $stream.Dispose()
+    }
+
+    $id = $nuspecXml.package.metadata.id
+    if ([string]::IsNullOrWhiteSpace($id)) {
+      throw "Package ID is missing in '$packageFilePath'."
+    }
+
+    return $id
+  } finally {
+    $archive.Dispose()
+  }
+}
+
+$packagesPath = (Resolve-Path -Path $packagesPath).Path
+$bucketRoot = Join-Path -Path $packagesPath -ChildPath 'publishBuckets'
+
+if (Test-Path -Path $bucketRoot) {
+  Remove-Item -Path $bucketRoot -Recurse -Force
+}
+
+$bucketDirectories = @{
+  packs       = (Join-Path -Path $bucketRoot -ChildPath 'packs')
+  manifests   = (Join-Path -Path $bucketRoot -ChildPath 'manifests')
+  workloadSet = (Join-Path -Path $bucketRoot -ChildPath 'workloadSet')
+}
+
+$bucketDirectories.Values | ForEach-Object {
+  $null = New-Item -Path $_ -ItemType Directory -Force
+}
+
+$allPackages = Get-ChildItem -Path $packagesPath -Recurse -Filter *.nupkg -File | Where-Object { $_.FullName -notlike "$bucketRoot*" }
+if (-not $allPackages) {
+  throw "No .nupkg files found under '$packagesPath'."
+}
+
+foreach ($package in $allPackages) {
+  $packageId = Get-PackageMetadata -packageFilePath $package.FullName
+
+  $packageIdLower = $packageId.ToLowerInvariant()
+
+  $bucketName = 'packs'
+  if ($packageIdLower.StartsWith($workloadSetPrefix)) {
+    $bucketName = 'workloadSet'
+  } elseif ($packageIdLower -match $manifestPattern) {
+    $bucketName = 'manifests'
+  }
+
+  Copy-Item -Path $package.FullName -Destination $bucketDirectories[$bucketName] -Force
+}
+
+Write-Host "Split packages under '$packagesPath' into publish buckets:"
+$bucketDirectories.GetEnumerator() | ForEach-Object {
+  $count = (Get-ChildItem -Path $_.Value -Filter *.nupkg -File).Count
+  Write-Host "  $($_.Key): $count"
+}

eng/wait-for-package-availability.ps1

@@ -0,0 +1,129 @@
+# This script waits for package versions to appear on a NuGet V3 feed.
+# It polls package versions sequentially using the interval configured below.
+
+param (
+  [Parameter(Mandatory = $true)] [string] $packagesPath,
+  [Parameter(Mandatory = $true)] [string] $feedIndexUrl,
+  [int] $pollingIntervalSeconds = 300,
+  [int] $maxAttempts = 60
+)
+
+Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+function Get-PackageMetadata {
+  param (
+    [Parameter(Mandatory = $true)] [string] $packageFilePath
+  )
+
+  $archive = [System.IO.Compression.ZipFile]::OpenRead($packageFilePath)
+  try {
+    $nuspecEntry = $archive.Entries | Where-Object { $_.Name.EndsWith('.nuspec', [System.StringComparison]::OrdinalIgnoreCase) } | Select-Object -First 1
+    if (-not $nuspecEntry) {
+      throw "Could not find .nuspec in package '$packageFilePath'."
+    }
+
+    $stream = $nuspecEntry.Open()
+    try {
+      $reader = New-Object System.IO.StreamReader($stream)
+      try {
+        $nuspecXml = [xml]$reader.ReadToEnd()
+      } finally {
+        $reader.Dispose()
+      }
+    } finally {
+      $stream.Dispose()
+    }
+
+    $id = $nuspecXml.package.metadata.id
+    $version = $nuspecXml.package.metadata.version
+    if ([string]::IsNullOrWhiteSpace($id) -or [string]::IsNullOrWhiteSpace($version)) {
+      throw "Package ID/version is missing in '$packageFilePath'."
+    }
+
+    return @{
+      Id      = $id
+      Version = $version
+    }
+  } finally {
+    $archive.Dispose()
+  }
+}
+
+function Get-PackageBaseAddress {
+  param (
+    [Parameter(Mandatory = $true)] [string] $indexUrl
+  )
+
+  $index = Invoke-RestMethod -Method Get -Uri $indexUrl
+  $resource = $index.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress/3.0.0*' } | Select-Object -First 1
+  if (-not $resource) {
+    throw "Could not find PackageBaseAddress in '$indexUrl'."
+  }
+
+  return $resource.'@id'.TrimEnd('/')
+}
+
+function Get-HttpStatusCodeFromException {
+  param (
+    [Parameter(Mandatory = $true)] $exception
+  )
+
+  if ($exception -and $exception.Response) {
+    try {
+      return [int]$exception.Response.StatusCode.value__
+    } catch {
+      return $null
+    }
+  }
+
+  return $null
+}
+
+$packagesPath = (Resolve-Path -Path $packagesPath).Path
+$packageFiles = Get-ChildItem -Path $packagesPath -Filter *.nupkg -File
+if (-not $packageFiles) {
+  throw "No .nupkg files found under '$packagesPath'."
+}
+
+$packageBaseAddress = Get-PackageBaseAddress -indexUrl $feedIndexUrl
+foreach ($packageFile in $packageFiles) {
+  $packageMetadata = Get-PackageMetadata -packageFilePath $packageFile.FullName
+  $idLower = $packageMetadata.Id.ToLowerInvariant()
+  $versionLower = $packageMetadata.Version.ToLowerInvariant()
+  $registrationUrl = "$packageBaseAddress/$idLower/index.json"
+
+  Write-Host "Waiting for $($packageMetadata.Id) $($packageMetadata.Version) on $feedIndexUrl"
+
+  $isAvailable = $false
+  for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) {
+    try {
+      $registrationIndex = Invoke-RestMethod -Method Get -Uri $registrationUrl
+      if ($registrationIndex.versions -contains $versionLower) {
+        Write-Host "  Available after attempt $attempt."
+        $isAvailable = $true
+        break
+      }
+
+      Write-Host "  Not available yet (attempt $attempt of $maxAttempts)."
+    } catch {
+      $statusCode = Get-HttpStatusCodeFromException -exception $_.Exception
+      if ($statusCode -in @(400, 401, 403, 409, 410, 422)) {
+        throw "Non-retryable HTTP status code $statusCode querying '$registrationUrl'. Error: $($_.Exception.Message)"
+      }
+
+      if ($statusCode) {
+        Write-Host "  Failed to query package index on attempt $attempt of $maxAttempts with HTTP $statusCode. Error: $($_.Exception.Message)"
+      } else {
+        Write-Host "  Failed to query package index on attempt $attempt of $maxAttempts. Error: $($_.Exception.Message)"
+      }
+    }
+
+    if ($attempt -lt $maxAttempts) {
+      Start-Sleep -Seconds $pollingIntervalSeconds
+    }
+  }
+
+  if (-not $isAvailable) {
+    throw "Timed out waiting for $($packageMetadata.Id) $($packageMetadata.Version) in '$feedIndexUrl'."
+  }
+}