diff --git a/eng/pipelines/official.yml b/eng/pipelines/official.yml index 42d0de864..8bb2fd6df 100644 --- a/eng/pipelines/official.yml +++ b/eng/pipelines/official.yml @@ -254,24 +254,82 @@ extends: inputs: artifactName: Artifacts targetPath: $(System.DefaultWorkingDirectory)/artifacts + - powershell: >- + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/split-package-publish-buckets.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages' + displayName: 🟣 Split packages into publish buckets # 1ES docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs/nuget-packages # DotNetCoreCLI@2 docs: https://learn.microsoft.com/en-us/azure/devops/pipelines/tasks/reference/dotnet-core-cli-v2 - ${{ if eq(parameters.publishToAzDO, true) }}: - task: 1ES.PublishNuget@1 - displayName: 🟣 Publish packages to AzDO + displayName: 🟣 Publish packs to AzDO inputs: useDotNetTask: true - packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/**/*.nupkg + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs/**/*.nupkg + packageParentPath: $(System.DefaultWorkingDirectory)/artifacts + publishVstsFeed: ${{ parameters.azDOPublishFeed }} + - powershell: >- + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs' + -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ replace(parameters.azDOPublishFeed, '/', '/_packaging/') }}/nuget/v3/index.json' + displayName: 🟣 Wait for packs in AzDO + - task: 1ES.PublishNuget@1 + displayName: 🟣 Publish manifests to AzDO + inputs: + useDotNetTask: true + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests/**/*.nupkg + packageParentPath: $(System.DefaultWorkingDirectory)/artifacts + publishVstsFeed: ${{ parameters.azDOPublishFeed }} + - powershell: >- + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests' + -feedIndexUrl 'https://pkgs.dev.azure.com/dnceng/${{ replace(parameters.azDOPublishFeed, '/', '/_packaging/') }}/nuget/v3/index.json' + displayName: 🟣 Wait for manifests in AzDO + - task: 1ES.PublishNuget@1 + displayName: 🟣 Publish workload set to AzDO + inputs: + useDotNetTask: true + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/workloadSet/**/*.nupkg packageParentPath: $(System.DefaultWorkingDirectory)/artifacts publishVstsFeed: ${{ parameters.azDOPublishFeed }} - ${{ if eq(parameters.publishToNuGet, true) }}: - task: 1ES.PublishNuget@1 - displayName: 🟣 Publish packages to NuGet.org + displayName: 🟣 Publish packs to NuGet.org + inputs: + useDotNetTask: false + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs/**/*.nupkg + packageParentPath: $(System.DefaultWorkingDirectory)/artifacts + nuGetFeedType: external + publishVstsFeed: https://api.nuget.org/v3/index.json + # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a + publishFeedCredentials: nuget.org (dotnetframework) + - powershell: >- + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/packs' + -feedIndexUrl 'https://api.nuget.org/v3/index.json' + displayName: 🟣 Wait for packs in NuGet.org + - task: 1ES.PublishNuget@1 + displayName: 🟣 Publish manifests to NuGet.org + inputs: + useDotNetTask: false + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests/**/*.nupkg + packageParentPath: $(System.DefaultWorkingDirectory)/artifacts + nuGetFeedType: external + publishVstsFeed: https://api.nuget.org/v3/index.json + # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a + publishFeedCredentials: nuget.org (dotnetframework) + - powershell: >- + $(System.DefaultWorkingDirectory)/artifacts/publishScripts/wait-for-package-availability.ps1 + -packagesPath '$(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/manifests' + -feedIndexUrl 'https://api.nuget.org/v3/index.json' + displayName: 🟣 Wait for manifests in NuGet.org + - task: 1ES.PublishNuget@1 + displayName: 🟣 Publish workload set to NuGet.org inputs: useDotNetTask: false - packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/**/*.nupkg + packagesToPush: $(System.DefaultWorkingDirectory)/artifacts/packages/publishBuckets/workloadSet/**/*.nupkg packageParentPath: $(System.DefaultWorkingDirectory)/artifacts nuGetFeedType: external publishVstsFeed: https://api.nuget.org/v3/index.json # Service connection: https://dev.azure.com/dnceng/internal/_settings/adminservices?resourceId=479fdc43-a27d-4f5f-b2fc-5cf19dce159a - publishFeedCredentials: nuget.org (dotnetframework) \ No newline at end of file + publishFeedCredentials: nuget.org (dotnetframework) diff --git a/eng/pipelines/templates/jobs/workload-build.yml b/eng/pipelines/templates/jobs/workload-build.yml index 0104ae9a7..2b1a1b2b9 100644 --- a/eng/pipelines/templates/jobs/workload-build.yml +++ b/eng/pipelines/templates/jobs/workload-build.yml @@ -115,6 +115,12 @@ jobs: displayName: 🟣 Build solution # Name is required to reference the variables created within this build step in other stages. name: BuildSolution + - powershell: | + $destDir = "$(Build.ArtifactStagingDirectory)/artifacts/publishScripts" + New-Item -ItemType Directory -Path $destDir -Force | Out-Null + Copy-Item "eng/split-package-publish-buckets.ps1" $destDir + Copy-Item "eng/wait-for-package-availability.ps1" $destDir + displayName: 🟣 Copy publish scripts to artifacts - ${{ if eq(parameters.createVSInsertion, true) }}: - task: 1ES.PublishPipelineArtifact@1 displayName: 🟣 Publish workload artifacts diff --git a/eng/split-package-publish-buckets.ps1 b/eng/split-package-publish-buckets.ps1 new file mode 100644 index 000000000..514b12579 --- /dev/null +++ b/eng/split-package-publish-buckets.ps1 @@ -0,0 +1,92 @@ +# This script splits package files into publish buckets. +# The buckets are: +# 1) packs: all non-manifest packages except the workload set package +# 2) manifests: package IDs containing 'manifest' +# 3) workloadSet: package IDs starting with Microsoft.NET.Workloads. (including architecture-specific MSI variants) + +param ( + [Parameter(Mandatory = $true)] [string] $packagesPath +) + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +$workloadSetPrefix = 'microsoft.net.workloads.' +$manifestPattern = 'manifest' + +function Get-PackageMetadata { + param ( + [Parameter(Mandatory = $true)] [string] $packageFilePath + ) + + $archive = [System.IO.Compression.ZipFile]::OpenRead($packageFilePath) + try { + $nuspecEntry = $archive.Entries | Where-Object { $_.Name.EndsWith('.nuspec', [System.StringComparison]::OrdinalIgnoreCase) } | Select-Object -First 1 + if (-not $nuspecEntry) { + throw "Could not find .nuspec in package '$packageFilePath'." + } + + $stream = $nuspecEntry.Open() + try { + $reader = New-Object System.IO.StreamReader($stream) + try { + $nuspecXml = [xml]$reader.ReadToEnd() + } finally { + $reader.Dispose() + } + } finally { + $stream.Dispose() + } + + $id = $nuspecXml.package.metadata.id + if ([string]::IsNullOrWhiteSpace($id)) { + throw "Package ID is missing in '$packageFilePath'." + } + + return $id + } finally { + $archive.Dispose() + } +} + +$packagesPath = (Resolve-Path -Path $packagesPath).Path +$bucketRoot = Join-Path -Path $packagesPath -ChildPath 'publishBuckets' + +if (Test-Path -Path $bucketRoot) { + Remove-Item -Path $bucketRoot -Recurse -Force +} + +$bucketDirectories = @{ + packs = (Join-Path -Path $bucketRoot -ChildPath 'packs') + manifests = (Join-Path -Path $bucketRoot -ChildPath 'manifests') + workloadSet = (Join-Path -Path $bucketRoot -ChildPath 'workloadSet') +} + +$bucketDirectories.Values | ForEach-Object { + $null = New-Item -Path $_ -ItemType Directory -Force +} + +$allPackages = Get-ChildItem -Path $packagesPath -Recurse -Filter *.nupkg -File | Where-Object { $_.FullName -notlike "$bucketRoot*" } +if (-not $allPackages) { + throw "No .nupkg files found under '$packagesPath'." +} + +foreach ($package in $allPackages) { + $packageId = Get-PackageMetadata -packageFilePath $package.FullName + + $packageIdLower = $packageId.ToLowerInvariant() + + $bucketName = 'packs' + if ($packageIdLower.StartsWith($workloadSetPrefix)) { + $bucketName = 'workloadSet' + } elseif ($packageIdLower -match $manifestPattern) { + $bucketName = 'manifests' + } + + Copy-Item -Path $package.FullName -Destination $bucketDirectories[$bucketName] -Force +} + +Write-Host "Split packages under '$packagesPath' into publish buckets:" +$bucketDirectories.GetEnumerator() | ForEach-Object { + $count = (Get-ChildItem -Path $_.Value -Filter *.nupkg -File).Count + Write-Host " $($_.Key): $count" +} diff --git a/eng/wait-for-package-availability.ps1 b/eng/wait-for-package-availability.ps1 new file mode 100644 index 000000000..fe542a062 --- /dev/null +++ b/eng/wait-for-package-availability.ps1 @@ -0,0 +1,129 @@ +# This script waits for package versions to appear on a NuGet V3 feed. +# It polls package versions sequentially using the interval configured below. + +param ( + [Parameter(Mandatory = $true)] [string] $packagesPath, + [Parameter(Mandatory = $true)] [string] $feedIndexUrl, + [int] $pollingIntervalSeconds = 300, + [int] $maxAttempts = 60 +) + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function Get-PackageMetadata { + param ( + [Parameter(Mandatory = $true)] [string] $packageFilePath + ) + + $archive = [System.IO.Compression.ZipFile]::OpenRead($packageFilePath) + try { + $nuspecEntry = $archive.Entries | Where-Object { $_.Name.EndsWith('.nuspec', [System.StringComparison]::OrdinalIgnoreCase) } | Select-Object -First 1 + if (-not $nuspecEntry) { + throw "Could not find .nuspec in package '$packageFilePath'." + } + + $stream = $nuspecEntry.Open() + try { + $reader = New-Object System.IO.StreamReader($stream) + try { + $nuspecXml = [xml]$reader.ReadToEnd() + } finally { + $reader.Dispose() + } + } finally { + $stream.Dispose() + } + + $id = $nuspecXml.package.metadata.id + $version = $nuspecXml.package.metadata.version + if ([string]::IsNullOrWhiteSpace($id) -or [string]::IsNullOrWhiteSpace($version)) { + throw "Package ID/version is missing in '$packageFilePath'." + } + + return @{ + Id = $id + Version = $version + } + } finally { + $archive.Dispose() + } +} + +function Get-PackageBaseAddress { + param ( + [Parameter(Mandatory = $true)] [string] $indexUrl + ) + + $index = Invoke-RestMethod -Method Get -Uri $indexUrl + $resource = $index.resources | Where-Object { $_.'@type' -like 'PackageBaseAddress/3.0.0*' } | Select-Object -First 1 + if (-not $resource) { + throw "Could not find PackageBaseAddress in '$indexUrl'." + } + + return $resource.'@id'.TrimEnd('/') +} + +function Get-HttpStatusCodeFromException { + param ( + [Parameter(Mandatory = $true)] $exception + ) + + if ($exception -and $exception.Response) { + try { + return [int]$exception.Response.StatusCode.value__ + } catch { + return $null + } + } + + return $null +} + +$packagesPath = (Resolve-Path -Path $packagesPath).Path +$packageFiles = Get-ChildItem -Path $packagesPath -Filter *.nupkg -File +if (-not $packageFiles) { + throw "No .nupkg files found under '$packagesPath'." +} + +$packageBaseAddress = Get-PackageBaseAddress -indexUrl $feedIndexUrl +foreach ($packageFile in $packageFiles) { + $packageMetadata = Get-PackageMetadata -packageFilePath $packageFile.FullName + $idLower = $packageMetadata.Id.ToLowerInvariant() + $versionLower = $packageMetadata.Version.ToLowerInvariant() + $registrationUrl = "$packageBaseAddress/$idLower/index.json" + + Write-Host "Waiting for $($packageMetadata.Id) $($packageMetadata.Version) on $feedIndexUrl" + + $isAvailable = $false + for ($attempt = 1; $attempt -le $maxAttempts; $attempt++) { + try { + $registrationIndex = Invoke-RestMethod -Method Get -Uri $registrationUrl + if ($registrationIndex.versions -contains $versionLower) { + Write-Host " Available after attempt $attempt." + $isAvailable = $true + break + } + + Write-Host " Not available yet (attempt $attempt of $maxAttempts)." + } catch { + $statusCode = Get-HttpStatusCodeFromException -exception $_.Exception + if ($statusCode -in @(400, 401, 403, 409, 410, 422)) { + throw "Non-retryable HTTP status code $statusCode querying '$registrationUrl'. Error: $($_.Exception.Message)" + } + + if ($statusCode) { + Write-Host " Failed to query package index on attempt $attempt of $maxAttempts with HTTP $statusCode. Error: $($_.Exception.Message)" + } else { + Write-Host " Failed to query package index on attempt $attempt of $maxAttempts. Error: $($_.Exception.Message)" + } + } + + if ($attempt -lt $maxAttempts) { + Start-Sleep -Seconds $pollingIntervalSeconds + } + } + + if (-not $isAvailable) { + throw "Timed out waiting for $($packageMetadata.Id) $($packageMetadata.Version) in '$feedIndexUrl'." + } +}