From 1f30844f508c5ef3300ac0c7e8a37fb907d75e18 Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:21:46 -0400 Subject: [PATCH 1/5] Begin to remove custom sparse checkout from standard check out of repo --- eng/README.md | 39 +++ eng/containers/ci.yml | 6 +- eng/pipelines/code-quality-reports.yml | 19 +- .../scripts/Invoke-Sparse-Checkout.ps1 | 153 +++++----- .../templates/jobs/build-validate-pom.yml | 17 +- eng/pipelines/templates/jobs/ci.yml | 39 ++- .../archetype-java-auto-release-batch.yml | 68 +++-- .../stages/archetype-java-release-batch.yml | 80 +++--- .../stages/archetype-java-release-patch.yml | 50 ++-- .../archetype-java-release-pom-only.yml | 19 +- .../stages/archetype-sdk-client-patch.yml | 38 +-- .../steps/initialize-test-environment.yml | 21 +- .../sparse-checkout-repo-initialized.yml | 56 ++-- eng/scripts/ci.yml | 4 + eng/scripts/tests/Sparse-Checkout.tests.ps1 | 267 ++++++++++++++++++ 15 files changed, 618 insertions(+), 258 deletions(-) create mode 100644 eng/scripts/tests/Sparse-Checkout.tests.ps1 diff --git a/eng/README.md b/eng/README.md index ac4f6478c24ab..efe345dd810f5 100644 --- a/eng/README.md +++ b/eng/README.md @@ -10,6 +10,45 @@ All the tools/utilities used in Microsoft Azure Java SDK's build config are defi - `lintingconfigs` - CheckStyle and SpotBugs rule configurations. +## Sparse Checkouts + +Java-owned pipeline jobs use the native Azure Pipelines +[`checkout` step](https://learn.microsoft.com/azure/devops/pipelines/yaml-schema/steps-checkout) +for both their initial sparse checkout and dependency-driven expansion. This requires agent version +3.253.0/4.253.0 or later and Git 2.25 or later. + +- Use `sparseCheckoutPatterns` for non-cone patterns, including file globs and exclusions. +- Preserve the base patterns `/* !/*/ /eng /.config` and the order of job-specific patterns. +- Use `fetchFilter: tree:0`, `fetchDepth: 0`, and `fetchTags: true` to retain the existing treeless fetch, full history, and tags. +- Set `path` explicitly when checking out Java alongside build-tools; it is relative to `$(Pipeline.Workspace)`. + +Java builds compute additional paths after generating project lists and updating POM files. Use +[pipelines/templates/steps/sparse-checkout-repo-initialized.yml](pipelines/templates/steps/sparse-checkout-repo-initialized.yml) +with `Paths: $(SparseCheckoutDirectories)` to expand these checkouts. The variable remains a JSON array. +The template prepares a space-separated `SparseCheckoutPatterns` variable containing the original patterns followed +by the additional paths, then passes it to a second native checkout. Both checkouts use `path: s` to retain the +existing source location. Empty path lists and full checkouts do not need a second checkout. + +The agent performs a forced checkout even with `clean: false`. Before checkout, the helper saves tracked-file changes +as a binary patch under `$(Agent.TempDirectory)`. After successful checkout it verifies that the source revision has +not changed and restores the patch, including changes to POM files generated during dependency discovery. +The second checkout uses `clean: false` to retain generated untracked files. Restoration failures fail the job and +leave the patch available for diagnosis; do not replace this template with an unprotected second checkout. + +Layered jobs keep private mirrored repositories on full checkouts so later expansion does not require persisted credentials. +Configurable external docs repositories still use the shared custom template because their repository names are selected +at runtime. Shared pipeline templates under `common` are maintained upstream in `azure-sdk-tools`. + +Run the expansion regression tests from the repository root using PowerShell 7 and Pester 5.7.1: + +```powershell +Import-Module Pester -RequiredVersion 5.7.1 +Invoke-Pester -Path eng/scripts/tests/Sparse-Checkout.tests.ps1 -Output Detailed +``` + +Before rolling out to an agent pool, validate PR, FromSource, private-mirror, and multi-repository publishing jobs on +Windows and Linux. Compare source revisions, checked-out files, preserved POM edits, and checkout time and transfer size. + --- For developer guides (building, testing, code quality, versioning), see the consolidated documentation hub: diff --git a/eng/containers/ci.yml b/eng/containers/ci.yml index 81232455fc503..3e734eeb20c93 100644 --- a/eng/containers/ci.yml +++ b/eng/containers/ci.yml @@ -45,7 +45,11 @@ extends: - $(containerRegistry).azurecr.io/$(imageRepository):$(stableTag) steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - task: AzureCLI@2 displayName: Login to $(containerRegistry) diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 551a561c20c5f..5d3f603ceb7ea 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -34,13 +34,15 @@ extends: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' - - 'sdk/tools/linting-extensions' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config **/*.xml + !sdk/**/test-recordings !sdk/**/session-records + sdk/tools/linting-extensions - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -77,9 +79,8 @@ extends: arguments: '--set-skip-linting-projects SkipLintingProjects --artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)' workingDirectory: '$(System.DefaultWorkingDirectory)' - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: - SkipCheckoutNone: true Paths: $(SparseCheckoutDirectories) - task: PowerShell@2 diff --git a/eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 b/eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 index db43a1fe21555..daa0ddd95a272 100644 --- a/eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 +++ b/eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 @@ -1,104 +1,103 @@ <# .SYNOPSIS -Invokes sparse checkout on the specified repositories. +Prepares or restores working-tree changes around a native sparse checkout. .DESCRIPTION -Invokes sparse checkout on the specified repositories. +Combines the initial sparse checkout patterns with paths computed by Java's dependency discovery. +Tracked-file changes are saved outside the repository and restored after the Azure Pipelines checkout +step resets them. The checkout must use clean: false to preserve generated untracked files. -This script is special to Java as it uses layered sparse checkout to reduce the amount of code to checkout. -The first run of sparse checkout is inlined into YAML as there is a chicken and egg problem where the script -to perform sparse checkout won't be available until after the checkout step has completed. +.PARAMETER PathsJson +JSON representation of the additional paths to checkout. -This script is used to reduce the size of YAML files as this is only called when the initial checkout has -already been completed. +.PARAMETER ChangesPath +Absolute path outside the repository for the patch containing tracked-file changes. -.PARAMETER PathsJson -JSON representation of the paths to checkout. +.PARAMETER SourceVersion +The original source revision, which must still be checked out before restoring changes. -.PARAMETER RepositoriesJson -JSON representation of the repositories to checkout from. +.PARAMETER Restore +Restore the saved changes after the native checkout has completed. #> +[CmdletBinding(DefaultParameterSetName = 'Prepare')] param( - [Parameter(Mandatory = $true)] - [string]$PathsJson, + [Parameter(Mandatory = $true, ParameterSetName = 'Prepare')] + [string]$PathsJson, + + [Parameter(Mandatory = $true)] + [string]$ChangesPath, - [Parameter(Mandatory = $true)] - [string]$RepositoriesJson + [Parameter(Mandatory = $true, ParameterSetName = 'Restore')] + [string]$SourceVersion, + + [Parameter(Mandatory = $true, ParameterSetName = 'Restore')] + [switch]$Restore ) -# Setting $PSNativeCommandArgumentPassing to 'Legacy' to use PowerShell -# 7.2 behavior for command argument passing. Newer behaviors will result -# in errors from git.exe. -$PSNativeCommandArgumentPassing = 'Legacy' +$ErrorActionPreference = 'Stop' -function SparseCheckout([Array]$paths, [Hashtable]$repository) -{ - $dir = $repository.WorkingDirectory - if (!$dir) { - $dir = "./$($repository.Name)" +if ($Restore) { + $currentVersion = git rev-parse HEAD + if ($LASTEXITCODE -ne 0 -or $currentVersion -ne $SourceVersion) { + throw "The native checkout changed the source revision. Patch retained at $ChangesPath." } - New-Item $dir -ItemType Directory -Force | Out-Null - Push-Location $dir - - if (Test-Path .git/info/sparse-checkout) { - $hasInitialized = $true - Write-Host "Repository $($repository.Name) has already been initialized. Skipping this step." - } else { - Write-Host "Repository $($repository.Name) is being initialized." - - if ($repository.Commitish -match '^refs/pull/\d+/merge$') { - Write-Host "git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)' https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 -c remote.origin.fetch=''+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)'' https://github.com/$($repository.Name) . - } else { - Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . + if ((Get-Item -LiteralPath $ChangesPath).Length -gt 0) { + git apply --whitespace=nowarn -- $ChangesPath + if ($LASTEXITCODE -ne 0) { + throw "Restoring checkout changes failed with exit code $LASTEXITCODE. Patch retained at $ChangesPath." } - - # Turn off git GC for sparse checkout. Note: The devops checkout task does this by default - Write-Host "git config gc.auto 0" - git config gc.auto 0 - - Write-Host "git sparse-checkout init" - git sparse-checkout init - - # Set non-cone mode otherwise path filters will not work in git >= 2.37.0 - # See https://github.blog/2022-06-27-highlights-from-git-2-37/#tidbits - Write-Host "git sparse-checkout set --no-cone '/*' '!/*/' '/eng'" - git sparse-checkout set --no-cone '/*' '!/*/' '/eng' } + Remove-Item -LiteralPath $ChangesPath + return +} - # Prevent wildcard expansion in Invoke-Expression (e.g. for checkout path '/*') - $quotedPaths = $paths | ForEach-Object { "'$_'" } - $gitsparsecmd = "git sparse-checkout add $quotedPaths" - Write-Host $gitsparsecmd - Invoke-Expression -Command $gitsparsecmd +Write-Output '##vso[task.setvariable variable=SparseCheckoutRequired]false' - Write-Host "Set sparse checkout paths to:" - Get-Content .git/info/sparse-checkout +# Paths may be sourced as a yaml object literal OR a dynamically generated variable json string. +# If the latter, convertToJson will wrap the 'string' in quotes, so remove them. +$paths = $PathsJson.Trim('"') | ConvertFrom-Json +if (@($paths).Count -eq 0) { + return +} - # sparse-checkout commands after initial checkout will auto-checkout again - if (!$hasInitialized) { - # Remove refs/heads/ prefix from branch names - $commitish = $repository.Commitish -replace '^refs/heads/', '' +$isWorkingTree = git rev-parse --is-inside-work-tree +if ($LASTEXITCODE -ne 0 -or $isWorkingTree -ne 'true') { + throw 'The repository is not an initialized Git working tree.' +} - # use -- to prevent git from interpreting the commitish as a path - Write-Host "git -c advice.detachedHead=false checkout $commitish --" +$isSparseCheckout = git config --type=bool --default=false --get core.sparseCheckout +if ($LASTEXITCODE -ne 0) { + throw 'Unable to determine sparse checkout mode.' +} +if ($isSparseCheckout -ne 'true') { + Write-Information 'The repository has a full checkout. Skipping expansion.' -InformationAction Continue + return +} - # This will use the default branch if repo.Commitish is empty - git -c advice.detachedHead=false checkout $commitish -- - } else { - Write-Host "Skipping checkout as repo has already been initialized" +$patternsPath = git rev-parse --git-path info/sparse-checkout +if ($LASTEXITCODE -ne 0) { + throw 'Unable to locate the initial sparse checkout patterns.' +} +$patterns = @(Get-Content -LiteralPath $patternsPath) + @($paths) +$quotedPatterns = foreach ($pattern in $patterns) { + if ($pattern -match "[`r`n]") { + throw 'Sparse checkout patterns cannot contain line breaks.' } - - Pop-Location + '"' + ($pattern -replace '(\\*)"', '$1$1\"' -replace '(\\+)$', '$1$1') + '"' } -# Paths may be sourced as a yaml object literal OR a dynamically generated variable json string. -# If the latter, convertToJson will wrap the 'string' in quotes, so remove them. -$paths = $PathsJson.Trim('"') | ConvertFrom-Json -# Replace windows backslash paths, as Azure Pipelines default directories are sometimes formatted like 'D:\a\1\s' -$repositories = $RepositoriesJson -replace '\\', '/' | ConvertFrom-Json -AsHashtable -foreach ($repo in $repositories) { - SparseCheckout $paths $repo +New-Item -ItemType Directory -Path (Split-Path -Parent $ChangesPath) -Force | Out-Null +$SourceVersion = git rev-parse HEAD +if ($LASTEXITCODE -ne 0) { + throw 'Unable to determine the original source revision.' } +git diff --binary --no-ext-diff --no-textconv --src-prefix=a/ --dst-prefix=b/ --output=$ChangesPath $SourceVersion -- +if ($LASTEXITCODE -ne 0) { + throw "Saving checkout changes failed with exit code $LASTEXITCODE." +} + +$patternsValue = ($quotedPatterns -join ' ').Replace('%', '%AZP25') +Write-Output "##vso[task.setvariable variable=SparseCheckoutPatterns]$patternsValue" +Write-Output "##vso[task.setvariable variable=SparseCheckoutSourceVersion]$SourceVersion" +Write-Output '##vso[task.setvariable variable=SparseCheckoutRequired]true' diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index bddc3dd582157..04a07aea5b7f2 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -24,14 +24,15 @@ jobs: os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - 'sdk/${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + sdk/${{ parameters.ServiceDirectory }} + **/*.xml **/*.md + !sdk/**/test-recordings !sdk/**/session-records - script: | echo "##vso[build.addbuildtag]Scheduled" diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 714fb140dbe31..68217a91f88b7 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -90,7 +90,7 @@ jobs: componentgovernance: enabled: true codeql: - binaryLanguages: java # Need to specify the language because we clone after the codeql initialize step + binaryLanguages: java compiled: enabled: true @@ -111,14 +111,15 @@ jobs: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/ci*.yml' # necessary for save-package-properties.yml - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + **/*.xml **/*.md **/ci*.yml + !sdk/**/test-recordings !sdk/**/session-records - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -151,7 +152,6 @@ jobs: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - ${{ parameters.PreBuildSteps }} @@ -321,15 +321,15 @@ jobs: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/*.yml' - - '.vscode/cspell.json' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + **/*.xml **/*.md **/*.yml .vscode/cspell.json + !sdk/**/test-recordings !sdk/**/session-records - task: PowerShell@2 displayName: 'Verify versions in POM files' @@ -365,7 +365,6 @@ jobs: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - task: UseNode@1 inputs: diff --git a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml index 15458ebc71785..8a7d1419fa52a 100644 --- a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml @@ -96,13 +96,14 @@ stages: image: $(LINUXVMIMAGE) os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - 'sdk/${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + sdk/${{ parameters.ServiceDirectory }} **/*.xml + !sdk/**/test-recordings !sdk/**/session-records - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -190,7 +191,11 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - template: /eng/common/pipelines/templates/steps/retain-run.yml - template: /eng/common/pipelines/templates/steps/login-to-github.yml parameters: @@ -259,13 +264,12 @@ stages: steps: - checkout: azure-sdk-build-tools path: azure-sdk-build-tools - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: Azure/azure-sdk-for-java - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - checkout: self + path: azure-sdk-for-java + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -369,14 +373,14 @@ stages: os: windows steps: - download: none - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + ${{ parameters.ServiceDirectory }} **/*.xml **/*.md + !sdk/**/test-recordings !sdk/**/session-records - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -415,11 +419,13 @@ stages: image: $(LINUXVMIMAGE) os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/${{ parameters.ServiceDirectory }}/**/*.md - - '!**/pom*.xml' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -501,7 +507,11 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - download: current displayName: 'Download Artifact: packages-signed-auto-release' diff --git a/eng/pipelines/templates/stages/archetype-java-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-release-batch.yml index 143df8382ef83..ea9cf7398293e 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-batch.yml @@ -115,13 +115,14 @@ stages: os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - 'sdk/${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + sdk/${{ parameters.ServiceDirectory }} **/*.xml + !sdk/**/test-recordings !sdk/**/session-records - download: current displayName: 'Download Artifact: packages' @@ -197,7 +198,11 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - template: /eng/common/pipelines/templates/steps/retain-run.yml # No pattern, just download everything that's been signed - download: current @@ -257,13 +262,12 @@ stages: steps: - checkout: azure-sdk-build-tools path: azure-sdk-build-tools - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: Azure/azure-sdk-for-java - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - checkout: self + path: azure-sdk-for-java + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - download: current displayName: 'Download Artifacts' artifact: packages-signed @@ -313,14 +317,14 @@ stages: steps: - download: none - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + ${{ parameters.ServiceDirectory }} **/*.xml **/*.md + !sdk/**/test-recordings !sdk/**/session-records - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -364,11 +368,13 @@ stages: os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/${{ parameters.ServiceDirectory }}/**/*.md - - '!**/pom*.xml' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml - download: current displayName: 'Download Artifact: packages' artifact: packages @@ -409,7 +415,11 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - download: current displayName: 'Download Artifact: packages-signed' artifact: packages-signed @@ -494,11 +504,13 @@ stages: image: $(LINUXVMIMAGE) os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/${{ parameters.ServiceDirectory }}/**/*.md - - '!**/pom*.xml' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml - download: current displayName: 'Download Artifacts' artifact: packages diff --git a/eng/pipelines/templates/stages/archetype-java-release-patch.yml b/eng/pipelines/templates/stages/archetype-java-release-patch.yml index c82a334643841..fbc2f17340687 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-patch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-patch.yml @@ -98,7 +98,11 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - template: /eng/common/pipelines/templates/steps/retain-run.yml # No pattern, just download everything that's been signed - download: current @@ -156,13 +160,12 @@ stages: steps: - checkout: azure-sdk-build-tools path: azure-sdk-build-tools - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: Azure/azure-sdk-for-java - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - checkout: self + path: azure-sdk-for-java + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - download: current displayName: 'Download Artifacts' artifact: packages-signed @@ -201,13 +204,13 @@ stages: steps: - download: none - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings/*' - - '!sdk/**/session-records/*' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config **/*.xml **/*.md + !sdk/**/test-recordings/* !sdk/**/session-records/* - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -253,11 +256,12 @@ stages: - template: /eng/pipelines/templates/variables/globals.yml steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/**/*.md - - '!**/pom*.xml' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config sdk/**/*.md !**/pom*.xml - download: current displayName: 'Download Artifact: packages' artifact: packages @@ -292,7 +296,11 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - download: current displayName: 'Download Artifact: packages-signed' artifact: packages-signed diff --git a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml index 3353db07e3da9..ce1477b8c1efe 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml @@ -95,7 +95,11 @@ stages: displayName: 'Download Artifact: packages-signed' artifact: packages-signed patterns: ${{artifact.groupId}}/${{artifact.name}}/** - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' - template: /eng/common/pipelines/templates/steps/retain-run.yml - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml parameters: @@ -145,13 +149,12 @@ stages: patterns: ${{artifact.groupId}}/${{artifact.name}}/** - checkout: azure-sdk-build-tools path: azure-sdk-build-tools - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: Azure/azure-sdk-for-java - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - checkout: self + path: azure-sdk-for-java + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: '/* !/*/ /eng /.config' # Setup Maven mirror settings and authenticate with Azure Artifacts - template: /eng/pipelines/templates/steps/maven-authenticate.yml parameters: diff --git a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml index f8ec5ec5843ff..ccfee7ea8065a 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml @@ -46,14 +46,15 @@ extends: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories as we require the GitHub # service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/*.yml' - - '!sdk/**/test-recordings/*' - - '!sdk/**/session-records/*' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + **/*.xml **/*.md **/*.yml + !sdk/**/test-recordings/* !sdk/**/session-records/* - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -94,7 +95,6 @@ extends: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - task: PipAuthenticate@1 displayName: 'Pip Authenticate to Azure Artifacts' @@ -208,15 +208,16 @@ extends: inputs: versionSpec: $(PythonVersion) - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/*.yml' # needed for package properties when setting the test pipeline version - - '.vscode/cspell.json' - - '!sdk/**/test-recordings/*' - - '!sdk/**/session-records/*' + - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config + **/*.xml **/*.md **/*.yml .vscode/cspell.json + !sdk/**/test-recordings/* !sdk/**/session-records/* - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -253,7 +254,6 @@ extends: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - template: /eng/common/pipelines/templates/steps/check-spelling.yml parameters: diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 27e2f04cdd7fc..f9348c25786bc 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -16,14 +16,18 @@ steps: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/ci*.yml' # necessary for save-package-properties.yml - - ${{ if not(parameters.CheckoutRecordings) }}: - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + ${{ if parameters.CheckoutRecordings }}: + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config **/*.xml **/ci*.yml + ${{ else }}: + sparseCheckoutPatterns: >- + /* !/*/ /eng /.config **/*.xml **/ci*.yml + !sdk/**/test-recordings !sdk/**/session-records - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -82,5 +86,4 @@ steps: - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: - SkipCheckoutNone: true Paths: $(SparseCheckoutDirectories) diff --git a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml index 07c75f94ff9e7..c791eaf655666 100644 --- a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml +++ b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml @@ -2,29 +2,39 @@ parameters: - name: Paths type: object default: [] - - name: Repositories - type: object - default: - - Name: $(Build.Repository.Name) - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(System.DefaultWorkingDirectory) - - name: SkipCheckoutNone - type: boolean - default: false steps: - - ${{ if not(parameters.SkipCheckoutNone) }}: - - checkout: none + - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: + - task: PowerShell@2 + displayName: 'Prepare computed sparse checkout paths' + inputs: + pwsh: true + filePath: eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 + arguments: >- + -PathsJson $env:SPARSE_CHECKOUT_PATHS + -ChangesPath "$(Agent.TempDirectory)/sparse-checkout-changes.patch" + workingDirectory: $(Build.SourcesDirectory) + env: + SPARSE_CHECKOUT_PATHS: ${{ convertToJson(parameters.Paths) }} + + - checkout: self + displayName: 'Checkout computed source paths' + path: s + clean: false + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: true + sparseCheckoutPatterns: $(SparseCheckoutPatterns) + condition: and(succeeded(), eq(variables['SparseCheckoutRequired'], 'true')) - - task: PowerShell@2 - ${{ if eq(length(parameters.Repositories), 1) }}: - displayName: 'Sparse checkout ${{ parameters.Repositories[0].Name }}' - ${{ else }}: - displayName: 'Sparse checkout repositories' - inputs: - pwsh: true - filePath: eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 - arguments: > - -PathsJson '${{ convertToJson(parameters.Paths) }}' - -RepositoriesJson '${{ convertToJson(parameters.Repositories) }}' - workingDirectory: $(System.DefaultWorkingDirectory) + - task: PowerShell@2 + displayName: 'Restore generated source-build changes' + condition: and(succeeded(), eq(variables['SparseCheckoutRequired'], 'true')) + inputs: + pwsh: true + filePath: eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 + arguments: >- + -Restore + -ChangesPath "$(Agent.TempDirectory)/sparse-checkout-changes.patch" + -SourceVersion "$(SparseCheckoutSourceVersion)" + workingDirectory: $(Build.SourcesDirectory) diff --git a/eng/scripts/ci.yml b/eng/scripts/ci.yml index fcecd646d6fc9..fc7e324e88b21 100644 --- a/eng/scripts/ci.yml +++ b/eng/scripts/ci.yml @@ -10,6 +10,8 @@ trigger: paths: include: - eng/scripts/* + - eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 + - eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml exclude: - eng/scripts/typespec @@ -23,6 +25,8 @@ pr: paths: include: - eng/scripts/* + - eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 + - eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml extends: template: /eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml diff --git a/eng/scripts/tests/Sparse-Checkout.tests.ps1 b/eng/scripts/tests/Sparse-Checkout.tests.ps1 new file mode 100644 index 0000000000000..d9b84518e159c --- /dev/null +++ b/eng/scripts/tests/Sparse-Checkout.tests.ps1 @@ -0,0 +1,267 @@ +BeforeAll { + $script:CheckoutScript = Join-Path $PSScriptRoot '../../pipelines/scripts/Invoke-Sparse-Checkout.ps1' + + function Invoke-TestGit { + param([string]$RepositoryPath, [string[]]$Arguments) + + $output = git -C $RepositoryPath @Arguments 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "Git command failed ($Arguments): $output" + } + return $output + } + + function Enable-TestSparseCheckout { + param([string]$RepositoryPath) + + Invoke-TestGit $RepositoryPath @('sparse-checkout', 'init', '--no-cone') | Out-Null + Invoke-TestGit $RepositoryPath @('sparse-checkout', 'set', '/*', '!/*/', '/eng', '/.config', '**/*.xml') | Out-Null + } + + function Invoke-CheckoutPreparation { + param([string]$RepositoryPath, [string]$PathsJson) + + Push-Location -LiteralPath $RepositoryPath + try { + & $script:CheckoutScript -PathsJson $PathsJson -ChangesPath $script:ChangesPath + } + finally { + Pop-Location + } + } + + function Restore-CheckoutState { + param([string]$RepositoryPath) + + Push-Location -LiteralPath $RepositoryPath + try { + & $script:CheckoutScript -Restore -ChangesPath $script:ChangesPath -SourceVersion $script:SourceVersion + } + finally { + Pop-Location + } + } + + function Invoke-TestNativeCheckout { + param([string]$RepositoryPath, [string[]]$PreparationOutput) + + $prefix = '##vso[task.setvariable variable=SparseCheckoutPatterns]' + $setting = @($PreparationOutput | Where-Object { $_.StartsWith($prefix) }) + $setting.Count | Should -Be 1 + $patterns = $setting[0].Substring($prefix.Length).Replace('%AZP25', '%') + + Invoke-TestGit $RepositoryPath @('sparse-checkout', 'init', '--no-cone') | Out-Null + $startInfo = [System.Diagnostics.ProcessStartInfo]::new() + $startInfo.FileName = (Get-Command git -CommandType Application).Source + $startInfo.WorkingDirectory = $RepositoryPath + $startInfo.UseShellExecute = $false + $startInfo.RedirectStandardOutput = $true + $startInfo.RedirectStandardError = $true + $startInfo.Arguments = "sparse-checkout set $patterns" + $process = [System.Diagnostics.Process]::Start($startInfo) + try { + $output = $process.StandardOutput.ReadToEnd() + $errorOutput = $process.StandardError.ReadToEnd() + $process.WaitForExit() + if ($process.ExitCode -ne 0) { + throw "Native sparse checkout failed: $output $errorOutput" + } + } + finally { + $process.Dispose() + } + Invoke-TestGit $RepositoryPath @('checkout', '--force', 'HEAD') | Out-Null + } +} + +Describe 'Native sparse checkout expansion' -Tag 'UnitTest' { + BeforeEach { + $repositoryPath = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $script:ChangesPath = Join-Path $TestDrive "$([guid]::NewGuid()).patch" + New-Item -ItemType Directory -Path $repositoryPath | Out-Null + $files = @( + 'pom.xml' + 'ClientFromSourcePom.xml' + 'eng/build.ps1' + '.config/settings.yml' + '.config/data.bin' + 'sdk/selected/src/Main.java' + 'sdk/unselected/src/Other.java' + 'sdk/space directory/src/Space.java' + "sdk/quoted'service/src/Quoted.java" + 'sdk/percent%service/src/Percent.java' + ) + foreach ($file in $files) { + $filePath = Join-Path $repositoryPath $file + New-Item -ItemType Directory -Path (Split-Path $filePath) -Force | Out-Null + Set-Content -LiteralPath $filePath -Value "$file baseline" -NoNewline + } + Invoke-TestGit $repositoryPath @('init', '--quiet') | Out-Null + Invoke-TestGit $repositoryPath @('add', '--all') | Out-Null + Invoke-TestGit $repositoryPath @( + '-c', 'user.name=Sparse Checkout Tests' + '-c', 'user.email=sparse-checkout-tests@example.invalid' + '-c', 'commit.gpgsign=false' + '-c', 'core.hooksPath=.git/hooks' + 'commit', '--quiet', '--message', 'Sparse checkout fixture' + ) | Out-Null + $script:SourceVersion = Invoke-TestGit $repositoryPath @('rev-parse', 'HEAD') + } + + It 'restores POM edits and retains generated files after a forced native checkout' { + Enable-TestSparseCheckout $repositoryPath + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeFalse + $pomPath = Join-Path $repositoryPath 'pom.xml' + Set-Content -LiteralPath $pomPath -Value 'updated version' -NoNewline + $sourcePomPath = Join-Path $repositoryPath 'ClientFromSourcePom.xml' + Set-Content -LiteralPath $sourcePomPath -Value 'source build modules' -NoNewline + $generatedPath = Join-Path $repositoryPath 'generated-pom.xml' + Set-Content -LiteralPath $generatedPath -Value 'generated POM' -NoNewline + $originalHead = Invoke-TestGit $repositoryPath @('rev-parse', 'HEAD') + $originalLocation = (Get-Location).Path + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '"["/sdk/selected"]"') + $preparation | Should -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]true' + $preparation | Should -Contain "##vso[task.setvariable variable=SparseCheckoutSourceVersion]$originalHead" + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeFalse + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'updated version' + Invoke-TestNativeCheckout $repositoryPath $preparation + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'pom.xml baseline' + Restore-CheckoutState $repositoryPath + + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'updated version' + Get-Content -LiteralPath $sourcePomPath -Raw | Should -BeExactly 'source build modules' + Get-Content -LiteralPath $generatedPath -Raw | Should -BeExactly 'generated POM' + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath 'sdk/unselected/src/Other.java') | Should -BeFalse + Test-Path (Join-Path $repositoryPath 'eng/build.ps1') | Should -BeTrue + Test-Path (Join-Path $repositoryPath '.config/settings.yml') | Should -BeTrue + Invoke-TestGit $repositoryPath @('sparse-checkout', 'list') | Should -Contain '**/*.xml' + Invoke-TestGit $repositoryPath @('rev-parse', 'HEAD') | Should -BeExactly $originalHead + (Get-Location).Path | Should -BeExactly $originalLocation + Test-Path $script:ChangesPath | Should -BeFalse + } + + It 'passes paths containing spaces, single quotes, and percent signs to the native task' { + Enable-TestSparseCheckout $repositoryPath + $pathsJson = ConvertTo-Json -InputObject @('/sdk/space directory', "/sdk/quoted'service", '/sdk/percent%service') -Compress + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath $pathsJson) + Invoke-TestNativeCheckout $repositoryPath $preparation + Restore-CheckoutState $repositoryPath + + Test-Path (Join-Path $repositoryPath 'sdk/space directory/src/Space.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath "sdk/quoted'service/src/Quoted.java") | Should -BeTrue + Test-Path (Join-Path $repositoryPath 'sdk/percent%service/src/Percent.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath 'sdk/unselected/src/Other.java') | Should -BeFalse + Test-Path $script:ChangesPath | Should -BeFalse + } + + It 'keeps existing patterns before additional paths and their exclusions' { + Enable-TestSparseCheckout $repositoryPath + $originalPatterns = Invoke-TestGit $repositoryPath @('sparse-checkout', 'list') + $paths = @('/sdk/selected', '!/sdk/selected/src/Main.java', '/sdk/space directory') + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath (ConvertTo-Json -InputObject $paths -Compress)) + Invoke-TestNativeCheckout $repositoryPath $preparation + Restore-CheckoutState $repositoryPath + + (Invoke-TestGit $repositoryPath @('sparse-checkout', 'list')) -join "`n" | + Should -BeExactly (($originalPatterns + $paths) -join "`n") + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeFalse + Test-Path (Join-Path $repositoryPath 'sdk/space directory/src/Space.java') | Should -BeTrue + } + + It 'leaves full checkouts and their edits intact' { + $pomPath = Join-Path $repositoryPath 'pom.xml' + Set-Content -LiteralPath $pomPath -Value 'updated version' -NoNewline + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + + $preparation | Should -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]false' + $preparation | Should -Not -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]true' + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'updated version' + Test-Path (Join-Path $repositoryPath 'sdk/unselected/src/Other.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath '.git/info/sparse-checkout') | Should -BeFalse + Test-Path $script:ChangesPath | Should -BeFalse + } + + It 'leaves sparse checkout patterns unchanged for an empty path list' { + Enable-TestSparseCheckout $repositoryPath + $originalPatterns = Invoke-TestGit $repositoryPath @('sparse-checkout', 'list') + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '[]') + + $preparation | Should -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]false' + $preparation | Should -Not -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]true' + (Invoke-TestGit $repositoryPath @('sparse-checkout', 'list')) -join "`n" | + Should -BeExactly ($originalPatterns -join "`n") + Test-Path $script:ChangesPath | Should -BeFalse + } + + It 'rejects uninitialized directories and restores the working directory' { + $emptyDirectory = Join-Path $TestDrive 'uninitialized' + New-Item -ItemType Directory -Path $emptyDirectory | Out-Null + $originalLocation = (Get-Location).Path + + { Invoke-CheckoutPreparation $emptyDirectory '["/sdk/selected"]' } | + Should -Throw + + (Get-Location).Path | Should -BeExactly $originalLocation + Test-Path (Join-Path $emptyDirectory '.git') | Should -BeFalse + } + + It 'restores binary changes and tracked-file deletions' { + Enable-TestSparseCheckout $repositoryPath + $binaryPath = Join-Path $repositoryPath '.config/data.bin' + $binaryContent = [byte[]]@(0, 255, 128, 13, 10, 42) + [System.IO.File]::WriteAllBytes($binaryPath, $binaryContent) + $deletedPath = Join-Path $repositoryPath '.config/settings.yml' + Remove-Item -LiteralPath $deletedPath + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + Invoke-TestNativeCheckout $repositoryPath $preparation + Test-Path $deletedPath | Should -BeTrue + Restore-CheckoutState $repositoryPath + + [Convert]::ToBase64String([System.IO.File]::ReadAllBytes($binaryPath)) | + Should -BeExactly ([Convert]::ToBase64String($binaryContent)) + Test-Path $deletedPath | Should -BeFalse + } + + It 'rejects restoring changes onto a different source revision' { + Enable-TestSparseCheckout $repositoryPath + $pomPath = Join-Path $repositoryPath 'pom.xml' + Set-Content -LiteralPath $pomPath -Value 'updated version' -NoNewline + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + Invoke-TestNativeCheckout $repositoryPath $preparation + Invoke-TestGit $repositoryPath @( + '-c', 'user.name=Sparse Checkout Tests' + '-c', 'user.email=sparse-checkout-tests@example.invalid' + '-c', 'commit.gpgsign=false' + '-c', 'core.hooksPath=.git/hooks' + 'commit', '--allow-empty', '--quiet', '--message', 'Different source revision' + ) | Out-Null + + { Restore-CheckoutState $repositoryPath } | Should -Throw '*changed the source revision*' + + Test-Path $script:ChangesPath | Should -BeTrue + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'pom.xml baseline' + } + + It 'fails without discarding the patch when changes cannot be restored' { + Enable-TestSparseCheckout $repositoryPath + $pomPath = Join-Path $repositoryPath 'pom.xml' + Set-Content -LiteralPath $pomPath -Value 'updated version' -NoNewline + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + Invoke-TestNativeCheckout $repositoryPath $preparation + Set-Content -LiteralPath $pomPath -Value 'conflicting version' -NoNewline + $originalLocation = (Get-Location).Path + + { Restore-CheckoutState $repositoryPath } | Should -Throw '*Restoring checkout changes failed*' + + Test-Path $script:ChangesPath | Should -BeTrue + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'conflicting version' + (Get-Location).Path | Should -BeExactly $originalLocation + } +} \ No newline at end of file From 4d450f74759a1d30dff563e4922a38551f3e78c8 Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:58:23 -0400 Subject: [PATCH 2/5] Minor cleanups, remove tag fetching where it isn't necessary --- eng/README.md | 9 ++- eng/containers/ci.yml | 8 ++- eng/pipelines/code-quality-reports.yml | 11 ++- .../templates/jobs/build-validate-pom.yml | 13 ++-- eng/pipelines/templates/jobs/ci.yml | 29 +++++--- .../archetype-java-auto-release-batch.yml | 61 +++++++++++----- .../stages/archetype-java-release-batch.yml | 71 +++++++++++++------ .../stages/archetype-java-release-patch.yml | 45 +++++++++--- .../archetype-java-release-pom-only.yml | 16 +++-- .../stages/archetype-sdk-client-patch.yml | 29 +++++--- .../steps/initialize-test-environment.yml | 19 +++-- .../sparse-checkout-repo-initialized.yml | 2 +- eng/scripts/tests/Sparse-Checkout.tests.ps1 | 53 ++++++++++++++ 13 files changed, 283 insertions(+), 83 deletions(-) diff --git a/eng/README.md b/eng/README.md index efe345dd810f5..fa9eaef957cd8 100644 --- a/eng/README.md +++ b/eng/README.md @@ -19,9 +19,16 @@ for both their initial sparse checkout and dependency-driven expansion. This req - Use `sparseCheckoutPatterns` for non-cone patterns, including file globs and exclusions. - Preserve the base patterns `/* !/*/ /eng /.config` and the order of job-specific patterns. -- Use `fetchFilter: tree:0`, `fetchDepth: 0`, and `fetchTags: true` to retain the existing treeless fetch, full history, and tags. +- Write literal patterns one per line in a folded YAML scalar (`>-`); YAML joins them with spaces for the checkout task. +- Use `fetchFilter: tree:0` and `fetchDepth: 0` to retain treeless fetches and full commit history. +- Set `fetchTags: false` explicitly to avoid unnecessary tag synchronization and pipeline-dependent defaults. - Set `path` explicitly when checking out Java alongside build-tools; it is relative to `$(Pipeline.Workspace)`. +Initial checkouts that can run test-pipeline versioning use `fetchTags: ${{ parameters.TestPipeline }}` instead. +[SetTestPipelineVersion.ps1](common/scripts/SetTestPipelineVersion.ps1) reads local tags to choose the version, so those +jobs still need tags when `TestPipeline` is enabled. Release creation checks and creates tags through the GitHub API +and does not require local tags. Expansion checkouts do not fetch tags again. + Java builds compute additional paths after generating project lists and updating POM files. Use [pipelines/templates/steps/sparse-checkout-repo-initialized.yml](pipelines/templates/steps/sparse-checkout-repo-initialized.yml) with `Paths: $(SparseCheckoutDirectories)` to expand these checkouts. The variable remains a JSON array. diff --git a/eng/containers/ci.yml b/eng/containers/ci.yml index 3e734eeb20c93..87c60b4220b26 100644 --- a/eng/containers/ci.yml +++ b/eng/containers/ci.yml @@ -48,8 +48,12 @@ extends: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - task: AzureCLI@2 displayName: Login to $(containerRegistry) diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 5d3f603ceb7ea..4b7731c33c0ba 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -38,10 +38,15 @@ extends: path: s fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config **/*.xml - !sdk/**/test-recordings !sdk/**/session-records + /* + !/*/ + /eng + /.config + **/*.xml + !sdk/**/test-recordings + !sdk/**/session-records sdk/tools/linting-extensions - task: UsePythonVersion@0 diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index 04a07aea5b7f2..2d797a58a134b 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -27,12 +27,17 @@ jobs: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config + /* + !/*/ + /eng + /.config sdk/${{ parameters.ServiceDirectory }} - **/*.xml **/*.md - !sdk/**/test-recordings !sdk/**/session-records + **/*.xml + **/*.md + !sdk/**/test-recordings + !sdk/**/session-records - script: | echo "##vso[build.addbuildtag]Scheduled" diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 68217a91f88b7..653703dbec6a9 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -115,11 +115,17 @@ jobs: path: s fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: ${{ parameters.TestPipeline }} sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - **/*.xml **/*.md **/ci*.yml - !sdk/**/test-recordings !sdk/**/session-records + /* + !/*/ + /eng + /.config + **/*.xml + **/*.md + **/ci*.yml + !sdk/**/test-recordings + !sdk/**/session-records - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -325,11 +331,18 @@ jobs: path: s fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: ${{ parameters.TestPipeline }} sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - **/*.xml **/*.md **/*.yml .vscode/cspell.json - !sdk/**/test-recordings !sdk/**/session-records + /* + !/*/ + /eng + /.config + **/*.xml + **/*.md + **/*.yml + .vscode/cspell.json + !sdk/**/test-recordings + !sdk/**/session-records - task: PowerShell@2 displayName: 'Verify versions in POM files' diff --git a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml index 8a7d1419fa52a..6098de789db4b 100644 --- a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml @@ -99,11 +99,16 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: ${{ parameters.TestPipeline }} sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - sdk/${{ parameters.ServiceDirectory }} **/*.xml - !sdk/**/test-recordings !sdk/**/session-records + /* + !/*/ + /eng + /.config + sdk/${{ parameters.ServiceDirectory }} + **/*.xml + !sdk/**/test-recordings + !sdk/**/session-records - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -194,8 +199,12 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - template: /eng/common/pipelines/templates/steps/retain-run.yml - template: /eng/common/pipelines/templates/steps/login-to-github.yml parameters: @@ -268,8 +277,12 @@ stages: path: azure-sdk-for-java fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -376,11 +389,17 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - ${{ parameters.ServiceDirectory }} **/*.xml **/*.md - !sdk/**/test-recordings !sdk/**/session-records + /* + !/*/ + /eng + /.config + ${{ parameters.ServiceDirectory }} + **/*.xml + **/*.md + !sdk/**/test-recordings + !sdk/**/session-records - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -422,10 +441,14 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml + /* + !/*/ + /eng + /.config + sdk/${{ parameters.ServiceDirectory }}/**/*.md + !**/pom*.xml - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -510,8 +533,12 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - download: current displayName: 'Download Artifact: packages-signed-auto-release' diff --git a/eng/pipelines/templates/stages/archetype-java-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-release-batch.yml index ea9cf7398293e..738b9845cee98 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-batch.yml @@ -118,11 +118,16 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: ${{ parameters.TestPipeline }} sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - sdk/${{ parameters.ServiceDirectory }} **/*.xml - !sdk/**/test-recordings !sdk/**/session-records + /* + !/*/ + /eng + /.config + sdk/${{ parameters.ServiceDirectory }} + **/*.xml + !sdk/**/test-recordings + !sdk/**/session-records - download: current displayName: 'Download Artifact: packages' @@ -201,8 +206,12 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - template: /eng/common/pipelines/templates/steps/retain-run.yml # No pattern, just download everything that's been signed - download: current @@ -266,8 +275,12 @@ stages: path: azure-sdk-for-java fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - download: current displayName: 'Download Artifacts' artifact: packages-signed @@ -320,11 +333,17 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - ${{ parameters.ServiceDirectory }} **/*.xml **/*.md - !sdk/**/test-recordings !sdk/**/session-records + /* + !/*/ + /eng + /.config + ${{ parameters.ServiceDirectory }} + **/*.xml + **/*.md + !sdk/**/test-recordings + !sdk/**/session-records - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -371,10 +390,14 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml + /* + !/*/ + /eng + /.config + sdk/${{ parameters.ServiceDirectory }}/**/*.md + !**/pom*.xml - download: current displayName: 'Download Artifact: packages' artifact: packages @@ -418,8 +441,12 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - download: current displayName: 'Download Artifact: packages-signed' artifact: packages-signed @@ -507,10 +534,14 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml + /* + !/*/ + /eng + /.config + sdk/${{ parameters.ServiceDirectory }}/**/*.md + !**/pom*.xml - download: current displayName: 'Download Artifacts' artifact: packages diff --git a/eng/pipelines/templates/stages/archetype-java-release-patch.yml b/eng/pipelines/templates/stages/archetype-java-release-patch.yml index fbc2f17340687..4ad78e280d74e 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-patch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-patch.yml @@ -101,8 +101,12 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - template: /eng/common/pipelines/templates/steps/retain-run.yml # No pattern, just download everything that's been signed - download: current @@ -164,8 +168,12 @@ stages: path: azure-sdk-for-java fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - download: current displayName: 'Download Artifacts' artifact: packages-signed @@ -207,10 +215,16 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config **/*.xml **/*.md - !sdk/**/test-recordings/* !sdk/**/session-records/* + /* + !/*/ + /eng + /.config + **/*.xml + **/*.md + !sdk/**/test-recordings/* + !sdk/**/session-records/* - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -259,9 +273,14 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: >- - /* !/*/ /eng /.config sdk/**/*.md !**/pom*.xml + /* + !/*/ + /eng + /.config + sdk/**/*.md + !**/pom*.xml - download: current displayName: 'Download Artifact: packages' artifact: packages @@ -299,8 +318,12 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - download: current displayName: 'Download Artifact: packages-signed' artifact: packages-signed diff --git a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml index ce1477b8c1efe..f5b0f6b90aa03 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml @@ -98,8 +98,12 @@ stages: - checkout: self fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config - template: /eng/common/pipelines/templates/steps/retain-run.yml - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml parameters: @@ -153,8 +157,12 @@ stages: path: azure-sdk-for-java fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true - sparseCheckoutPatterns: '/* !/*/ /eng /.config' + fetchTags: false + sparseCheckoutPatterns: >- + /* + !/*/ + /eng + /.config # Setup Maven mirror settings and authenticate with Azure Artifacts - template: /eng/pipelines/templates/steps/maven-authenticate.yml parameters: diff --git a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml index ccfee7ea8065a..d8c958ea9b0bc 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml @@ -50,11 +50,17 @@ extends: path: s fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: ${{ parameters.TestPipeline }} sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - **/*.xml **/*.md **/*.yml - !sdk/**/test-recordings/* !sdk/**/session-records/* + /* + !/*/ + /eng + /.config + **/*.xml + **/*.md + **/*.yml + !sdk/**/test-recordings/* + !sdk/**/session-records/* - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -213,11 +219,18 @@ extends: path: s fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: ${{ parameters.TestPipeline }} sparseCheckoutPatterns: >- - /* !/*/ /eng /.config - **/*.xml **/*.md **/*.yml .vscode/cspell.json - !sdk/**/test-recordings/* !sdk/**/session-records/* + /* + !/*/ + /eng + /.config + **/*.xml + **/*.md + **/*.yml + .vscode/cspell.json + !sdk/**/test-recordings/* + !sdk/**/session-records/* - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index f9348c25786bc..2e39c9e715d61 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -20,14 +20,25 @@ steps: path: s fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false ${{ if parameters.CheckoutRecordings }}: sparseCheckoutPatterns: >- - /* !/*/ /eng /.config **/*.xml **/ci*.yml + /* + !/*/ + /eng + /.config + **/*.xml + **/ci*.yml ${{ else }}: sparseCheckoutPatterns: >- - /* !/*/ /eng /.config **/*.xml **/ci*.yml - !sdk/**/test-recordings !sdk/**/session-records + /* + !/*/ + /eng + /.config + **/*.xml + **/ci*.yml + !sdk/**/test-recordings + !sdk/**/session-records - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' diff --git a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml index c791eaf655666..9b3f09e3ddb96 100644 --- a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml +++ b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml @@ -23,7 +23,7 @@ steps: clean: false fetchFilter: tree:0 fetchDepth: 0 - fetchTags: true + fetchTags: false sparseCheckoutPatterns: $(SparseCheckoutPatterns) condition: and(succeeded(), eq(variables['SparseCheckoutRequired'], 'true')) diff --git a/eng/scripts/tests/Sparse-Checkout.tests.ps1 b/eng/scripts/tests/Sparse-Checkout.tests.ps1 index d9b84518e159c..25c9beaaff0b7 100644 --- a/eng/scripts/tests/Sparse-Checkout.tests.ps1 +++ b/eng/scripts/tests/Sparse-Checkout.tests.ps1 @@ -264,4 +264,57 @@ Describe 'Native sparse checkout expansion' -Tag 'UnitTest' { Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'conflicting version' (Get-Location).Path | Should -BeExactly $originalLocation } +} + +Describe 'Native sparse checkout tag policy' -Tag 'UnitTest' { + BeforeAll { + $script:EngineeringRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path + . "$script:EngineeringRoot/common/scripts/Helpers/PSModule-Helpers.ps1" + Install-ModuleIfNotInstalled 'powershell-yaml' '0.4.7' | Import-Module + + function Get-SparseCheckout { + param($Node, [string]$JobName) + + if ($Node -is [System.Collections.IDictionary]) { + if ($Node.job) { + $JobName = $Node.job + } + if ($Node.checkout -eq 'self' -and $Node.fetchFilter -eq 'tree:0') { + return @{ JobName = $JobName; Step = $Node } + } + foreach ($value in $Node.Values) { + Get-SparseCheckout -Node $value -JobName $JobName + } + } + elseif ($Node -is [System.Collections.IList]) { + foreach ($item in $Node) { + Get-SparseCheckout -Node $item -JobName $JobName + } + } + } + } + + It 'only fetches tags for test-versioning in ' -TestCases @( + @{ File = 'pipelines/templates/jobs/ci.yml'; Count = 2; TagJobs = @('Build', 'Analyze') } + @{ File = 'pipelines/templates/jobs/build-validate-pom.yml'; Count = 1; TagJobs = @() } + @{ File = 'pipelines/templates/stages/archetype-sdk-client-patch.yml'; Count = 2; TagJobs = @('Build', 'AnalyzeAndVerify') } + @{ File = 'pipelines/templates/stages/archetype-java-release-batch.yml'; Count = 7; TagJobs = @('VerifyReleaseVersion') } + @{ File = 'pipelines/templates/stages/archetype-java-auto-release-batch.yml'; Count = 6; TagJobs = @('VerifyReleaseVersion') } + @{ File = 'pipelines/templates/stages/archetype-java-release-patch.yml'; Count = 5; TagJobs = @() } + @{ File = 'pipelines/templates/stages/archetype-java-release-pom-only.yml'; Count = 2; TagJobs = @() } + @{ File = 'pipelines/templates/steps/initialize-test-environment.yml'; Count = 1; TagJobs = @() } + @{ File = 'pipelines/templates/steps/sparse-checkout-repo-initialized.yml'; Count = 1; TagJobs = @() } + @{ File = 'pipelines/code-quality-reports.yml'; Count = 1; TagJobs = @() } + @{ File = 'containers/ci.yml'; Count = 1; TagJobs = @() } + ) { + param($File, $Count, $TagJobs) + + $yaml = Get-Content (Join-Path $script:EngineeringRoot $File) -Raw | ConvertFrom-Yaml -Ordered + $checkouts = @(Get-SparseCheckout -Node $yaml) + $checkouts.Count | Should -Be $Count + foreach ($checkout in $checkouts) { + $expectedTags = if ($TagJobs -contains $checkout.JobName) { '${{ parameters.TestPipeline }}' } else { $false } + $checkout.Step.fetchTags | Should -BeExactly $expectedTags + } + } } \ No newline at end of file From 844d9637450712f2a8a147a5ea808841a9d47d7a Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:39:10 -0400 Subject: [PATCH 3/5] Make checkout quiet --- eng/README.md | 1 + eng/containers/ci.yml | 2 ++ eng/pipelines/code-quality-reports.yml | 2 ++ .../templates/jobs/build-validate-pom.yml | 2 ++ eng/pipelines/templates/jobs/ci.yml | 4 ++++ .../stages/archetype-java-auto-release-batch.yml | 12 ++++++++++++ .../stages/archetype-java-release-batch.yml | 14 ++++++++++++++ .../stages/archetype-java-release-patch.yml | 10 ++++++++++ .../stages/archetype-java-release-pom-only.yml | 4 ++++ .../stages/archetype-sdk-client-patch.yml | 4 ++++ .../steps/initialize-test-environment.yml | 2 ++ .../steps/sparse-checkout-repo-initialized.yml | 2 ++ eng/scripts/tests/Sparse-Checkout.tests.ps1 | 5 +++-- 13 files changed, 62 insertions(+), 2 deletions(-) diff --git a/eng/README.md b/eng/README.md index fa9eaef957cd8..31438503e2b6e 100644 --- a/eng/README.md +++ b/eng/README.md @@ -22,6 +22,7 @@ for both their initial sparse checkout and dependency-driven expansion. This req - Write literal patterns one per line in a folded YAML scalar (`>-`); YAML joins them with spaces for the checkout task. - Use `fetchFilter: tree:0` and `fetchDepth: 0` to retain treeless fetches and full commit history. - Set `fetchTags: false` explicitly to avoid unnecessary tag synchronization and pipeline-dependent defaults. +- Set `AGENT_SOURCE_CHECKOUT_QUIET: 'true'` in each checkout step's `env` to suppress verbose checkout progress output. - Set `path` explicitly when checking out Java alongside build-tools; it is relative to `$(Pipeline.Workspace)`. Initial checkouts that can run test-pipeline versioning use `fetchTags: ${{ parameters.TestPipeline }}` instead. diff --git a/eng/containers/ci.yml b/eng/containers/ci.yml index 87c60b4220b26..83e745b192f0d 100644 --- a/eng/containers/ci.yml +++ b/eng/containers/ci.yml @@ -49,6 +49,8 @@ extends: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 4b7731c33c0ba..78a38358a6172 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -39,6 +39,8 @@ extends: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index 2d797a58a134b..b198565ed4076 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -28,6 +28,8 @@ jobs: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 653703dbec6a9..de6ed4707ca3b 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -116,6 +116,8 @@ jobs: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -332,6 +334,8 @@ jobs: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml index 6098de789db4b..6315748c48f3b 100644 --- a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml @@ -100,6 +100,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -200,6 +202,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -278,6 +282,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -390,6 +396,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -442,6 +450,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -534,6 +544,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/templates/stages/archetype-java-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-release-batch.yml index 738b9845cee98..b6ee91c1b89e6 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-batch.yml @@ -119,6 +119,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -207,6 +209,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -276,6 +280,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -334,6 +340,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -391,6 +399,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -442,6 +452,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -535,6 +547,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/templates/stages/archetype-java-release-patch.yml b/eng/pipelines/templates/stages/archetype-java-release-patch.yml index 4ad78e280d74e..3aba453afa700 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-patch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-patch.yml @@ -102,6 +102,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -169,6 +171,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -216,6 +220,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -274,6 +280,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -319,6 +327,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml index f5b0f6b90aa03..7d54fcfba5756 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml @@ -99,6 +99,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -158,6 +160,8 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml index d8c958ea9b0bc..c32f86230d88c 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml @@ -51,6 +51,8 @@ extends: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ @@ -220,6 +222,8 @@ extends: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: >- /* !/*/ diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 2e39c9e715d61..1f14a25ed9fbd 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -21,6 +21,8 @@ steps: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' ${{ if parameters.CheckoutRecordings }}: sparseCheckoutPatterns: >- /* diff --git a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml index 9b3f09e3ddb96..53a84f127306a 100644 --- a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml +++ b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml @@ -24,6 +24,8 @@ steps: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' sparseCheckoutPatterns: $(SparseCheckoutPatterns) condition: and(succeeded(), eq(variables['SparseCheckoutRequired'], 'true')) diff --git a/eng/scripts/tests/Sparse-Checkout.tests.ps1 b/eng/scripts/tests/Sparse-Checkout.tests.ps1 index 25c9beaaff0b7..e8f161e511687 100644 --- a/eng/scripts/tests/Sparse-Checkout.tests.ps1 +++ b/eng/scripts/tests/Sparse-Checkout.tests.ps1 @@ -266,7 +266,7 @@ Describe 'Native sparse checkout expansion' -Tag 'UnitTest' { } } -Describe 'Native sparse checkout tag policy' -Tag 'UnitTest' { +Describe 'Native sparse checkout settings' -Tag 'UnitTest' { BeforeAll { $script:EngineeringRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path . "$script:EngineeringRoot/common/scripts/Helpers/PSModule-Helpers.ps1" @@ -294,7 +294,7 @@ Describe 'Native sparse checkout tag policy' -Tag 'UnitTest' { } } - It 'only fetches tags for test-versioning in ' -TestCases @( + It 'uses quiet checkout and only fetches tags for test-versioning in ' -TestCases @( @{ File = 'pipelines/templates/jobs/ci.yml'; Count = 2; TagJobs = @('Build', 'Analyze') } @{ File = 'pipelines/templates/jobs/build-validate-pom.yml'; Count = 1; TagJobs = @() } @{ File = 'pipelines/templates/stages/archetype-sdk-client-patch.yml'; Count = 2; TagJobs = @('Build', 'AnalyzeAndVerify') } @@ -315,6 +315,7 @@ Describe 'Native sparse checkout tag policy' -Tag 'UnitTest' { foreach ($checkout in $checkouts) { $expectedTags = if ($TagJobs -contains $checkout.JobName) { '${{ parameters.TestPipeline }}' } else { $false } $checkout.Step.fetchTags | Should -BeExactly $expectedTags + $checkout.Step.env.AGENT_SOURCE_CHECKOUT_QUIET | Should -BeExactly 'true' } } } \ No newline at end of file From 322c5833f1670dac953a648b636ec4632b4b04d7 Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:12:41 -0400 Subject: [PATCH 4/5] Attempt to fix tests --- eng/scripts/tests/Sparse-Checkout.tests.ps1 | 22 ++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/eng/scripts/tests/Sparse-Checkout.tests.ps1 b/eng/scripts/tests/Sparse-Checkout.tests.ps1 index e8f161e511687..9de246dd784d4 100644 --- a/eng/scripts/tests/Sparse-Checkout.tests.ps1 +++ b/eng/scripts/tests/Sparse-Checkout.tests.ps1 @@ -52,7 +52,7 @@ BeforeAll { Invoke-TestGit $RepositoryPath @('sparse-checkout', 'init', '--no-cone') | Out-Null $startInfo = [System.Diagnostics.ProcessStartInfo]::new() - $startInfo.FileName = (Get-Command git -CommandType Application).Source + $startInfo.FileName = (Get-Command git -CommandType Application | Select-Object -First 1).Source $startInfo.WorkingDirectory = $RepositoryPath $startInfo.UseShellExecute = $false $startInfo.RedirectStandardOutput = $true @@ -157,6 +157,26 @@ Describe 'Native sparse checkout expansion' -Tag 'UnitTest' { Test-Path $script:ChangesPath | Should -BeFalse } + It 'uses the first Git executable when command discovery returns multiple matches' { + Enable-TestSparseCheckout $repositoryPath + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + $script:GitExecutablePath = (Get-Command git -CommandType Application | Select-Object -First 1).Source + + Mock Get-Command { + [pscustomobject]@{ Source = $script:GitExecutablePath } + [pscustomobject]@{ Source = 'nonexistent-secondary-git' } + } -ParameterFilter { $Name -eq 'git' -and $CommandType -eq 'Application' } + + Invoke-TestNativeCheckout $repositoryPath $preparation + Restore-CheckoutState $repositoryPath + + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath 'sdk/unselected/src/Other.java') | Should -BeFalse + Should -Invoke Get-Command -Times 1 -Exactly -ParameterFilter { + $Name -eq 'git' -and $CommandType -eq 'Application' + } + } + It 'keeps existing patterns before additional paths and their exclusions' { Enable-TestSparseCheckout $repositoryPath $originalPatterns = Invoke-TestGit $repositoryPath @('sparse-checkout', 'list') From ede9e3d9d54a392f28681d9170763bf7a4412b31 Mon Sep 17 00:00:00 2001 From: alzimmermsft <48699787+alzimmermsft@users.noreply.github.com> Date: Mon, 14 Sep 2026 14:26:06 -0400 Subject: [PATCH 5/5] Minor tyding --- eng/containers/ci.yml | 6 +- eng/pipelines/code-quality-reports.yml | 10 +--- .../templates/jobs/build-validate-pom.yml | 11 +--- eng/pipelines/templates/jobs/ci.yml | 23 +------- .../archetype-java-auto-release-batch.yml | 47 ++-------------- .../stages/archetype-java-release-batch.yml | 55 +++---------------- .../stages/archetype-java-release-patch.yml | 36 ++---------- .../archetype-java-release-pom-only.yml | 12 +--- .../stages/archetype-sdk-client-patch.yml | 23 +------- .../steps/initialize-test-environment.yml | 22 ++------ .../sparse-checkout-repo-initialized.yml | 2 +- 11 files changed, 32 insertions(+), 215 deletions(-) diff --git a/eng/containers/ci.yml b/eng/containers/ci.yml index 83e745b192f0d..89b300f0de13d 100644 --- a/eng/containers/ci.yml +++ b/eng/containers/ci.yml @@ -49,13 +49,9 @@ extends: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - task: AzureCLI@2 displayName: Login to $(containerRegistry) diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 78a38358a6172..eec4962f0ee7e 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -39,17 +39,9 @@ extends: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml !sdk/**/test-recordings !sdk/**/session-records sdk/tools/linting-extensions env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - **/*.xml - !sdk/**/test-recordings - !sdk/**/session-records - sdk/tools/linting-extensions - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index b198565ed4076..9fb2e6b17d570 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -28,18 +28,9 @@ jobs: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }} **/*.xml **/*.md !sdk/**/test-recordings !sdk/**/session-records env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - sdk/${{ parameters.ServiceDirectory }} - **/*.xml - **/*.md - !sdk/**/test-recordings - !sdk/**/session-records - script: | echo "##vso[build.addbuildtag]Scheduled" diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index de6ed4707ca3b..9bb412ad665f7 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -116,18 +116,9 @@ jobs: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md **/ci*.yml !sdk/**/test-recordings !sdk/**/session-records env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - **/*.xml - **/*.md - **/ci*.yml - !sdk/**/test-recordings - !sdk/**/session-records - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -334,19 +325,9 @@ jobs: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md **/*.yml .vscode/cspell.json !sdk/**/test-recordings !sdk/**/session-records env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - **/*.xml - **/*.md - **/*.yml - .vscode/cspell.json - !sdk/**/test-recordings - !sdk/**/session-records - task: PowerShell@2 displayName: 'Verify versions in POM files' diff --git a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml index 6315748c48f3b..7b09c0a11ebec 100644 --- a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml @@ -100,17 +100,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }} **/*.xml !sdk/**/test-recordings !sdk/**/session-records env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - sdk/${{ parameters.ServiceDirectory }} - **/*.xml - !sdk/**/test-recordings - !sdk/**/session-records - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -202,13 +194,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - template: /eng/common/pipelines/templates/steps/retain-run.yml - template: /eng/common/pipelines/templates/steps/login-to-github.yml parameters: @@ -282,13 +270,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -396,18 +380,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config ${{ parameters.ServiceDirectory }} **/*.xml **/*.md !sdk/**/test-recordings !sdk/**/session-records env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - ${{ parameters.ServiceDirectory }} - **/*.xml - **/*.md - !sdk/**/test-recordings - !sdk/**/session-records - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -450,15 +425,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - sdk/${{ parameters.ServiceDirectory }}/**/*.md - !**/pom*.xml - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -544,13 +513,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - download: current displayName: 'Download Artifact: packages-signed-auto-release' diff --git a/eng/pipelines/templates/stages/archetype-java-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-release-batch.yml index b6ee91c1b89e6..db12a12a23722 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-batch.yml @@ -119,17 +119,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }} **/*.xml !sdk/**/test-recordings !sdk/**/session-records env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - sdk/${{ parameters.ServiceDirectory }} - **/*.xml - !sdk/**/test-recordings - !sdk/**/session-records - download: current displayName: 'Download Artifact: packages' @@ -209,13 +201,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - template: /eng/common/pipelines/templates/steps/retain-run.yml # No pattern, just download everything that's been signed - download: current @@ -280,13 +268,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - download: current displayName: 'Download Artifacts' artifact: packages-signed @@ -340,18 +324,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config ${{ parameters.ServiceDirectory }} **/*.xml **/*.md !sdk/**/test-recordings !sdk/**/session-records env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - ${{ parameters.ServiceDirectory }} - **/*.xml - **/*.md - !sdk/**/test-recordings - !sdk/**/session-records - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -399,15 +374,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - sdk/${{ parameters.ServiceDirectory }}/**/*.md - !**/pom*.xml - download: current displayName: 'Download Artifact: packages' artifact: packages @@ -452,13 +421,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - download: current displayName: 'Download Artifact: packages-signed' artifact: packages-signed @@ -547,15 +512,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - sdk/${{ parameters.ServiceDirectory }}/**/*.md - !**/pom*.xml - download: current displayName: 'Download Artifacts' artifact: packages diff --git a/eng/pipelines/templates/stages/archetype-java-release-patch.yml b/eng/pipelines/templates/stages/archetype-java-release-patch.yml index 3aba453afa700..12783f080ea2b 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-patch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-patch.yml @@ -102,13 +102,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - template: /eng/common/pipelines/templates/steps/retain-run.yml # No pattern, just download everything that's been signed - download: current @@ -171,13 +167,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - download: current displayName: 'Download Artifacts' artifact: packages-signed @@ -220,17 +212,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md !sdk/**/test-recordings/* !sdk/**/session-records/* env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - **/*.xml - **/*.md - !sdk/**/test-recordings/* - !sdk/**/session-records/* - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -280,15 +264,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/**/*.md !**/pom*.xml env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - sdk/**/*.md - !**/pom*.xml - download: current displayName: 'Download Artifact: packages' artifact: packages @@ -327,13 +305,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - download: current displayName: 'Download Artifact: packages-signed' artifact: packages-signed diff --git a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml index 7d54fcfba5756..de795da61b8d9 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml @@ -99,13 +99,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - template: /eng/common/pipelines/templates/steps/retain-run.yml - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml parameters: @@ -160,13 +156,9 @@ stages: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config # Setup Maven mirror settings and authenticate with Azure Artifacts - template: /eng/pipelines/templates/steps/maven-authenticate.yml parameters: diff --git a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml index c32f86230d88c..4af1b4f23e156 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml @@ -51,18 +51,9 @@ extends: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md **/*.yml !sdk/**/test-recordings/* !sdk/**/session-records/* env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - **/*.xml - **/*.md - **/*.yml - !sdk/**/test-recordings/* - !sdk/**/session-records/* - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -222,19 +213,9 @@ extends: fetchFilter: tree:0 fetchDepth: 0 fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md **/*.yml .vscode/cspell.json !sdk/**/test-recordings/* !sdk/**/session-records/* env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - **/*.xml - **/*.md - **/*.yml - .vscode/cspell.json - !sdk/**/test-recordings/* - !sdk/**/session-records/* - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 1f14a25ed9fbd..5162a70ac20c6 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -21,26 +21,12 @@ steps: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false - env: - AGENT_SOURCE_CHECKOUT_QUIET: 'true' ${{ if parameters.CheckoutRecordings }}: - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - **/*.xml - **/ci*.yml + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/ci*.yml ${{ else }}: - sparseCheckoutPatterns: >- - /* - !/*/ - /eng - /.config - **/*.xml - **/ci*.yml - !sdk/**/test-recordings - !sdk/**/session-records + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/ci*.yml !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' diff --git a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml index 53a84f127306a..554efe68c6790 100644 --- a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml +++ b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml @@ -24,9 +24,9 @@ steps: fetchFilter: tree:0 fetchDepth: 0 fetchTags: false + sparseCheckoutPatterns: $(SparseCheckoutPatterns) env: AGENT_SOURCE_CHECKOUT_QUIET: 'true' - sparseCheckoutPatterns: $(SparseCheckoutPatterns) condition: and(succeeded(), eq(variables['SparseCheckoutRequired'], 'true')) - task: PowerShell@2