diff --git a/eng/README.md b/eng/README.md index ac4f6478c24ab..31438503e2b6e 100644 --- a/eng/README.md +++ b/eng/README.md @@ -10,6 +10,53 @@ All the tools/utilities used in Microsoft Azure Java SDK's build config are defi - `lintingconfigs` - CheckStyle and SpotBugs rule configurations. +## Sparse Checkouts + +Java-owned pipeline jobs use the native Azure Pipelines +[`checkout` step](https://learn.microsoft.com/azure/devops/pipelines/yaml-schema/steps-checkout) +for both their initial sparse checkout and dependency-driven expansion. This requires agent version +3.253.0/4.253.0 or later and Git 2.25 or later. + +- Use `sparseCheckoutPatterns` for non-cone patterns, including file globs and exclusions. +- Preserve the base patterns `/* !/*/ /eng /.config` and the order of job-specific patterns. +- Write literal patterns one per line in a folded YAML scalar (`>-`); YAML joins them with spaces for the checkout task. +- Use `fetchFilter: tree:0` and `fetchDepth: 0` to retain treeless fetches and full commit history. +- Set `fetchTags: false` explicitly to avoid unnecessary tag synchronization and pipeline-dependent defaults. +- Set `AGENT_SOURCE_CHECKOUT_QUIET: 'true'` in each checkout step's `env` to suppress verbose checkout progress output. +- Set `path` explicitly when checking out Java alongside build-tools; it is relative to `$(Pipeline.Workspace)`. + +Initial checkouts that can run test-pipeline versioning use `fetchTags: ${{ parameters.TestPipeline }}` instead. +[SetTestPipelineVersion.ps1](common/scripts/SetTestPipelineVersion.ps1) reads local tags to choose the version, so those +jobs still need tags when `TestPipeline` is enabled. Release creation checks and creates tags through the GitHub API +and does not require local tags. Expansion checkouts do not fetch tags again. + +Java builds compute additional paths after generating project lists and updating POM files. Use +[pipelines/templates/steps/sparse-checkout-repo-initialized.yml](pipelines/templates/steps/sparse-checkout-repo-initialized.yml) +with `Paths: $(SparseCheckoutDirectories)` to expand these checkouts. The variable remains a JSON array. +The template prepares a space-separated `SparseCheckoutPatterns` variable containing the original patterns followed +by the additional paths, then passes it to a second native checkout. Both checkouts use `path: s` to retain the +existing source location. Empty path lists and full checkouts do not need a second checkout. + +The agent performs a forced checkout even with `clean: false`. Before checkout, the helper saves tracked-file changes +as a binary patch under `$(Agent.TempDirectory)`. After successful checkout it verifies that the source revision has +not changed and restores the patch, including changes to POM files generated during dependency discovery. +The second checkout uses `clean: false` to retain generated untracked files. Restoration failures fail the job and +leave the patch available for diagnosis; do not replace this template with an unprotected second checkout. + +Layered jobs keep private mirrored repositories on full checkouts so later expansion does not require persisted credentials. +Configurable external docs repositories still use the shared custom template because their repository names are selected +at runtime. Shared pipeline templates under `common` are maintained upstream in `azure-sdk-tools`. + +Run the expansion regression tests from the repository root using PowerShell 7 and Pester 5.7.1: + +```powershell +Import-Module Pester -RequiredVersion 5.7.1 +Invoke-Pester -Path eng/scripts/tests/Sparse-Checkout.tests.ps1 -Output Detailed +``` + +Before rolling out to an agent pool, validate PR, FromSource, private-mirror, and multi-repository publishing jobs on +Windows and Linux. Compare source revisions, checked-out files, preserved POM edits, and checkout time and transfer size. + --- For developer guides (building, testing, code quality, versioning), see the consolidated documentation hub: diff --git a/eng/containers/ci.yml b/eng/containers/ci.yml index 81232455fc503..89b300f0de13d 100644 --- a/eng/containers/ci.yml +++ b/eng/containers/ci.yml @@ -45,7 +45,13 @@ extends: - $(containerRegistry).azurecr.io/$(imageRepository):$(stableTag) steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - task: AzureCLI@2 displayName: Login to $(containerRegistry) diff --git a/eng/pipelines/code-quality-reports.yml b/eng/pipelines/code-quality-reports.yml index 551a561c20c5f..eec4962f0ee7e 100644 --- a/eng/pipelines/code-quality-reports.yml +++ b/eng/pipelines/code-quality-reports.yml @@ -34,13 +34,14 @@ extends: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' - - 'sdk/tools/linting-extensions' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml !sdk/**/test-recordings !sdk/**/session-records sdk/tools/linting-extensions + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -77,9 +78,8 @@ extends: arguments: '--set-skip-linting-projects SkipLintingProjects --artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)' workingDirectory: '$(System.DefaultWorkingDirectory)' - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: - SkipCheckoutNone: true Paths: $(SparseCheckoutDirectories) - task: PowerShell@2 diff --git a/eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 b/eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 index db43a1fe21555..daa0ddd95a272 100644 --- a/eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 +++ b/eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 @@ -1,104 +1,103 @@ <# .SYNOPSIS -Invokes sparse checkout on the specified repositories. +Prepares or restores working-tree changes around a native sparse checkout. .DESCRIPTION -Invokes sparse checkout on the specified repositories. +Combines the initial sparse checkout patterns with paths computed by Java's dependency discovery. +Tracked-file changes are saved outside the repository and restored after the Azure Pipelines checkout +step resets them. The checkout must use clean: false to preserve generated untracked files. -This script is special to Java as it uses layered sparse checkout to reduce the amount of code to checkout. -The first run of sparse checkout is inlined into YAML as there is a chicken and egg problem where the script -to perform sparse checkout won't be available until after the checkout step has completed. +.PARAMETER PathsJson +JSON representation of the additional paths to checkout. -This script is used to reduce the size of YAML files as this is only called when the initial checkout has -already been completed. +.PARAMETER ChangesPath +Absolute path outside the repository for the patch containing tracked-file changes. -.PARAMETER PathsJson -JSON representation of the paths to checkout. +.PARAMETER SourceVersion +The original source revision, which must still be checked out before restoring changes. -.PARAMETER RepositoriesJson -JSON representation of the repositories to checkout from. +.PARAMETER Restore +Restore the saved changes after the native checkout has completed. #> +[CmdletBinding(DefaultParameterSetName = 'Prepare')] param( - [Parameter(Mandatory = $true)] - [string]$PathsJson, + [Parameter(Mandatory = $true, ParameterSetName = 'Prepare')] + [string]$PathsJson, + + [Parameter(Mandatory = $true)] + [string]$ChangesPath, - [Parameter(Mandatory = $true)] - [string]$RepositoriesJson + [Parameter(Mandatory = $true, ParameterSetName = 'Restore')] + [string]$SourceVersion, + + [Parameter(Mandatory = $true, ParameterSetName = 'Restore')] + [switch]$Restore ) -# Setting $PSNativeCommandArgumentPassing to 'Legacy' to use PowerShell -# 7.2 behavior for command argument passing. Newer behaviors will result -# in errors from git.exe. -$PSNativeCommandArgumentPassing = 'Legacy' +$ErrorActionPreference = 'Stop' -function SparseCheckout([Array]$paths, [Hashtable]$repository) -{ - $dir = $repository.WorkingDirectory - if (!$dir) { - $dir = "./$($repository.Name)" +if ($Restore) { + $currentVersion = git rev-parse HEAD + if ($LASTEXITCODE -ne 0 -or $currentVersion -ne $SourceVersion) { + throw "The native checkout changed the source revision. Patch retained at $ChangesPath." } - New-Item $dir -ItemType Directory -Force | Out-Null - Push-Location $dir - - if (Test-Path .git/info/sparse-checkout) { - $hasInitialized = $true - Write-Host "Repository $($repository.Name) has already been initialized. Skipping this step." - } else { - Write-Host "Repository $($repository.Name) is being initialized." - - if ($repository.Commitish -match '^refs/pull/\d+/merge$') { - Write-Host "git clone --no-checkout --filter=tree:0 -c remote.origin.fetch='+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)' https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 -c remote.origin.fetch=''+$($repository.Commitish):refs/remotes/origin/$($repository.Commitish)'' https://github.com/$($repository.Name) . - } else { - Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . + if ((Get-Item -LiteralPath $ChangesPath).Length -gt 0) { + git apply --whitespace=nowarn -- $ChangesPath + if ($LASTEXITCODE -ne 0) { + throw "Restoring checkout changes failed with exit code $LASTEXITCODE. Patch retained at $ChangesPath." } - - # Turn off git GC for sparse checkout. Note: The devops checkout task does this by default - Write-Host "git config gc.auto 0" - git config gc.auto 0 - - Write-Host "git sparse-checkout init" - git sparse-checkout init - - # Set non-cone mode otherwise path filters will not work in git >= 2.37.0 - # See https://github.blog/2022-06-27-highlights-from-git-2-37/#tidbits - Write-Host "git sparse-checkout set --no-cone '/*' '!/*/' '/eng'" - git sparse-checkout set --no-cone '/*' '!/*/' '/eng' } + Remove-Item -LiteralPath $ChangesPath + return +} - # Prevent wildcard expansion in Invoke-Expression (e.g. for checkout path '/*') - $quotedPaths = $paths | ForEach-Object { "'$_'" } - $gitsparsecmd = "git sparse-checkout add $quotedPaths" - Write-Host $gitsparsecmd - Invoke-Expression -Command $gitsparsecmd +Write-Output '##vso[task.setvariable variable=SparseCheckoutRequired]false' - Write-Host "Set sparse checkout paths to:" - Get-Content .git/info/sparse-checkout +# Paths may be sourced as a yaml object literal OR a dynamically generated variable json string. +# If the latter, convertToJson will wrap the 'string' in quotes, so remove them. +$paths = $PathsJson.Trim('"') | ConvertFrom-Json +if (@($paths).Count -eq 0) { + return +} - # sparse-checkout commands after initial checkout will auto-checkout again - if (!$hasInitialized) { - # Remove refs/heads/ prefix from branch names - $commitish = $repository.Commitish -replace '^refs/heads/', '' +$isWorkingTree = git rev-parse --is-inside-work-tree +if ($LASTEXITCODE -ne 0 -or $isWorkingTree -ne 'true') { + throw 'The repository is not an initialized Git working tree.' +} - # use -- to prevent git from interpreting the commitish as a path - Write-Host "git -c advice.detachedHead=false checkout $commitish --" +$isSparseCheckout = git config --type=bool --default=false --get core.sparseCheckout +if ($LASTEXITCODE -ne 0) { + throw 'Unable to determine sparse checkout mode.' +} +if ($isSparseCheckout -ne 'true') { + Write-Information 'The repository has a full checkout. Skipping expansion.' -InformationAction Continue + return +} - # This will use the default branch if repo.Commitish is empty - git -c advice.detachedHead=false checkout $commitish -- - } else { - Write-Host "Skipping checkout as repo has already been initialized" +$patternsPath = git rev-parse --git-path info/sparse-checkout +if ($LASTEXITCODE -ne 0) { + throw 'Unable to locate the initial sparse checkout patterns.' +} +$patterns = @(Get-Content -LiteralPath $patternsPath) + @($paths) +$quotedPatterns = foreach ($pattern in $patterns) { + if ($pattern -match "[`r`n]") { + throw 'Sparse checkout patterns cannot contain line breaks.' } - - Pop-Location + '"' + ($pattern -replace '(\\*)"', '$1$1\"' -replace '(\\+)$', '$1$1') + '"' } -# Paths may be sourced as a yaml object literal OR a dynamically generated variable json string. -# If the latter, convertToJson will wrap the 'string' in quotes, so remove them. -$paths = $PathsJson.Trim('"') | ConvertFrom-Json -# Replace windows backslash paths, as Azure Pipelines default directories are sometimes formatted like 'D:\a\1\s' -$repositories = $RepositoriesJson -replace '\\', '/' | ConvertFrom-Json -AsHashtable -foreach ($repo in $repositories) { - SparseCheckout $paths $repo +New-Item -ItemType Directory -Path (Split-Path -Parent $ChangesPath) -Force | Out-Null +$SourceVersion = git rev-parse HEAD +if ($LASTEXITCODE -ne 0) { + throw 'Unable to determine the original source revision.' } +git diff --binary --no-ext-diff --no-textconv --src-prefix=a/ --dst-prefix=b/ --output=$ChangesPath $SourceVersion -- +if ($LASTEXITCODE -ne 0) { + throw "Saving checkout changes failed with exit code $LASTEXITCODE." +} + +$patternsValue = ($quotedPatterns -join ' ').Replace('%', '%AZP25') +Write-Output "##vso[task.setvariable variable=SparseCheckoutPatterns]$patternsValue" +Write-Output "##vso[task.setvariable variable=SparseCheckoutSourceVersion]$SourceVersion" +Write-Output '##vso[task.setvariable variable=SparseCheckoutRequired]true' diff --git a/eng/pipelines/templates/jobs/build-validate-pom.yml b/eng/pipelines/templates/jobs/build-validate-pom.yml index bddc3dd582157..9fb2e6b17d570 100644 --- a/eng/pipelines/templates/jobs/build-validate-pom.yml +++ b/eng/pipelines/templates/jobs/build-validate-pom.yml @@ -24,14 +24,13 @@ jobs: os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - 'sdk/${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }} **/*.xml **/*.md !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - script: | echo "##vso[build.addbuildtag]Scheduled" diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 714fb140dbe31..9bb412ad665f7 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -90,7 +90,7 @@ jobs: componentgovernance: enabled: true codeql: - binaryLanguages: java # Need to specify the language because we clone after the codeql initialize step + binaryLanguages: java compiled: enabled: true @@ -111,14 +111,14 @@ jobs: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/ci*.yml' # necessary for save-package-properties.yml - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md **/ci*.yml !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -151,7 +151,6 @@ jobs: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - ${{ parameters.PreBuildSteps }} @@ -321,15 +320,14 @@ jobs: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/*.yml' - - '.vscode/cspell.json' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md **/*.yml .vscode/cspell.json !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - task: PowerShell@2 displayName: 'Verify versions in POM files' @@ -365,7 +363,6 @@ jobs: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - task: UseNode@1 inputs: diff --git a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml index 15458ebc71785..7b09c0a11ebec 100644 --- a/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-auto-release-batch.yml @@ -96,13 +96,13 @@ stages: image: $(LINUXVMIMAGE) os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - 'sdk/${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }} **/*.xml !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -190,7 +190,13 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - template: /eng/common/pipelines/templates/steps/retain-run.yml - template: /eng/common/pipelines/templates/steps/login-to-github.yml parameters: @@ -259,13 +265,14 @@ stages: steps: - checkout: azure-sdk-build-tools path: azure-sdk-build-tools - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: Azure/azure-sdk-for-java - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - checkout: self + path: azure-sdk-for-java + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -369,14 +376,13 @@ stages: os: windows steps: - download: none - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config ${{ parameters.ServiceDirectory }} **/*.xml **/*.md !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -415,11 +421,13 @@ stages: image: $(LINUXVMIMAGE) os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/${{ parameters.ServiceDirectory }}/**/*.md - - '!**/pom*.xml' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages-signed-auto-release' @@ -501,7 +509,13 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages-signed-auto-release' diff --git a/eng/pipelines/templates/stages/archetype-java-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-release-batch.yml index 143df8382ef83..db12a12a23722 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-batch.yml @@ -115,13 +115,13 @@ stages: os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - 'sdk/${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }} **/*.xml !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages' @@ -197,7 +197,13 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - template: /eng/common/pipelines/templates/steps/retain-run.yml # No pattern, just download everything that's been signed - download: current @@ -257,13 +263,14 @@ stages: steps: - checkout: azure-sdk-build-tools path: azure-sdk-build-tools - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: Azure/azure-sdk-for-java - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - checkout: self + path: azure-sdk-for-java + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifacts' artifact: packages-signed @@ -313,14 +320,13 @@ stages: steps: - download: none - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '${{ parameters.ServiceDirectory }}' - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config ${{ parameters.ServiceDirectory }} **/*.xml **/*.md !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -364,11 +370,13 @@ stages: os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/${{ parameters.ServiceDirectory }}/**/*.md - - '!**/pom*.xml' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages' artifact: packages @@ -409,7 +417,13 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages-signed' artifact: packages-signed @@ -494,11 +508,13 @@ stages: image: $(LINUXVMIMAGE) os: linux steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/${{ parameters.ServiceDirectory }}/**/*.md - - '!**/pom*.xml' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/${{ parameters.ServiceDirectory }}/**/*.md !**/pom*.xml + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifacts' artifact: packages diff --git a/eng/pipelines/templates/stages/archetype-java-release-patch.yml b/eng/pipelines/templates/stages/archetype-java-release-patch.yml index c82a334643841..12783f080ea2b 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-patch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-patch.yml @@ -98,7 +98,13 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - template: /eng/common/pipelines/templates/steps/retain-run.yml # No pattern, just download everything that's been signed - download: current @@ -156,13 +162,14 @@ stages: steps: - checkout: azure-sdk-build-tools path: azure-sdk-build-tools - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: Azure/azure-sdk-for-java - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - checkout: self + path: azure-sdk-for-java + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifacts' artifact: packages-signed @@ -201,13 +208,13 @@ stages: steps: - download: none - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '!sdk/**/test-recordings/*' - - '!sdk/**/session-records/*' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md !sdk/**/test-recordings/* !sdk/**/session-records/* + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -253,11 +260,13 @@ stages: - template: /eng/pipelines/templates/variables/globals.yml steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/**/*.md - - '!**/pom*.xml' + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config sdk/**/*.md !**/pom*.xml + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages' artifact: packages @@ -292,7 +301,13 @@ stages: image: $(WINDOWSVMIMAGE) os: windows steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - download: current displayName: 'Download Artifact: packages-signed' artifact: packages-signed diff --git a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml index 3353db07e3da9..de795da61b8d9 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-pom-only.yml @@ -95,7 +95,13 @@ stages: displayName: 'Download Artifact: packages-signed' artifact: packages-signed patterns: ${{artifact.groupId}}/${{artifact.name}}/** - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - checkout: self + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - template: /eng/common/pipelines/templates/steps/retain-run.yml - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml parameters: @@ -145,13 +151,14 @@ stages: patterns: ${{artifact.groupId}}/${{artifact.name}}/** - checkout: azure-sdk-build-tools path: azure-sdk-build-tools - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - SkipCheckoutNone: true - Repositories: - - Name: Azure/azure-sdk-for-java - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(Pipeline.Workspace)/azure-sdk-for-java + - checkout: self + path: azure-sdk-for-java + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: /* !/*/ /eng /.config + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' # Setup Maven mirror settings and authenticate with Azure Artifacts - template: /eng/pipelines/templates/steps/maven-authenticate.yml parameters: diff --git a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml index f8ec5ec5843ff..4af1b4f23e156 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client-patch.yml @@ -46,14 +46,14 @@ extends: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories as we require the GitHub # service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/*.yml' - - '!sdk/**/test-recordings/*' - - '!sdk/**/session-records/*' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md **/*.yml !sdk/**/test-recordings/* !sdk/**/session-records/* + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -94,7 +94,6 @@ extends: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - task: PipAuthenticate@1 displayName: 'Pip Authenticate to Azure Artifacts' @@ -208,15 +207,15 @@ extends: inputs: versionSpec: $(PythonVersion) - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/*.md' - - '**/*.yml' # needed for package properties when setting the test pipeline version - - '.vscode/cspell.json' - - '!sdk/**/test-recordings/*' - - '!sdk/**/session-records/*' + - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: ${{ parameters.TestPipeline }} + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/*.md **/*.yml .vscode/cspell.json !sdk/**/test-recordings/* !sdk/**/session-records/* + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml parameters: @@ -253,7 +252,6 @@ extends: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: Paths: $(SparseCheckoutDirectories) - SkipCheckoutNone: true - template: /eng/common/pipelines/templates/steps/check-spelling.yml parameters: diff --git a/eng/pipelines/templates/steps/initialize-test-environment.yml b/eng/pipelines/templates/steps/initialize-test-environment.yml index 27e2f04cdd7fc..5162a70ac20c6 100644 --- a/eng/pipelines/templates/steps/initialize-test-environment.yml +++ b/eng/pipelines/templates/steps/initialize-test-environment.yml @@ -16,14 +16,17 @@ steps: # Skip sparse checkout for the `azure-sdk-for--pr` private mirrored repositories # as we require the GitHub service connection to be loaded. - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - '**/*.xml' - - '**/ci*.yml' # necessary for save-package-properties.yml - - ${{ if not(parameters.CheckoutRecordings) }}: - - '!sdk/**/test-recordings' - - '!sdk/**/session-records' + - checkout: self + path: s + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + ${{ if parameters.CheckoutRecordings }}: + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/ci*.yml + ${{ else }}: + sparseCheckoutPatterns: /* !/*/ /eng /.config **/*.xml **/ci*.yml !sdk/**/test-recordings !sdk/**/session-records + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' - task: UsePythonVersion@0 displayName: 'Use Python $(PythonVersion)' @@ -82,5 +85,4 @@ steps: - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: - template: /eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml parameters: - SkipCheckoutNone: true Paths: $(SparseCheckoutDirectories) diff --git a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml index 07c75f94ff9e7..554efe68c6790 100644 --- a/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml +++ b/eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml @@ -2,29 +2,41 @@ parameters: - name: Paths type: object default: [] - - name: Repositories - type: object - default: - - Name: $(Build.Repository.Name) - Commitish: $(Build.SourceVersion) - WorkingDirectory: $(System.DefaultWorkingDirectory) - - name: SkipCheckoutNone - type: boolean - default: false steps: - - ${{ if not(parameters.SkipCheckoutNone) }}: - - checkout: none + - ${{ if not(contains(variables['Build.DefinitionName'], 'java-pr')) }}: + - task: PowerShell@2 + displayName: 'Prepare computed sparse checkout paths' + inputs: + pwsh: true + filePath: eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 + arguments: >- + -PathsJson $env:SPARSE_CHECKOUT_PATHS + -ChangesPath "$(Agent.TempDirectory)/sparse-checkout-changes.patch" + workingDirectory: $(Build.SourcesDirectory) + env: + SPARSE_CHECKOUT_PATHS: ${{ convertToJson(parameters.Paths) }} + + - checkout: self + displayName: 'Checkout computed source paths' + path: s + clean: false + fetchFilter: tree:0 + fetchDepth: 0 + fetchTags: false + sparseCheckoutPatterns: $(SparseCheckoutPatterns) + env: + AGENT_SOURCE_CHECKOUT_QUIET: 'true' + condition: and(succeeded(), eq(variables['SparseCheckoutRequired'], 'true')) - - task: PowerShell@2 - ${{ if eq(length(parameters.Repositories), 1) }}: - displayName: 'Sparse checkout ${{ parameters.Repositories[0].Name }}' - ${{ else }}: - displayName: 'Sparse checkout repositories' - inputs: - pwsh: true - filePath: eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 - arguments: > - -PathsJson '${{ convertToJson(parameters.Paths) }}' - -RepositoriesJson '${{ convertToJson(parameters.Repositories) }}' - workingDirectory: $(System.DefaultWorkingDirectory) + - task: PowerShell@2 + displayName: 'Restore generated source-build changes' + condition: and(succeeded(), eq(variables['SparseCheckoutRequired'], 'true')) + inputs: + pwsh: true + filePath: eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 + arguments: >- + -Restore + -ChangesPath "$(Agent.TempDirectory)/sparse-checkout-changes.patch" + -SourceVersion "$(SparseCheckoutSourceVersion)" + workingDirectory: $(Build.SourcesDirectory) diff --git a/eng/scripts/ci.yml b/eng/scripts/ci.yml index fcecd646d6fc9..fc7e324e88b21 100644 --- a/eng/scripts/ci.yml +++ b/eng/scripts/ci.yml @@ -10,6 +10,8 @@ trigger: paths: include: - eng/scripts/* + - eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 + - eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml exclude: - eng/scripts/typespec @@ -23,6 +25,8 @@ pr: paths: include: - eng/scripts/* + - eng/pipelines/scripts/Invoke-Sparse-Checkout.ps1 + - eng/pipelines/templates/steps/sparse-checkout-repo-initialized.yml extends: template: /eng/common/pipelines/templates/stages/archetype-sdk-tool-pwsh.yml diff --git a/eng/scripts/tests/Sparse-Checkout.tests.ps1 b/eng/scripts/tests/Sparse-Checkout.tests.ps1 new file mode 100644 index 0000000000000..9de246dd784d4 --- /dev/null +++ b/eng/scripts/tests/Sparse-Checkout.tests.ps1 @@ -0,0 +1,341 @@ +BeforeAll { + $script:CheckoutScript = Join-Path $PSScriptRoot '../../pipelines/scripts/Invoke-Sparse-Checkout.ps1' + + function Invoke-TestGit { + param([string]$RepositoryPath, [string[]]$Arguments) + + $output = git -C $RepositoryPath @Arguments 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "Git command failed ($Arguments): $output" + } + return $output + } + + function Enable-TestSparseCheckout { + param([string]$RepositoryPath) + + Invoke-TestGit $RepositoryPath @('sparse-checkout', 'init', '--no-cone') | Out-Null + Invoke-TestGit $RepositoryPath @('sparse-checkout', 'set', '/*', '!/*/', '/eng', '/.config', '**/*.xml') | Out-Null + } + + function Invoke-CheckoutPreparation { + param([string]$RepositoryPath, [string]$PathsJson) + + Push-Location -LiteralPath $RepositoryPath + try { + & $script:CheckoutScript -PathsJson $PathsJson -ChangesPath $script:ChangesPath + } + finally { + Pop-Location + } + } + + function Restore-CheckoutState { + param([string]$RepositoryPath) + + Push-Location -LiteralPath $RepositoryPath + try { + & $script:CheckoutScript -Restore -ChangesPath $script:ChangesPath -SourceVersion $script:SourceVersion + } + finally { + Pop-Location + } + } + + function Invoke-TestNativeCheckout { + param([string]$RepositoryPath, [string[]]$PreparationOutput) + + $prefix = '##vso[task.setvariable variable=SparseCheckoutPatterns]' + $setting = @($PreparationOutput | Where-Object { $_.StartsWith($prefix) }) + $setting.Count | Should -Be 1 + $patterns = $setting[0].Substring($prefix.Length).Replace('%AZP25', '%') + + Invoke-TestGit $RepositoryPath @('sparse-checkout', 'init', '--no-cone') | Out-Null + $startInfo = [System.Diagnostics.ProcessStartInfo]::new() + $startInfo.FileName = (Get-Command git -CommandType Application | Select-Object -First 1).Source + $startInfo.WorkingDirectory = $RepositoryPath + $startInfo.UseShellExecute = $false + $startInfo.RedirectStandardOutput = $true + $startInfo.RedirectStandardError = $true + $startInfo.Arguments = "sparse-checkout set $patterns" + $process = [System.Diagnostics.Process]::Start($startInfo) + try { + $output = $process.StandardOutput.ReadToEnd() + $errorOutput = $process.StandardError.ReadToEnd() + $process.WaitForExit() + if ($process.ExitCode -ne 0) { + throw "Native sparse checkout failed: $output $errorOutput" + } + } + finally { + $process.Dispose() + } + Invoke-TestGit $RepositoryPath @('checkout', '--force', 'HEAD') | Out-Null + } +} + +Describe 'Native sparse checkout expansion' -Tag 'UnitTest' { + BeforeEach { + $repositoryPath = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $script:ChangesPath = Join-Path $TestDrive "$([guid]::NewGuid()).patch" + New-Item -ItemType Directory -Path $repositoryPath | Out-Null + $files = @( + 'pom.xml' + 'ClientFromSourcePom.xml' + 'eng/build.ps1' + '.config/settings.yml' + '.config/data.bin' + 'sdk/selected/src/Main.java' + 'sdk/unselected/src/Other.java' + 'sdk/space directory/src/Space.java' + "sdk/quoted'service/src/Quoted.java" + 'sdk/percent%service/src/Percent.java' + ) + foreach ($file in $files) { + $filePath = Join-Path $repositoryPath $file + New-Item -ItemType Directory -Path (Split-Path $filePath) -Force | Out-Null + Set-Content -LiteralPath $filePath -Value "$file baseline" -NoNewline + } + Invoke-TestGit $repositoryPath @('init', '--quiet') | Out-Null + Invoke-TestGit $repositoryPath @('add', '--all') | Out-Null + Invoke-TestGit $repositoryPath @( + '-c', 'user.name=Sparse Checkout Tests' + '-c', 'user.email=sparse-checkout-tests@example.invalid' + '-c', 'commit.gpgsign=false' + '-c', 'core.hooksPath=.git/hooks' + 'commit', '--quiet', '--message', 'Sparse checkout fixture' + ) | Out-Null + $script:SourceVersion = Invoke-TestGit $repositoryPath @('rev-parse', 'HEAD') + } + + It 'restores POM edits and retains generated files after a forced native checkout' { + Enable-TestSparseCheckout $repositoryPath + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeFalse + $pomPath = Join-Path $repositoryPath 'pom.xml' + Set-Content -LiteralPath $pomPath -Value 'updated version' -NoNewline + $sourcePomPath = Join-Path $repositoryPath 'ClientFromSourcePom.xml' + Set-Content -LiteralPath $sourcePomPath -Value 'source build modules' -NoNewline + $generatedPath = Join-Path $repositoryPath 'generated-pom.xml' + Set-Content -LiteralPath $generatedPath -Value 'generated POM' -NoNewline + $originalHead = Invoke-TestGit $repositoryPath @('rev-parse', 'HEAD') + $originalLocation = (Get-Location).Path + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '"["/sdk/selected"]"') + $preparation | Should -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]true' + $preparation | Should -Contain "##vso[task.setvariable variable=SparseCheckoutSourceVersion]$originalHead" + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeFalse + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'updated version' + Invoke-TestNativeCheckout $repositoryPath $preparation + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'pom.xml baseline' + Restore-CheckoutState $repositoryPath + + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'updated version' + Get-Content -LiteralPath $sourcePomPath -Raw | Should -BeExactly 'source build modules' + Get-Content -LiteralPath $generatedPath -Raw | Should -BeExactly 'generated POM' + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath 'sdk/unselected/src/Other.java') | Should -BeFalse + Test-Path (Join-Path $repositoryPath 'eng/build.ps1') | Should -BeTrue + Test-Path (Join-Path $repositoryPath '.config/settings.yml') | Should -BeTrue + Invoke-TestGit $repositoryPath @('sparse-checkout', 'list') | Should -Contain '**/*.xml' + Invoke-TestGit $repositoryPath @('rev-parse', 'HEAD') | Should -BeExactly $originalHead + (Get-Location).Path | Should -BeExactly $originalLocation + Test-Path $script:ChangesPath | Should -BeFalse + } + + It 'passes paths containing spaces, single quotes, and percent signs to the native task' { + Enable-TestSparseCheckout $repositoryPath + $pathsJson = ConvertTo-Json -InputObject @('/sdk/space directory', "/sdk/quoted'service", '/sdk/percent%service') -Compress + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath $pathsJson) + Invoke-TestNativeCheckout $repositoryPath $preparation + Restore-CheckoutState $repositoryPath + + Test-Path (Join-Path $repositoryPath 'sdk/space directory/src/Space.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath "sdk/quoted'service/src/Quoted.java") | Should -BeTrue + Test-Path (Join-Path $repositoryPath 'sdk/percent%service/src/Percent.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath 'sdk/unselected/src/Other.java') | Should -BeFalse + Test-Path $script:ChangesPath | Should -BeFalse + } + + It 'uses the first Git executable when command discovery returns multiple matches' { + Enable-TestSparseCheckout $repositoryPath + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + $script:GitExecutablePath = (Get-Command git -CommandType Application | Select-Object -First 1).Source + + Mock Get-Command { + [pscustomobject]@{ Source = $script:GitExecutablePath } + [pscustomobject]@{ Source = 'nonexistent-secondary-git' } + } -ParameterFilter { $Name -eq 'git' -and $CommandType -eq 'Application' } + + Invoke-TestNativeCheckout $repositoryPath $preparation + Restore-CheckoutState $repositoryPath + + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath 'sdk/unselected/src/Other.java') | Should -BeFalse + Should -Invoke Get-Command -Times 1 -Exactly -ParameterFilter { + $Name -eq 'git' -and $CommandType -eq 'Application' + } + } + + It 'keeps existing patterns before additional paths and their exclusions' { + Enable-TestSparseCheckout $repositoryPath + $originalPatterns = Invoke-TestGit $repositoryPath @('sparse-checkout', 'list') + $paths = @('/sdk/selected', '!/sdk/selected/src/Main.java', '/sdk/space directory') + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath (ConvertTo-Json -InputObject $paths -Compress)) + Invoke-TestNativeCheckout $repositoryPath $preparation + Restore-CheckoutState $repositoryPath + + (Invoke-TestGit $repositoryPath @('sparse-checkout', 'list')) -join "`n" | + Should -BeExactly (($originalPatterns + $paths) -join "`n") + Test-Path (Join-Path $repositoryPath 'sdk/selected/src/Main.java') | Should -BeFalse + Test-Path (Join-Path $repositoryPath 'sdk/space directory/src/Space.java') | Should -BeTrue + } + + It 'leaves full checkouts and their edits intact' { + $pomPath = Join-Path $repositoryPath 'pom.xml' + Set-Content -LiteralPath $pomPath -Value 'updated version' -NoNewline + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + + $preparation | Should -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]false' + $preparation | Should -Not -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]true' + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'updated version' + Test-Path (Join-Path $repositoryPath 'sdk/unselected/src/Other.java') | Should -BeTrue + Test-Path (Join-Path $repositoryPath '.git/info/sparse-checkout') | Should -BeFalse + Test-Path $script:ChangesPath | Should -BeFalse + } + + It 'leaves sparse checkout patterns unchanged for an empty path list' { + Enable-TestSparseCheckout $repositoryPath + $originalPatterns = Invoke-TestGit $repositoryPath @('sparse-checkout', 'list') + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '[]') + + $preparation | Should -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]false' + $preparation | Should -Not -Contain '##vso[task.setvariable variable=SparseCheckoutRequired]true' + (Invoke-TestGit $repositoryPath @('sparse-checkout', 'list')) -join "`n" | + Should -BeExactly ($originalPatterns -join "`n") + Test-Path $script:ChangesPath | Should -BeFalse + } + + It 'rejects uninitialized directories and restores the working directory' { + $emptyDirectory = Join-Path $TestDrive 'uninitialized' + New-Item -ItemType Directory -Path $emptyDirectory | Out-Null + $originalLocation = (Get-Location).Path + + { Invoke-CheckoutPreparation $emptyDirectory '["/sdk/selected"]' } | + Should -Throw + + (Get-Location).Path | Should -BeExactly $originalLocation + Test-Path (Join-Path $emptyDirectory '.git') | Should -BeFalse + } + + It 'restores binary changes and tracked-file deletions' { + Enable-TestSparseCheckout $repositoryPath + $binaryPath = Join-Path $repositoryPath '.config/data.bin' + $binaryContent = [byte[]]@(0, 255, 128, 13, 10, 42) + [System.IO.File]::WriteAllBytes($binaryPath, $binaryContent) + $deletedPath = Join-Path $repositoryPath '.config/settings.yml' + Remove-Item -LiteralPath $deletedPath + + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + Invoke-TestNativeCheckout $repositoryPath $preparation + Test-Path $deletedPath | Should -BeTrue + Restore-CheckoutState $repositoryPath + + [Convert]::ToBase64String([System.IO.File]::ReadAllBytes($binaryPath)) | + Should -BeExactly ([Convert]::ToBase64String($binaryContent)) + Test-Path $deletedPath | Should -BeFalse + } + + It 'rejects restoring changes onto a different source revision' { + Enable-TestSparseCheckout $repositoryPath + $pomPath = Join-Path $repositoryPath 'pom.xml' + Set-Content -LiteralPath $pomPath -Value 'updated version' -NoNewline + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + Invoke-TestNativeCheckout $repositoryPath $preparation + Invoke-TestGit $repositoryPath @( + '-c', 'user.name=Sparse Checkout Tests' + '-c', 'user.email=sparse-checkout-tests@example.invalid' + '-c', 'commit.gpgsign=false' + '-c', 'core.hooksPath=.git/hooks' + 'commit', '--allow-empty', '--quiet', '--message', 'Different source revision' + ) | Out-Null + + { Restore-CheckoutState $repositoryPath } | Should -Throw '*changed the source revision*' + + Test-Path $script:ChangesPath | Should -BeTrue + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'pom.xml baseline' + } + + It 'fails without discarding the patch when changes cannot be restored' { + Enable-TestSparseCheckout $repositoryPath + $pomPath = Join-Path $repositoryPath 'pom.xml' + Set-Content -LiteralPath $pomPath -Value 'updated version' -NoNewline + $preparation = @(Invoke-CheckoutPreparation $repositoryPath '["/sdk/selected"]') + Invoke-TestNativeCheckout $repositoryPath $preparation + Set-Content -LiteralPath $pomPath -Value 'conflicting version' -NoNewline + $originalLocation = (Get-Location).Path + + { Restore-CheckoutState $repositoryPath } | Should -Throw '*Restoring checkout changes failed*' + + Test-Path $script:ChangesPath | Should -BeTrue + Get-Content -LiteralPath $pomPath -Raw | Should -BeExactly 'conflicting version' + (Get-Location).Path | Should -BeExactly $originalLocation + } +} + +Describe 'Native sparse checkout settings' -Tag 'UnitTest' { + BeforeAll { + $script:EngineeringRoot = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path + . "$script:EngineeringRoot/common/scripts/Helpers/PSModule-Helpers.ps1" + Install-ModuleIfNotInstalled 'powershell-yaml' '0.4.7' | Import-Module + + function Get-SparseCheckout { + param($Node, [string]$JobName) + + if ($Node -is [System.Collections.IDictionary]) { + if ($Node.job) { + $JobName = $Node.job + } + if ($Node.checkout -eq 'self' -and $Node.fetchFilter -eq 'tree:0') { + return @{ JobName = $JobName; Step = $Node } + } + foreach ($value in $Node.Values) { + Get-SparseCheckout -Node $value -JobName $JobName + } + } + elseif ($Node -is [System.Collections.IList]) { + foreach ($item in $Node) { + Get-SparseCheckout -Node $item -JobName $JobName + } + } + } + } + + It 'uses quiet checkout and only fetches tags for test-versioning in ' -TestCases @( + @{ File = 'pipelines/templates/jobs/ci.yml'; Count = 2; TagJobs = @('Build', 'Analyze') } + @{ File = 'pipelines/templates/jobs/build-validate-pom.yml'; Count = 1; TagJobs = @() } + @{ File = 'pipelines/templates/stages/archetype-sdk-client-patch.yml'; Count = 2; TagJobs = @('Build', 'AnalyzeAndVerify') } + @{ File = 'pipelines/templates/stages/archetype-java-release-batch.yml'; Count = 7; TagJobs = @('VerifyReleaseVersion') } + @{ File = 'pipelines/templates/stages/archetype-java-auto-release-batch.yml'; Count = 6; TagJobs = @('VerifyReleaseVersion') } + @{ File = 'pipelines/templates/stages/archetype-java-release-patch.yml'; Count = 5; TagJobs = @() } + @{ File = 'pipelines/templates/stages/archetype-java-release-pom-only.yml'; Count = 2; TagJobs = @() } + @{ File = 'pipelines/templates/steps/initialize-test-environment.yml'; Count = 1; TagJobs = @() } + @{ File = 'pipelines/templates/steps/sparse-checkout-repo-initialized.yml'; Count = 1; TagJobs = @() } + @{ File = 'pipelines/code-quality-reports.yml'; Count = 1; TagJobs = @() } + @{ File = 'containers/ci.yml'; Count = 1; TagJobs = @() } + ) { + param($File, $Count, $TagJobs) + + $yaml = Get-Content (Join-Path $script:EngineeringRoot $File) -Raw | ConvertFrom-Yaml -Ordered + $checkouts = @(Get-SparseCheckout -Node $yaml) + $checkouts.Count | Should -Be $Count + foreach ($checkout in $checkouts) { + $expectedTags = if ($TagJobs -contains $checkout.JobName) { '${{ parameters.TestPipeline }}' } else { $false } + $checkout.Step.fetchTags | Should -BeExactly $expectedTags + $checkout.Step.env.AGENT_SOURCE_CHECKOUT_QUIET | Should -BeExactly 'true' + } + } +} \ No newline at end of file