From d0362dc87a1a68bae85f690f1dfd62dfc23d18ab Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 19:09:46 +0000 Subject: [PATCH 1/2] Test: run-fresh-clone-example.ps1 takes -ExpectDownload as typed with pwsh -File The new Pester test runs the script through pwsh -File with -ExpectDownload as a person types it, bare or left out, and runs the two calls the Examples from a fresh clone workflow makes. A stand-in for java on the PATH prints results, and the download line when asked, so no build or data file is needed. The CI script tests workflow now also runs when that workflow changes. The -File cases fail until the parameter is a switch. --- .github/workflows/ci-script-tests.yml | 5 + ci/run-fresh-clone-example.Tests.ps1 | 154 ++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 ci/run-fresh-clone-example.Tests.ps1 diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml index 56b7b3aaf..c7ca4a2c8 100644 --- a/.github/workflows/ci-script-tests.yml +++ b/.github/workflows/ci-script-tests.yml @@ -2,12 +2,16 @@ name: CI script tests # Runs the Pester tests of the scripts in ci on Linux and Windows, where the # builds run them. The environment variable names workflow runs its own test. +# The test of run-fresh-clone-example.ps1 also runs the calls in +# examples-fresh-clone.yml, so a change to that workflow runs the tests too. on: pull_request: paths: - 'ci/**' - '.github/workflows/ci-script-tests.yml' + # run-fresh-clone-example.Tests.ps1 runs the calls this workflow makes. + - '.github/workflows/examples-fresh-clone.yml' push: branches: - main @@ -15,6 +19,7 @@ on: paths: - 'ci/**' - '.github/workflows/ci-script-tests.yml' + - '.github/workflows/examples-fresh-clone.yml' workflow_dispatch: permissions: diff --git a/ci/run-fresh-clone-example.Tests.ps1 b/ci/run-fresh-clone-example.Tests.ps1 new file mode 100644 index 000000000..edc3e7176 --- /dev/null +++ b/ci/run-fresh-clone-example.Tests.ps1 @@ -0,0 +1,154 @@ +# Tests for ci/run-fresh-clone-example.ps1. +# +# Each case runs the script in a new pwsh process, with -File as a person +# would type it at a shell, or with -Command as the workflow's pwsh step does. +# The process runs in a folder that holds an empty jar where the script looks +# for one, and a stand-in for java comes first on the PATH. The stand-in +# prints results, and also the line an example logs when it downloads its data +# file when it is given the example name Downloads. + +BeforeDiscovery { + # Each call the workflow makes, with the job it is in, so that a change to + # the workflow's call is tested as well. + $workflow = Join-Path $PSScriptRoot '../.github/workflows/examples-fresh-clone.yml' + $lines = @(Get-Content -LiteralPath $workflow) + $script:WorkflowCalls = @() + for ($i = 0; $i -lt $lines.Count - 1; $i++) { + if ($lines[$i].Trim() -ne 'run: >-' -or + $lines[$i + 1] -notmatch 'run-fresh-clone-example\.ps1') { + continue + } + $parts = @() + for ($j = $i + 1; $j -lt $lines.Count -and $lines[$j].Trim() -ne ''; $j++) { + $parts += $lines[$j].Trim() + } + $job = $null + for ($k = $i; $k -ge 0 -and $null -eq $job; $k--) { + if ($lines[$k] -match '^ ([A-Za-z0-9_-]+):\s*$') { + $job = $Matches[1] + } + } + $script:WorkflowCalls += @{ Job = $job; Command = $parts -join ' ' } + } +} + +BeforeAll { + $script:Script = Join-Path $PSScriptRoot 'run-fresh-clone-example.ps1' + $script:Pwsh = (Get-Process -Id $PID).Path + + $script:Root = Join-Path $TestDrive 'repository' + $target = Join-Path $script:Root 'examples/device-detection/console/target' + New-Item -ItemType Directory $target -Force | Out-Null + New-Item -ItemType File (Join-Path $target 'example-jar-with-dependencies.jar') | Out-Null + + $script:Bin = Join-Path $TestDrive 'bin' + New-Item -ItemType Directory $script:Bin | Out-Null + $missing = "'device-detection/51Degrees-LiteV4.1.hash' is missing, so it is being downloaded by running the script" + if ($IsWindows) { + # The arguments are -cp, the jar and the example. + [IO.File]::WriteAllText((Join-Path $script:Bin 'java.cmd'), + "@echo off`r`nif ""%~3""==""Downloads"" echo $missing`r`necho Results:`r`nexit /b 0`r`n") + } else { + $java = Join-Path $script:Bin 'java' + [IO.File]::WriteAllText($java, + "#!/bin/sh`nif [ ""`$3"" = Downloads ]; then echo ""$missing""; fi`necho Results:`nexit 0`n") + chmod +x $java + } + + # Runs pwsh with the arguments in the repository folder, with the stand-in + # for java first on the PATH, and returns the exit code and the output. + function Invoke-Pwsh([string[]]$Arguments) { + $path = $env:PATH + Push-Location $script:Root + try { + $env:PATH = $script:Bin + [IO.Path]::PathSeparator + $path + $output = & $script:Pwsh -NoProfile -NonInteractive @Arguments 2>&1 | + ForEach-Object { "$_" } + [pscustomobject]@{ Code = $LASTEXITCODE; Text = $output -join "`n" } + } finally { + $env:PATH = $path + Pop-Location + } + } + + function Invoke-File([string]$Example, [string[]]$Flag) { + Invoke-Pwsh (@('-File', $script:Script, '-Product', 'device-detection', + '-Example', $Example) + $Flag) + } +} + +Describe 'run-fresh-clone-example.ps1 called with pwsh -File' { + + It 'passes when the example downloads, given ' -ForEach @( + @{ Flag = @('-ExpectDownload') } + @{ Flag = @('-ExpectDownload:$true') } + ) { + $run = Invoke-File 'Downloads' $Flag + $run.Text | Should -Match 'Results:' + $run.Text | Should -Not -Match '::error::' + $run.Code | Should -Be 0 + } + + It 'fails when the example does not download, given ' -ForEach @( + @{ Flag = @('-ExpectDownload') } + @{ Flag = @('-ExpectDownload:$true') } + ) { + $run = Invoke-File 'FromCache' $Flag + $run.Text | Should -Match '::error::The example did not download its free data file\.' + $run.Code | Should -Be 1 + } + + It 'passes when the example does not download, given ' -ForEach @( + @{ Name = 'no flag'; Flag = @() } + @{ Name = '-ExpectDownload:$false'; Flag = @('-ExpectDownload:$false') } + ) { + $run = Invoke-File 'FromCache' $Flag + $run.Text | Should -Match 'Results:' + $run.Text | Should -Not -Match '::error::' + $run.Code | Should -Be 0 + } + + It 'fails when the example downloads, given ' -ForEach @( + @{ Name = 'no flag'; Flag = @() } + @{ Name = '-ExpectDownload:$false'; Flag = @('-ExpectDownload:$false') } + ) { + $run = Invoke-File 'Downloads' $Flag + $run.Text | Should -Match '::error::The example downloaded its data file' + $run.Code | Should -Be 1 + } +} + +Describe 'run-fresh-clone-example.ps1 called as the workflow calls it' { + + It 'finds both of the workflow''s calls' -ForEach @( + @{ Found = $script:WorkflowCalls.Count } + ) { + $Found | Should -Be 2 + } + + # The download job expects the example to download its file, and the + # other jobs expect it to be there already. + # The command is wrapped as GitHub's pwsh shell wraps a step, which stops + # on an error and exits with the last exit code. + It 'passes the job''s call only when the example does what that job expects' -ForEach $script:WorkflowCalls { + $quoted = $script:Script.Replace("'", "''") + $call = $Command.Replace('./ci/run-fresh-clone-example.ps1', "& '$quoted'") + $call = $call.Replace('${{ matrix.product }}', 'device-detection') + $call = "`$ErrorActionPreference = 'stop'`n$call`n" + + "if ((Test-Path -LiteralPath variable:\LASTEXITCODE)) { exit `$LASTEXITCODE }" + $expected, $other = if ($Job -eq 'download') { + 'Downloads', 'FromCache' + } else { + 'FromCache', 'Downloads' + } + + $run = Invoke-Pwsh @('-Command', $call.Replace('${{ matrix.example }}', $expected)) + $run.Text | Should -Match 'Results:' + $run.Text | Should -Not -Match '::error::' + $run.Code | Should -Be 0 + + $run = Invoke-Pwsh @('-Command', $call.Replace('${{ matrix.example }}', $other)) + $run.Text | Should -Match '::error::' + $run.Code | Should -Be 1 + } +} From 075ffb8671d8ad4d5dbb0e4f7916ddc71b2cc902 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 19:11:03 +0000 Subject: [PATCH 2/2] Fix: run-fresh-clone-example.ps1 takes -ExpectDownload as a switch -ExpectDownload was a [bool], which pwsh -File cannot bind from anything a person types, as -File passes true, 1 and $true as strings. It is now a switch: given when the example has to download its data file, left out or given as -ExpectDownload:$false when the file is in place. The Examples from a fresh clone workflow calls it that way, and ci/README.md and AGENTS.md show the command to run it by hand. --- .github/workflows/examples-fresh-clone.yml | 4 ++-- AGENTS.md | 5 ++++- ci/README.md | 12 +++++++++++- ci/run-fresh-clone-example.ps1 | 8 +++++--- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/examples-fresh-clone.yml b/.github/workflows/examples-fresh-clone.yml index e4eb3185b..eb7d8e1ad 100644 --- a/.github/workflows/examples-fresh-clone.yml +++ b/.github/workflows/examples-fresh-clone.yml @@ -62,7 +62,7 @@ jobs: shell: pwsh run: >- ./ci/run-fresh-clone-example.ps1 -Product '${{ matrix.product }}' - -Example '${{ matrix.example }}' -ExpectDownload $true + -Example '${{ matrix.example }}' -ExpectDownload # The key follows the link, so a new free file gets a new entry, and # the same file is kept once rather than once a run. @@ -121,4 +121,4 @@ jobs: shell: pwsh run: >- ./ci/run-fresh-clone-example.ps1 -Product '${{ matrix.product }}' - -Example '${{ matrix.example }}' -ExpectDownload $false + -Example '${{ matrix.example }}' -ExpectDownload:$false diff --git a/AGENTS.md b/AGENTS.md index 2e9a7a210..6c8c47196 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -230,7 +230,10 @@ example. The IP Intelligence cloud examples read their key from settings. On Ubuntu the free data file is downloaded, as from a fresh clone, and the Windows and macOS jobs take that file from the run's cache, so each file is downloaded once a run. It runs on each push to `main` and - `rc`, daily, and on a pull request that changes the examples. + `rc`, daily, and on a pull request that changes the examples. Its check + runs by hand from the repository root after the build, with + `-ExpectDownload` only when the data file is still a link: + `pwsh -File ./ci/run-fresh-clone-example.ps1 -Product device-detection -Example fiftyone.devicedetection.examples.console.GettingStartedOnPrem -ExpectDownload`. ## Environment variable names diff --git a/ci/README.md b/ci/README.md index 1c3d50fc1..0d117061f 100644 --- a/ci/README.md +++ b/ci/README.md @@ -55,7 +55,17 @@ repositories used to do for one package tree each. downloads its free data file exactly when the job expects it to. That workflow's Ubuntu job downloads each product's free data file once a run, as a test of the download, and its other jobs take the file from the cache - the Ubuntu job saves. + the Ubuntu job saves. To run it by hand, build the console examples as + [AGENTS.md](../AGENTS.md#build) says, then from the repository root give + `-ExpectDownload` when the free data file is only a link, and leave it out + when the file is there. + + ``` + pwsh -File ./ci/run-fresh-clone-example.ps1 -Product device-detection -Example fiftyone.devicedetection.examples.console.GettingStartedOnPrem -ExpectDownload + ``` + + [run-fresh-clone-example.Tests.ps1](run-fresh-clone-example.Tests.ps1) + tests it with `pwsh -File` and with the workflow's own calls. - [run-performance-tests.ps1](run-performance-tests.ps1) measures the Pipeline API, Device Detection and IP Intelligence and writes one results file per configuration. The Device Detection and IP Intelligence metrics are prefixed diff --git a/ci/run-fresh-clone-example.ps1 b/ci/run-fresh-clone-example.ps1 index 5c1e2102b..137feec70 100644 --- a/ci/run-fresh-clone-example.ps1 +++ b/ci/run-fresh-clone-example.ps1 @@ -1,9 +1,11 @@ param ( [Parameter(Mandatory)][string]$Product, [Parameter(Mandatory)][string]$Example, - # True where the clone holds only the data file's link, so the example - # has to download the file. False where the file is already in place. - [Parameter(Mandatory)][bool]$ExpectDownload + # Given where the clone holds only the data file's link, so the example + # has to download the file. Left out, or given as -ExpectDownload:$false, + # where the file is already in place. A switch, so that it works the same + # with pwsh -File, which passes every other value as a string. + [switch]$ExpectDownload ) # Runs a product's console example jar from the repository root with no