diff --git a/ci/device-detection-performance.Tests.ps1 b/ci/device-detection-performance.Tests.ps1 new file mode 100644 index 00000000..01571ec6 --- /dev/null +++ b/ci/device-detection-performance.Tests.ps1 @@ -0,0 +1,83 @@ +# Tests for device-detection/ci/run-performance-tests.ps1. +# +# common-ci runs the script from its own folder, which holds the assets, with +# the product folder as RepoName. Each test lays out such a folder with empty +# stand-ins for the data files, and mocks python, so that what is checked is +# which data file the script measures on, not the measurement itself. + +BeforeAll { + $script:Script = Join-Path $PSScriptRoot '..' 'device-detection' 'ci' ` + 'run-performance-tests.ps1' + + # The name of the file given to the console benchmark as --data_file. + function Get-DataFileArgument([object[]]$Arguments) { + $index = [Array]::IndexOf($Arguments, '--data_file') + if ($index -lt 0) { + return $null + } + return Split-Path -Leaf $Arguments[$index + 1] + } +} + +Describe 'device-detection run-performance-tests.ps1' { + +BeforeEach { + $script:CommonCi = Join-Path ([IO.Path]::GetTempPath()) ` + ([Guid]::NewGuid().ToString()) + $script:Assets = New-Item -ItemType Directory -Force ` + (Join-Path $script:CommonCi 'assets') + $script:Product = (New-Item -ItemType Directory -Force ` + (Join-Path $script:CommonCi 'device-detection')).FullName + foreach ($name in '51Degrees-LiteV4.1.hash', '20000 User Agents.csv') { + Set-Content (Join-Path $script:Assets $name) '' + } + + # The console benchmark writes its summary where --json_output says. + Mock python { + $index = [Array]::IndexOf($args, '--json_output') + if ($index -ge 0) { + '{ "DetectionsPerSecond": 1, "MsPerDetection": 1 }' | + Set-Content $args[$index + 1] + } + } + + Push-Location $script:CommonCi +} + +AfterEach { + Pop-Location + Remove-Item -Recurse -Force $script:CommonCi +} + +It 'measures on the free Lite file when the TAC file is absent' { + $output = & $script:Script -RepoName $script:Product -Name 'test' | + Out-String + + Should -Invoke python -Times 1 -Exactly -ParameterFilter { + (Get-DataFileArgument $args) -eq '51Degrees-LiteV4.1.hash' + } + $output | Should -Match 'on 51Degrees-LiteV4\.1\.hash' + Join-Path $script:Product 'test-results' 'performance-summary' ` + 'results_test.json' | Should -Exist +} + +It 'measures on the TAC file when it is there' { + Set-Content (Join-Path $script:Assets 'TAC-HashV41.hash') '' + + $output = & $script:Script -RepoName $script:Product -Name 'test' | + Out-String + + Should -Invoke python -Times 1 -Exactly -ParameterFilter { + (Get-DataFileArgument $args) -eq 'TAC-HashV41.hash' + } + $output | Should -Match 'on TAC-HashV41\.hash' +} + +It 'runs nothing when performance is turned off' { + & $script:Script -RepoName $script:Product -Name 'test' ` + -RunPerformance $false + + Should -Invoke python -Times 0 -Exactly +} + +} diff --git a/device-detection/ci/run-performance-tests.ps1 b/device-detection/ci/run-performance-tests.ps1 index 22884d5f..b6ad085c 100644 --- a/device-detection/ci/run-performance-tests.ps1 +++ b/device-detection/ci/run-performance-tests.ps1 @@ -9,9 +9,18 @@ param ( if (!$RunPerformance) { Write-Output "Skipping performance tests" return -} elseif (!(Test-Path assets/TAC-HashV41.hash)) { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No On-Premise Data File::On-Premise Data File wasn't found, so performance tests will not run." - return +} + +# Every example and test runs on the free data files, so without the licensed +# TAC file the console benchmark measures on the Lite file rather than not at +# all. fetch-assets.ps1 always fetches the Lite file, so its absence is an +# error, not a reason to skip. +$dataFileName = 'TAC-HashV41.hash' +if (!(Test-Path "assets/$dataFileName")) { + $dataFileName = '51Degrees-LiteV4.1.hash' + if (!(Test-Path "assets/$dataFileName")) { + throw "Neither TAC-HashV41.hash nor $dataFileName is in the assets, so the performance tests cannot run" + } } $perfSummary = New-Item -ItemType directory -Path $RepoName/test-results/performance-summary -Force @@ -26,8 +35,8 @@ try { Write-Output "Running Web performance tests" python fiftyone_devicedetection_examples/performance-tests/test_performance.py "$assets/51Degrees-LiteV4.1.hash" "$assets/20000 User Agents.csv" --output || $(throw "benchmark failed with code: $LASTEXITCODE") - Write-Output "Running Console performance tests" - python fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py --data_file "$assets/TAC-HashV41.hash" --user_agents_file "$assets/20000 User Agents.csv" --json_output $pwd/summary.json || $(throw "performance tests failed with $LASTEXITCODE") + Write-Output "Running Console performance tests on $dataFileName" + python fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/onpremise/performance.py --data_file "$assets/$dataFileName" --user_agents_file "$assets/20000 User Agents.csv" --json_output $pwd/summary.json || $(throw "performance tests failed with $LASTEXITCODE") $Results = Get-Content $pwd/summary.json | ConvertFrom-Json @{