Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions ci/dispatch.Tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,16 @@ BeforeAll {
"param ([Parameter(Mandatory)][string]`$RepoName)`n$Body"
}

# The body of a product's run-performance-tests step that writes the
# summary of the configuration 'test' with the given figures.
function Get-SummaryStep([hashtable]$Higher, [hashtable]$Lower) {
$json = @{ HigherIsBetter = $Higher; LowerIsBetter = $Lower } |
ConvertTo-Json -Compress
return '$dir = New-Item -ItemType Directory -Force ' +
'(Join-Path $RepoName "test-results" "performance-summary"); ' +
"Set-Content (Join-Path `$dir 'results_test.json') '$json'"
}

# A folder laid out like a common-ci checkout, which the dispatcher runs in.
function New-CommonCi {
$path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString())
Expand Down Expand Up @@ -118,6 +128,38 @@ Describe 'Invoke-ProductStep' {
Test-Path (Join-Path $script:Repo 'alpha' 'test-results') | Should -BeFalse
}

It 'keeps every product''s figures in the one summary of a configuration' {
# common-ci compares one results_<Name>.json for each configuration.
Set-ProductStep $script:Repo 'alpha' 'run-performance-tests' `
(Get-SummaryStep @{ alpha_PerSecond = 1 } @{ alpha_Ms = 2 })
Set-ProductStep $script:Repo 'beta' 'run-performance-tests' `
(Get-SummaryStep @{ beta_PerSecond = 3 } @{ beta_Ms = 4 })

Invoke-ProductStep 'run-performance-tests' $script:Repo @{
RepoName = $script:Repo }

$results = Get-Content (Join-Path $script:Repo 'test-results' `
'performance-summary' 'results_test.json') | ConvertFrom-Json
$results.HigherIsBetter.alpha_PerSecond | Should -Be 1
$results.HigherIsBetter.beta_PerSecond | Should -Be 3
$results.LowerIsBetter.alpha_Ms | Should -Be 2
$results.LowerIsBetter.beta_Ms | Should -Be 4
}

It 'fails the step when two products report the same figure' {
Set-ProductStep $script:Repo 'alpha' 'run-performance-tests' `
(Get-SummaryStep @{ PerSecond = 1 } @{})
Set-ProductStep $script:Repo 'beta' 'run-performance-tests' `
(Get-SummaryStep @{ PerSecond = 3 } @{})

{ Invoke-ProductStep 'run-performance-tests' $script:Repo @{
RepoName = $script:Repo } } | Should -Throw '*failed for: beta'

$results = Get-Content (Join-Path $script:Repo 'test-results' `
'performance-summary' 'results_test.json') | ConvertFrom-Json
$results.HigherIsBetter.PerSecond | Should -Be 1
}

It 'fetches assets when called without a repository name' {
# common-ci's prefetch_assets action runs ci/fetch-assets.ps1 directly
# with only the keys, from the common-ci checkout that holds this
Expand Down
45 changes: 43 additions & 2 deletions ci/dispatch.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,14 @@ function Invoke-ProductStep([string]$Step, [string]$RepoName,
Write-Output "::error title=$Step failed::$product`: $_"
$failed.Add($product)
} finally {
Move-ProductTestResults $RepoName $product
try {
Move-ProductTestResults $RepoName $product
} catch {
Write-Output "::error title=$Step failed::$product`: $_"
if ($failed.Contains($product) -eq $false) {
$failed.Add($product)
}
}
Write-Output "::endgroup::"
}
}
Expand All @@ -51,6 +58,8 @@ function Invoke-ProductStep([string]$Step, [string]$RepoName,
# common-ci collects test results from <repository>/test-results, and a
# product's own scripts write them under its folder, where they would not be
# found. Move them up, keeping the unit/integration/performance folders.
# common-ci compares one performance summary for each configuration, so a
# summary another product already wrote gets this product's figures added.
function Move-ProductTestResults([string]$RepoName, [string]$Product) {
$results = Join-Path $RepoName $Product 'test-results'
if ((Test-Path $results) -eq $false) {
Expand All @@ -59,7 +68,39 @@ function Move-ProductTestResults([string]$RepoName, [string]$Product) {
foreach ($kind in (Get-ChildItem -Directory $results)) {
$target = New-Item -ItemType Directory -Force `
(Join-Path $RepoName 'test-results' $kind.Name)
Get-ChildItem $kind.FullName | Move-Item -Destination $target -Force
foreach ($file in (Get-ChildItem $kind.FullName)) {
$existing = Join-Path $target $file.Name
if ($kind.Name -eq 'performance-summary' -and
$file.Extension -eq '.json' -and
(Test-Path $existing -PathType Leaf) -eq $true) {
Add-PerformanceFigures $file.FullName $existing
} else {
Move-Item $file.FullName -Destination $target -Force
}
}
}
Remove-Item -Recurse -Force $results
}

# Adds the HigherIsBetter and LowerIsBetter figures of the summary at $Source
# to the one at $Destination. A figure already in $Destination fails, because
# one of the two would be lost.
function Add-PerformanceFigures([string]$Source, [string]$Destination) {
$from = Get-Content -Raw $Source | ConvertFrom-Json -AsHashtable
$into = Get-Content -Raw $Destination | ConvertFrom-Json -AsHashtable
foreach ($section in 'HigherIsBetter', 'LowerIsBetter') {
if ($null -eq $from[$section]) {
continue
}
if ($null -eq $into[$section]) {
$into[$section] = [ordered]@{}
}
foreach ($metric in $from[$section].Keys) {
if ($into[$section].Contains($metric) -eq $true) {
throw "$section.$metric is in both '$Source' and '$Destination'"
}
$into[$section][$metric] = $from[$section][$metric]
}
}
$into | ConvertTo-Json -Depth 10 | Set-Content $Destination
}
121 changes: 121 additions & 0 deletions ci/ip-intelligence-performance.Tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# Tests for ip-intelligence/ci/run-performance-tests.ps1.
#
# common-ci runs the script from its own folder, which holds the assets, with
# the product folder as RepoName. Each test lays out such a folder with empty
# stand-ins for the free Lite data file and the evidence, and mocks python, so
# that what is checked is what the script measures and on which file, not the
# measurement itself.

BeforeAll {
$script:Script = Join-Path $PSScriptRoot '..' 'ip-intelligence' 'ci' `
'run-performance-tests.ps1'

# The free data file and the evidence, as common-ci's assets name them.
$script:LiteFile = '51Degrees-LiteIpiV41.ipi'
$script:EvidenceFile = 'ip-intelligence-evidence.yml'

# The web benchmark, in the same place as Device Detection's, and the
# console example, relative to the product folder.
$script:WebBenchmark =
'fiftyone_ipintelligence_examples/performance-tests/test_performance.py'
$script:ConsoleBenchmark = 'fiftyone_ipintelligence_examples/src/' +
'fiftyone_ipintelligence_examples/onpremise/performance_console.py'

# The name of the file that follows $Option in the arguments, or null.
function Get-OptionFile([object[]]$Arguments, [string]$Option) {
$index = [Array]::IndexOf($Arguments, $Option)
if ($index -lt 0) {
return $null
}
return Split-Path -Leaf $Arguments[$index + 1]
}
}

Describe 'ip-intelligence run-performance-tests.ps1' {

BeforeEach {
$script:CommonCi = Join-Path ([IO.Path]::GetTempPath()) `
([Guid]::NewGuid().ToString())
$script:Assets = New-Item -ItemType Directory -Force `
(Join-Path $script:CommonCi 'assets')
$script:Product = (New-Item -ItemType Directory -Force `
(Join-Path $script:CommonCi 'ip-intelligence')).FullName
foreach ($name in $script:LiteFile, $script:EvidenceFile) {
Set-Content (Join-Path $script:Assets $name) ''
}

# The console benchmark writes its summary where --json_output says.
Mock python {
$index = [Array]::IndexOf($args, '--json_output')
if ($index -ge 0) {
'{ "DetectionsPerSecond": 1, "MsPerDetection": 1 }' |
Set-Content $args[$index + 1]
}
}

Push-Location $script:CommonCi
}

AfterEach {
Pop-Location
Remove-Item -Recurse -Force $script:CommonCi
}

It 'has a web benchmark in the same form as Device Detection' {
Join-Path $PSScriptRoot '..' 'ip-intelligence' $script:WebBenchmark |
Should -Exist
}

It 'runs the web benchmark on the free Lite file and the evidence' {
& $script:Script -RepoName $script:Product -Name 'test' | Out-Null

Should -Invoke python -Times 1 -Exactly -ParameterFilter {
$args[0] -eq $script:WebBenchmark -and
(Split-Path -Leaf $args[1]) -eq $script:LiteFile -and
(Split-Path -Leaf $args[2]) -eq $script:EvidenceFile -and
$args -contains '--output'
}
}

It 'runs the console benchmark on the free Lite file and the evidence' {
$output = & $script:Script -RepoName $script:Product -Name 'test' |
Out-String

Should -Invoke python -Times 1 -Exactly -ParameterFilter {
$args[0] -eq $script:ConsoleBenchmark -and
(Get-OptionFile $args '--data_file') -eq $script:LiteFile -and
(Get-OptionFile $args '--evidence_file') -eq $script:EvidenceFile
}
$output | Should -Match 'on 51Degrees-LiteIpiV41\.ipi'
}

It 'writes the summary the performance step reports' {
& $script:Script -RepoName $script:Product -Name 'test' | Out-Null

# common-ci compares results_<Name>.json, one file for each configuration,
# which also holds Device Detection's figures, so these carry the product.
$folder = Join-Path $script:Product 'test-results' 'performance-summary'
Get-ChildItem $folder | ForEach-Object Name | Should -Be 'results_test.json'
$results = Get-Content (Join-Path $folder 'results_test.json') |
ConvertFrom-Json
$results.HigherIsBetter.IpIntelligence_DetectionsPerSecond | Should -Be 1
$results.LowerIsBetter.IpIntelligence_AvgMillisecsPerDetection |
Should -Be 1
}

It 'fails rather than skipping when the free Lite file is absent' {
Remove-Item (Join-Path $script:Assets $script:LiteFile)

{ & $script:Script -RepoName $script:Product -Name 'test' } |
Should -Throw "*$($script:LiteFile)*"
Should -Invoke python -Times 0 -Exactly
}

It 'runs nothing when performance is turned off' {
& $script:Script -RepoName $script:Product -Name 'test' `
-RunPerformance $false

Should -Invoke python -Times 0 -Exactly
}

}
53 changes: 49 additions & 4 deletions ip-intelligence/ci/run-performance-tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,53 @@ param (
[string]$Name,
[boolean]$RunPerformance = $True
)
$ErrorActionPreference = 'Stop'

# Performance tests are not yet implemented for IP Intelligence. They should
# mirror the device detection performance tests once a stable data file
# source is available in CI.
Write-Output "Skipping performance tests - not yet implemented for IP Intelligence"
if (!$RunPerformance) {
Write-Output "Skipping performance tests"
return
}

# Every example and test runs on the free data files, so the benchmarks
# measure on the free Lite file, which needs no licence. fetch-assets.ps1
# always fetches it and the evidence, so either one missing is an error, not
# a reason to skip. The Lite file is used rather than the smaller Asn file
# because it has RegisteredCountry, the property the benchmarks read, so each
# lookup returns a value as it does with a paid file.
$dataFileName = '51Degrees-LiteIpiV41.ipi'
$evidenceFileName = 'ip-intelligence-evidence.yml'
foreach ($file in $dataFileName, $evidenceFileName) {
if (!(Test-Path "assets/$file")) {
throw "$file is not in the assets, so the performance tests cannot run"
}
}

$perfSummary = New-Item -ItemType directory -Path $RepoName/test-results/performance-summary -Force

# The assets are in the common-ci folder this runs from. The tests below run
# from the product's folder, so the path is resolved first.
$assets = (Resolve-Path assets).Path

Push-Location $RepoName
try {
Write-Output "Running Web performance tests on $dataFileName"
python fiftyone_ipintelligence_examples/performance-tests/test_performance.py "$assets/$dataFileName" "$assets/$evidenceFileName" --output || $(throw "benchmark failed with code: $LASTEXITCODE")

Write-Output "Running Console performance tests on $dataFileName"
python fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py --data_file "$assets/$dataFileName" --evidence_file "$assets/$evidenceFileName" --json_output $pwd/summary.json || $(throw "performance tests failed with $LASTEXITCODE")

# The root step puts every product's figures in the one summary of the
# configuration, and Device Detection measures the same things, so these
# carry the product's name.
$Results = Get-Content $pwd/summary.json | ConvertFrom-Json
@{
HigherIsBetter = @{
IpIntelligence_DetectionsPerSecond = $Results.DetectionsPerSecond
};
LowerIsBetter = @{
IpIntelligence_AvgMillisecsPerDetection = $Results.MsPerDetection
}
} | ConvertTo-Json | Out-File $perfSummary/results_$Name.json
} finally {
Pop-Location
}
Loading
Loading