From 9c94b2b2480998dbd685643ab3ba5f78d4355828 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 19:02:47 +0000 Subject: [PATCH 1/4] Test: the IP Intelligence performance step measures on the free Lite file The IP Intelligence specification says its examples follow the device detection rules for on-premise examples (ip-intelligence-specification/required-examples.md lines 80 and 81 at 6570727d2af6967390d3c4fbec77b9cf0540d3f3), which say examples MUST run out of the box with the free data files (device-detection-specification/required-examples.md line 91 at the same commit), and every example and test runs on the free files. ip-intelligence/ci/run-performance-tests.ps1 measures nothing: it prints that performance tests are not yet implemented, so the performance step has no IP Intelligence figures at all. ci/ip-intelligence-performance.Tests.ps1 runs the step with only the free Lite file and the evidence in the assets. It expects a web benchmark at fiftyone_ipintelligence_examples/performance-tests/test_performance.py, in the same place and form as Device Detection's, to be run on the Lite file and the evidence with --output, the console example performance_console.py to be run on the same files, the output to name the Lite file, and a results file to be written with DetectionsPerSecond and AvgMillisecsPerDetection. Without the Lite file the step fails rather than skipping, and with performance turned off it runs nothing. test_web_performance_benchmark.py runs the web benchmark on the free Asn file and 50 evidence records, and checks the JSON it prints. PF-2. --- ci/ip-intelligence-performance.Tests.ps1 | 118 ++++++++++++++++++ .../tests/test_web_performance_benchmark.py | 117 +++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 ci/ip-intelligence-performance.Tests.ps1 create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/tests/test_web_performance_benchmark.py diff --git a/ci/ip-intelligence-performance.Tests.ps1 b/ci/ip-intelligence-performance.Tests.ps1 new file mode 100644 index 00000000..0b8dbc2f --- /dev/null +++ b/ci/ip-intelligence-performance.Tests.ps1 @@ -0,0 +1,118 @@ +# Tests for ip-intelligence/ci/run-performance-tests.ps1. +# +# common-ci runs the script from its own folder, which holds the assets, with +# the product folder as RepoName. Each test lays out such a folder with empty +# stand-ins for the free Lite data file and the evidence, and mocks python, so +# that what is checked is what the script measures and on which file, not the +# measurement itself. + +BeforeAll { + $script:Script = Join-Path $PSScriptRoot '..' 'ip-intelligence' 'ci' ` + 'run-performance-tests.ps1' + + # The free data file and the evidence, as common-ci's assets name them. + $script:LiteFile = '51Degrees-LiteIpiV41.ipi' + $script:EvidenceFile = 'ip-intelligence-evidence.yml' + + # The web benchmark, in the same place as Device Detection's, and the + # console example, relative to the product folder. + $script:WebBenchmark = + 'fiftyone_ipintelligence_examples/performance-tests/test_performance.py' + $script:ConsoleBenchmark = 'fiftyone_ipintelligence_examples/src/' + + 'fiftyone_ipintelligence_examples/onpremise/performance_console.py' + + # The name of the file that follows $Option in the arguments, or null. + function Get-OptionFile([object[]]$Arguments, [string]$Option) { + $index = [Array]::IndexOf($Arguments, $Option) + if ($index -lt 0) { + return $null + } + return Split-Path -Leaf $Arguments[$index + 1] + } +} + +Describe 'ip-intelligence run-performance-tests.ps1' { + +BeforeEach { + $script:CommonCi = Join-Path ([IO.Path]::GetTempPath()) ` + ([Guid]::NewGuid().ToString()) + $script:Assets = New-Item -ItemType Directory -Force ` + (Join-Path $script:CommonCi 'assets') + $script:Product = (New-Item -ItemType Directory -Force ` + (Join-Path $script:CommonCi 'ip-intelligence')).FullName + foreach ($name in $script:LiteFile, $script:EvidenceFile) { + Set-Content (Join-Path $script:Assets $name) '' + } + + # The console benchmark writes its summary where --json_output says. + Mock python { + $index = [Array]::IndexOf($args, '--json_output') + if ($index -ge 0) { + '{ "DetectionsPerSecond": 1, "MsPerDetection": 1 }' | + Set-Content $args[$index + 1] + } + } + + Push-Location $script:CommonCi +} + +AfterEach { + Pop-Location + Remove-Item -Recurse -Force $script:CommonCi +} + +It 'has a web benchmark in the same form as Device Detection' { + Join-Path $PSScriptRoot '..' 'ip-intelligence' $script:WebBenchmark | + Should -Exist +} + +It 'runs the web benchmark on the free Lite file and the evidence' { + & $script:Script -RepoName $script:Product -Name 'test' | Out-Null + + Should -Invoke python -Times 1 -Exactly -ParameterFilter { + $args[0] -eq $script:WebBenchmark -and + (Split-Path -Leaf $args[1]) -eq $script:LiteFile -and + (Split-Path -Leaf $args[2]) -eq $script:EvidenceFile -and + $args -contains '--output' + } +} + +It 'runs the console benchmark on the free Lite file and the evidence' { + $output = & $script:Script -RepoName $script:Product -Name 'test' | + Out-String + + Should -Invoke python -Times 1 -Exactly -ParameterFilter { + $args[0] -eq $script:ConsoleBenchmark -and + (Get-OptionFile $args '--data_file') -eq $script:LiteFile -and + (Get-OptionFile $args '--evidence_file') -eq $script:EvidenceFile + } + $output | Should -Match 'on 51Degrees-LiteIpiV41\.ipi' +} + +It 'writes the summary the performance step reports' { + & $script:Script -RepoName $script:Product -Name 'test' | Out-Null + + $summary = Join-Path $script:Product 'test-results' 'performance-summary' ` + 'results_test.json' + $summary | Should -Exist + $results = Get-Content $summary | ConvertFrom-Json + $results.HigherIsBetter.DetectionsPerSecond | Should -Be 1 + $results.LowerIsBetter.AvgMillisecsPerDetection | Should -Be 1 +} + +It 'fails rather than skipping when the free Lite file is absent' { + Remove-Item (Join-Path $script:Assets $script:LiteFile) + + { & $script:Script -RepoName $script:Product -Name 'test' } | + Should -Throw "*$($script:LiteFile)*" + Should -Invoke python -Times 0 -Exactly +} + +It 'runs nothing when performance is turned off' { + & $script:Script -RepoName $script:Product -Name 'test' ` + -RunPerformance $false + + Should -Invoke python -Times 0 -Exactly +} + +} diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_web_performance_benchmark.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_web_performance_benchmark.py new file mode 100644 index 00000000..03e8e414 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_web_performance_benchmark.py @@ -0,0 +1,117 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +Runs the web performance benchmark the performance step runs, as that step +runs it, with python and the --output option, on a free data file and a short +file of evidence so that it takes seconds. The step gives it the free Lite +file. This test gives it the free Asn file, which the other example tests use +and which is far smaller, because the benchmark measures any data file the +same way. +""" + +import itertools +import json +import os +import subprocess +import sys +import tempfile +import unittest + +from ruamel.yaml import YAML + +from fiftyone_ipintelligence_examples.example_utils import ExampleUtils +from fiftyone_ipintelligence_shared.example_constants import \ + ASN_DATAFILE_NAME, EVIDENCE_FILE_NAME + +# The benchmark, in the same place as Device Detection's. +BENCHMARK = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "performance-tests", "test_performance.py") + +# The evidence records given to the benchmark, enough to time lookups +# without making the test slow. +RECORDS = 50 + +# Stops a run that never finishes. A run takes seconds, so this does not +# limit how fast the benchmark has to be. +HANG_TIMEOUT_SECONDS = 300 + + +class WebPerformanceBenchmarkTests(unittest.TestCase): + + def setUp(self): + self.data_file = ExampleUtils.find_file(ASN_DATAFILE_NAME) + self.evidence_file = ExampleUtils.find_file(EVIDENCE_FILE_NAME) + self.assertIsNotNone( + self.data_file, f"{ASN_DATAFILE_NAME} was not found") + self.assertIsNotNone( + self.evidence_file, f"{EVIDENCE_FILE_NAME} was not found") + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.folder = folder.name + + def short_evidence(self, count): + """The path of a file holding the first count records of the free + evidence file, each as a document of its own.""" + yaml = YAML(typ="safe") + with open(self.evidence_file, "r") as source: + records = [dict(record) for record in itertools.islice( + (record for record in yaml.load_all(source) if record), + count)] + self.assertEqual(count, len(records)) + evidence = os.path.join(self.folder, "evidence.yml") + with open(evidence, "w") as target: + yaml.dump_all(records, target) + return evidence + + def test_times_every_record_and_writes_json(self): + evidence = self.short_evidence(RECORDS) + + finished = subprocess.run( + [sys.executable, BENCHMARK, self.data_file, evidence, "--output"], + capture_output=True, text=True, timeout=HANG_TIMEOUT_SECONDS) + + self.assertEqual(0, finished.returncode, + finished.stdout + finished.stderr) + results = json.loads(finished.stdout) + self.assertEqual(RECORDS, results["Detections"]) + self.assertGreater(results["DetectionsPerSecond"], 0) + self.assertGreater(results["RuntimeSeconds"], 0) + self.assertGreater(results["AvgMillisecsPerDetection"], 0) + + def test_prints_nothing_without_the_output_option(self): + evidence = self.short_evidence(RECORDS) + + finished = subprocess.run( + [sys.executable, BENCHMARK, self.data_file, evidence], + capture_output=True, text=True, timeout=HANG_TIMEOUT_SECONDS) + + self.assertEqual(0, finished.returncode, + finished.stdout + finished.stderr) + self.assertEqual("", finished.stdout) + self.assertIn(f"Benchmarking with {RECORDS} IP addresses", + finished.stderr) + + +if __name__ == "__main__": + unittest.main() From d751511b50ed2fb315effc5885918406b32f7339 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 19:03:56 +0000 Subject: [PATCH 2/4] Fix: measure IP Intelligence in the performance step on the free Lite file ip-intelligence/ci/run-performance-tests.ps1 now runs two benchmarks on 51Degrees-LiteIpiV41.ipi and the evidence, which fetch-assets.ps1 always fetches, as the rule that every example and test runs on the free files and ip-intelligence-specification/required-examples.md lines 80 and 81 with device-detection-specification/required-examples.md line 91 at 6570727d2af6967390d3c4fbec77b9cf0540d3f3 require. The root ci/run-performance-tests.ps1 runs it for IP Intelligence as it does for Device Detection. The web benchmark, fiftyone_ipintelligence_examples/performance-tests/test_performance.py, is in the same form as Device Detection's: it makes each IP address of the evidence the client address of a mock Flask request, processes the request's evidence with the on-premise pipeline on MaxPerformance, and with --output prints the same JSON fields. The console benchmark is the existing performance_console.py example, whose results become results_.json with DetectionsPerSecond and AvgMillisecsPerDetection, as Device Detection's do. The step names the file it measured on. When the Lite file or the evidence is missing it fails rather than skipping, and with performance turned off it runs nothing. The Lite file is used rather than the smaller Asn file because it has RegisteredCountry, the property both benchmarks read. PF-2. --- ip-intelligence/ci/run-performance-tests.ps1 | 50 ++++++- .../performance-tests/test_performance.py | 123 ++++++++++++++++++ 2 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 ip-intelligence/fiftyone_ipintelligence_examples/performance-tests/test_performance.py diff --git a/ip-intelligence/ci/run-performance-tests.ps1 b/ip-intelligence/ci/run-performance-tests.ps1 index c9a3dd82..659551ce 100644 --- a/ip-intelligence/ci/run-performance-tests.ps1 +++ b/ip-intelligence/ci/run-performance-tests.ps1 @@ -5,8 +5,50 @@ param ( [string]$Name, [boolean]$RunPerformance = $True ) +$ErrorActionPreference = 'Stop' -# Performance tests are not yet implemented for IP Intelligence. They should -# mirror the device detection performance tests once a stable data file -# source is available in CI. -Write-Output "Skipping performance tests - not yet implemented for IP Intelligence" +if (!$RunPerformance) { + Write-Output "Skipping performance tests" + return +} + +# Every example and test runs on the free data files, so the benchmarks +# measure on the free Lite file, which needs no licence. fetch-assets.ps1 +# always fetches it and the evidence, so either one missing is an error, not +# a reason to skip. The Lite file is used rather than the smaller Asn file +# because it has RegisteredCountry, the property the benchmarks read, so each +# lookup returns a value as it does with a paid file. +$dataFileName = '51Degrees-LiteIpiV41.ipi' +$evidenceFileName = 'ip-intelligence-evidence.yml' +foreach ($name in $dataFileName, $evidenceFileName) { + if (!(Test-Path "assets/$name")) { + throw "$name is not in the assets, so the performance tests cannot run" + } +} + +$perfSummary = New-Item -ItemType directory -Path $RepoName/test-results/performance-summary -Force + +# The assets are in the common-ci folder this runs from. The tests below run +# from the product's folder, so the path is resolved first. +$assets = (Resolve-Path assets).Path + +Push-Location $RepoName +try { + Write-Output "Running Web performance tests on $dataFileName" + python fiftyone_ipintelligence_examples/performance-tests/test_performance.py "$assets/$dataFileName" "$assets/$evidenceFileName" --output || $(throw "benchmark failed with code: $LASTEXITCODE") + + Write-Output "Running Console performance tests on $dataFileName" + python fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py --data_file "$assets/$dataFileName" --evidence_file "$assets/$evidenceFileName" --json_output $pwd/summary.json || $(throw "performance tests failed with $LASTEXITCODE") + + $Results = Get-Content $pwd/summary.json | ConvertFrom-Json + @{ + HigherIsBetter = @{ + DetectionsPerSecond = $Results.DetectionsPerSecond + }; + LowerIsBetter = @{ + AvgMillisecsPerDetection = $Results.MsPerDetection + } + } | ConvertTo-Json | Out-File $perfSummary/results_$Name.json +} finally { + Pop-Location +} diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/performance-tests/test_performance.py b/ip-intelligence/fiftyone_ipintelligence_examples/performance-tests/test_performance.py new file mode 100644 index 00000000..9193f94e --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_examples/performance-tests/test_performance.py @@ -0,0 +1,123 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# Times IP Intelligence lookups made from web requests, in the same form as +# the Device Detection benchmark in +# device-detection/fiftyone_devicedetection_examples/performance-tests. Each +# IP address in the evidence file is made the client address of a mock Flask +# request, whose evidence the pipeline then processes. The performance step +# runs it on the free Lite data file, so it needs no licensed file. + +import sys +import json +import argparse +from fiftyone_ipintelligence_onpremise.ipintelligence_onpremise_pipelinebuilder import IpIntelligenceOnPremisePipelineBuilder +from fiftyone_pipeline_core.web import webevidence +from flask import Flask, request +from ruamel.yaml import YAML +from timeit import timeit + +# The evidence file gives each IP address under this key. +CLIENT_IP = "server.client-ip" + +# Helper function to get a property value if it exists and return +# the reason why if it doesn't +def getValueHelper(flowdata, engine, propertyKey): + engineProperties = getattr(flowdata, engine) + try: + propertyValue = getattr(engineProperties, propertyKey) + if propertyValue.has_value(): + return propertyValue.value() + else: + return propertyValue.no_value_message() + except: + return "Not found in datafile" + +def detect(app, pipeline, ip_addresses): + # The IP address is the client address of the mock request, which + # webevidence gives as server.client-ip. + with app.test_request_context( + "/process", environ_base={"REMOTE_ADDR": next(ip_addresses)}): + + # The FlowData is closed when the function returns, which releases + # the results of the lookup straight away. + with pipeline.create_flowdata() as flowdata: + + # Add any information from the request (headers, cookies and additional + # client side provided information) + flowdata.evidence.add_from_dict(webevidence(request)) + + # Process the flowdata + flowdata.process() + + return getValueHelper(flowdata, "ip", "registeredcountry") + +def load_ip_addresses(evidence_file): + # Each record in the evidence file is a YAML document of its own. + yaml = YAML(typ="safe") + with open(evidence_file) as f: + return [str(record[CLIENT_IP]) for record in yaml.load_all(f) + if record and CLIENT_IP in record] + +def benchmark(data_file, evidence_file, output=False): + app = Flask(__name__) + + # Create IP Intelligence pipeline using datafile + pipeline = IpIntelligenceOnPremisePipelineBuilder( + data_file_path = data_file, + licence_keys = "", + performance_profile = "MaxPerformance", + auto_update = False, + update_on_start = False, + # The data file is used in place, rather than through a temporary + # copy of a file of hundreds of megabytes. + create_temp_data_copy = False, + usage_sharing = False, + settings = { "add_javascript_builder": False }, + ).build() + + # The pipeline is closed once the lookups have been timed, which + # releases the native data it holds. + with pipeline: + ip_addresses_list = load_ip_addresses(evidence_file) + ip_addresses = iter(ip_addresses_list) + + print(f"Benchmarking with {len(ip_addresses_list)} IP addresses", file=sys.stderr) + environment = {"detect": detect, "app": app, "pipeline": pipeline, "ip_addresses": ip_addresses} + time = timeit("detect(app, pipeline, ip_addresses)", globals=environment, number=len(ip_addresses_list)) + + if output: + json.dump({ + "Detections": len(ip_addresses_list), + "DetectionsPerSecond": len(ip_addresses_list) / time, + "RuntimeSeconds": time, + "AvgMillisecsPerDetection": time / len(ip_addresses_list) * 1000, + }, sys.stdout, indent=4) + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description='Run IP Intelligence lookup benchmark.') + ap.add_argument('data_file', help='Path to data file') + ap.add_argument('evidence_file', help='Path to YAML evidence file of IP addresses') + ap.add_argument('-o', '--output', action='store_true', help='Output results in JSON format') + args = ap.parse_args() + + benchmark(args.data_file, args.evidence_file, args.output) From 4e4bb0f24f595562a09f1b1643a5f0042b1c3d9e Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 03:21:57 +0000 Subject: [PATCH 3/4] Test: IP Intelligence figures reach the configuration's performance summary ci/ip-intelligence-performance.Tests.ps1 now checks that the only file in the performance summary folder is results_test.json, for the configuration named test, and that its figures are IpIntelligence_DetectionsPerSecond and IpIntelligence_AvgMillisecsPerDetection. At d751511 the step writes results_ip-intelligence-evidence.yml.json instead. The loop over the asset names in ip-intelligence/ci/run-performance-tests.ps1 uses $name, which PowerShell treats as the same variable as the $Name parameter, because variable names are not case sensitive, so the last asset name replaces the configuration name. common-ci's steps/compare-performance.ps1 reads only results_.json, so the IP Intelligence figures were uploaded but never compared. ci/dispatch.Tests.ps1 gets two cases for the root step. When two products write the summary of the same configuration, the one file keeps the figures of both, and when both report the same figure the step fails rather than losing one. Move-ProductTestResults moves each product's files with Move-Item -Force, so the product that runs later replaces the summary of the one before. Device Detection runs before IP Intelligence and also writes results_.json, so correcting the name alone would replace Device Detection's figures with IP Intelligence's. --- ci/dispatch.Tests.ps1 | 42 ++++++++++++++++++++++++ ci/ip-intelligence-performance.Tests.ps1 | 15 +++++---- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/ci/dispatch.Tests.ps1 b/ci/dispatch.Tests.ps1 index 352e667b..092ec732 100644 --- a/ci/dispatch.Tests.ps1 +++ b/ci/dispatch.Tests.ps1 @@ -46,6 +46,16 @@ BeforeAll { "param ([Parameter(Mandatory)][string]`$RepoName)`n$Body" } + # The body of a product's run-performance-tests step that writes the + # summary of the configuration 'test' with the given figures. + function Get-SummaryStep([hashtable]$Higher, [hashtable]$Lower) { + $json = @{ HigherIsBetter = $Higher; LowerIsBetter = $Lower } | + ConvertTo-Json -Compress + return '$dir = New-Item -ItemType Directory -Force ' + + '(Join-Path $RepoName "test-results" "performance-summary"); ' + + "Set-Content (Join-Path `$dir 'results_test.json') '$json'" + } + # A folder laid out like a common-ci checkout, which the dispatcher runs in. function New-CommonCi { $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) @@ -118,6 +128,38 @@ Describe 'Invoke-ProductStep' { Test-Path (Join-Path $script:Repo 'alpha' 'test-results') | Should -BeFalse } + It 'keeps every product''s figures in the one summary of a configuration' { + # common-ci compares one results_.json for each configuration. + Set-ProductStep $script:Repo 'alpha' 'run-performance-tests' ` + (Get-SummaryStep @{ alpha_PerSecond = 1 } @{ alpha_Ms = 2 }) + Set-ProductStep $script:Repo 'beta' 'run-performance-tests' ` + (Get-SummaryStep @{ beta_PerSecond = 3 } @{ beta_Ms = 4 }) + + Invoke-ProductStep 'run-performance-tests' $script:Repo @{ + RepoName = $script:Repo } + + $results = Get-Content (Join-Path $script:Repo 'test-results' ` + 'performance-summary' 'results_test.json') | ConvertFrom-Json + $results.HigherIsBetter.alpha_PerSecond | Should -Be 1 + $results.HigherIsBetter.beta_PerSecond | Should -Be 3 + $results.LowerIsBetter.alpha_Ms | Should -Be 2 + $results.LowerIsBetter.beta_Ms | Should -Be 4 + } + + It 'fails the step when two products report the same figure' { + Set-ProductStep $script:Repo 'alpha' 'run-performance-tests' ` + (Get-SummaryStep @{ PerSecond = 1 } @{}) + Set-ProductStep $script:Repo 'beta' 'run-performance-tests' ` + (Get-SummaryStep @{ PerSecond = 3 } @{}) + + { Invoke-ProductStep 'run-performance-tests' $script:Repo @{ + RepoName = $script:Repo } } | Should -Throw '*failed for: beta' + + $results = Get-Content (Join-Path $script:Repo 'test-results' ` + 'performance-summary' 'results_test.json') | ConvertFrom-Json + $results.HigherIsBetter.PerSecond | Should -Be 1 + } + It 'fetches assets when called without a repository name' { # common-ci's prefetch_assets action runs ci/fetch-assets.ps1 directly # with only the keys, from the common-ci checkout that holds this diff --git a/ci/ip-intelligence-performance.Tests.ps1 b/ci/ip-intelligence-performance.Tests.ps1 index 0b8dbc2f..161e7c2d 100644 --- a/ci/ip-intelligence-performance.Tests.ps1 +++ b/ci/ip-intelligence-performance.Tests.ps1 @@ -92,12 +92,15 @@ It 'runs the console benchmark on the free Lite file and the evidence' { It 'writes the summary the performance step reports' { & $script:Script -RepoName $script:Product -Name 'test' | Out-Null - $summary = Join-Path $script:Product 'test-results' 'performance-summary' ` - 'results_test.json' - $summary | Should -Exist - $results = Get-Content $summary | ConvertFrom-Json - $results.HigherIsBetter.DetectionsPerSecond | Should -Be 1 - $results.LowerIsBetter.AvgMillisecsPerDetection | Should -Be 1 + # common-ci compares results_.json, one file for each configuration, + # which also holds Device Detection's figures, so these carry the product. + $folder = Join-Path $script:Product 'test-results' 'performance-summary' + Get-ChildItem $folder | ForEach-Object Name | Should -Be 'results_test.json' + $results = Get-Content (Join-Path $folder 'results_test.json') | + ConvertFrom-Json + $results.HigherIsBetter.IpIntelligence_DetectionsPerSecond | Should -Be 1 + $results.LowerIsBetter.IpIntelligence_AvgMillisecsPerDetection | + Should -Be 1 } It 'fails rather than skipping when the free Lite file is absent' { From 4d47dcdf5198cd7fee98f27d052de98181ac52f4 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Tue, 17 Nov 2026 03:28:04 +0000 Subject: [PATCH 4/4] Fix: put the IP Intelligence figures in the configuration's performance summary ip-intelligence/ci/run-performance-tests.ps1 names its loop over the asset names $file rather than $name, so the $Name parameter keeps the configuration name and the summary is results_.json, the file common-ci's steps/compare-performance.ps1 reads for each configuration. Its figures are now IpIntelligence_DetectionsPerSecond and IpIntelligence_AvgMillisecsPerDetection, because Device Detection reports DetectionsPerSecond and AvgMillisecsPerDetection in the same file. 51degrees-java's ci/run-performance-tests.ps1 prefixes the Device Detection and IP Intelligence figures in its one file in the same way. Move-ProductTestResults in ci/dispatch.ps1 adds a product's figures to a summary another product already wrote for the configuration, rather than replacing that file. A figure both products report fails the step for the later product, and the summary keeps the earlier figure. Invoke-ProductStep records that failure as it records a failing script, so every product still runs. --- ci/dispatch.ps1 | 45 +++++++++++++++++++- ip-intelligence/ci/run-performance-tests.ps1 | 13 +++--- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/ci/dispatch.ps1 b/ci/dispatch.ps1 index 06ba77ef..f06d2a35 100644 --- a/ci/dispatch.ps1 +++ b/ci/dispatch.ps1 @@ -39,7 +39,14 @@ function Invoke-ProductStep([string]$Step, [string]$RepoName, Write-Output "::error title=$Step failed::$product`: $_" $failed.Add($product) } finally { - Move-ProductTestResults $RepoName $product + try { + Move-ProductTestResults $RepoName $product + } catch { + Write-Output "::error title=$Step failed::$product`: $_" + if ($failed.Contains($product) -eq $false) { + $failed.Add($product) + } + } Write-Output "::endgroup::" } } @@ -51,6 +58,8 @@ function Invoke-ProductStep([string]$Step, [string]$RepoName, # common-ci collects test results from /test-results, and a # product's own scripts write them under its folder, where they would not be # found. Move them up, keeping the unit/integration/performance folders. +# common-ci compares one performance summary for each configuration, so a +# summary another product already wrote gets this product's figures added. function Move-ProductTestResults([string]$RepoName, [string]$Product) { $results = Join-Path $RepoName $Product 'test-results' if ((Test-Path $results) -eq $false) { @@ -59,7 +68,39 @@ function Move-ProductTestResults([string]$RepoName, [string]$Product) { foreach ($kind in (Get-ChildItem -Directory $results)) { $target = New-Item -ItemType Directory -Force ` (Join-Path $RepoName 'test-results' $kind.Name) - Get-ChildItem $kind.FullName | Move-Item -Destination $target -Force + foreach ($file in (Get-ChildItem $kind.FullName)) { + $existing = Join-Path $target $file.Name + if ($kind.Name -eq 'performance-summary' -and + $file.Extension -eq '.json' -and + (Test-Path $existing -PathType Leaf) -eq $true) { + Add-PerformanceFigures $file.FullName $existing + } else { + Move-Item $file.FullName -Destination $target -Force + } + } } Remove-Item -Recurse -Force $results } + +# Adds the HigherIsBetter and LowerIsBetter figures of the summary at $Source +# to the one at $Destination. A figure already in $Destination fails, because +# one of the two would be lost. +function Add-PerformanceFigures([string]$Source, [string]$Destination) { + $from = Get-Content -Raw $Source | ConvertFrom-Json -AsHashtable + $into = Get-Content -Raw $Destination | ConvertFrom-Json -AsHashtable + foreach ($section in 'HigherIsBetter', 'LowerIsBetter') { + if ($null -eq $from[$section]) { + continue + } + if ($null -eq $into[$section]) { + $into[$section] = [ordered]@{} + } + foreach ($metric in $from[$section].Keys) { + if ($into[$section].Contains($metric) -eq $true) { + throw "$section.$metric is in both '$Source' and '$Destination'" + } + $into[$section][$metric] = $from[$section][$metric] + } + } + $into | ConvertTo-Json -Depth 10 | Set-Content $Destination +} diff --git a/ip-intelligence/ci/run-performance-tests.ps1 b/ip-intelligence/ci/run-performance-tests.ps1 index 659551ce..45735340 100644 --- a/ip-intelligence/ci/run-performance-tests.ps1 +++ b/ip-intelligence/ci/run-performance-tests.ps1 @@ -20,9 +20,9 @@ if (!$RunPerformance) { # lookup returns a value as it does with a paid file. $dataFileName = '51Degrees-LiteIpiV41.ipi' $evidenceFileName = 'ip-intelligence-evidence.yml' -foreach ($name in $dataFileName, $evidenceFileName) { - if (!(Test-Path "assets/$name")) { - throw "$name is not in the assets, so the performance tests cannot run" +foreach ($file in $dataFileName, $evidenceFileName) { + if (!(Test-Path "assets/$file")) { + throw "$file is not in the assets, so the performance tests cannot run" } } @@ -40,13 +40,16 @@ try { Write-Output "Running Console performance tests on $dataFileName" python fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/onpremise/performance_console.py --data_file "$assets/$dataFileName" --evidence_file "$assets/$evidenceFileName" --json_output $pwd/summary.json || $(throw "performance tests failed with $LASTEXITCODE") + # The root step puts every product's figures in the one summary of the + # configuration, and Device Detection measures the same things, so these + # carry the product's name. $Results = Get-Content $pwd/summary.json | ConvertFrom-Json @{ HigherIsBetter = @{ - DetectionsPerSecond = $Results.DetectionsPerSecond + IpIntelligence_DetectionsPerSecond = $Results.DetectionsPerSecond }; LowerIsBetter = @{ - AvgMillisecsPerDetection = $Results.MsPerDetection + IpIntelligence_AvgMillisecsPerDetection = $Results.MsPerDetection } } | ConvertTo-Json | Out-File $perfSummary/results_$Name.json } finally {