diff --git a/ci/run-integration-tests.Tests.ps1 b/ci/run-integration-tests.Tests.ps1 new file mode 100644 index 00000000..d4677bf4 --- /dev/null +++ b/ci/run-integration-tests.Tests.ps1 @@ -0,0 +1,163 @@ +# Tests for the integration test step when no resource key is passed. +# +# Each product's ci/run-integration-tests.ps1 is run as common-ci runs it, +# through ci/dispatch.ps1 and common-ci's steps/run-script.ps1, from a folder +# laid out like a common-ci checkout. Its python/run-integration-tests.ps1 is +# a stand-in that runs no tests and records, for each call, the packages it +# was given and the PYTEST_ADDOPTS the tests would see. + +BeforeAll { + . (Join-Path $PSScriptRoot 'dispatch.ps1') + + # common-ci checked out beside this repository, or wherever COMMON_CI says. + $commonCi = $env:COMMON_CI + if ([string]::IsNullOrEmpty($commonCi) -eq $true) { + $commonCi = Join-Path $PSScriptRoot '..' '..' 'common-ci' + } + $script:Steps = Join-Path $commonCi 'steps' + if ((Test-Path (Join-Path $script:Steps 'run-script.ps1')) -eq $false) { + throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it" + } + + $script:Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + + # The products whose integration tests call the cloud service. + $script:Products = 'device-detection', 'ip-intelligence', 'location', + 'pipeline' + + # The environment variables the scripts set, kept to be put back. + $script:Saved = '_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY', + 'PYTEST_ADDOPTS', 'TEMP' + + # The stand-in for common-ci's python/run-integration-tests.ps1. It + # writes one line of JSON for each call to calls.jsonl in the folder it + # runs from. + $script:StandIn = @' +param ( + [Parameter(Mandatory=$true)][string]$RepoName, + [Parameter(Mandatory=$true)][string[]]$Packages, + [Parameter(Mandatory=$true)][hashtable]$Keys +) +@{ + Product = Split-Path $RepoName -Leaf + Packages = @($Packages) + AddOpts = "$env:PYTEST_ADDOPTS" +} | ConvertTo-Json -Compress | Add-Content (Join-Path $PWD 'calls.jsonl') +$global:LASTEXITCODE = 0 +'@ + + # A folder laid out like a common-ci checkout, which the step runs in. + function New-CommonCi { + $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + $null = New-Item -ItemType Directory -Force (Join-Path $path 'python') + Copy-Item -Recurse $script:Steps (Join-Path $path 'steps') + Set-Content (Join-Path $path 'python' 'run-integration-tests.ps1') ` + $script:StandIn + return $path + } + + # The calls the stand-in recorded for a product. + function Get-Calls([string]$Product) { + $file = Join-Path $script:CommonCi 'calls.jsonl' + if ((Test-Path $file) -eq $false) { + return @() + } + return @(Get-Content $file | ForEach-Object { $_ | ConvertFrom-Json } | + Where-Object Product -eq $Product) + } +} + +Describe 'run-integration-tests.ps1' { + +BeforeEach { + $script:Environment = @{} + foreach ($name in $script:Saved) { + $script:Environment[$name] = [Environment]::GetEnvironmentVariable($name) + } + $script:CommonCi = New-CommonCi + Push-Location $script:CommonCi +} + +AfterEach { + Pop-Location + Remove-Item -Recurse -Force $script:CommonCi + foreach ($name in $script:Saved) { + [Environment]::SetEnvironmentVariable($name, $script:Environment[$name]) + } +} + +Context 'with no TestResourceKey' { + BeforeEach { + $script:Output = Invoke-ProductStep 'run-integration-tests' $script:Repo ` + @{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 | + Out-String + } + + It 'runs the tests of every product' { + foreach ($product in $script:Products) { + (Get-Calls $product).Count | Should -BeGreaterThan 0 -Because $product + } + } + + It 'leaves out only the tests marked live, which need a resource key' { + foreach ($product in $script:Products) { + foreach ($call in (Get-Calls $product)) { + $call.AddOpts | Should -Be '-m "not live"' -Because $product + } + } + } + + It 'says for every product that the tests needing a key are skipped' { + foreach ($product in $script:Products) { + # What the step printed for this product, between its group + # markers. + $group = ($script:Output -split '::group::') | + Where-Object { $_ -like "run-integration-tests for $product*" } + $group | Should -Match ("No TestResourceKey was passed, so the " + + "tests that need a resource key are skipped") -Because $product + } + } + + It 'runs the tests of the device detection and IP intelligence examples' { + (Get-Calls 'device-detection').Packages | + Should -Contain 'fiftyone_devicedetection_examples' + (Get-Calls 'device-detection').Packages | + Should -Contain 'fiftyone_devicedetection_cloud' + (Get-Calls 'ip-intelligence').Packages | + Should -Contain 'fiftyone_ipintelligence_examples' + } + + It 'runs the cloud request engine and its examples, and skips the 51Did live tests' { + $packages = (Get-Calls 'pipeline').Packages + $packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine' + $packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine/examples' + $packages | Should -Not -Contain 'fiftyone_pipeline_did' + $script:Output | Should -Match 'fiftyone_pipeline_did.*skipped' + } + + It 'skips the Selenium contract against the cloud example and saying so' { + $script:Output | Should -Match 'Selenium contract tests will not run against the cloud example' + Test-Path (Join-Path $script:CommonCi 'selenium-api-tests') | + Should -BeFalse + } +} + +Context 'with a TestResourceKey' { + It 'runs every test, the ones marked live included' { + Invoke-ProductStep 'run-integration-tests' $script:Repo @{ + RepoName = $script:Repo + Keys = @{ TestResourceKey = 'key' } + SeleniumContract = $false + } 'device-detection', 'location' *>&1 | Out-Null + + foreach ($product in 'device-detection', 'location') { + $calls = Get-Calls $product + $calls.Count | Should -BeGreaterThan 0 -Because $product + foreach ($call in $calls) { + $call.AddOpts | Should -Not -Match 'live' -Because $product + } + } + } +} + +} diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index ed7a17d2..c465b293 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -16,10 +16,13 @@ if ($IsWindows) { $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples" +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = '-m "not live"' } # The cloud package runs without the CSV file too, because only its property # coverage tests read the file, and they say so when it is missing. @@ -44,7 +47,11 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey # files from _51DEGREES_LICENSE_KEY, which common-ci sets as license_key. $env:_51DEGREES_LICENSE_KEY = $Keys.DeviceDetection -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} $failures = @() if ($LASTEXITCODE -ne 0) { $failures += "the Python integration tests exited with code $LASTEXITCODE" @@ -58,6 +65,25 @@ if (-not $SeleniumContract) { return } +# The contract tests run against the cloud example when there is a resource +# key, and against the on-premise example when there is the TAC data file, +# which is only fetched when a licence is given. +$tac = "$PWD/assets/TAC-HashV41.hash" +$runCloud = [bool]$Keys.TestResourceKey +$runOnPremise = Test-Path $tac +if (-not $runCloud) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No resource key::No TestResourceKey was passed, so the Selenium contract tests will not run against the cloud example." +} +if (-not $runOnPremise) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example." +} +if (-not ($runCloud -or $runOnPremise)) { + if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" + } + return +} + # The shared Selenium contract tests are run against the cloud example and the # on-premise example in turn. Each result is also written as a trx file next # to the other integration results, so it is published with them. @@ -141,22 +167,21 @@ function Test-Example([string]$Name, [string]$Module, [int]$Port) { } # The cloud example, pointed at the live cloud. -$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey -$env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/" -$failures += Test-Example -Name 'cloud' ` - -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097 +if ($runCloud) { + $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey + $env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/" + $failures += Test-Example -Name 'cloud' ` + -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097 +} # The on-premise example. The contract tests need the device type and the # JavaScript properties, which the Lite data file does not have, so it runs -# against the TAC data file, which is only fetched when a licence is given. -$tac = "$PWD/assets/TAC-HashV41.hash" -if (Test-Path $tac) { +# against the TAC data file. +if ($runOnPremise) { $env:_51DEGREES_DD_PATH = (Resolve-Path $tac).Path $failures += Test-Example -Name 'onpremise' ` -Module 'fiftyone_devicedetection_examples.onpremise.gettingstarted_web' -Port 8098 Remove-Item Env:_51DEGREES_DD_PATH -} else { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example." } # A failure anywhere above must fail the job. This script is dot-sourced by diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index 99dfb369..be1117ab 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -43,7 +43,9 @@ the Enterprise data file. The cloud tests need a valid 51Degrees resource key (see above), and fail without one. They read it from the environment variable -`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. +`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. They are +marked `live`, so `python -m pytest -m "not live"` runs every other test, +which is what CI does when no resource key is passed. For example, use following command to set the resource key on Linux: `export _51DEGREES_RESOURCE_KEY=MY-RESOURCE-KEY` diff --git a/device-detection/fiftyone_devicedetection_examples/tests/conftest.py b/device-detection/fiftyone_devicedetection_examples/tests/conftest.py index fc53a6d0..db192e0e 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/conftest.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/conftest.py @@ -82,6 +82,16 @@ def _describe(names): "statement, so their native data is released straight away.") +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + def pytest_runtest_logreport(report): if report.failed: _failed_tests.add(report.nodeid) diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py index 97b50e11..1e5afbb6 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py @@ -24,6 +24,7 @@ import json5 import os from pathlib import Path +import pytest import subprocess import sys import unittest @@ -37,6 +38,8 @@ from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the device product, and the hardware product for the lookups", "device detection properties, and for the hardware profiles the TAC " diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py index 1ac2b995..4852ed0c 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py @@ -34,10 +34,14 @@ from contextlib import redirect_stdout from unittest import mock +import pytest + from fiftyone_devicedetection_examples.example_utils import ExampleUtils from .live_cloud import answered_without, find_resource_key, no_resource_key +pytestmark = pytest.mark.live + MODULE = "fiftyone_devicedetection_examples.cloud.failuretomatch" NO_KEY = no_resource_key("the device product", "the IsMobile property") diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py index 440f3a81..04bfb6cf 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py @@ -20,26 +20,51 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -import flask_unittest import re import unittest + +import pytest + from fiftyone_pipeline_core.logger import Logger -from fiftyone_devicedetection_examples.example_utils import ExampleUtils from fiftyone_devicedetection_examples.cloud.gettingstarted_web.app import GettingStartedWeb -class CloudGettingStartedWebTests(flask_unittest.ClientTestCase): +from .live_cloud import find_resource_key, no_resource_key + +pytestmark = pytest.mark.live + +NO_KEY = no_resource_key( + "the device product", "device detection properties such as HardwareName") + +class CloudGettingStartedWebTests(unittest.TestCase): # The test client sends no User-Agent of its own, so supply one that detection can # resolve to a hardware profile. CHROME_USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") - # Assign the `Flask` app object - resource_key = ExampleUtils.get_resource_key() - logger = Logger() - app = GettingStartedWeb().build(resource_key, logger).app + # The pipeline is built when the tests run rather than when they are + # collected, because building it with no resource key raises, and the + # tests are collected whether or not they are run. + resource_key = find_resource_key() + + @classmethod + def setUpClass(cls): + super().setUpClass() + if cls.resource_key: + GettingStartedWeb().build(cls.resource_key, Logger()) + + @classmethod + def tearDownClass(cls): + if cls.resource_key: + GettingStartedWeb.pipeline.close() + super().tearDownClass() + + def setUp(self): + if not self.resource_key: + self.fail(NO_KEY) + self.client = GettingStartedWeb.app.test_client() - def test_cloud_getting_started_web(self, client): - response = client.get('/') + def test_cloud_getting_started_web(self): + response = self.client.get('/') self.assertEqual(200, response.status_code) self.assertIn(b"Find out more", response.data) @@ -47,8 +72,8 @@ def test_cloud_getting_started_web(self, client): # inlined client-side script. Now that the script is served separately, check that the # page renders a real one of its own, and that detection actually found a profile # rather than returning the all-zero id. - def test_cloud_getting_started_web_device_id(self, client): - response = client.get('/', headers={"User-Agent": self.CHROME_USER_AGENT}) + def test_cloud_getting_started_web_device_id(self): + response = self.client.get('/', headers={"User-Agent": self.CHROME_USER_AGENT}) self.assertEqual(200, response.status_code) device_ids = re.findall(r"\d+-\d+-\d+-\d+", response.get_data(as_text=True)) self.assertTrue(device_ids, "No device id was rendered on the page") @@ -57,8 +82,8 @@ def test_cloud_getting_started_web_device_id(self, client): # The page references the client-side script by the '/51Degrees.core.js' name used # by the web integrations in the other Pipeline APIs, so check that the route # returns the bundle rather than, for example, falling through to the page. - def test_cloud_getting_started_web_core_js(self, client): - response = client.get('/51Degrees.core.js') + def test_cloud_getting_started_web_core_js(self): + response = self.client.get('/51Degrees.core.js') self.assertEqual(200, response.status_code) self.assertEqual("application/x-javascript", response.headers["Content-Type"]) self.assertIn(b"fiftyoneDegreesManager", response.data) diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py index afb5e23c..e00f9d97 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py @@ -29,6 +29,8 @@ import re import unittest +import pytest + from fiftyone_pipeline_core.logger import Logger from fiftyone_devicedetection_examples.cloud.useragentclienthints_web.app \ @@ -36,6 +38,8 @@ from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the device product", "device detection properties such as BrowserName") diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 4b9fc088..e759e8e9 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -45,6 +45,9 @@ pass_env = # running the examples. GITHUB_ACTIONS GITHUB_REPOSITORY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] deps = diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py b/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py index fc53a6d0..db192e0e 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py @@ -82,6 +82,16 @@ def _describe(names): "statement, so their native data is released straight away.") +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + def pytest_runtest_logreport(report): if report.failed: _failed_tests.add(report.nodeid) diff --git a/ip-intelligence/ci/run-integration-tests.ps1 b/ip-intelligence/ci/run-integration-tests.ps1 index a64f611e..c55559a4 100644 --- a/ip-intelligence/ci/run-integration-tests.ps1 +++ b/ip-intelligence/ci/run-integration-tests.ps1 @@ -13,10 +13,13 @@ if ($IsWindows) { $packages = , "fiftyone_ipintelligence_examples" +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = '-m "not live"' } # The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci @@ -28,7 +31,9 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey # only, never the key. Found through this script's own folder, because # common-ci runs it from the common-ci checkout, where # './ci/report-key-products.ps1' does not exist. -& (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey +if ($Keys.TestResourceKey) { + & (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey +} # nightly-publish-main workflow doesn't create the examples package, so # install-package.ps1 won't install it and its test dependencies won't get @@ -38,6 +43,10 @@ if ($env:GITHUB_JOB -eq "Test") { pip install $RepoName/fiftyone_ipintelligence_examples || $(throw "pip install failed") } -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py index fc53a6d0..db192e0e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py @@ -82,6 +82,16 @@ def _describe(names): "statement, so their native data is released straight away.") +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + def pytest_runtest_logreport(report): if report.failed: _failed_tests.add(report.nodeid) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py index 40a52678..d8a8f4e4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py @@ -27,11 +27,15 @@ import unittest +import pytest + from fiftyone_ipintelligence_examples.cloud.gettingstarted_web \ import app as example from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the ip product", "IP Intelligence properties such as RegisteredName") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index 79a09daa..85ff1661 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -23,6 +23,8 @@ import os import unittest +import pytest + from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_examples.example_utils import ExampleUtils @@ -174,6 +176,7 @@ def test_onpremise_compare_console(self): # header + up to 20 records self.assertGreater(len(output.getvalue().splitlines()), 1) + @pytest.mark.live def test_cloud_gettingstarted_console(self): if not resource_key: self.fail(NO_IP_KEY) @@ -181,6 +184,7 @@ def test_cloud_gettingstarted_console(self): example = GettingStartedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) + @pytest.mark.live def test_cloud_metadata_console(self): if not resource_key: self.fail(NO_IP_KEY) @@ -188,6 +192,7 @@ def test_cloud_metadata_console(self): example = MetadataConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) + @pytest.mark.live def test_cloud_getallproperties_console(self): if not resource_key: self.fail(NO_IP_KEY) @@ -203,6 +208,7 @@ def test_onpremise_mixed_console(self): example = MixedConsole() example.run(hash_file, data_file, Logger(min_level="error"), lambda x: None) + @pytest.mark.live @unittest.skipUnless( device_detection_available, "device detection package not available") def test_cloud_mixed_console(self): @@ -212,6 +218,7 @@ def test_cloud_mixed_console(self): example = MixedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) + @pytest.mark.live def test_cloud_countries_console(self): if not resource_key: self.fail(NO_COUNTRIES_KEY) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index dbce9c11..0fa22930 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -60,6 +60,9 @@ pass_env = # running the examples. GITHUB_ACTIONS GITHUB_REPOSITORY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] # The examples package itself must be installed, because its tests import diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py index fc53a6d0..db192e0e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py @@ -82,6 +82,16 @@ def _describe(names): "statement, so their native data is released straight away.") +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + def pytest_runtest_logreport(report): if report.failed: _failed_tests.add(report.nodeid) diff --git a/location/ci/run-integration-tests.ps1 b/location/ci/run-integration-tests.ps1 index 01ae9b5c..31cac9b0 100644 --- a/location/ci/run-integration-tests.ps1 +++ b/location/ci/run-integration-tests.ps1 @@ -5,16 +5,23 @@ param ( [Hashtable]$Keys ) +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = '-m "not live"' } # The examples and tests read the resource key from _51DEGREES_RESOURCE_KEY, # and common-ci sets only the older resource_key, so the key is set here. $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/location/tests/conftest.py b/location/tests/conftest.py new file mode 100644 index 00000000..48020cdf --- /dev/null +++ b/location/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. The tests with no marker need no key, so they run either way. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/location/tests/test_examples.py b/location/tests/test_examples.py index d01a5bee..4a6920e5 100644 --- a/location/tests/test_examples.py +++ b/location/tests/test_examples.py @@ -25,13 +25,21 @@ import unittest from pathlib import Path -from .live_cloud import read_environment +import pytest + +from .live_cloud import no_resource_key, read_environment + +pytestmark = pytest.mark.live + +NO_KEY = no_resource_key( + "the location product", "the country at a latitude and longitude") class LocationExampleTests(unittest.TestCase): def test_cloud_getting_started(self): - if read_environment("_51DEGREES_RESOURCE_KEY", "resource_key"): - # this is required to import the example code, - # since pytest sets sys.path to the test directory - sys.path.append(str(Path(__file__).parent.parent.absolute() / "examples")) - import cloud.gettingstarted + if not read_environment("_51DEGREES_RESOURCE_KEY", "resource_key"): + self.fail(NO_KEY) + # this is required to import the example code, + # since pytest sets sys.path to the test directory + sys.path.append(str(Path(__file__).parent.parent.absolute() / "examples")) + import cloud.gettingstarted diff --git a/location/tests/test_properties51D.py b/location/tests/test_properties51D.py index a31468c2..a33a0dc2 100644 --- a/location/tests/test_properties51D.py +++ b/location/tests/test_properties51D.py @@ -23,36 +23,55 @@ import os import unittest +import pytest + from fiftyone_location.location_pipelinebuilder import LocationPipelineBuilder from .test_helper import * -from .live_cloud import read_environment +from .live_cloud import no_resource_key, read_environment + +pytestmark = pytest.mark.live location_LAT = '51.4578261'; location_LON = '-0.975922996290084'; -resource_key = read_environment("_51DEGREES_RESOURCE_KEY", "resource_key") -if not resource_key: - raise Exception("To run the cloud tests, please set a valid 51Degrees " - "cloud resource key as the _51DEGREES_RESOURCE_KEY environment variable.") - -# Create a simple pipeline to access the engine with and process it with flow data -pipeline = LocationPipelineBuilder(resource_key = resource_key, location_provider = "fiftyonedegrees").build() - -location_engine = pipeline.get_element("location") -flowData = pipeline.create_flowdata(); -flowData.evidence.add('query.51D_Pos_latitude', location_LAT); -flowData.evidence.add('query.51D_Pos_longitude', location_LON); -flowData.process() +NO_KEY = no_resource_key( + "the location product", "the location properties at a latitude and " + "longitude", several=True) class PropertyTests(unittest.TestCase): + # The pipeline is built and the position processed when the tests run + # rather than when they are collected, because the tests are collected + # whether or not they are run. + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.resource_key = read_environment( + "_51DEGREES_RESOURCE_KEY", "resource_key") + if not cls.resource_key: + return + + # Create a simple pipeline to access the engine with and process it with flow data + pipeline = LocationPipelineBuilder(resource_key = cls.resource_key, location_provider = "fiftyonedegrees").build() + + cls.location_engine = pipeline.get_element("location") + cls.flowData = pipeline.create_flowdata(); + cls.flowData.evidence.add('query.51D_Pos_latitude', location_LAT); + cls.flowData.evidence.add('query.51D_Pos_longitude', location_LON); + cls.flowData.process() + + def setUp(self): + if not self.resource_key: + self.fail(NO_KEY) + def test_available_properties_fiftyonedegrees(self): """! Tests value types of the properties present present in the engine """ - location = flowData.get(location_engine.datakey) + location_engine = self.location_engine + location = self.flowData.get(location_engine.datakey) # Get list of all the properties in the engine properties_list = location_engine.get_properties() @@ -72,7 +91,8 @@ def test_value_types_fiftyonedegrees(self): Tests whether the all the properties present in the engine when initialised with a resource key are accessible. """ - location = flowData.get(location_engine.datakey) + location_engine = self.location_engine + location = self.flowData.get(location_engine.datakey) # Get list of all the properties in the engine properties_list = location_engine.get_properties() diff --git a/location/tests/test_web_example.py b/location/tests/test_web_example.py index 1fafe50b..de01960f 100644 --- a/location/tests/test_web_example.py +++ b/location/tests/test_web_example.py @@ -37,6 +37,8 @@ from pathlib import Path from unittest import mock +import pytest + from .live_cloud import answered_without, find_resource_key, no_resource_key EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "cloud" / "web.py" @@ -108,6 +110,7 @@ def load_example(resource_key): return module +@pytest.mark.live class LocationWebExampleTests(unittest.TestCase): resource_key = find_resource_key() diff --git a/location/tox.ini b/location/tox.ini index f8d1cda1..685c974b 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -28,6 +28,9 @@ pass_env = resource_key _51DEGREES_LICENSE_KEY license_key + # The marker selection of ci/run-integration-tests.ps1, which leaves the + # live tests out when no resource key is passed. + PYTEST_ADDOPTS # Tests the packages just built, so it takes the test tools from [common] # and every package from the requirements file, never the editable copies diff --git a/pipeline/ci/run-integration-tests.ps1 b/pipeline/ci/run-integration-tests.ps1 index a4845edd..372b8ef1 100644 --- a/pipeline/ci/run-integration-tests.ps1 +++ b/pipeline/ci/run-integration-tests.ps1 @@ -5,12 +5,6 @@ param ( [Hashtable]$Keys ) -if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." -} - # The examples call the cloud service, so they are integration tests rather # than unit tests. Listing them here runs them on every Python version and # operating system in options.json, and in the nightly, which a standalone @@ -26,23 +20,47 @@ if (!$Keys.TestResourceKey) { # The 51Did package runs only its tests marked live here, as its others ran # in the unit step. Its tier tests need a key for another tier, which only # the Cloud 51Did workflow has. Only a package whose tox.ini passes -# PYTEST_ADDOPTS sees the selection. Its build needs the OWID source, which -# build-project.ps1 copies, and the publish workflow's test job does not run -# that script, so it is copied here too. -& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName -$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples", "fiftyone_pipeline_did" +# PYTEST_ADDOPTS sees the selection, so the 51Did package is run on its own, +# after the others. Its build needs the OWID source, which build-project.ps1 +# copies, and the publish workflow's test job does not run that script, so +# it is copied here too. +# +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. That leaves nothing of the 51Did +# package, so it is not run. A live test still fails when it finds no key, so +# a key that is passed but does not reach it is not missed. +$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples" $pipelineKeys = $Keys.Clone() $pipelineKeys.Remove('DeviceDetection') # The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci # sets only the older resource_key, so the key is set here. $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey +$failed = $false $addopts = $env:PYTEST_ADDOPTS -$env:PYTEST_ADDOPTS = '-m "live and not tier"' try { + if (!$Keys.TestResourceKey) { + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. fiftyone_pipeline_did runs only live tests here, so it is skipped. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = '-m "not live"' + } ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys + if ($LASTEXITCODE -ne 0) { $failed = $true } + + if ($Keys.TestResourceKey) { + # common-ci's script combines the coverage of the packages it ran + # into .coverage, and a second run replaces that file, so the first + # run's is kept under a name the second run's combine reads. + $coverage = Join-Path $RepoName '.coverage' + if (Test-Path $coverage) { + Move-Item -Force $coverage (Join-Path $RepoName '.coverage.cloudrequestengine') + } + & "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName + $env:PYTEST_ADDOPTS = '-m "live and not tier"' + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages "fiftyone_pipeline_did" -Keys $pipelineKeys + if ($LASTEXITCODE -ne 0) { $failed = $true } + } } finally { $env:PYTEST_ADDOPTS = $addopts } -exit $LASTEXITCODE +exit ($failed ? 1 : 0) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 5dc64d97..d7f2d61f 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -93,14 +93,16 @@ then open . Set `PORT` to listen on another port. The tests that call the cloud service with a resource key fail when none is set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY` -and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. The +and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. They are marked +`live`, so `python -m pytest -m "not live"` runs every other test. The tests that use a licence key skip themselves when none is set. In CI the examples are run by [ci/run-integration-tests.ps1](../../ci/run-integration-tests.ps1) through common-ci, on every Python version and operating system in [ci/options.json](../../ci/options.json), with the resource key the pipeline -holds. No licence key is passed to the pipeline, so the licence-key tests -skip themselves there. That uses the [tox.ini](tox.ini) in this folder, +holds. When no resource key is passed, the script leaves out the tests +marked `live`, says so, and runs the rest. No licence key is passed to the +pipeline, so the licence-key tests skip themselves there. That uses the [tox.ini](tox.ini) in this folder, which can also be run by hand with `python -m tox -e py` from here. ## Cloud engine diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py new file mode 100644 index 00000000..48020cdf --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. The tests with no marker need no key, so they run either way. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py index 60ec92d8..90549679 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py @@ -38,12 +38,15 @@ # The ways a test can reach the cloud service, as the keyword arguments the # examples' create_pipeline takes, each passed through credentials(). A test -# with a resource key fails when none is set. A test with a licence key is -# skipped when none is set, because CI has no cloud licence key to give it. +# with a resource key is marked live, which ci/run-integration-tests.ps1 +# leaves out when no resource key is passed, and fails when none is set. A +# test with a licence key is skipped when none is set, because CI has no +# cloud licence key to give it. KEYS = [ pytest.param( {"resource_key": find_resource_key()}, - id="resource key"), + id="resource key", + marks=pytest.mark.live), pytest.param( {"license_key": ExampleUtils.get_license_key()}, id="licence key", diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index cd1bd54a..578dd0e9 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -46,6 +46,9 @@ pass_env = cloud_endpoint _51DEGREES_CLOUD_REQUEST_ORIGIN cloud_request_origin + # The marker selection of ../../ci/run-integration-tests.ps1, which + # leaves the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] deps = diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py new file mode 100644 index 00000000..48020cdf --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. The tests with no marker need no key, so they run either way. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py index 831734c7..069794f3 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py @@ -25,6 +25,8 @@ import unittest from urllib.parse import urlencode +import pytest + from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import CloudRequestEngine from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import CloudRequestException from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine @@ -33,6 +35,7 @@ from .live_cloud import read_environment +@pytest.mark.live class CloudEngineTests(unittest.TestCase): def setUp(self): self.resource_key = read_environment( @@ -136,6 +139,11 @@ def test_cloud_get_request_with_sequence_evidence(self): json_response = json.loads(json_response) self.assertNotIn("errors", json_response) + +class CloudErrorTests(unittest.TestCase): + """Calls the live cloud service with a resource key that does not + exist, so these need no key of their own.""" + def test_HttpDataSetInException(self): """! Check that errors from the cloud service will cause the diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini index 6c759a83..f7312330 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini @@ -23,6 +23,9 @@ pass_env = resource_key _51DEGREES_LICENSE_KEY license_key + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] package = skip