From c6d02d4d5e333ad182d1ad538a301c9804e3de2f Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 21:55:07 +0000 Subject: [PATCH 1/2] Test: with no resource key the integration step runs the tests that need none ci/run-integration-tests.ps1 stops before running anything when no TestResourceKey is passed, for every product, because each product's own script throws "No TestResourceKey was passed". Two macOS runs stopped there, so none of the integration tests that need no key ran on them. ci/run-integration-tests.Tests.ps1 runs the step as common-ci does, through ci/dispatch.ps1 and common-ci's steps/run-script.ps1, for Device Detection, IP Intelligence, location and pipeline, with a stand-in for common-ci's python/run-integration-tests.ps1 that records what each product asked it to test. With no TestResourceKey it expects: - every product to run its tests, the step not failing; - each run to leave out only the tests marked live, which need a resource key, through PYTEST_ADDOPTS; - each product to say that the tests needing a resource key are skipped; - the Device Detection cloud and examples packages and the IP Intelligence examples to be run; - pipeline to run the cloud request engine and its examples, and to skip the 51Did package, whose integration run has only live tests, saying so; - the Selenium contract against the cloud example to be skipped, saying so, with nothing cloned for it. With a TestResourceKey, Device Detection and location run every test, the live ones included. CI-1. --- ci/run-integration-tests.Tests.ps1 | 163 +++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 ci/run-integration-tests.Tests.ps1 diff --git a/ci/run-integration-tests.Tests.ps1 b/ci/run-integration-tests.Tests.ps1 new file mode 100644 index 00000000..d4677bf4 --- /dev/null +++ b/ci/run-integration-tests.Tests.ps1 @@ -0,0 +1,163 @@ +# Tests for the integration test step when no resource key is passed. +# +# Each product's ci/run-integration-tests.ps1 is run as common-ci runs it, +# through ci/dispatch.ps1 and common-ci's steps/run-script.ps1, from a folder +# laid out like a common-ci checkout. Its python/run-integration-tests.ps1 is +# a stand-in that runs no tests and records, for each call, the packages it +# was given and the PYTEST_ADDOPTS the tests would see. + +BeforeAll { + . (Join-Path $PSScriptRoot 'dispatch.ps1') + + # common-ci checked out beside this repository, or wherever COMMON_CI says. + $commonCi = $env:COMMON_CI + if ([string]::IsNullOrEmpty($commonCi) -eq $true) { + $commonCi = Join-Path $PSScriptRoot '..' '..' 'common-ci' + } + $script:Steps = Join-Path $commonCi 'steps' + if ((Test-Path (Join-Path $script:Steps 'run-script.ps1')) -eq $false) { + throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it" + } + + $script:Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path + + # The products whose integration tests call the cloud service. + $script:Products = 'device-detection', 'ip-intelligence', 'location', + 'pipeline' + + # The environment variables the scripts set, kept to be put back. + $script:Saved = '_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY', + 'PYTEST_ADDOPTS', 'TEMP' + + # The stand-in for common-ci's python/run-integration-tests.ps1. It + # writes one line of JSON for each call to calls.jsonl in the folder it + # runs from. + $script:StandIn = @' +param ( + [Parameter(Mandatory=$true)][string]$RepoName, + [Parameter(Mandatory=$true)][string[]]$Packages, + [Parameter(Mandatory=$true)][hashtable]$Keys +) +@{ + Product = Split-Path $RepoName -Leaf + Packages = @($Packages) + AddOpts = "$env:PYTEST_ADDOPTS" +} | ConvertTo-Json -Compress | Add-Content (Join-Path $PWD 'calls.jsonl') +$global:LASTEXITCODE = 0 +'@ + + # A folder laid out like a common-ci checkout, which the step runs in. + function New-CommonCi { + $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) + $null = New-Item -ItemType Directory -Force (Join-Path $path 'python') + Copy-Item -Recurse $script:Steps (Join-Path $path 'steps') + Set-Content (Join-Path $path 'python' 'run-integration-tests.ps1') ` + $script:StandIn + return $path + } + + # The calls the stand-in recorded for a product. + function Get-Calls([string]$Product) { + $file = Join-Path $script:CommonCi 'calls.jsonl' + if ((Test-Path $file) -eq $false) { + return @() + } + return @(Get-Content $file | ForEach-Object { $_ | ConvertFrom-Json } | + Where-Object Product -eq $Product) + } +} + +Describe 'run-integration-tests.ps1' { + +BeforeEach { + $script:Environment = @{} + foreach ($name in $script:Saved) { + $script:Environment[$name] = [Environment]::GetEnvironmentVariable($name) + } + $script:CommonCi = New-CommonCi + Push-Location $script:CommonCi +} + +AfterEach { + Pop-Location + Remove-Item -Recurse -Force $script:CommonCi + foreach ($name in $script:Saved) { + [Environment]::SetEnvironmentVariable($name, $script:Environment[$name]) + } +} + +Context 'with no TestResourceKey' { + BeforeEach { + $script:Output = Invoke-ProductStep 'run-integration-tests' $script:Repo ` + @{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 | + Out-String + } + + It 'runs the tests of every product' { + foreach ($product in $script:Products) { + (Get-Calls $product).Count | Should -BeGreaterThan 0 -Because $product + } + } + + It 'leaves out only the tests marked live, which need a resource key' { + foreach ($product in $script:Products) { + foreach ($call in (Get-Calls $product)) { + $call.AddOpts | Should -Be '-m "not live"' -Because $product + } + } + } + + It 'says for every product that the tests needing a key are skipped' { + foreach ($product in $script:Products) { + # What the step printed for this product, between its group + # markers. + $group = ($script:Output -split '::group::') | + Where-Object { $_ -like "run-integration-tests for $product*" } + $group | Should -Match ("No TestResourceKey was passed, so the " + + "tests that need a resource key are skipped") -Because $product + } + } + + It 'runs the tests of the device detection and IP intelligence examples' { + (Get-Calls 'device-detection').Packages | + Should -Contain 'fiftyone_devicedetection_examples' + (Get-Calls 'device-detection').Packages | + Should -Contain 'fiftyone_devicedetection_cloud' + (Get-Calls 'ip-intelligence').Packages | + Should -Contain 'fiftyone_ipintelligence_examples' + } + + It 'runs the cloud request engine and its examples, and skips the 51Did live tests' { + $packages = (Get-Calls 'pipeline').Packages + $packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine' + $packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine/examples' + $packages | Should -Not -Contain 'fiftyone_pipeline_did' + $script:Output | Should -Match 'fiftyone_pipeline_did.*skipped' + } + + It 'skips the Selenium contract against the cloud example and saying so' { + $script:Output | Should -Match 'Selenium contract tests will not run against the cloud example' + Test-Path (Join-Path $script:CommonCi 'selenium-api-tests') | + Should -BeFalse + } +} + +Context 'with a TestResourceKey' { + It 'runs every test, the ones marked live included' { + Invoke-ProductStep 'run-integration-tests' $script:Repo @{ + RepoName = $script:Repo + Keys = @{ TestResourceKey = 'key' } + SeleniumContract = $false + } 'device-detection', 'location' *>&1 | Out-Null + + foreach ($product in 'device-detection', 'location') { + $calls = Get-Calls $product + $calls.Count | Should -BeGreaterThan 0 -Because $product + foreach ($call in $calls) { + $call.AddOpts | Should -Not -Match 'live' -Because $product + } + } + } +} + +} From 892c4338643c2a825141b7d1046c1c27edd43cc5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 16 Nov 2026 22:00:25 +0000 Subject: [PATCH 2/2] Fix: with no resource key the integration step runs the tests that need none Each product's ci/run-integration-tests.ps1 now runs when no TestResourceKey is passed. It prints a warning that the tests needing a resource key are skipped, sets PYTEST_ADDOPTS to -m "not live" so they are left out, and runs every other integration test. With a key nothing is left out, and a live test that finds no key still fails, as before. - Device Detection, IP Intelligence and location set the selection around their one run. IP Intelligence reports what the key carries only when there is a key. - Pipeline runs the cloud request engine and its examples first, then the 51Did package on its own with its "live and not tier" selection, because both runs now see PYTEST_ADDOPTS. The first run's coverage is kept under a name the second run's combine reads. With no key the 51Did package has no test to run, so it is skipped, saying so, and the OWID source is not copied. - Device Detection skips the Selenium contract against the cloud example when there is no key, saying so, and clones and sets up nothing when neither example can be run. - The tox.ini of the Device Detection and IP Intelligence examples, location, the cloud request engine and its examples pass PYTEST_ADDOPTS. - The tests that need a resource key are marked live: the Device Detection cloud example tests, the IP Intelligence cloud console and web tests, location's example, property and web example tests, the cloud request engine's CloudEngineTests and the pipeline examples' resource key cases. Each folder registers the marker in its conftest.py, and the four copies of the on-premise conftest.py stay the same. - Two test modules built a cloud pipeline when they were collected, which raises with no key, so collecting them failed even when they were left out. The Device Detection cloud getting started web test now builds it in setUpClass and fails with the no key message, as the other web tests do, and location's property tests do the same. - location's cloud example test passed without a key. It now fails, naming the variables, like the other live tests. - The cloud request engine's test of the error a wrong key gets needs no key, so it moves out of CloudEngineTests into a class of its own. - The licence key tests already skip themselves without a licence key, and are unchanged. The readmes of the Device Detection and pipeline examples say how to run the tests that need no key. CI-1. --- device-detection/ci/run-integration-tests.ps1 | 51 +++++++++++++----- .../readme.md | 4 +- .../tests/conftest.py | 10 ++++ .../tests/test_cloudexamples.py | 3 ++ .../tests/test_cloudfailuretomatch.py | 4 ++ .../tests/test_cloudgettingstartedweb.py | 51 +++++++++++++----- .../test_clouduseragentclienthintsweb.py | 4 ++ .../fiftyone_devicedetection_examples/tox.ini | 3 ++ .../tests/conftest.py | 10 ++++ ip-intelligence/ci/run-integration-tests.ps1 | 19 +++++-- .../tests/conftest.py | 10 ++++ .../tests/test_cloudgettingstartedweb.py | 4 ++ .../tests/test_examples.py | 7 +++ .../fiftyone_ipintelligence_examples/tox.ini | 3 ++ .../tests/conftest.py | 10 ++++ location/ci/run-integration-tests.ps1 | 15 ++++-- location/tests/conftest.py | 32 ++++++++++++ location/tests/test_examples.py | 20 ++++--- location/tests/test_properties51D.py | 52 +++++++++++++------ location/tests/test_web_example.py | 3 ++ location/tox.ini | 3 ++ pipeline/ci/run-integration-tests.ps1 | 44 +++++++++++----- .../examples/readme.md | 8 +-- .../examples/tests/conftest.py | 32 ++++++++++++ .../examples/tests/helpers.py | 9 ++-- .../examples/tox.ini | 3 ++ .../tests/conftest.py | 32 ++++++++++++ .../tests/test_cloudengine.py | 8 +++ .../tox.ini | 3 ++ 29 files changed, 380 insertions(+), 77 deletions(-) create mode 100644 location/tests/conftest.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py create mode 100644 pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index ed7a17d2..c465b293 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -16,10 +16,13 @@ if ($IsWindows) { $packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples" +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = '-m "not live"' } # The cloud package runs without the CSV file too, because only its property # coverage tests read the file, and they say so when it is missing. @@ -44,7 +47,11 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey # files from _51DEGREES_LICENSE_KEY, which common-ci sets as license_key. $env:_51DEGREES_LICENSE_KEY = $Keys.DeviceDetection -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} $failures = @() if ($LASTEXITCODE -ne 0) { $failures += "the Python integration tests exited with code $LASTEXITCODE" @@ -58,6 +65,25 @@ if (-not $SeleniumContract) { return } +# The contract tests run against the cloud example when there is a resource +# key, and against the on-premise example when there is the TAC data file, +# which is only fetched when a licence is given. +$tac = "$PWD/assets/TAC-HashV41.hash" +$runCloud = [bool]$Keys.TestResourceKey +$runOnPremise = Test-Path $tac +if (-not $runCloud) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No resource key::No TestResourceKey was passed, so the Selenium contract tests will not run against the cloud example." +} +if (-not $runOnPremise) { + Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example." +} +if (-not ($runCloud -or $runOnPremise)) { + if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" + } + return +} + # The shared Selenium contract tests are run against the cloud example and the # on-premise example in turn. Each result is also written as a trx file next # to the other integration results, so it is published with them. @@ -141,22 +167,21 @@ function Test-Example([string]$Name, [string]$Module, [int]$Port) { } # The cloud example, pointed at the live cloud. -$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey -$env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/" -$failures += Test-Example -Name 'cloud' ` - -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097 +if ($runCloud) { + $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey + $env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/" + $failures += Test-Example -Name 'cloud' ` + -Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097 +} # The on-premise example. The contract tests need the device type and the # JavaScript properties, which the Lite data file does not have, so it runs -# against the TAC data file, which is only fetched when a licence is given. -$tac = "$PWD/assets/TAC-HashV41.hash" -if (Test-Path $tac) { +# against the TAC data file. +if ($runOnPremise) { $env:_51DEGREES_DD_PATH = (Resolve-Path $tac).Path $failures += Test-Example -Name 'onpremise' ` -Module 'fiftyone_devicedetection_examples.onpremise.gettingstarted_web' -Port 8098 Remove-Item Env:_51DEGREES_DD_PATH -} else { - Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example." } # A failure anywhere above must fail the job. This script is dot-sourced by diff --git a/device-detection/fiftyone_devicedetection_examples/readme.md b/device-detection/fiftyone_devicedetection_examples/readme.md index 99dfb369..be1117ab 100644 --- a/device-detection/fiftyone_devicedetection_examples/readme.md +++ b/device-detection/fiftyone_devicedetection_examples/readme.md @@ -43,7 +43,9 @@ the Enterprise data file. The cloud tests need a valid 51Degrees resource key (see above), and fail without one. They read it from the environment variable -`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. +`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. They are +marked `live`, so `python -m pytest -m "not live"` runs every other test, +which is what CI does when no resource key is passed. For example, use following command to set the resource key on Linux: `export _51DEGREES_RESOURCE_KEY=MY-RESOURCE-KEY` diff --git a/device-detection/fiftyone_devicedetection_examples/tests/conftest.py b/device-detection/fiftyone_devicedetection_examples/tests/conftest.py index fc53a6d0..db192e0e 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/conftest.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/conftest.py @@ -82,6 +82,16 @@ def _describe(names): "statement, so their native data is released straight away.") +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + def pytest_runtest_logreport(report): if report.failed: _failed_tests.add(report.nodeid) diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py index 97b50e11..1e5afbb6 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudexamples.py @@ -24,6 +24,7 @@ import json5 import os from pathlib import Path +import pytest import subprocess import sys import unittest @@ -37,6 +38,8 @@ from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the device product, and the hardware product for the lookups", "device detection properties, and for the hardware profiles the TAC " diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py index 1ac2b995..4852ed0c 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudfailuretomatch.py @@ -34,10 +34,14 @@ from contextlib import redirect_stdout from unittest import mock +import pytest + from fiftyone_devicedetection_examples.example_utils import ExampleUtils from .live_cloud import answered_without, find_resource_key, no_resource_key +pytestmark = pytest.mark.live + MODULE = "fiftyone_devicedetection_examples.cloud.failuretomatch" NO_KEY = no_resource_key("the device product", "the IsMobile property") diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py index 440f3a81..04bfb6cf 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_cloudgettingstartedweb.py @@ -20,26 +20,51 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* -import flask_unittest import re import unittest + +import pytest + from fiftyone_pipeline_core.logger import Logger -from fiftyone_devicedetection_examples.example_utils import ExampleUtils from fiftyone_devicedetection_examples.cloud.gettingstarted_web.app import GettingStartedWeb -class CloudGettingStartedWebTests(flask_unittest.ClientTestCase): +from .live_cloud import find_resource_key, no_resource_key + +pytestmark = pytest.mark.live + +NO_KEY = no_resource_key( + "the device product", "device detection properties such as HardwareName") + +class CloudGettingStartedWebTests(unittest.TestCase): # The test client sends no User-Agent of its own, so supply one that detection can # resolve to a hardware profile. CHROME_USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") - # Assign the `Flask` app object - resource_key = ExampleUtils.get_resource_key() - logger = Logger() - app = GettingStartedWeb().build(resource_key, logger).app + # The pipeline is built when the tests run rather than when they are + # collected, because building it with no resource key raises, and the + # tests are collected whether or not they are run. + resource_key = find_resource_key() + + @classmethod + def setUpClass(cls): + super().setUpClass() + if cls.resource_key: + GettingStartedWeb().build(cls.resource_key, Logger()) + + @classmethod + def tearDownClass(cls): + if cls.resource_key: + GettingStartedWeb.pipeline.close() + super().tearDownClass() + + def setUp(self): + if not self.resource_key: + self.fail(NO_KEY) + self.client = GettingStartedWeb.app.test_client() - def test_cloud_getting_started_web(self, client): - response = client.get('/') + def test_cloud_getting_started_web(self): + response = self.client.get('/') self.assertEqual(200, response.status_code) self.assertIn(b"Find out more", response.data) @@ -47,8 +72,8 @@ def test_cloud_getting_started_web(self, client): # inlined client-side script. Now that the script is served separately, check that the # page renders a real one of its own, and that detection actually found a profile # rather than returning the all-zero id. - def test_cloud_getting_started_web_device_id(self, client): - response = client.get('/', headers={"User-Agent": self.CHROME_USER_AGENT}) + def test_cloud_getting_started_web_device_id(self): + response = self.client.get('/', headers={"User-Agent": self.CHROME_USER_AGENT}) self.assertEqual(200, response.status_code) device_ids = re.findall(r"\d+-\d+-\d+-\d+", response.get_data(as_text=True)) self.assertTrue(device_ids, "No device id was rendered on the page") @@ -57,8 +82,8 @@ def test_cloud_getting_started_web_device_id(self, client): # The page references the client-side script by the '/51Degrees.core.js' name used # by the web integrations in the other Pipeline APIs, so check that the route # returns the bundle rather than, for example, falling through to the page. - def test_cloud_getting_started_web_core_js(self, client): - response = client.get('/51Degrees.core.js') + def test_cloud_getting_started_web_core_js(self): + response = self.client.get('/51Degrees.core.js') self.assertEqual(200, response.status_code) self.assertEqual("application/x-javascript", response.headers["Content-Type"]) self.assertIn(b"fiftyoneDegreesManager", response.data) diff --git a/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py b/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py index afb5e23c..e00f9d97 100644 --- a/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py +++ b/device-detection/fiftyone_devicedetection_examples/tests/test_clouduseragentclienthintsweb.py @@ -29,6 +29,8 @@ import re import unittest +import pytest + from fiftyone_pipeline_core.logger import Logger from fiftyone_devicedetection_examples.cloud.useragentclienthints_web.app \ @@ -36,6 +38,8 @@ from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the device product", "device detection properties such as BrowserName") diff --git a/device-detection/fiftyone_devicedetection_examples/tox.ini b/device-detection/fiftyone_devicedetection_examples/tox.ini index 4b9fc088..e759e8e9 100644 --- a/device-detection/fiftyone_devicedetection_examples/tox.ini +++ b/device-detection/fiftyone_devicedetection_examples/tox.ini @@ -45,6 +45,9 @@ pass_env = # running the examples. GITHUB_ACTIONS GITHUB_REPOSITORY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] deps = diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py b/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py index fc53a6d0..db192e0e 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/conftest.py @@ -82,6 +82,16 @@ def _describe(names): "statement, so their native data is released straight away.") +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + def pytest_runtest_logreport(report): if report.failed: _failed_tests.add(report.nodeid) diff --git a/ip-intelligence/ci/run-integration-tests.ps1 b/ip-intelligence/ci/run-integration-tests.ps1 index a64f611e..c55559a4 100644 --- a/ip-intelligence/ci/run-integration-tests.ps1 +++ b/ip-intelligence/ci/run-integration-tests.ps1 @@ -13,10 +13,13 @@ if ($IsWindows) { $packages = , "fiftyone_ipintelligence_examples" +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = '-m "not live"' } # The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci @@ -28,7 +31,9 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey # only, never the key. Found through this script's own folder, because # common-ci runs it from the common-ci checkout, where # './ci/report-key-products.ps1' does not exist. -& (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey +if ($Keys.TestResourceKey) { + & (Join-Path $PSScriptRoot 'report-key-products.ps1') -ResourceKey $Keys.TestResourceKey +} # nightly-publish-main workflow doesn't create the examples package, so # install-package.ps1 won't install it and its test dependencies won't get @@ -38,6 +43,10 @@ if ($env:GITHUB_JOB -eq "Test") { pip install $RepoName/fiftyone_ipintelligence_examples || $(throw "pip install failed") } -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py index fc53a6d0..db192e0e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/conftest.py @@ -82,6 +82,16 @@ def _describe(names): "statement, so their native data is released straight away.") +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + def pytest_runtest_logreport(report): if report.failed: _failed_tests.add(report.nodeid) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py index 40a52678..d8a8f4e4 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_cloudgettingstartedweb.py @@ -27,11 +27,15 @@ import unittest +import pytest + from fiftyone_ipintelligence_examples.cloud.gettingstarted_web \ import app as example from .live_cloud import find_resource_key, no_resource_key +pytestmark = pytest.mark.live + NO_KEY = no_resource_key( "the ip product", "IP Intelligence properties such as RegisteredName") diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py index 79a09daa..85ff1661 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tests/test_examples.py @@ -23,6 +23,8 @@ import os import unittest +import pytest + from fiftyone_pipeline_core.logger import Logger from fiftyone_ipintelligence_examples.example_utils import ExampleUtils @@ -174,6 +176,7 @@ def test_onpremise_compare_console(self): # header + up to 20 records self.assertGreater(len(output.getvalue().splitlines()), 1) + @pytest.mark.live def test_cloud_gettingstarted_console(self): if not resource_key: self.fail(NO_IP_KEY) @@ -181,6 +184,7 @@ def test_cloud_gettingstarted_console(self): example = GettingStartedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) + @pytest.mark.live def test_cloud_metadata_console(self): if not resource_key: self.fail(NO_IP_KEY) @@ -188,6 +192,7 @@ def test_cloud_metadata_console(self): example = MetadataConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) + @pytest.mark.live def test_cloud_getallproperties_console(self): if not resource_key: self.fail(NO_IP_KEY) @@ -203,6 +208,7 @@ def test_onpremise_mixed_console(self): example = MixedConsole() example.run(hash_file, data_file, Logger(min_level="error"), lambda x: None) + @pytest.mark.live @unittest.skipUnless( device_detection_available, "device detection package not available") def test_cloud_mixed_console(self): @@ -212,6 +218,7 @@ def test_cloud_mixed_console(self): example = MixedConsole() example.run(resource_key, Logger(min_level="error"), lambda x: None) + @pytest.mark.live def test_cloud_countries_console(self): if not resource_key: self.fail(NO_COUNTRIES_KEY) diff --git a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini index dbce9c11..0fa22930 100644 --- a/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini +++ b/ip-intelligence/fiftyone_ipintelligence_examples/tox.ini @@ -60,6 +60,9 @@ pass_env = # running the examples. GITHUB_ACTIONS GITHUB_REPOSITORY + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] # The examples package itself must be installed, because its tests import diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py index fc53a6d0..db192e0e 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/conftest.py @@ -82,6 +82,16 @@ def _describe(names): "statement, so their native data is released straight away.") +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") + + def pytest_runtest_logreport(report): if report.failed: _failed_tests.add(report.nodeid) diff --git a/location/ci/run-integration-tests.ps1 b/location/ci/run-integration-tests.ps1 index 01ae9b5c..31cac9b0 100644 --- a/location/ci/run-integration-tests.ps1 +++ b/location/ci/run-integration-tests.ps1 @@ -5,16 +5,23 @@ param ( [Hashtable]$Keys ) +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. A live test still fails when it finds +# no key, so a key that is passed but does not reach it is not missed. +$addopts = $env:PYTEST_ADDOPTS if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = '-m "not live"' } # The examples and tests read the resource key from _51DEGREES_RESOURCE_KEY, # and common-ci sets only the older resource_key, so the key is set here. $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey -./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys +try { + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages . -Keys $Keys +} finally { + $env:PYTEST_ADDOPTS = $addopts +} exit $LASTEXITCODE diff --git a/location/tests/conftest.py b/location/tests/conftest.py new file mode 100644 index 00000000..48020cdf --- /dev/null +++ b/location/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. The tests with no marker need no key, so they run either way. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/location/tests/test_examples.py b/location/tests/test_examples.py index d01a5bee..4a6920e5 100644 --- a/location/tests/test_examples.py +++ b/location/tests/test_examples.py @@ -25,13 +25,21 @@ import unittest from pathlib import Path -from .live_cloud import read_environment +import pytest + +from .live_cloud import no_resource_key, read_environment + +pytestmark = pytest.mark.live + +NO_KEY = no_resource_key( + "the location product", "the country at a latitude and longitude") class LocationExampleTests(unittest.TestCase): def test_cloud_getting_started(self): - if read_environment("_51DEGREES_RESOURCE_KEY", "resource_key"): - # this is required to import the example code, - # since pytest sets sys.path to the test directory - sys.path.append(str(Path(__file__).parent.parent.absolute() / "examples")) - import cloud.gettingstarted + if not read_environment("_51DEGREES_RESOURCE_KEY", "resource_key"): + self.fail(NO_KEY) + # this is required to import the example code, + # since pytest sets sys.path to the test directory + sys.path.append(str(Path(__file__).parent.parent.absolute() / "examples")) + import cloud.gettingstarted diff --git a/location/tests/test_properties51D.py b/location/tests/test_properties51D.py index a31468c2..a33a0dc2 100644 --- a/location/tests/test_properties51D.py +++ b/location/tests/test_properties51D.py @@ -23,36 +23,55 @@ import os import unittest +import pytest + from fiftyone_location.location_pipelinebuilder import LocationPipelineBuilder from .test_helper import * -from .live_cloud import read_environment +from .live_cloud import no_resource_key, read_environment + +pytestmark = pytest.mark.live location_LAT = '51.4578261'; location_LON = '-0.975922996290084'; -resource_key = read_environment("_51DEGREES_RESOURCE_KEY", "resource_key") -if not resource_key: - raise Exception("To run the cloud tests, please set a valid 51Degrees " - "cloud resource key as the _51DEGREES_RESOURCE_KEY environment variable.") - -# Create a simple pipeline to access the engine with and process it with flow data -pipeline = LocationPipelineBuilder(resource_key = resource_key, location_provider = "fiftyonedegrees").build() - -location_engine = pipeline.get_element("location") -flowData = pipeline.create_flowdata(); -flowData.evidence.add('query.51D_Pos_latitude', location_LAT); -flowData.evidence.add('query.51D_Pos_longitude', location_LON); -flowData.process() +NO_KEY = no_resource_key( + "the location product", "the location properties at a latitude and " + "longitude", several=True) class PropertyTests(unittest.TestCase): + # The pipeline is built and the position processed when the tests run + # rather than when they are collected, because the tests are collected + # whether or not they are run. + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.resource_key = read_environment( + "_51DEGREES_RESOURCE_KEY", "resource_key") + if not cls.resource_key: + return + + # Create a simple pipeline to access the engine with and process it with flow data + pipeline = LocationPipelineBuilder(resource_key = cls.resource_key, location_provider = "fiftyonedegrees").build() + + cls.location_engine = pipeline.get_element("location") + cls.flowData = pipeline.create_flowdata(); + cls.flowData.evidence.add('query.51D_Pos_latitude', location_LAT); + cls.flowData.evidence.add('query.51D_Pos_longitude', location_LON); + cls.flowData.process() + + def setUp(self): + if not self.resource_key: + self.fail(NO_KEY) + def test_available_properties_fiftyonedegrees(self): """! Tests value types of the properties present present in the engine """ - location = flowData.get(location_engine.datakey) + location_engine = self.location_engine + location = self.flowData.get(location_engine.datakey) # Get list of all the properties in the engine properties_list = location_engine.get_properties() @@ -72,7 +91,8 @@ def test_value_types_fiftyonedegrees(self): Tests whether the all the properties present in the engine when initialised with a resource key are accessible. """ - location = flowData.get(location_engine.datakey) + location_engine = self.location_engine + location = self.flowData.get(location_engine.datakey) # Get list of all the properties in the engine properties_list = location_engine.get_properties() diff --git a/location/tests/test_web_example.py b/location/tests/test_web_example.py index 1fafe50b..de01960f 100644 --- a/location/tests/test_web_example.py +++ b/location/tests/test_web_example.py @@ -37,6 +37,8 @@ from pathlib import Path from unittest import mock +import pytest + from .live_cloud import answered_without, find_resource_key, no_resource_key EXAMPLE = Path(__file__).resolve().parents[1] / "examples" / "cloud" / "web.py" @@ -108,6 +110,7 @@ def load_example(resource_key): return module +@pytest.mark.live class LocationWebExampleTests(unittest.TestCase): resource_key = find_resource_key() diff --git a/location/tox.ini b/location/tox.ini index f8d1cda1..685c974b 100644 --- a/location/tox.ini +++ b/location/tox.ini @@ -28,6 +28,9 @@ pass_env = resource_key _51DEGREES_LICENSE_KEY license_key + # The marker selection of ci/run-integration-tests.ps1, which leaves the + # live tests out when no resource key is passed. + PYTEST_ADDOPTS # Tests the packages just built, so it takes the test tools from [common] # and every package from the requirements file, never the editable copies diff --git a/pipeline/ci/run-integration-tests.ps1 b/pipeline/ci/run-integration-tests.ps1 index a4845edd..372b8ef1 100644 --- a/pipeline/ci/run-integration-tests.ps1 +++ b/pipeline/ci/run-integration-tests.ps1 @@ -5,12 +5,6 @@ param ( [Hashtable]$Keys ) -if (!$Keys.TestResourceKey) { - throw "No TestResourceKey was passed, so the cloud tests cannot call " + - "the cloud service. The workflows set it from the " + - "_51DEGREES_RESOURCE_KEY_BESPOKE secret." -} - # The examples call the cloud service, so they are integration tests rather # than unit tests. Listing them here runs them on every Python version and # operating system in options.json, and in the nightly, which a standalone @@ -26,23 +20,47 @@ if (!$Keys.TestResourceKey) { # The 51Did package runs only its tests marked live here, as its others ran # in the unit step. Its tier tests need a key for another tier, which only # the Cloud 51Did workflow has. Only a package whose tox.ini passes -# PYTEST_ADDOPTS sees the selection. Its build needs the OWID source, which -# build-project.ps1 copies, and the publish workflow's test job does not run -# that script, so it is copied here too. -& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName -$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples", "fiftyone_pipeline_did" +# PYTEST_ADDOPTS sees the selection, so the 51Did package is run on its own, +# after the others. Its build needs the OWID source, which build-project.ps1 +# copies, and the publish workflow's test job does not run that script, so +# it is copied here too. +# +# Without a resource key the tests that need one, which are marked live, are +# left out and every other test runs. That leaves nothing of the 51Did +# package, so it is not run. A live test still fails when it finds no key, so +# a key that is passed but does not reach it is not missed. +$packages = "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_cloudrequestengine/examples" $pipelineKeys = $Keys.Clone() $pipelineKeys.Remove('DeviceDetection') # The tests read the resource key from _51DEGREES_RESOURCE_KEY, and common-ci # sets only the older resource_key, so the key is set here. $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey +$failed = $false $addopts = $env:PYTEST_ADDOPTS -$env:PYTEST_ADDOPTS = '-m "live and not tier"' try { + if (!$Keys.TestResourceKey) { + Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. fiftyone_pipeline_did runs only live tests here, so it is skipped. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret." + $env:PYTEST_ADDOPTS = '-m "not live"' + } ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $pipelineKeys + if ($LASTEXITCODE -ne 0) { $failed = $true } + + if ($Keys.TestResourceKey) { + # common-ci's script combines the coverage of the packages it ran + # into .coverage, and a second run replaces that file, so the first + # run's is kept under a name the second run's combine reads. + $coverage = Join-Path $RepoName '.coverage' + if (Test-Path $coverage) { + Move-Item -Force $coverage (Join-Path $RepoName '.coverage.cloudrequestengine') + } + & "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName + $env:PYTEST_ADDOPTS = '-m "live and not tier"' + ./python/run-integration-tests.ps1 -RepoName $RepoName -Packages "fiftyone_pipeline_did" -Keys $pipelineKeys + if ($LASTEXITCODE -ne 0) { $failed = $true } + } } finally { $env:PYTEST_ADDOPTS = $addopts } -exit $LASTEXITCODE +exit ($failed ? 1 : 0) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md index 5dc64d97..d7f2d61f 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/readme.md @@ -93,14 +93,16 @@ then open . Set `PORT` to listen on another port. The tests that call the cloud service with a resource key fail when none is set, naming the variables they read, which are `_51DEGREES_RESOURCE_KEY` -and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. The +and `_51DEGREES_RESOURCE_KEY_BESPOKE`, in that order. They are marked +`live`, so `python -m pytest -m "not live"` runs every other test. The tests that use a licence key skip themselves when none is set. In CI the examples are run by [ci/run-integration-tests.ps1](../../ci/run-integration-tests.ps1) through common-ci, on every Python version and operating system in [ci/options.json](../../ci/options.json), with the resource key the pipeline -holds. No licence key is passed to the pipeline, so the licence-key tests -skip themselves there. That uses the [tox.ini](tox.ini) in this folder, +holds. When no resource key is passed, the script leaves out the tests +marked `live`, says so, and runs the rest. No licence key is passed to the +pipeline, so the licence-key tests skip themselves there. That uses the [tox.ini](tox.ini) in this folder, which can also be run by hand with `python -m tox -e py` from here. ## Cloud engine diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py new file mode 100644 index 00000000..48020cdf --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. The tests with no marker need no key, so they run either way. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py index 60ec92d8..90549679 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tests/helpers.py @@ -38,12 +38,15 @@ # The ways a test can reach the cloud service, as the keyword arguments the # examples' create_pipeline takes, each passed through credentials(). A test -# with a resource key fails when none is set. A test with a licence key is -# skipped when none is set, because CI has no cloud licence key to give it. +# with a resource key is marked live, which ci/run-integration-tests.ps1 +# leaves out when no resource key is passed, and fails when none is set. A +# test with a licence key is skipped when none is set, because CI has no +# cloud licence key to give it. KEYS = [ pytest.param( {"resource_key": find_resource_key()}, - id="resource key"), + id="resource key", + marks=pytest.mark.live), pytest.param( {"license_key": ExampleUtils.get_license_key()}, id="licence key", diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini index cd1bd54a..578dd0e9 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/examples/tox.ini @@ -46,6 +46,9 @@ pass_env = cloud_endpoint _51DEGREES_CLOUD_REQUEST_ORIGIN cloud_request_origin + # The marker selection of ../../ci/run-integration-tests.ps1, which + # leaves the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] deps = diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py new file mode 100644 index 00000000..48020cdf --- /dev/null +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/conftest.py @@ -0,0 +1,32 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +# The marker of the tests that call the live cloud service, which the +# product's ci/run-integration-tests.ps1 leaves out when no resource key is +# passed. The tests with no marker need no key, so they run either way. + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live: calls the live 51Degrees cloud service and needs a resource " + "key") diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py index 831734c7..069794f3 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tests/test_cloudengine.py @@ -25,6 +25,8 @@ import unittest from urllib.parse import urlencode +import pytest + from fiftyone_pipeline_cloudrequestengine.cloudrequestengine import CloudRequestEngine from fiftyone_pipeline_cloudrequestengine.cloudrequestexception import CloudRequestException from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine @@ -33,6 +35,7 @@ from .live_cloud import read_environment +@pytest.mark.live class CloudEngineTests(unittest.TestCase): def setUp(self): self.resource_key = read_environment( @@ -136,6 +139,11 @@ def test_cloud_get_request_with_sequence_evidence(self): json_response = json.loads(json_response) self.assertNotIn("errors", json_response) + +class CloudErrorTests(unittest.TestCase): + """Calls the live cloud service with a resource key that does not + exist, so these need no key of their own.""" + def test_HttpDataSetInException(self): """! Check that errors from the cloud service will cause the diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini index 6c759a83..f7312330 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/tox.ini @@ -23,6 +23,9 @@ pass_env = resource_key _51DEGREES_LICENSE_KEY license_key + # The marker selection of ../ci/run-integration-tests.ps1, which leaves + # the live tests out when no resource key is passed. + PYTEST_ADDOPTS [testenv:pre-publish] package = skip