Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 163 additions & 0 deletions ci/run-integration-tests.Tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
# Tests for the integration test step when no resource key is passed.
#
# Each product's ci/run-integration-tests.ps1 is run as common-ci runs it,
# through ci/dispatch.ps1 and common-ci's steps/run-script.ps1, from a folder
# laid out like a common-ci checkout. Its python/run-integration-tests.ps1 is
# a stand-in that runs no tests and records, for each call, the packages it
# was given and the PYTEST_ADDOPTS the tests would see.

BeforeAll {
. (Join-Path $PSScriptRoot 'dispatch.ps1')

# common-ci checked out beside this repository, or wherever COMMON_CI says.
$commonCi = $env:COMMON_CI
if ([string]::IsNullOrEmpty($commonCi) -eq $true) {
$commonCi = Join-Path $PSScriptRoot '..' '..' 'common-ci'
}
$script:Steps = Join-Path $commonCi 'steps'
if ((Test-Path (Join-Path $script:Steps 'run-script.ps1')) -eq $false) {
throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it"
}

$script:Repo = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path

# The products whose integration tests call the cloud service.
$script:Products = 'device-detection', 'ip-intelligence', 'location',
'pipeline'

# The environment variables the scripts set, kept to be put back.
$script:Saved = '_51DEGREES_RESOURCE_KEY', '_51DEGREES_LICENSE_KEY',
'PYTEST_ADDOPTS', 'TEMP'

# The stand-in for common-ci's python/run-integration-tests.ps1. It
# writes one line of JSON for each call to calls.jsonl in the folder it
# runs from.
$script:StandIn = @'
param (
[Parameter(Mandatory=$true)][string]$RepoName,
[Parameter(Mandatory=$true)][string[]]$Packages,
[Parameter(Mandatory=$true)][hashtable]$Keys
)
@{
Product = Split-Path $RepoName -Leaf
Packages = @($Packages)
AddOpts = "$env:PYTEST_ADDOPTS"
} | ConvertTo-Json -Compress | Add-Content (Join-Path $PWD 'calls.jsonl')
$global:LASTEXITCODE = 0
'@

# A folder laid out like a common-ci checkout, which the step runs in.
function New-CommonCi {
$path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString())
$null = New-Item -ItemType Directory -Force (Join-Path $path 'python')
Copy-Item -Recurse $script:Steps (Join-Path $path 'steps')
Set-Content (Join-Path $path 'python' 'run-integration-tests.ps1') `
$script:StandIn
return $path
}

# The calls the stand-in recorded for a product.
function Get-Calls([string]$Product) {
$file = Join-Path $script:CommonCi 'calls.jsonl'
if ((Test-Path $file) -eq $false) {
return @()
}
return @(Get-Content $file | ForEach-Object { $_ | ConvertFrom-Json } |
Where-Object Product -eq $Product)
}
}

Describe 'run-integration-tests.ps1' {

BeforeEach {
$script:Environment = @{}
foreach ($name in $script:Saved) {
$script:Environment[$name] = [Environment]::GetEnvironmentVariable($name)
}
$script:CommonCi = New-CommonCi
Push-Location $script:CommonCi
}

AfterEach {
Pop-Location
Remove-Item -Recurse -Force $script:CommonCi
foreach ($name in $script:Saved) {
[Environment]::SetEnvironmentVariable($name, $script:Environment[$name])
}
}

Context 'with no TestResourceKey' {
BeforeEach {
$script:Output = Invoke-ProductStep 'run-integration-tests' $script:Repo `
@{ RepoName = $script:Repo; Keys = @{} } $script:Products *>&1 |
Out-String
}

It 'runs the tests of every product' {
foreach ($product in $script:Products) {
(Get-Calls $product).Count | Should -BeGreaterThan 0 -Because $product
}
}

It 'leaves out only the tests marked live, which need a resource key' {
foreach ($product in $script:Products) {
foreach ($call in (Get-Calls $product)) {
$call.AddOpts | Should -Be '-m "not live"' -Because $product
}
}
}

It 'says for every product that the tests needing a key are skipped' {
foreach ($product in $script:Products) {
# What the step printed for this product, between its group
# markers.
$group = ($script:Output -split '::group::') |
Where-Object { $_ -like "run-integration-tests for $product*" }
$group | Should -Match ("No TestResourceKey was passed, so the " +
"tests that need a resource key are skipped") -Because $product
}
}

It 'runs the tests of the device detection and IP intelligence examples' {
(Get-Calls 'device-detection').Packages |
Should -Contain 'fiftyone_devicedetection_examples'
(Get-Calls 'device-detection').Packages |
Should -Contain 'fiftyone_devicedetection_cloud'
(Get-Calls 'ip-intelligence').Packages |
Should -Contain 'fiftyone_ipintelligence_examples'
}

It 'runs the cloud request engine and its examples, and skips the 51Did live tests' {
$packages = (Get-Calls 'pipeline').Packages
$packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine'
$packages | Should -Contain 'fiftyone_pipeline_cloudrequestengine/examples'
$packages | Should -Not -Contain 'fiftyone_pipeline_did'
$script:Output | Should -Match 'fiftyone_pipeline_did.*skipped'
}

It 'skips the Selenium contract against the cloud example and saying so' {
$script:Output | Should -Match 'Selenium contract tests will not run against the cloud example'
Test-Path (Join-Path $script:CommonCi 'selenium-api-tests') |
Should -BeFalse
}
}

Context 'with a TestResourceKey' {
It 'runs every test, the ones marked live included' {
Invoke-ProductStep 'run-integration-tests' $script:Repo @{
RepoName = $script:Repo
Keys = @{ TestResourceKey = 'key' }
SeleniumContract = $false
} 'device-detection', 'location' *>&1 | Out-Null

foreach ($product in 'device-detection', 'location') {
$calls = Get-Calls $product
$calls.Count | Should -BeGreaterThan 0 -Because $product
foreach ($call in $calls) {
$call.AddOpts | Should -Not -Match 'live' -Because $product
}
}
}
}

}
51 changes: 38 additions & 13 deletions device-detection/ci/run-integration-tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@ if ($IsWindows) {

$packages = "fiftyone_devicedetection_cloud", "fiftyone_devicedetection_examples"

# Without a resource key the tests that need one, which are marked live, are
# left out and every other test runs. A live test still fails when it finds
# no key, so a key that is passed but does not reach it is not missed.
$addopts = $env:PYTEST_ADDOPTS
if (!$Keys.TestResourceKey) {
throw "No TestResourceKey was passed, so the cloud tests cannot call " +
"the cloud service. The workflows set it from the " +
"_51DEGREES_RESOURCE_KEY_BESPOKE secret."
Write-Output "::warning title=No resource key::No TestResourceKey was passed, so the tests that need a resource key are skipped. They are marked live, and the other integration tests run. The workflows set it from the _51DEGREES_RESOURCE_KEY_BESPOKE secret."
$env:PYTEST_ADDOPTS = '-m "not live"'
}
# The cloud package runs without the CSV file too, because only its property
# coverage tests read the file, and they say so when it is missing.
Expand All @@ -44,7 +47,11 @@ $env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
# files from _51DEGREES_LICENSE_KEY, which common-ci sets as license_key.
$env:_51DEGREES_LICENSE_KEY = $Keys.DeviceDetection

./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys
try {
./python/run-integration-tests.ps1 -RepoName $RepoName -Packages $packages -Keys $Keys
} finally {
$env:PYTEST_ADDOPTS = $addopts
}
$failures = @()
if ($LASTEXITCODE -ne 0) {
$failures += "the Python integration tests exited with code $LASTEXITCODE"
Expand All @@ -58,6 +65,25 @@ if (-not $SeleniumContract) {
return
}

# The contract tests run against the cloud example when there is a resource
# key, and against the on-premise example when there is the TAC data file,
# which is only fetched when a licence is given.
$tac = "$PWD/assets/TAC-HashV41.hash"
$runCloud = [bool]$Keys.TestResourceKey
$runOnPremise = Test-Path $tac
if (-not $runCloud) {
Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No resource key::No TestResourceKey was passed, so the Selenium contract tests will not run against the cloud example."
}
if (-not $runOnPremise) {
Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example."
}
if (-not ($runCloud -or $runOnPremise)) {
if ($failures.Count -gt 0) {
throw "Integration tests failed: $($failures -join '; ')"
}
return
}

# The shared Selenium contract tests are run against the cloud example and the
# on-premise example in turn. Each result is also written as a trx file next
# to the other integration results, so it is published with them.
Expand Down Expand Up @@ -141,22 +167,21 @@ function Test-Example([string]$Name, [string]$Module, [int]$Port) {
}

# The cloud example, pointed at the live cloud.
$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
$env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/"
$failures += Test-Example -Name 'cloud' `
-Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097
if ($runCloud) {
$env:_51DEGREES_RESOURCE_KEY = $Keys.TestResourceKey
$env:_51DEGREES_CLOUD_ENDPOINT = "https://cloud.51degrees.com/api/v4/"
$failures += Test-Example -Name 'cloud' `
-Module 'fiftyone_devicedetection_examples.cloud.gettingstarted_web' -Port 8097
}

# The on-premise example. The contract tests need the device type and the
# JavaScript properties, which the Lite data file does not have, so it runs
# against the TAC data file, which is only fetched when a licence is given.
$tac = "$PWD/assets/TAC-HashV41.hash"
if (Test-Path $tac) {
# against the TAC data file.
if ($runOnPremise) {
$env:_51DEGREES_DD_PATH = (Resolve-Path $tac).Path
$failures += Test-Example -Name 'onpremise' `
-Module 'fiftyone_devicedetection_examples.onpremise.gettingstarted_web' -Port 8098
Remove-Item Env:_51DEGREES_DD_PATH
} else {
Write-Output "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No TAC Data File::The TAC data file wasn't found, so the Selenium tests will not run against the on-premise example."
}

# A failure anywhere above must fail the job. This script is dot-sourced by
Expand Down
4 changes: 3 additions & 1 deletion device-detection/fiftyone_devicedetection_examples/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@ the Enterprise data file.

The cloud tests need a valid 51Degrees resource key (see above), and fail
without one. They read it from the environment variable
`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`.
`_51DEGREES_RESOURCE_KEY`, then `_51DEGREES_RESOURCE_KEY_BESPOKE`. They are
marked `live`, so `python -m pytest -m "not live"` runs every other test,
which is what CI does when no resource key is passed.

For example, use following command to set the resource key on Linux:
`export _51DEGREES_RESOURCE_KEY=MY-RESOURCE-KEY`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,16 @@ def _describe(names):
"statement, so their native data is released straight away.")


# The marker of the tests that call the live cloud service, which the
# product's ci/run-integration-tests.ps1 leaves out when no resource key is
# passed.
def pytest_configure(config):
config.addinivalue_line(
"markers",
"live: calls the live 51Degrees cloud service and needs a resource "
"key")


def pytest_runtest_logreport(report):
if report.failed:
_failed_tests.add(report.nodeid)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import json5
import os
from pathlib import Path
import pytest
import subprocess
import sys
import unittest
Expand All @@ -37,6 +38,8 @@

from .live_cloud import find_resource_key, no_resource_key

pytestmark = pytest.mark.live

NO_KEY = no_resource_key(
"the device product, and the hardware product for the lookups",
"device detection properties, and for the hardware profiles the TAC "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,14 @@
from contextlib import redirect_stdout
from unittest import mock

import pytest

from fiftyone_devicedetection_examples.example_utils import ExampleUtils

from .live_cloud import answered_without, find_resource_key, no_resource_key

pytestmark = pytest.mark.live

MODULE = "fiftyone_devicedetection_examples.cloud.failuretomatch"

NO_KEY = no_resource_key("the device product", "the IsMobile property")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,35 +20,60 @@
# such notice(s) shall fulfill the requirements of that article.
# *********************************************************************

import flask_unittest
import re
import unittest

import pytest

from fiftyone_pipeline_core.logger import Logger
from fiftyone_devicedetection_examples.example_utils import ExampleUtils
from fiftyone_devicedetection_examples.cloud.gettingstarted_web.app import GettingStartedWeb

class CloudGettingStartedWebTests(flask_unittest.ClientTestCase):
from .live_cloud import find_resource_key, no_resource_key

pytestmark = pytest.mark.live

NO_KEY = no_resource_key(
"the device product", "device detection properties such as HardwareName")

class CloudGettingStartedWebTests(unittest.TestCase):
# The test client sends no User-Agent of its own, so supply one that detection can
# resolve to a hardware profile.
CHROME_USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")

# Assign the `Flask` app object
resource_key = ExampleUtils.get_resource_key()
logger = Logger()
app = GettingStartedWeb().build(resource_key, logger).app
# The pipeline is built when the tests run rather than when they are
# collected, because building it with no resource key raises, and the
# tests are collected whether or not they are run.
resource_key = find_resource_key()

@classmethod
def setUpClass(cls):
super().setUpClass()
if cls.resource_key:
GettingStartedWeb().build(cls.resource_key, Logger())

@classmethod
def tearDownClass(cls):
if cls.resource_key:
GettingStartedWeb.pipeline.close()
super().tearDownClass()

def setUp(self):
if not self.resource_key:
self.fail(NO_KEY)
self.client = GettingStartedWeb.app.test_client()

def test_cloud_getting_started_web(self, client):
response = client.get('/')
def test_cloud_getting_started_web(self):
response = self.client.get('/')
self.assertEqual(200, response.status_code)
self.assertIn(b"Find out more", response.data)

# The device id used to appear on the page only by accident, inside the JSON of the
# inlined client-side script. Now that the script is served separately, check that the
# page renders a real one of its own, and that detection actually found a profile
# rather than returning the all-zero id.
def test_cloud_getting_started_web_device_id(self, client):
response = client.get('/', headers={"User-Agent": self.CHROME_USER_AGENT})
def test_cloud_getting_started_web_device_id(self):
response = self.client.get('/', headers={"User-Agent": self.CHROME_USER_AGENT})
self.assertEqual(200, response.status_code)
device_ids = re.findall(r"\d+-\d+-\d+-\d+", response.get_data(as_text=True))
self.assertTrue(device_ids, "No device id was rendered on the page")
Expand All @@ -57,8 +82,8 @@ def test_cloud_getting_started_web_device_id(self, client):
# The page references the client-side script by the '/51Degrees.core.js' name used
# by the web integrations in the other Pipeline APIs, so check that the route
# returns the bundle rather than, for example, falling through to the page.
def test_cloud_getting_started_web_core_js(self, client):
response = client.get('/51Degrees.core.js')
def test_cloud_getting_started_web_core_js(self):
response = self.client.get('/51Degrees.core.js')
self.assertEqual(200, response.status_code)
self.assertEqual("application/x-javascript", response.headers["Content-Type"])
self.assertIn(b"fiftyoneDegreesManager", response.data)
Loading
Loading