diff --git a/.github/workflows/ci-script-tests.yml b/.github/workflows/ci-script-tests.yml new file mode 100644 index 00000000..8f68b5e0 --- /dev/null +++ b/.github/workflows/ci-script-tests.yml @@ -0,0 +1,55 @@ +name: CI script tests + +# Runs the Pester tests of the scripts in ci/, which choose what a pull request +# builds and what a publish releases. common-ci's main is checked out beside +# this repository, where ci/dispatch.Tests.ps1 looks for it. +on: + pull_request: + paths: + - 'ci/**' + - '.github/workflows/ci-script-tests.yml' + push: + branches: + - main + - rc + paths: + - 'ci/**' + - '.github/workflows/ci-script-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + pester: + name: CI script tests on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: 51degrees-python + persist-credentials: false + + - name: Checkout common-ci + uses: actions/checkout@v4 + with: + repository: 51Degrees/common-ci + ref: main + path: common-ci + persist-credentials: false + + - name: Run the tests + shell: pwsh + working-directory: 51degrees-python + run: | + $configuration = New-PesterConfiguration + $configuration.Run.Path = @(Get-ChildItem ci -Filter *.Tests.ps1 | + ForEach-Object FullName) + $configuration.Run.Exit = $true + $configuration.Output.Verbosity = 'Detailed' + Invoke-Pester -Configuration $configuration diff --git a/.github/workflows/cloud-51did.yml b/.github/workflows/cloud-51did.yml index 5f258dc1..2a2e6ee2 100644 --- a/.github/workflows/cloud-51did.yml +++ b/.github/workflows/cloud-51did.yml @@ -29,7 +29,7 @@ on: schedule: - cron: '30 5 * * *' pull_request: - branches: [main] + branches: [main, rc] paths: - 'pipeline/fiftyone_pipeline_did/**' - 'pipeline/owid-python/**' diff --git a/device-detection/.github/workflows/example-assets-update.yml b/.github/workflows/example-assets-update.yml similarity index 73% rename from device-detection/.github/workflows/example-assets-update.yml rename to .github/workflows/example-assets-update.yml index dfbd3e5d..7c7b1851 100644 --- a/device-detection/.github/workflows/example-assets-update.yml +++ b/.github/workflows/example-assets-update.yml @@ -3,12 +3,14 @@ name: Example Assets Update # Refreshes the shared web-example CSS/JS (examples-main) committed in this repo # from the latest examples-assets release, via the common-ci reusable workflow. # The assets change rarely, so this runs weekly and only opens a PR if they -# changed. common-ci is not a submodule; it is referenced as a reusable workflow. +# changed. It calls common-ci as a reusable workflow, not a submodule. on: workflow_dispatch: + # Mondays at 03:45 UTC, so it starts apart from the other repositories' + # example asset updates. schedule: - - cron: '0 3 * * 1' + - cron: '45 3 * * 1' jobs: Example_Assets_Update: diff --git a/.github/workflows/nightly-pipeline.yml b/.github/workflows/nightly-pipeline.yml index e44f9d29..f0455882 100644 --- a/.github/workflows/nightly-pipeline.yml +++ b/.github/workflows/nightly-pipeline.yml @@ -15,60 +15,59 @@ on: default: '' schedule: # The 51degrees-cxx nightly starts at 00:35 and ends by about 02:35, so - # its release reaches this submodule update the same night. - - cron: '0 3 * * *' + # its release reaches this submodule update the same night. The Node.js + # nightly starts at 02:30 and the Java nightly at 03:00, and both want + # macOS runners too, so this one starts after them. + - cron: '0 5 * * *' jobs: - SubmoduleUpdate: - name: Nightly Submodule Update - uses: 51Degrees/common-ci/.github/workflows/nightly-submodule-update.yml@main + # The publish builds main, which the pull requests into rc do not change, so + # it runs first and has the runners before the pull request builds do. + Publish: + name: Nightly Publish + uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} + cache-assets: true secrets: token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} + PypiToken: ${{ secrets.PYPI_TOKEN }} - PullRequests: - if: ${{ !cancelled() }} - needs: SubmoduleUpdate - name: Nightly Pull Requests - uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main + SubmoduleUpdate: + name: Nightly Submodule Update + uses: 51Degrees/common-ci/.github/workflows/nightly-submodule-update.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} - skip-lite-tests: ${{ inputs.skip-lite-tests || false }} - cache-assets: true secrets: token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} - IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} - IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - Publish: + PullRequests: if: ${{ !cancelled() }} - needs: PullRequests - name: Nightly Publish - uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main + needs: [Publish, SubmoduleUpdate] + name: Nightly Pull Requests + uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main with: repo-name: ${{ github.event.repository.name }} org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} + skip-lite-tests: ${{ inputs.skip-lite-tests || false }} cache-assets: true + # One configuration that fails does not cancel the others. + fail-fast: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - PypiToken: ${{ secrets.PYPI_TOKEN }} - TestPypiToken: ${{ secrets.TEST_PYPI_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 195969ae..b75b3038 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,6 +9,12 @@ on: common-ci-ref: type: string default: '' + # common-ci refuses a release less than a week after the last unless + # this is set, which is only for a release agreed to go out early. + allow-early-release: + description: 'Publish even though a package was released less than a week ago' + type: boolean + default: false jobs: Publish: @@ -19,14 +25,12 @@ jobs: org-name: ${{ github.event.repository.owner.login }} dryrun: ${{ inputs.dryrun || false }} common-ci-ref: ${{ inputs.common-ci-ref || '' }} + allow-early-release: ${{ inputs.allow-early-release || false }} cache-assets: true secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} PypiToken: ${{ secrets.PYPI_TOKEN }} - TestPypiToken: ${{ secrets.TEST_PYPI_TOKEN }} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 0d600a54..e154435f 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -13,6 +13,10 @@ on: common-ci-ref: type: string default: '' + full-matrix: + description: 'Build every configuration, not only those the changes need' + type: boolean + default: false jobs: PullRequests: @@ -25,11 +29,15 @@ jobs: common-ci-ref: ${{ inputs.common-ci-ref || '' }} skip-lite-tests: ${{ inputs.skip-lite-tests || false }} cache-assets: true + # Each pull request is built on the configurations its changes need, + # as ci/select-options.ps1 chooses, and one that fails does not cancel + # the others. With nothing eligible, the nightly builds rc instead. + full-matrix: ${{ inputs.full-matrix || false }} + fail-fast: false + build-without-pull-requests: false secrets: token: ${{ secrets.ACCESS_TOKEN }} DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} diff --git a/.github/workflows/release-pull-request.yml b/.github/workflows/release-pull-request.yml new file mode 100644 index 00000000..6faa5356 --- /dev/null +++ b/.github/workflows/release-pull-request.yml @@ -0,0 +1,30 @@ +name: Release Pull Request + +# Builds and tests the release pull request, rc into main, on every entry in +# ci/options.json, and never merges it. A pull request into rc is built only on +# the entries its changes need, so this is the full build of a release before +# it reaches main. See common-ci .github/workflows/release-pull-request.yml. +on: + pull_request: + branches: [main] + +# A push to rc while the release is open builds it again, in place of the +# build of the older rc. +concurrency: + group: release-pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + Release: + name: Release Pull Request + uses: 51Degrees/common-ci/.github/workflows/release-pull-request.yml@main + with: + repo-name: ${{ github.event.repository.name }} + org-name: ${{ github.event.repository.owner.login }} + cache-assets: true + secrets: + token: ${{ secrets.ACCESS_TOKEN }} + DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} + IpIntelligence: ${{ secrets.IP_INTELLIGENCE_KEY }} + IpIntelligenceUrl: ${{ secrets.IPI_DATA_FILE_URL }} + TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} diff --git a/.gitmodules b/.gitmodules index af600379..c031260b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "cxx"] path = cxx url = ../51degrees-cxx +[submodule "pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties"] + path = pipeline/fiftyone_pipeline_derivedproperty/src/fiftyone_pipeline_derivedproperty/derived-properties + url = ../derived-properties diff --git a/AGENTS.md b/AGENTS.md index 1b697ff7..af06e625 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,10 +10,10 @@ layout and how to get the code are in the [README](README.md). Device Detection's on-premise `gettingstarted_console` is the quickest example to show, because its data file is about 35 MB. IP Intelligence's -on-premise `gettingstarted_console` runs on the free Asn data file, which is -about 6 MB, and shows registration values once it is given the Lite file, -which is a download of about 230 MB. Run each command from the repository root -unless the step says otherwise. +on-premise `gettingstarted_console` runs on the free Lite data file, which is a +download of about 230 MB and has the registration values. The free Asn data +file, about 6 MB, has only the network properties. Run each command from the +repository root unless the step says otherwise. The on-premise engines are C and C++ code that is compiled when the package is installed. PyPI has only source distributions of @@ -150,16 +150,19 @@ macOS (`find_file` in python -m fiftyone_devicedetection_examples.onpremise.gettingstarted_console ``` -The IP Intelligence example with no argument looks for the Asn file in the -folder it runs in and in its `assets` and `cxx/data/ip-intelligence` folders, -then does the same in each of the seven folders above it, and downloads the -file into `cxx/data/ip-intelligence` when it is missing. To see registration -values, download the Lite file first, as "Data files" says, and give its path -as the first argument. +The IP Intelligence example with no argument looks for the Lite file, +`51Degrees-LiteV41.ipi`, in the folder it runs in and in its `assets`, +`cxx/data/ip-intelligence` and `cxx/data/device-detection` folders, then does +the same in each of the seven folders above it. When none of them has the +file, it downloads the file into the `cxx/data/ip-intelligence` folder that +holds the file's `.link` (`find_file` in +`ip-intelligence/fiftyone_ipintelligence_examples/src/fiftyone_ipintelligence_examples/example_utils.py`). +To run it on the smaller Asn file, download that file first, as "Data files" +says, and give its path as the first argument. ``` python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console -python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console cxx/data/ip-intelligence/51Degrees-LiteV41.ipi +python -m fiftyone_ipintelligence_examples.onpremise.gettingstarted_console cxx/data/ip-intelligence/51Degrees-IPIV4AsnIpiV41.ipi ``` Either example uses a path given as the first argument as it is, so a file @@ -187,11 +190,12 @@ The IP Intelligence example logs `Using a '' data file created ''`, then for each of four IP addresses, starting with `116.154.188.222`, prints `Input values:` with `query.client-ip`, then `Results:` with a line for each of 16 properties, starting with `RegisteredName`, `RegisteredOwner` and -`RegisteredCountry`. A property the data file lacks reads -`This property is not present in the data file.` With the Asn file that is -every registration and location property, as the examples' README says. The -Lite file has RegisteredName, RegisteredOwner and RegisteredCountry, so with it -those three lines are read from the file. +`RegisteredCountry`. The Lite file has RegisteredName, RegisteredOwner and +RegisteredCountry, so those three lines are read from the file. Each property +the data file lacks reads `needs the Enterprise data file`, and after the last +address the example prints `Some of these properties need the Enterprise data +file. Contact us for a copy at` followed by a link. With the Asn file the +registration properties need the Enterprise data file as well. The paid data files have many more properties, as the [pricing page](https://51degrees.com/pricing?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=correct-output) @@ -202,10 +206,10 @@ describes. The cloud examples need a resource key, which a person creates in the configurator, so ask for one if none is set. A key with the properties the Device Detection cloud example uses can be made -[here](https://configure.51degrees.com/1QWJwHxl?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples), +[here](https://51degrees.com/configure/1QWJwHxl?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples), which is the configuration the example's own message names, and a key for IP Intelligence in the -[configurator](https://configure.51degrees.com/?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples). +[configuration page](https://51degrees.com/configure/?utm_source=github&utm_medium=docs&utm_campaign=51degrees-python&utm_content=agents.md&utm_term=cloud-examples). The Device Detection and IP Intelligence cloud examples take the key as their first argument, or from `_51DEGREES_RESOURCE_KEY`. @@ -250,6 +254,19 @@ example logs an error that names the variable and the configurator. assets, and the IP Intelligence example tests use the Asn file and throw their output away. +## Environment variable names + +[common-ci's list of environment variable names](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.md) +holds every environment variable name that the packages, tests, examples and +CI scripts of every language read, with what each name holds and the +deprecated names each replaced. The list decides. Use the names it gives, and +when a change needs a new name, or reads an old name again, change +[`environment-variables.json`](https://github.com/51Degrees/common-ci/blob/d8a9c596eabf69dd5581a08e9360cdd1c9b3d9e8/environment-variables.json) +in common-ci in the same piece of work. This repository's own name +check, [`ci/check-environment-variable-names.ps1`](ci/check-environment-variable-names.ps1), still keeps a +list of its own. Where the two differ, common-ci's list is the one to +follow. + ## Products keep their own CI scripts Each product folder still has the `ci/` scripts it had as its own repository, @@ -262,16 +279,33 @@ product is built or tested belongs in that product's `ci/`, not in the root. A change touches a product when it is in that product's folder, or in a package another product depends on; a change outside every product folder touches all of them ([ci/packages.ps1](ci/packages.ps1), -`Resolve-AffectedProducts`). +`Resolve-AffectedProducts`). A pull request's changes are measured from the +branch it merges into, `rc`. + +[ci/select-options.ps1](ci/select-options.ps1) chooses the entries a pull +request runs. The standard set is Python 3.14 and 3.10 on Linux, with the +Selenium contract on 3.14 alone. A native change adds Windows, a packaging +change adds the other Python versions on Linux, and a change outside the +product folders or to a `ci/` folder adds every entry but macOS. macOS and +performance run only when `rc` itself or the release pull request is built, +which `.github/workflows/release-pull-request.yml` does on every entry. ## Versions are per package Every package is versioned and tagged on its own, `/`. A -package is released when its folders changed since its last tag, or when a -package it depends on is released. The examples packages are tested but never -published. common-ci's publish carries one version; here it is only a label, -and the release plan in `ci/packages.ps1` decides what is built, published -and tagged. Never push a plain `4.5.x` tag: nothing here reads one. +package is released when a file that counts in its folders changed since its +last tag, or when a package it depends on is released. Documentation +(`*.md`, `docs/`), tests (`tests/`, `test/`), examples (`examples/`) and +`LICENSE` files do not count, nor do files a package's `Ignore` globs in +`ci/packages.json` match, unless it sets `IgnoreDefaults` to false. A +`setup.py`, `setup.cfg` or `pyproject.toml` whose change is only to the +description, keywords, urls, classifiers, readme, authors or maintainers does +not count either, which `ci/release-plan-filter.ps1` works out with Python. +The examples packages are tested but never published. common-ci's publish +carries one version, which here is only a label, and the release plan +decides what is built, published and tagged. The release plan is common-ci's +`steps/release-plan.ps1`, which `ci/packages.ps1` loads, and common-ci tests +it. Never push a plain `4.5.x` tag, because nothing here reads one. `ci/packages.json` is the list of packages and what each depends on. Add a package there, with a `FirstVersion` if it has never been released, or it is @@ -313,13 +347,17 @@ A package's `ExtraPaths` in `ci/packages.json` cannot be a folder inside cxx update therefore releases no package, even when it brings new translations in `cxx/data/ip-intelligence`. -## Releases - -Pull requests go to `rc`, not `main`, and stay drafts until they have been -reviewed, because the nightly merges any pull request into `rc` that is not a -draft and has no review requested. When a release is wanted, one pull request -merges `rc` into `main`, and the "Only rc into main" check fails a pull request -into `main` from any other branch. +`pipeline/fiftyone_pipeline_derivedproperty` runs the scripts of +[derived-properties](https://github.com/51Degrees/derived-properties), which is +its submodule `src/fiftyone_pipeline_derivedproperty/derived-properties`, inside +the package so that the wheel carries its `scripts` folder (`MANIFEST.in`) and +nothing else from it. Its tests run every case file in the submodule's `tests` +folder, so moving the submodule moves the cases the tests run. `parse.py`, +`validate.py`, `evaluate.py` and `canonical.py` follow the reference tools in +the submodule's `tools` folder function for function, with `jsvalues.py` +holding the JavaScript rules they rely on, so a change there is made here the +same way. The package is `"Publish": false` in `ci/packages.json`, so it is +tested in every build and not published. ## Releases @@ -334,3 +372,11 @@ into `main` from any other branch. The four repositories were brought in with `git subtree add`, so commit SHAs match the originals. Use `git log --follow` to trace a file back through the import. + +## Tests that fail on timing + +A test that fails on a time limit, a performance threshold or a timeout is +fixed, not run again until it passes. Either the limit is set to fit the +slowest runner the test runs on, with each runner's measurement behind it, or +the reason it takes too long is found and fixed. A failure that comes and goes +costs everyone who meets it the time to look into it. diff --git a/README.md b/README.md index 04fb4d05..23688d15 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ python -m pip install fiftyone-ipintelligence The table below lists the other packages. To build from source, see [Getting the code](#getting-the-code). -Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://configure.51degrees.com/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install). +Try the [User-Agent tester](https://51degrees.com/developers/user-agent-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install) and [IP tester](https://51degrees.com/developers/ip-tester?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install). For cloud use, [create a Resource Key](https://51degrees.com/configure/?utm_source=github&utm_medium=readme&utm_campaign=51degrees-python&utm_content=readme.md&utm_term=install). Created from `pipeline-python`, `device-detection-python`, and `location-python`, this repository continues their work. @@ -31,7 +31,7 @@ own folder, which has its own README. | Product | Folder | PyPI packages | | --- | --- | --- | -| Pipeline API | [pipeline](pipeline/readme.md) | `fiftyone_pipeline_core`, `_engines`, `_engines_fiftyone`, `_cloudrequestengine`, `_translation`, `_did` | +| Pipeline API | [pipeline](pipeline/readme.md) | `fiftyone_pipeline_core`, `_engines`, `_engines_fiftyone`, `_cloudrequestengine`, `_translation`, `_did`, `_derivedproperty` (not yet published) | | Device Detection | [device-detection](device-detection/readme.md) | `fiftyone_devicedetection`, `_shared`, `_cloud`, `_onpremise` | | IP Intelligence | [ip-intelligence](ip-intelligence/readme.md) | `fiftyone_ipintelligence`, `_shared`, `_cloud`, `_onpremise`, `_translation` | | Location | [location](location/readme.md) | `fiftyone_location` | diff --git a/ci/build-project.ps1 b/ci/build-project.ps1 index 3461d091..639d7435 100644 --- a/ci/build-project.ps1 +++ b/ci/build-project.ps1 @@ -1,5 +1,8 @@ param ( - [Parameter(Mandatory)][string]$RepoName + [Parameter(Mandatory)][string]$RepoName, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/dispatch.Tests.ps1 b/ci/dispatch.Tests.ps1 index 3d75e0a2..352e667b 100644 --- a/ci/dispatch.Tests.ps1 +++ b/ci/dispatch.Tests.ps1 @@ -2,8 +2,8 @@ # # Each test builds a throwaway repository with two products, 'alpha' and # 'beta', whose ci/ scripts are written by the test, and runs the dispatcher -# from a folder laid out like a common-ci checkout, with common-ci's real -# steps/run-script.ps1, because that is how common-ci calls it. +# from a folder laid out like a common-ci checkout, with a copy of common-ci's +# real steps folder, because that is how common-ci calls it. BeforeAll { . (Join-Path $PSScriptRoot 'dispatch.ps1') @@ -17,6 +17,10 @@ BeforeAll { if ((Test-Path $script:RunScript) -eq $false) { throw "common-ci not found at '$commonCi'; set COMMON_CI to a checkout of it" } + # The whole steps folder is copied, because ci/packages.ps1 loads the + # release plan from the common-ci checkout that holds this repository's + # folder, and the release plan may load other steps beside it. + $script:Steps = Join-Path $commonCi 'steps' # A repository with packages.json naming one package in each product. function New-ProductRepo { @@ -45,8 +49,8 @@ BeforeAll { # A folder laid out like a common-ci checkout, which the dispatcher runs in. function New-CommonCi { $path = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - $steps = New-Item -ItemType Directory -Force (Join-Path $path 'steps') - Copy-Item $script:RunScript $steps + $null = New-Item -ItemType Directory -Force $path + Copy-Item -Recurse $script:Steps (Join-Path $path 'steps') return $path } } diff --git a/ci/dispatch.ps1 b/ci/dispatch.ps1 index 4249d40b..06ba77ef 100644 --- a/ci/dispatch.ps1 +++ b/ci/dispatch.ps1 @@ -10,12 +10,15 @@ # Runs $Step for each affected product, or for each of $Products when given. # Every product is run even when an earlier one fails, so one run reports -# every broken product, and the step fails at the end if any did. +# every broken product, and the step fails at the end if any did. A pull +# request's changes are measured from the Branch in $Options, the branch it +# merges into. function Invoke-ProductStep([string]$Step, [string]$RepoName, [Collections.IDictionary]$Options, [string[]]$Products) { $failed = [Collections.Generic.List[string]]::new() if (-not $Products) { - $Products = Get-AffectedProducts $RepoName + $branch = $Options.Branch ? $Options.Branch : 'main' + $Products = Get-AffectedProducts $RepoName $branch } foreach ($product in $Products) { $script = Join-Path $RepoName $product 'ci' "$Step.ps1" diff --git a/ci/fetch-assets.ps1 b/ci/fetch-assets.ps1 index 802476f1..62209ec2 100644 --- a/ci/fetch-assets.ps1 +++ b/ci/fetch-assets.ps1 @@ -5,10 +5,11 @@ param ( # common-ci checkout, which is where both callers run it from. [string]$RepoName = (Split-Path -Leaf (Split-Path -Parent $PSScriptRoot)), [string]$DeviceDetection, - [string]$DeviceDetectionUrl, - [string]$CsvUrl, [string]$IpIntelligence, - [string]$IpIntelligenceUrl + [string]$IpIntelligenceUrl, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/get-next-package-version.Tests.ps1 b/ci/get-next-package-version.Tests.ps1 new file mode 100644 index 00000000..706e6f0c --- /dev/null +++ b/ci/get-next-package-version.Tests.ps1 @@ -0,0 +1,46 @@ +# Tests for ci/get-next-package-version.ps1, which labels a publish with the +# highest version in the release plan. +# +# Each test copies the script to a throwaway folder beside a packages.ps1 +# whose release plan is the versions the test gives, so the test states the +# plan it labels. + +Describe 'get-next-package-version.ps1' { + BeforeAll { + $script:Label = Join-Path $PSScriptRoot 'get-next-package-version.ps1' + + function Get-Label([string[]]$Versions) { + $folder = Join-Path ([IO.Path]::GetTempPath()) ` + ([Guid]::NewGuid().ToString()) + $null = New-Item -ItemType Directory $folder + try { + Copy-Item $script:Label $folder + $list = ($Versions | ForEach-Object { "'$_'" }) -join ', ' + Set-Content (Join-Path $folder 'packages.ps1') @" +function Get-ReleasePlan(`$RepoPath) { + @(@($list) | ForEach-Object { + [pscustomobject]@{ Name = "package-`$_"; Version = `$_ } + }) +} +"@ + & (Join-Path $folder 'get-next-package-version.ps1') ` + -RepoName 'repository' -VariableName 'TestLabel' + return $global:TestLabel + } finally { + Remove-Item -Recurse -Force $folder + } + } + } + + It 'labels a plan of one package with its whole version' { + Get-Label @('4.5.3') | Should -Be '4.5.3' + } + + It 'labels a plan of several packages with the highest version' { + Get-Label @('4.5.3', '4.5.10', '4.5.9') | Should -Be '4.5.10' + } + + It 'gives a valid version when nothing is released' { + Get-Label @() | Should -Be '4.5.0' + } +} diff --git a/ci/get-next-package-version.ps1 b/ci/get-next-package-version.ps1 index d94aa4d7..0456e988 100644 --- a/ci/get-next-package-version.ps1 +++ b/ci/get-next-package-version.ps1 @@ -13,7 +13,7 @@ $ErrorActionPreference = "Stop" # build-package.ps1 and publish-package.ps1, all from the same release plan. $plan = @(Get-ReleasePlan $RepoName) if ($plan.Count -gt 0) { - $label = ($plan.Version | Sort-Object { [version]$_ })[-1] + $label = @($plan.Version | Sort-Object { [version]$_ })[-1] } else { # Nothing to release. Any valid version will do, because # package-update-required.ps1 stops the publish before it is used. diff --git a/ci/packages.Tests.ps1 b/ci/packages.Tests.ps1 index 766d1380..63066049 100644 --- a/ci/packages.Tests.ps1 +++ b/ci/packages.Tests.ps1 @@ -1,9 +1,11 @@ -# Tests for ci/packages.ps1. +# Tests for ci/packages.ps1 and ci/release-plan-filter.ps1. # # The selection rules are tested against a small package list written here, so -# a test states exactly which graph it relies on. The release plan reads tags -# and diffs, so those tests build a throwaway git repository. One test reads the -# real packages.json, to catch a typo in a dependency name or a cycle. +# a test states exactly which graph it relies on. The release plan, which +# packages.ps1 loads from common-ci, is tested there. The tests of this +# repository's filter and of the build plan read tags and diffs, so they build +# a throwaway git repository. Some tests read the real packages.json, to catch +# a typo in a dependency name or a cycle, and to check its Ignore globs. BeforeAll { . (Join-Path $PSScriptRoot 'packages.ps1') @@ -75,40 +77,6 @@ AfterEach { $script:Repo = $null } -Describe 'Get-DependentClosure' { - It 'adds every package that depends on a changed one, through others' { - $closure = Get-DependentClosure $script:Graph @('core') - $closure | Sort-Object | Should -Be @( - 'core', 'dd', 'engines', 'examples', 'ipi', 'location') - } - - It 'adds nothing for a package nothing depends on' { - Get-DependentClosure $script:Graph @('did') | Should -Be @('did') - } - - It 'is empty when given nothing' { - @(Get-DependentClosure $script:Graph @()).Count | Should -Be 0 - } -} - -Describe 'Sort-ByDependency' { - It 'puts every package after the packages it depends on' { - $order = Sort-ByDependency $script:Graph @('dd', 'engines', 'core') - $order | Should -Be @('core', 'engines', 'dd') - } - - It 'ignores dependencies that are not being sorted' { - Sort-ByDependency $script:Graph @('dd') | Should -Be @('dd') - } - - It 'refuses a cycle rather than looping' { - $cycle = @( - New-Package 'a' 'p' 'p/a' @('b') - New-Package 'b' 'p' 'p/b' @('a')) - { Sort-ByDependency $cycle @('a', 'b') } | Should -Throw '*cycle*' - } -} - Describe 'Resolve-AffectedProducts' { It 'tests only the product of a package nothing depends on' { Resolve-AffectedProducts $script:Graph @('device/dd/a.py') | @@ -147,105 +115,204 @@ Describe 'Resolve-AffectedProducts' { } } -Describe 'Get-ReleasePlan' { - It 'releases nothing but first releases when nothing changed since the tags' { - $script:Repo = New-Repo $script:AllFiles $script:AllTags - Add-Commit $script:Repo @() - $plan = @(Get-ReleasePlan $script:Repo $script:Graph) - $plan.Name | Should -Be @('ipi') - $plan[0].Version | Should -Be '4.5.0' - $plan[0].Reason | Should -Be 'first release' - } - - It 'releases a changed package at the next patch, and its dependents' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('pipeline/engines/a.py') - $plan = @(Get-ReleasePlan $script:Repo $script:Graph) - $plan.Name | Should -Be @('engines', 'dd', 'ipi') - ($plan | Where-Object Name -eq 'engines').Version | Should -Be '4.5.39' - ($plan | Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' - ($plan | Where-Object Name -eq 'dd').Reason | - Should -Be 'a package it depends on is released' - ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.1' - } - - It 'releases a vendoring package when the vendored folder changes' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('pipeline/owid/a.py') - @(Get-ReleasePlan $script:Repo $script:Graph).Name | Should -Be @('did') - } - - It 'releases nothing for a change to an unpublished package' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/examples/a.py') - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'releases nothing for a change outside every package' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('ci/build.ps1', 'device/docs/a.md') - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'releases nothing a second time from a commit already released' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/a.py') - git -C $script:Repo tag 'dd/4.5.97' - @(Get-ReleasePlan $script:Repo $script:Graph).Count | Should -Be 0 - } - - It 'continues from the highest tag, not the latest one made' { - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + @( - 'ipi/4.5.0', 'dd/4.5.9')) - Add-Commit $script:Repo @('device/dd/a.py') - (@(Get-ReleasePlan $script:Repo $script:Graph) | - Where-Object Name -eq 'dd').Version | Should -Be '4.5.97' - } - - It 'does not read another package''s tags as its own' { - # 'ipi' must not see tags of a package whose name starts with 'ipi'. - $graph = $script:Graph + (New-Package 'ipi_cloud' 'ipi' 'ipi/cloud' -FirstVersion '4.5.0') - $script:Repo = New-Repo ($script:AllFiles + 'ipi/cloud/a.py') ( - $script:AllTags + 'ipi_cloud/4.5.7') - $plan = @(Get-ReleasePlan $script:Repo $graph) - ($plan | Where-Object Name -eq 'ipi').Version | Should -Be '4.5.0' - } +Describe 'release-plan-filter.ps1' { + BeforeAll { + # A setup.py, setup.cfg and pyproject.toml like the ones here, with + # the metadata the filter leaves out and settings it does not. Line + # endings are made LF, whatever the checkout gave this file. + $script:SetupPy = @' +import setuptools + +setuptools.setup( + name="dd", + version=read("version.txt"), + author="51Degrees Engineering", + author_email="engineering@51degrees.com", + project_urls={"Source": "https://github.com/51Degrees/51degrees-python"}, + description=("Device detection."), + keywords=["51degrees", "device-detection"], + long_description=read("readme.md"), + long_description_content_type='text/markdown', + python_requires=">=3.10", + install_requires=["fiftyone_pipeline_core"], + classifiers=["Programming Language :: Python"], +) +'@.Replace("`r`n", "`n") + $script:SetupCfg = @' +[metadata] +name = dd +description = Device detection. +url = https://github.com/51Degrees/51degrees-python + +[options] +install_requires = + fiftyone_pipeline_core +'@.Replace("`r`n", "`n") + "`n" + $script:Pyproject = @' +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "dd" +description = "Device detection." +keywords = [ + "51degrees", + "device-detection", +] +dependencies = ["fiftyone_pipeline_core"] + +[project.urls] +Source = "https://github.com/51Degrees/51degrees-python" +'@.Replace("`r`n", "`n") + "`n" + + # A repository with the given files, holding the filter as this + # repository does, with the tags of the small graph and the tag of + # dd on the commit that adds the files. + function New-FilterRepo([hashtable]$Files) { + $repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') + $null = New-Item -ItemType Directory -Force (Join-Path $repo 'ci') + foreach ($name in 'release-plan-filter.ps1', 'release-plan-filter.py') { + Copy-Item (Join-Path $PSScriptRoot $name) (Join-Path $repo 'ci') + } + Set-Files $repo $Files + git -C $repo tag -f 'dd/4.5.96' | Out-Null + return $repo + } - It 'plans a changed package for a caller that stops on native errors' { - # publish-package.ps1 stops on a failing native command; 'git diff - # --quiet' exiting 1 for a changed package once failed the publish. - $script:Repo = New-Repo $script:AllFiles ($script:AllTags + 'ipi/4.5.0') - Add-Commit $script:Repo @('device/dd/a.py') - $plan = & { - $ErrorActionPreference = 'Stop' - $PSNativeCommandUseErrorActionPreference = $true - @(Get-ReleasePlan $script:Repo $script:Graph) + # Commits the files with exactly the text given. + function Set-Files([string]$Repo, [hashtable]$Files) { + foreach ($file in $Files.Keys) { + $full = Join-Path $Repo $file + $null = New-Item -ItemType Directory -Force (Split-Path $full) + [IO.File]::WriteAllText($full, $Files[$file]) + } + git -C $Repo add -A + git -C $Repo -c user.name=test -c user.email=test@example.com ` + commit --quiet --allow-empty -m 'Change' } - $plan.Name | Should -Be @('dd') - } - - It 'reads the tags whatever folder it runs from' { - # PowerShell expands a wildcard in a native command's argument into - # matching files, so a tag pattern run from a folder that holds a - # folder of the same name, as the repository root holds 'location', - # once found no tags. - $script:Repo = New-Repo $script:AllFiles $script:AllTags - $here = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().ToString()) - $null = New-Item -ItemType Directory -Force (Join-Path $here 'location') - $null = New-Item -ItemType File (Join-Path $here 'location' 'setup.py') - Push-Location $here - try { - (Get-ReleasedVersions $script:Repo 'location')[-1] | Should -Be ([version]'4.5.4') - } finally { - Pop-Location - Remove-Item -Recurse -Force $here + + function Get-DdPlan([string]$Repo) { + return @(Get-ReleasePlan $Repo $script:Graph | + Where-Object Name -eq 'dd') } } - It 'refuses a never-released package with no FirstVersion' { - $graph = @(New-Package 'new' 'p' 'p/new') - $script:Repo = New-Repo @('p/new/a.py') @() - { Get-ReleasePlan $script:Repo $graph } | Should -Throw '*FirstVersion*' + It 'does not release for a setup.py change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + $changed = $script:SetupPy + $changed = $changed.Replace('Device detection.', 'Device detection for Python.') + $changed = $changed.Replace('"device-detection"]', '"device-detection", "user-agent"]') + $changed = $changed.Replace('51degrees-python"}', '51degrees-python", "Docs": "x"}') + $changed = $changed.Replace('51Degrees Engineering', '51Degrees') + $changed = $changed.Replace('"Programming Language :: Python"]', + '"Programming Language :: Python", "Typing :: Typed"]') + $changed = $changed.Replace("'text/markdown'", "'text/x-rst'") + Set-Files $script:Repo @{ 'device/dd/setup.py' = $changed + "# A note.`n" } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a setup.py change to anything else' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + $changed = $script:SetupPy.Replace('["fiftyone_pipeline_core"]', + '["fiftyone_pipeline_core>=4.5.40"]') + Set-Files $script:Repo @{ 'device/dd/setup.py' = $changed } + $plan = Get-DdPlan $script:Repo + $plan.Count | Should -Be 1 + $plan[0].Reason | Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'counts the code changed beside a setup.py change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + Set-Files $script:Repo @{ + 'device/dd/setup.py' = $script:SetupPy.Replace( + 'Device detection.', 'Device detection for Python.') + 'device/dd/src/a.py' = 'print(1)' + } + (Get-DdPlan $script:Repo)[0].Reason | + Should -Be 'changed since 4.5.96 (1 file)' + } + + It 'does not release for a setup.cfg change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.cfg' = $script:SetupCfg } + $changed = $script:SetupCfg.Replace('Device detection.', 'Device detection for Python.') + $changed = $changed.Replace('51degrees-python', '51degrees-python/tree/main') + Set-Files $script:Repo @{ 'device/dd/setup.cfg' = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a setup.cfg change to its options' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.cfg' = $script:SetupCfg } + Set-Files $script:Repo @{ 'device/dd/setup.cfg' = + $script:SetupCfg + " fiftyone_pipeline_engines`n" } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'does not release for a pyproject.toml change only to metadata' { + $script:Repo = New-FilterRepo @{ 'device/dd/pyproject.toml' = $script:Pyproject } + $changed = $script:Pyproject.Replace('"Device detection."', '"Device detection for Python."') + $changed = $changed.Replace(" `"device-detection`",`n", + " `"device-detection`",`n `"user-agent`",`n") + $changed = $changed.Replace('Source = ', "Docs = `"x`"`nSource = ") + $changed += "`n[[project.authors]]`nname = `"51Degrees`"`n" + Set-Files $script:Repo @{ 'device/dd/pyproject.toml' = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'does not release for a change only to the maintainers in ' -ForEach @( + @{ Case = 'setup.py'; File = 'setup.py'; Start = 'SetupPy' + Was = 'author_email="engineering@51degrees.com",' + Changed = "author_email=`"engineering@51degrees.com`",`n maintainer=`"51Degrees`",`n maintainer_email=`"support@51degrees.com`"," } + @{ Case = 'setup.cfg'; File = 'setup.cfg'; Start = 'SetupCfg' + Was = "name = dd`n" + Changed = "name = dd`nmaintainer = 51Degrees`nmaintainer_email = support@51degrees.com`n" } + @{ Case = 'a pyproject.toml key'; File = 'pyproject.toml'; Start = 'Pyproject' + Was = 'dependencies = ' + Changed = "maintainers = [{ name = `"51Degrees`" }]`ndependencies = " } + @{ Case = 'a pyproject.toml table'; File = 'pyproject.toml'; Start = 'Pyproject' + Was = '[project.urls]' + Changed = "[[project.maintainers]]`nname = `"51Degrees`"`n`n[project.urls]" } + ) { + $text = @{ + SetupPy = $script:SetupPy + SetupCfg = $script:SetupCfg + Pyproject = $script:Pyproject + }[$Start] + $changed = $text.Replace($Was, $Changed) + $changed | Should -Not -Be $text + $script:Repo = New-FilterRepo @{ "device/dd/$File" = $text } + Set-Files $script:Repo @{ "device/dd/$File" = $changed } + (Get-DdPlan $script:Repo).Count | Should -Be 0 + } + + It 'releases for a pyproject.toml change to ' -ForEach @( + @{ Was = '["fiftyone_pipeline_core"]'; Changed = '["fiftyone_pipeline_core>=4.5"]' } + @{ Was = '"setuptools>=42"'; Changed = '"setuptools>=61"' } + ) { + $script:Repo = New-FilterRepo @{ 'device/dd/pyproject.toml' = $script:Pyproject } + Set-Files $script:Repo @{ 'device/dd/pyproject.toml' = + $script:Pyproject.Replace($Was, $Changed) } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest that is new since the last tag' { + $script:Repo = New-FilterRepo @{} + Set-Files $script:Repo @{ 'device/dd/setup.py' = $script:SetupPy } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest it cannot read' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + Set-Files $script:Repo @{ 'device/dd/setup.py' = 'setup(' } + (Get-DdPlan $script:Repo).Count | Should -Be 1 + } + + It 'releases for a manifest that is not UTF-8' { + $script:Repo = New-FilterRepo @{ 'device/dd/setup.py' = $script:SetupPy } + [IO.File]::WriteAllBytes((Join-Path $script:Repo 'device/dd/setup.py'), + [byte[]](0x23, 0xff, 0xfe, 0x0a)) + Set-Files $script:Repo @{} + (Get-DdPlan $script:Repo).Count | Should -Be 1 } } @@ -320,6 +387,40 @@ Describe 'the real packages.json' { Get-ToxGaps (Split-Path $PSScriptRoot) $script:Real | Should -BeNullOrEmpty } + + Context 'a change to a package''s tox.ini' { + BeforeAll { + $published = @($script:Real | Where-Object Publish -eq $true) + $script:RealTags = @($published | ForEach-Object { "$($_.Name)/4.5.1" }) + $script:RealFiles = @($script:Real | ForEach-Object { + "$($_.Path)/src/a.py" + "$($_.Path)/tox.ini" + }) + } + + It 'releases nothing, whichever package it sits beside' { + $script:Repo = New-Repo $script:RealFiles $script:RealTags + Add-Commit $script:Repo ($script:Real | ForEach-Object { "$($_.Path)/tox.ini" }) + @(Get-ReleasePlan $script:Repo $script:Real).Count | Should -Be 0 + } + + It 'still releases a package for a change to its source' { + $script:Repo = New-Repo $script:RealFiles $script:RealTags + Add-Commit $script:Repo @( + 'pipeline/fiftyone_pipeline_core/tox.ini', + 'pipeline/fiftyone_pipeline_core/src/a.py') + $plan = @(Get-ReleasePlan $script:Repo $script:Real) + $plan.Name | Should -Contain 'fiftyone_pipeline_core' + ($plan | Where-Object Name -eq 'fiftyone_pipeline_core').Reason | + Should -Be 'changed since 4.5.1 (1 file)' + } + + It 'is ignored by every package, so one added later counts for nothing' { + foreach ($package in $script:Real) { + $package.Ignore | Should -Contain '**/tox.ini' -Because $package.Name + } + } + } } } diff --git a/ci/packages.json b/ci/packages.json index deb06ce9..b95ea458 100644 --- a/ci/packages.json +++ b/ci/packages.json @@ -3,6 +3,7 @@ "Name": "fiftyone_pipeline_core", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_core", + "Ignore": ["**/tox.ini"], "DependsOn": [], "Publish": true }, @@ -10,6 +11,7 @@ "Name": "fiftyone_pipeline_engines", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_engines", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, @@ -17,6 +19,7 @@ "Name": "fiftyone_pipeline_engines_fiftyone", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_engines_fiftyone", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_engines"], "Publish": true }, @@ -24,6 +27,7 @@ "Name": "fiftyone_pipeline_cloudrequestengine", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_cloudrequestengine", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_engines"], "Publish": true }, @@ -31,13 +35,23 @@ "Name": "fiftyone_pipeline_translation", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_translation", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_pipeline_core"], "Publish": true }, + { + "Name": "fiftyone_pipeline_derivedproperty", + "Product": "pipeline", + "Path": "pipeline/fiftyone_pipeline_derivedproperty", + "Ignore": ["**/tox.ini"], + "DependsOn": ["fiftyone_pipeline_core"], + "Publish": false + }, { "Name": "fiftyone_pipeline_did", "Product": "pipeline", "Path": "pipeline/fiftyone_pipeline_did", + "Ignore": ["**/tox.ini"], "ExtraPaths": ["pipeline/owid-python"], "DependsOn": [], "Publish": true @@ -46,6 +60,7 @@ "Name": "fiftyone_pipeline_examples", "Product": "pipeline", "Path": "pipeline/examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -57,6 +72,7 @@ "Name": "fiftyone_devicedetection_shared", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_shared", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -69,6 +85,7 @@ "Name": "fiftyone_devicedetection_cloud", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_cloud", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_devicedetection_shared"], "Publish": true }, @@ -76,6 +93,7 @@ "Name": "fiftyone_devicedetection_onpremise", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_onpremise", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_shared", "fiftyone_pipeline_engines_fiftyone" @@ -87,6 +105,7 @@ "Name": "fiftyone_devicedetection", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_shared", "fiftyone_devicedetection_cloud", @@ -98,6 +117,7 @@ "Name": "fiftyone_devicedetection_examples", "Product": "device-detection", "Path": "device-detection/fiftyone_devicedetection_examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_devicedetection_cloud", "fiftyone_devicedetection" @@ -108,6 +128,7 @@ "Name": "fiftyone_location", "Product": "location", "Path": "location", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -119,6 +140,7 @@ "Name": "fiftyone_ipintelligence_shared", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_shared", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_engines", @@ -132,6 +154,7 @@ "Name": "fiftyone_ipintelligence_cloud", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_cloud", + "Ignore": ["**/tox.ini"], "DependsOn": ["fiftyone_ipintelligence_shared"], "Publish": true, "FirstVersion": "4.5.0" @@ -140,6 +163,7 @@ "Name": "fiftyone_ipintelligence_onpremise", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_onpremise", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_pipeline_engines_fiftyone" @@ -152,6 +176,7 @@ "Name": "fiftyone_ipintelligence_translation", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_translation", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_pipeline_core", "fiftyone_pipeline_translation" @@ -163,6 +188,7 @@ "Name": "fiftyone_ipintelligence", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence_shared", "fiftyone_ipintelligence_cloud", @@ -175,6 +201,7 @@ "Name": "fiftyone_ipintelligence_examples", "Product": "ip-intelligence", "Path": "ip-intelligence/fiftyone_ipintelligence_examples", + "Ignore": ["**/tox.ini"], "DependsOn": [ "fiftyone_ipintelligence", "fiftyone_ipintelligence_translation" diff --git a/ci/packages.ps1 b/ci/packages.ps1 index 429c43ff..17f73ca2 100644 --- a/ci/packages.ps1 +++ b/ci/packages.ps1 @@ -1,34 +1,26 @@ -# Shared by the scripts in this folder, which dot-source it. Reads the -# packages this repository publishes from packages.json, works out which -# products a pull request touches, and which packages a publish releases at -# which version. +# Shared by the scripts in this folder, which dot-source it. Works out which +# products a pull request touches, what a build of HEAD makes, and checks each +# package's tox.ini. # -# Each package has its own version, tagged '/'. A package is -# released when a file under its folder changed since its last tag, or when a -# package it depends on is being released, because its requirement on that -# package moves. This is the scheme in 51Degrees/rust#205, which the per -# language repositories follow. - -# Reads packages.json. Kept as a function so the tests can give their own list. -function Get-Packages([string]$RepoPath) { - return Get-Content -Raw (Join-Path $RepoPath 'ci/packages.json') | - ConvertFrom-Json -} - -# The folders whose changes count as a change to the package: its own folder, -# and any folder it takes files from at build time, such as a submodule it -# vendors. -function Get-PackagePaths($Package) { - $paths = @($Package.Path) - if ($Package.ExtraPaths) { - $paths += $Package.ExtraPaths - } - return $paths -} - -function Test-PathUnder([string]$Path, [string]$Folder) { - return $Path -eq $Folder -or $Path.StartsWith("$Folder/") +# Which packages a publish releases, and at which version, is worked out by +# common-ci's steps/release-plan.ps1 from ci/packages.json. common-ci runs +# these scripts from its own root, with this repository checked out inside +# it. A checkout of common-ci beside this repository is found too, or one +# elsewhere can be named with COMMON_CI. +$releasePlan = @( + $env:COMMON_CI, + (Join-Path $PSScriptRoot '..' '..'), + (Join-Path $PSScriptRoot '..' '..' 'common-ci')) | + Where-Object { $_ } | + ForEach-Object { Join-Path $_ 'steps' 'release-plan.ps1' } | + Where-Object { Test-Path $_ } | + Select-Object -First 1 +if ($null -eq $releasePlan) { + throw "common-ci's steps/release-plan.ps1 was not found. Set COMMON_CI " + + "to a checkout of common-ci." } +. $releasePlan +Remove-Variable releasePlan # Each tox.ini that does not install every package its package depends on, # directly or through another, from this checkout, as a line naming what is @@ -75,56 +67,6 @@ function Get-ToxGaps([string]$RepoPath, $Packages = $null) { return $gaps } -# The names of the given packages and of every package that depends on one of -# them, directly or through another. -function Get-DependentClosure($Packages, [string[]]$Names) { - $closure = [Collections.Generic.HashSet[string]]::new() - foreach ($name in @($Names | Where-Object { $_ })) { - $null = $closure.Add($name) - } - $grew = $true - while ($grew -eq $true) { - $grew = $false - foreach ($package in $Packages) { - if ($closure.Contains($package.Name) -eq $true) { - continue - } - foreach ($dependency in $package.DependsOn) { - if ($closure.Contains($dependency) -eq $true) { - $null = $closure.Add($package.Name) - $grew = $true - break - } - } - } - } - return @($closure) -} - -# The named packages, each after every package it depends on, so a release -# never publishes a package before the version it requires exists. -function Sort-ByDependency($Packages, [string[]]$Names) { - $remaining = [Collections.Generic.List[string]]::new([string[]]@($Names)) - $sorted = [Collections.Generic.List[string]]::new() - while ($remaining.Count -gt 0) { - $ready = @($remaining | Where-Object { - $name = $_ - $package = $Packages | Where-Object { $_.Name -eq $name } - @($package.DependsOn | Where-Object { - $remaining.Contains($_) -eq $true - }).Count -eq 0 - } | Sort-Object) - if ($ready.Count -eq 0) { - throw "packages.json has a dependency cycle among: $($remaining -join ', ')" - } - foreach ($name in $ready) { - $sorted.Add($name) - $null = $remaining.Remove($name) - } - } - return @($sorted) -} - # The products a set of changed files touches. A file in a package's folders # touches that package's product and the product of every package depending on # it, because a change to pipeline can break device-detection. @@ -169,14 +111,14 @@ function Resolve-AffectedProducts($Packages, [string[]]$Changed) { # The products a pull request changes, or every product when the checkout is # not a pull request (a publish, or a run on main), as 51degrees-cxx does. -function Get-AffectedProducts([string]$RepoPath) { - # merge-base fails where there is no origin/main, which is an answer here. +function Get-AffectedProducts([string]$RepoPath, [string]$Branch = 'main') { + # merge-base fails where there is no such branch, which is an answer here. # Set for this function only, so a caller that stops on a failing native # command, as publish-package.ps1 does, is not stopped by it. $PSNativeCommandUseErrorActionPreference = $false $packages = Get-Packages $RepoPath $head = git -C $RepoPath rev-parse HEAD - $base = git -C $RepoPath merge-base origin/main HEAD 2>$null + $base = git -C $RepoPath merge-base "origin/$Branch" HEAD 2>$null if ($LASTEXITCODE -ne 0 -or $null -eq $base -or $base -eq $head) { $global:LASTEXITCODE = 0 return @($packages.Product | Select-Object -Unique) @@ -185,114 +127,6 @@ function Get-AffectedProducts([string]$RepoPath) { return Resolve-AffectedProducts $packages $changed } -# The parts of a version such as 4.5.38, or $null for anything else. -function ConvertTo-VersionParts([string]$Text) { - $match = [regex]::Match($Text, '^(\d+)\.(\d+)\.(\d+)$') - if ($match.Success -eq $false) { - return $null - } - return [version]::new([int]$match.Groups[1].Value, - [int]$match.Groups[2].Value, [int]$match.Groups[3].Value) -} - -# The package's released versions, read from its '/' tags, -# highest last. The '/' after the name keeps fiftyone_ipintelligence from -# reading the tags of fiftyone_ipintelligence_cloud. -function Get-ReleasedVersions([string]$RepoPath, [string]$Name, - [switch]$AtHead) { - # Tags are found by prefix, not with a 'name/*' pattern: PowerShell on - # Linux and macOS expands a '*' in an argument to a native command into - # matching files, so from a folder holding a 'location' folder git would - # be handed file names and find no tags. - $arguments = @('-C', $RepoPath, 'for-each-ref', - '--format=%(refname:strip=2)', "refs/tags/$Name/") - if ($AtHead -eq $true) { - $arguments += @('--points-at', 'HEAD') - } - $versions = foreach ($tag in (& git @arguments)) { - ConvertTo-VersionParts $tag.Substring($Name.Length + 1) - } - return @($versions | Where-Object { $null -ne $_ } | Sort-Object) -} - -# What a publish of HEAD releases: one entry per package, in the order to -# publish them, each with the version it is released at and why. -# -# A package already tagged at HEAD was released from this commit, so it is -# left out, which makes a second publish of the same commit release nothing. -# A package with no tag has never been released and starts at its FirstVersion -# from packages.json. Packages that are not published, the examples, are never -# in the plan; a change to them is tested but releases nothing. -function Get-ReleasePlan([string]$RepoPath, $Packages = $null) { - # 'git diff --quiet' exits 1 to say a package changed. That is an answer, - # not a failure, but a caller that stops on a failing native command, as - # publish-package.ps1 does, stopped there and failed the publish. Set for - # this function only. - $PSNativeCommandUseErrorActionPreference = $false - if ($null -eq $Packages) { - $Packages = Get-Packages $RepoPath - } - $published = @($Packages | Where-Object { $_.Publish -eq $true }) - - $previous = @{} - $reasons = @{} - foreach ($package in $published) { - $atHead = Get-ReleasedVersions $RepoPath $package.Name -AtHead - if ($atHead.Count -gt 0) { - continue - } - $released = Get-ReleasedVersions $RepoPath $package.Name - if ($released.Count -eq 0) { - if ([string]::IsNullOrEmpty($package.FirstVersion) -eq $true) { - throw "$($package.Name) has never been released and " + - "packages.json gives it no FirstVersion." - } - $previous[$package.Name] = $null - $reasons[$package.Name] = 'first release' - continue - } - $last = $released[-1] - $previous[$package.Name] = $last - $tag = "$($package.Name)/$last" - & git -C $RepoPath diff --quiet $tag HEAD -- (Get-PackagePaths $package) - if ($LASTEXITCODE -eq 1) { - $reasons[$package.Name] = "changed since $last" - } - $global:LASTEXITCODE = 0 - } - - # A dependent is released with its dependency, as long as it was not - # already released from this commit. - $candidates = @($published.Name | Where-Object { - $previous.ContainsKey($_) -eq $true - }) - $changed = @($reasons.Keys) - $toRelease = @(Get-DependentClosure $published $changed | Where-Object { - $candidates -contains $_ - }) - - foreach ($name in (Sort-ByDependency $published $toRelease)) { - $package = $published | Where-Object { $_.Name -eq $name } - $last = $previous[$name] - if ($null -eq $last) { - $version = $package.FirstVersion - } else { - $version = "$($last.Major).$($last.Minor).$($last.Build + 1)" - } - $reason = $reasons[$name] - if ($null -eq $reason) { - $reason = 'a package it depends on is released' - } - [pscustomobject]@{ - Name = $name - Path = $package.Path - Version = $version - Previous = $last - Reason = $reason - } - } -} - # What a build of HEAD makes: the release plan, or, when nothing changed since # the last release, every published package at its released version. Nothing # is published then, because package-update-required.ps1 finds the release diff --git a/ci/release-plan-filter.ps1 b/ci/release-plan-filter.ps1 new file mode 100644 index 00000000..dcd316aa --- /dev/null +++ b/ci/release-plan-filter.ps1 @@ -0,0 +1,39 @@ +param ( + [Parameter(Mandatory)][string]$RepoPath, + [Parameter(Mandatory)]$Package, + [Parameter(Mandatory)][string]$From, + [Parameter(Mandatory)][string[]]$Files +) +$ErrorActionPreference = "Stop" + +# Called by the release plan for each package with changed files left once +# the Ignore globs have left theirs out, and returns the files that count. +# A pyproject.toml, setup.py or setup.cfg whose change since $From is only to +# the description, keywords, urls, classifiers, readme, authors or maintainers +# does not count, which release-plan-filter.py works out. Every other file +# counts. +# +# Python is required rather than optional, because a job that counted every +# file would plan different releases from a job that could compare them. + +$manifests = 'pyproject.toml', 'setup.py', 'setup.cfg' +$compare = @($Files | Where-Object { (Split-Path -Leaf $_) -in $manifests }) +$counted = @($Files | Where-Object { (Split-Path -Leaf $_) -notin $manifests }) + +if ($compare.Count -gt 0) { + $python = Get-Command python, python3 -CommandType Application ` + -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($null -eq $python) { + throw "Python is needed to tell whether a change to " + + "$($compare -join ', ') is only to metadata, and none was found." + } + $PSNativeCommandUseErrorActionPreference = $false + $script = Join-Path $PSScriptRoot 'release-plan-filter.py' + $changed = @(& $python.Source -I $script $RepoPath $From @compare) + if ($LASTEXITCODE -ne 0) { + throw "release-plan-filter.py failed for $($Package.Name) since $From." + } + $counted += $changed | Where-Object { $_ } +} + +$counted diff --git a/ci/release-plan-filter.py b/ci/release-plan-filter.py new file mode 100644 index 00000000..94191c90 --- /dev/null +++ b/ci/release-plan-filter.py @@ -0,0 +1,194 @@ +"""Prints the manifest files whose change since a tag is more than metadata. + +Usage: release-plan-filter.py [ ...] + +Each file is a pyproject.toml, setup.py or setup.cfg, as a path from the +repository root. Its text at the tag and at HEAD is compared with the +description, keywords, urls, classifiers, readme, authors and maintainers +left out, so a file is printed when anything else changed, and left out when +only those did. A file that is missing on either side, or that cannot be read as what +its name says, is printed, so a release is never skipped by mistake. + +Only the standard library is used, and the same way on every Python this +repository supports, so every job of a publish works out the same plan. +""" + +import ast +import configparser +import os +import re +import subprocess +import sys + +# The setup() arguments and setup.cfg [metadata] options that are metadata. +SETUP_METADATA = frozenset(( + "description", "long_description", "long_description_content_type", + "keywords", "url", "project_urls", "download_url", "classifiers", + "author", "author_email", "maintainer", "maintainer_email")) + +# The pyproject.toml [project] keys that are metadata. +PROJECT_METADATA = frozenset(( + "description", "keywords", "urls", "classifiers", "readme", "authors", + "maintainers")) + +# A table header, such as [project.urls] or [[project.maintainers]]. +HEADER = re.compile(r"^\[\[?\s*([^\]]+?)\s*\]\]?\s*(#.*)?$") + +# The start of a key and value. The first part of a dotted key is the one +# that says what it is, as urls in urls.Homepage. +ASSIGNMENT = re.compile( + r"^(\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+)" + r"(?:\s*\.\s*(?:\"[^\"]*\"|'[^']*'|[A-Za-z0-9_-]+))*\s*=") + + +def bytes_at(repository, revision, path): + """The file's content at the revision, or None when it is not there.""" + found = subprocess.run( + ["git", "-C", repository, "show", revision + ":" + path], + capture_output=True) + if found.returncode != 0: + return None + return found.stdout + + +class StripSetupMetadata(ast.NodeTransformer): + """Removes the metadata arguments from every call of setup().""" + + def visit_Call(self, node): + self.generic_visit(node) + name = node.func.attr if isinstance(node.func, ast.Attribute) \ + else getattr(node.func, "id", None) + if name == "setup": + node.keywords = [ + keyword for keyword in node.keywords + if keyword.arg not in SETUP_METADATA] + return node + + +def setup_py(text): + # The syntax tree leaves out comments and layout as well. + return ast.dump(StripSetupMetadata().visit(ast.parse(text))) + + +def setup_cfg(text): + parser = configparser.RawConfigParser() + parser.optionxform = str + parser.read_string(text) + return { + section: { + key: value for key, value in parser.items(section) + if section != "metadata" or key not in SETUP_METADATA} + for section in parser.sections()} + + +def toml_lines(text): + """The lines of a TOML file that are not metadata, each with the table + and key it belongs to, leaving out blank lines and comments. + + A small reader rather than tomllib, which Python 3.10 does not have. It + follows tables, keys, values that run over several lines in brackets or + in triple quotes, and comments, which is enough to place every line. + """ + kept = [] + table = "" + key = None + depth = 0 + quote = None + for line in text.splitlines(): + stripped = line.strip() + if quote is None and depth == 0: + if stripped == "" or stripped.startswith("#"): + continue + header = HEADER.match(stripped) + if header: + table = header.group(1).replace(" ", "") + key = None + if not is_metadata(table, None): + kept.append((table, None, stripped)) + continue + assignment = ASSIGNMENT.match(stripped) + if assignment is None: + # Not TOML this reader knows, so it counts unless the whole + # table is metadata. + if not is_metadata(table, "?"): + kept.append((table, "?", stripped)) + continue + key = assignment.group(1).strip("\"'") + value = stripped[assignment.end():] + else: + value = stripped + depth, quote = scan(value, depth, quote) + if not is_metadata(table, key): + kept.append((table, key, stripped)) + return kept + + +def scan(value, depth, quote): + """The bracket depth and the open triple quote after the value.""" + position = 0 + while position < len(value): + if quote is not None: + end = value.find(quote, position) + if end < 0: + return depth, quote + position = end + 3 + quote = None + continue + character = value[position] + if value.startswith('"""', position) or \ + value.startswith("'''", position): + quote = value[position:position + 3] + position += 3 + continue + if character in "\"'": + end = value.find(character, position + 1) + position = len(value) if end < 0 else end + 1 + continue + if character == "#": + break + if character in "[{": + depth += 1 + elif character in "]}": + depth -= 1 + position += 1 + return depth, quote + + +def is_metadata(table, key): + if table == "project": + return key in PROJECT_METADATA + return table in ( + "project.urls", "project.authors", "project.maintainers", + "project.readme") + + +READERS = { + "setup.py": setup_py, + "setup.cfg": setup_cfg, + "pyproject.toml": toml_lines, +} + + +def counts(repository, tag, path): + """Whether the change to the file since the tag is more than metadata.""" + reader = READERS.get(os.path.basename(path)) + before = bytes_at(repository, tag, path) + after = bytes_at(repository, "HEAD", path) + if reader is None or before is None or after is None: + return True + try: + return reader(before.decode("utf-8")) != reader(after.decode("utf-8")) + except (SyntaxError, ValueError, configparser.Error, + UnicodeDecodeError): + return True + + +def main(arguments): + repository, tag = arguments[0], arguments[1] + for path in arguments[2:]: + if counts(repository, tag, path): + print(path) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/ci/run-integration-tests.ps1 b/ci/run-integration-tests.ps1 index ae1c20f8..59885cae 100644 --- a/ci/run-integration-tests.ps1 +++ b/ci/run-integration-tests.ps1 @@ -1,6 +1,12 @@ param ( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][hashtable]$Keys + [Parameter(Mandatory)][hashtable]$Keys, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main', + # False, from select-options.ps1, on every entry of a pull request but + # one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/run-performance-tests.ps1 b/ci/run-performance-tests.ps1 index 58401f22..4fa56758 100644 --- a/ci/run-performance-tests.ps1 +++ b/ci/run-performance-tests.ps1 @@ -1,7 +1,10 @@ param ( [Parameter(Mandatory)][string]$RepoName, [string]$Name, - [bool]$RunPerformance + [bool]$RunPerformance, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/run-unit-tests.ps1 b/ci/run-unit-tests.ps1 index e74a60ff..38e3ee6e 100644 --- a/ci/run-unit-tests.ps1 +++ b/ci/run-unit-tests.ps1 @@ -1,5 +1,8 @@ param ( - [Parameter(Mandatory)][string]$RepoName + [Parameter(Mandatory)][string]$RepoName, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/ci/select-options.Tests.ps1 b/ci/select-options.Tests.ps1 new file mode 100644 index 00000000..9e0042dc --- /dev/null +++ b/ci/select-options.Tests.ps1 @@ -0,0 +1,69 @@ +# Tests for ci/select-options.ps1, with the entries of the real options.json +# read as common-ci's steps/select-options.ps1 reads them, as hashtables. + +BeforeAll { + $script:Select = Join-Path $PSScriptRoot 'select-options.ps1' + + # The names of the entries chosen for the changed files given, and the + # entries themselves in $script:Chosen. + function Get-Chosen([string[]]$ChangedFiles, [string[]]$CommentOnlyFiles = @()) { + $options = @(Get-Content -Raw (Join-Path $PSScriptRoot 'options.json') | + ConvertFrom-Json -AsHashtable) + $script:Chosen = @(& $script:Select -Options $options ` + -ChangedFiles $ChangedFiles -CommentOnlyFiles $CommentOnlyFiles 6>$null) + return @($script:Chosen | ForEach-Object { $_.Name } | Sort-Object) + } + + $script:StandardSet = @('Ubuntu_Python_3.10', 'Ubuntu_Python_3.14') +} + +Describe 'select-options.ps1' { + It 'builds the standard set for a change to a package''s code' { + Get-Chosen @('pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py') | + Should -Be $script:StandardSet + } + + It 'runs the Selenium contract on Python 3.14 alone, and no performance tests' { + $null = Get-Chosen @('location/src/fiftyone_location/location_cloud.py') + foreach ($entry in $script:Chosen) { + $entry.SeleniumContract | Should -Be ($entry.Name -eq 'Ubuntu_Python_3.14') + $entry.RunPerformance | Should -BeFalse + } + } + + It 'adds Windows for a change to an on-premise package' { + Get-Chosen @('device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py') | + Should -Be (@($script:StandardSet + 'Windows_Python_3.14') | Sort-Object) + } + + It 'adds Windows for a change to the cxx submodule' { + Get-Chosen @('cxx') | + Should -Be (@($script:StandardSet + 'Windows_Python_3.14') | Sort-Object) + } + + It 'leaves Windows out for a change only to the comments of an on-premise file' { + $file = 'ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py' + Get-Chosen @($file) @($file) | Should -Be $script:StandardSet + } + + It 'adds Python 3.11 to 3.13 on Linux for a change to a setup.py' { + Get-Chosen @('location/setup.py') | Should -Be (@($script:StandardSet + + 'Ubuntu_Python_3.11' + 'Ubuntu_Python_3.12' + 'Ubuntu_Python_3.13') | + Sort-Object) + } + + It 'builds every entry but macOS for a change to ' -ForEach @( + @{ Case = 'a ci folder'; File = 'device-detection/ci/build-project.ps1' } + @{ Case = 'the root ci folder'; File = 'ci/packages.ps1' } + @{ Case = 'a file outside the product folders'; File = '.github/workflows/pull-requests.yml' } + ) { + $chosen = Get-Chosen @($File) + $chosen.Count | Should -Be 10 + $chosen | Where-Object { $_ -like 'macOS*' } | Should -BeNullOrEmpty + } + + It 'never builds macOS on a pull request' { + Get-Chosen @('cxx', 'ci/packages.ps1', 'location/setup.py') | + Where-Object { $_ -like 'macOS*' } | Should -BeNullOrEmpty + } +} diff --git a/ci/select-options.ps1 b/ci/select-options.ps1 new file mode 100644 index 00000000..cc22871f --- /dev/null +++ b/ci/select-options.ps1 @@ -0,0 +1,73 @@ +param( + # Every entry of options.json. + [Parameter(Mandatory)][object[]]$Options, + # The files the pull request changes, less documentation. + [string[]]$ChangedFiles, + # Those of the changed files whose change touches only comments. + [string[]]$CommentOnlyFiles +) +# Chooses the options.json entries a pull request is built on, for common-ci's +# steps/select-options.ps1. The branch itself and a release build every entry. +# Within each entry, packages.ps1 still builds only the products the change +# touches. +# +# The standard set is Python 3.14 and 3.10 on Linux, the newest and the oldest, +# with the Selenium contract on 3.14 alone. A change in more than its comments +# to an on-premise package, its data or the cxx submodule adds Windows, where +# the native module is built differently. A change to a setup.py, setup.cfg or +# pyproject.toml adds Python 3.11 to 3.13 on Linux, and a change outside the +# product folders or to a ci folder adds every entry. Only five macOS jobs run +# at once across the organisation, and the releases need them, so macOS is left +# to the branch and release builds, as is performance. + +$Standard = @('Ubuntu_Python_3.14', 'Ubuntu_Python_3.10') +$Contract = 'Ubuntu_Python_3.14' +$Native = @('Windows_Python_3.14') +$Versions = @('Ubuntu_Python_3.13', 'Ubuntu_Python_3.12', 'Ubuntu_Python_3.11') +$ProductFolders = @('pipeline/', 'device-detection/', 'ip-intelligence/', + 'location/') +# The data files come in through the cxx submodule, so a change to them is a +# change to cxx. +$NativePaths = @('device-detection/fiftyone_devicedetection_onpremise/', + 'ip-intelligence/fiftyone_ipintelligence_onpremise/') + +function Test-Under([string]$Path, [string[]]$Prefixes) { + return @($Prefixes | Where-Object { + $Path -eq $_.TrimEnd('/') -or $Path.StartsWith($_.TrimEnd('/') + '/') + }).Count -gt 0 +} + +# PowerShell names are not case sensitive, so these must not share a name with +# the lists above, such as $Native. +$code = @($ChangedFiles | Where-Object { $CommentOnlyFiles -notcontains $_ }) +$nativeChanged = @($code | Where-Object { + $_ -eq 'cxx' -or (Test-Under $_ $NativePaths) +}).Count -gt 0 +$setupChanged = @($code | Where-Object { + $_ -match '(^|/)(setup\.py|setup\.cfg|pyproject\.toml)$' +}).Count -gt 0 +$otherChanged = @($code | Where-Object { + $_ -ne 'cxx' -and (!(Test-Under $_ $ProductFolders) -or $_ -match '(^|/)ci/') +}).Count -gt 0 +Write-Host ("Native: $nativeChanged. Setup: $setupChanged. " + + "Outside the product folders or in a ci folder: $otherChanged.") + +foreach ($entry in $Options) { + $name = $entry.Name + if ($entry.Image -like 'macos*') { + Write-Host "$name is left to the branch and release builds" + continue + } + $why = if ($Standard -contains $name) { 'the standard set' } + elseif ($otherChanged) { 'a change outside the product folders or to CI' } + elseif ($nativeChanged -and $Native -contains $name) { 'a native change' } + elseif ($setupChanged -and $Versions -contains $name) { 'a packaging change' } + if (!$why) { + Write-Host "$name is not needed for this change" + continue + } + $entry.SeleniumContract = $name -eq $Contract + $entry.RunPerformance = $false + Write-Host "$name runs for $why" + $entry +} diff --git a/ci/setup-environment.ps1 b/ci/setup-environment.ps1 index 932f5d81..24c1b8bd 100644 --- a/ci/setup-environment.ps1 +++ b/ci/setup-environment.ps1 @@ -1,6 +1,9 @@ param ( [Parameter(Mandatory)][string]$RepoName, - [Parameter(Mandatory)][string]$LanguageVersion + [Parameter(Mandatory)][string]$LanguageVersion, + # The branch a pull request merges into, which its changes are measured + # against. + [string]$Branch = 'main' ) $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'dispatch.ps1') diff --git a/cxx b/cxx index 7992af70..178371ac 160000 --- a/cxx +++ b/cxx @@ -1 +1 @@ -Subproject commit 7992af7010b7654c115510938a2f7626204bf936 +Subproject commit 178371ac8d5dec71e625fcdcb833d1dcd1ce3480 diff --git a/device-detection/.github/workflows/monthly-copyright-update.yml b/device-detection/.github/workflows/monthly-copyright-update.yml deleted file mode 100644 index 8a8536f6..00000000 --- a/device-detection/.github/workflows/monthly-copyright-update.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Monthly Copyright Update - -on: - workflow_dispatch: - inputs: - dryrun: - default: false - type: boolean - description: Dry Run - schedule: - # The copyright year only changes once a year, so this runs on 1 January. - - cron: '0 0 1 1 *' - -jobs: - Monthly_Copyright_Update: - uses: 51Degrees/common-ci/.github/workflows/monthly-copyright-update.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} diff --git a/device-detection/.github/workflows/nightly-pipeline.yml b/device-detection/.github/workflows/nightly-pipeline.yml deleted file mode 100644 index dba1de47..00000000 --- a/device-detection/.github/workflows/nightly-pipeline.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Nightly Pipeline - -on: - workflow_dispatch: - inputs: - dryrun: - type: boolean - default: false - schedule: - - cron: '0 2 * * *' - -jobs: - SubmoduleUpdate: - name: Nightly Submodule Update - uses: 51Degrees/common-ci/.github/workflows/nightly-submodule-update.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - - PullRequests: - if: ${{ !cancelled() }} - needs: SubmoduleUpdate - name: Nightly Pull Requests - uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} - - Publish: - if: ${{ !cancelled() }} - needs: PullRequests - name: Nightly Publish - uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} - PypiToken: ${{ secrets.PYPI_TOKEN }} - TestPypiToken: ${{ secrets.TEST_PYPI_TOKEN }} diff --git a/device-detection/.github/workflows/publish.yml b/device-detection/.github/workflows/publish.yml deleted file mode 100644 index cf1eb28a..00000000 --- a/device-detection/.github/workflows/publish.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Publish - -on: - workflow_dispatch: - inputs: - dryrun: - type: boolean - default: false - -jobs: - Publish: - name: Publish - uses: 51Degrees/common-ci/.github/workflows/nightly-publish.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} - PypiToken: ${{ secrets.PYPI_TOKEN }} - TestPypiToken: ${{ secrets.TEST_PYPI_TOKEN }} diff --git a/device-detection/.github/workflows/pull-requests.yml b/device-detection/.github/workflows/pull-requests.yml deleted file mode 100644 index 434e7918..00000000 --- a/device-detection/.github/workflows/pull-requests.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Pull Requests - -on: - workflow_dispatch: - inputs: - dryrun: - type: boolean - default: false - -jobs: - PullRequests: - name: Pull Requests - uses: 51Degrees/common-ci/.github/workflows/nightly-pull-requests.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - cache-assets: true - secrets: - token: ${{ secrets.ACCESS_TOKEN }} - DeviceDetection: ${{ secrets.DEVICE_DETECTION_KEY }} - TestResourceKey: ${{ secrets._51DEGREES_RESOURCE_KEY_BESPOKE }} - DeviceDetectionUrl: ${{ secrets.DEVICE_DETECTION_URL }} - CsvUrl: ${{ secrets.CSV_URL }} diff --git a/device-detection/.github/workflows/submodule-update.yml b/device-detection/.github/workflows/submodule-update.yml deleted file mode 100644 index 76c78c0f..00000000 --- a/device-detection/.github/workflows/submodule-update.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Submodule Update - -on: - workflow_dispatch: - inputs: - dryrun: - type: boolean - default: false - -jobs: - SubmoduleUpdate: - name: Submodule Update - uses: 51Degrees/common-ci/.github/workflows/nightly-submodule-update.yml@main - with: - repo-name: ${{ github.event.repository.name }} - org-name: ${{ github.event.repository.owner.login }} - dryrun: ${{ inputs.dryrun || false }} - secrets: - token: ${{ secrets.ACCESS_TOKEN }} diff --git a/device-detection/.github/workflows/utm-link-lint.yml b/device-detection/.github/workflows/utm-link-lint.yml deleted file mode 100644 index 2edc6722..00000000 --- a/device-detection/.github/workflows/utm-link-lint.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: UTM link lint - -on: - pull_request: - push: - branches: - - main - workflow_dispatch: - -jobs: - utm-link-lint: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Checkout common-ci - uses: actions/checkout@v4 - with: - repository: 51Degrees/common-ci - path: utm-lint-common-ci - - - name: Lint 51degrees.com links - shell: pwsh - run: ./utm-lint-common-ci/scripts/utm-lint.ps1 -RepoRoot . diff --git a/device-detection/ci/README.md b/device-detection/ci/README.md index 76a328d6..570aa052 100644 --- a/device-detection/ci/README.md +++ b/device-detection/ci/README.md @@ -10,7 +10,3 @@ The following secrets are required: * Example: `V3RYL0NGR4ND0M57R1NG` * `PYPI_TOKEN` - [PyPI token](https://pypi.org/help/#apitoken) for publishing packages * Example: `pypi-sUp3r-l0nG_r4nd0m-s7r1Ng` - -The following secrets are optional: -* `DEVICE_DETECTION_URL` - URL for downloading the enterprise TAC hashes file - * Default: `https://distributor.51degrees.com/api/v2/download?LicenseKeys=DEVICE_DETECTION_KEY&Type=HashV41&Download=True&Product=V4TAC` diff --git a/device-detection/ci/fetch-assets.ps1 b/device-detection/ci/fetch-assets.ps1 index a2501740..78b11657 100644 --- a/device-detection/ci/fetch-assets.ps1 +++ b/device-detection/ci/fetch-assets.ps1 @@ -1,7 +1,5 @@ param ( - [string]$DeviceDetection, - [string]$DeviceDetectionUrl, - [string]$CsvUrl + [string]$DeviceDetection ) $ErrorActionPreference = 'Stop' @@ -21,7 +19,7 @@ if ($DeviceDetection) { Write-Host "::warning file=$($MyInvocation.ScriptName),line=$($MyInvocation.ScriptLineNumber),title=No License Key::A device detection license was not provided, so On-Premise Data file and 51Degrees.csv will not be downloaded." } -./steps/fetch-assets.ps1 -DeviceDetection:$DeviceDetection -DeviceDetectionUrl:$DeviceDetectionUrl -CsvUrl:$CsvUrl -Assets $assets +./steps/fetch-assets.ps1 -DeviceDetection:$DeviceDetection -Assets $assets foreach ($asset in $assets) { switch -Exact -CaseSensitive ($asset) { 'TAC-HashV41.hash' {New-Item -ItemType SymbolicLink -Force -Target "$PWD/assets/$_" -Path "$deviceDetectionData/Enterprise-HashV41.hash"} diff --git a/device-detection/ci/run-integration-tests.ps1 b/device-detection/ci/run-integration-tests.ps1 index 9c45928e..ed7a17d2 100644 --- a/device-detection/ci/run-integration-tests.ps1 +++ b/device-detection/ci/run-integration-tests.ps1 @@ -2,7 +2,10 @@ param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] - [Hashtable]$Keys + [Hashtable]$Keys, + # False, from the root select-options.ps1, on every entry of a pull + # request but one, so the browser contract runs once. + [bool]$SeleniumContract = $true ) if ($IsWindows) { @@ -47,6 +50,14 @@ if ($LASTEXITCODE -ne 0) { $failures += "the Python integration tests exited with code $LASTEXITCODE" } +if (-not $SeleniumContract) { + Write-Output "Another entry runs the Selenium contract tests for this pull request." + if ($failures.Count -gt 0) { + throw "Integration tests failed: $($failures -join '; ')" + } + return +} + # The shared Selenium contract tests are run against the cloud example and the # on-premise example in turn. Each result is also written as a trx file next # to the other integration results, so it is published with them. diff --git a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py index b439c3d3..b17ecaa5 100644 --- a/device-detection/fiftyone_devicedetection/examples/gettingstarted.py +++ b/device-detection/fiftyone_devicedetection/examples/gettingstarted.py @@ -21,11 +21,36 @@ # ********************************************************************* from fiftyone_devicedetection.devicedetection_pipelinebuilder import DeviceDetectionPipelineBuilder +from fiftyone_devicedetection_examples.example_utils import ExampleUtils + +# Where to ask for a copy of the Enterprise data file, tagged with this +# example. +CONTACT_US = "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-examples-gettingstarted.py&utm_term=enterprise-data-file" + +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-examples-gettingstarted.py&utm_term=find-out-more-device-detection"), + ("Enterprise data file", + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-examples-gettingstarted.py&utm_term=find-out-more-enterprise-data-file"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection-examples-gettingstarted.py&utm_term=find-out-more-resource-keys"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), + ("Device detection engine", + "https://github.com/51Degrees/51degrees-cxx"), +] # First create the device detection pipeline with the desired settings. -data_file = "../../cxx/data/device-detection/51Degrees-LiteV4.1.hash" -resource_key = "!!YOUR_RESOURCE_KEY!!" +# The data file named in the _51DEGREES_DD_PATH environment variable, or +# else the free Lite data file, found in the folder this runs in or a folder +# above it, or downloaded when it is missing. +data_file = (ExampleUtils.get_data_file_path() + or ExampleUtils.find_file("51Degrees-LiteV4.1.hash")) +# The resource key for the Cloud line below, from the +# _51DEGREES_RESOURCE_KEY environment variable. +resource_key = ExampleUtils.get_resource_key() or "!!YOUR_RESOURCE_KEY!!" # Uncommment the line below 'On-Premise' and comment the line below 'Cloud' to # switch to an On-Premise installation @@ -81,3 +106,18 @@ else: # Output why the value isn't meaningful print(flowdata2.device.ismobile.no_value_message()) + + # The hardware vendor is only in the Enterprise data file, so the + # free Lite data file says so and where to ask for a copy. + engine = pipeline.get_element("device") + vendor = ExampleUtils.get_enterprise_value( + engine, flowdata2.device, "hardwarevendor") + print("Who made the device? ") + print(vendor) + if vendor == ExampleUtils.NEEDS_ENTERPRISE: + print(ExampleUtils.get_enterprise_note(CONTACT_US)) + +print("=" * 79) +print("Find out more:") +for label, url in FIND_OUT_MORE: + print(f"\t{label}: {url}") diff --git a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py index 1a735f87..e227fdd2 100644 --- a/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection/src/fiftyone_devicedetection/devicedetection_pipelinebuilder.py @@ -80,7 +80,10 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type restricted_properties: list @param restricted_properties: List of property keys to limit the engine to @type license_keys: string - @param licence_keys: licencekeys to use for the data file update service + @param licence_keys: licence keys to use for the data file update service. + Needed only when auto_update or update_on_start is on with the default + data_update_url, and the engine refuses to build without one then. The + file system watcher needs no key. @type download: bool @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service @type max_matched_useragent_length: int diff --git a/device-detection/fiftyone_devicedetection_cloud/readme.md b/device-detection/fiftyone_devicedetection_cloud/readme.md index 07923fdc..c41fda5c 100644 --- a/device-detection/fiftyone_devicedetection_cloud/readme.md +++ b/device-detection/fiftyone_devicedetection_cloud/readme.md @@ -12,7 +12,7 @@ You can confirm this is working with the following micro-example. -* Create a resource key for free with the 51Degrees [configurator](https://configure.51degrees.com/np5M4nlF?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=from-pypi). This defines the properties you want to access. +* Create a resource key for free with the 51Degrees [configuration page](https://51degrees.com/configure/np5M4nlF?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_cloud&utm_term=from-pypi). This defines the properties you want to access. * On the 'implement' page of the configurator, copy the resource key and replace YOUR_RESOURCE_KEY in the example below. Save this as exampledd.py * Run the example with `python exampledd.py` * Feel free to try different user-agents and property values. diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py index 8380803d..a50325e7 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/devicedetection_cloud_pipelinebuilder.py @@ -38,9 +38,9 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): """! - Constructor for Device Detection Pipeline Builder - Depending on parameters passed through, builds a pipeline - with either the cloud or on premise device detection engines + Constructor for the Device Detection Cloud Pipeline Builder, which + builds a pipeline with the cloud request engine and the device + detection cloud engine. General pipeline settings: @@ -67,47 +67,6 @@ def __init__(self, settings = None, usage_sharing = True, **kwargs): @type cloud_request_origin: string @param cloud_request_origin: The value to set the Origin header to when making requests to the cloud service - On Premise Engine Settings - - @type usage_sharing : bool - @param usage_sharing : Whether to enable usage sharing if using the on premise engine - @type data_file_path: string - @param data_file_path: path to the data file - @type auto_update: bool - @param auto_update: whether to automatically update the datafile when required - @type restricted_properties: list - @param restricted_properties: List of property keys to limit the engine to - @type license_keys: string - @param licence_keys: licencekeys to use for the data file update service - @type download: bool - @param download: whether to download the datafile or keep it in memory when it is returned from the datafile update service - @type drift: int - @param drift: Set maximum drift in hash position to allow when processing HTTP headers - @type difference: int - @param difference: set the maximum difference to allow when processing HTTP headers. The difference is the difference in hash value between the hash that was found, and the hash that is being searched for. By default this is 0. - @type allow_unmatched : bool - @param allow_unmatched: True if there should be at least one matched node in order for the results to be considered valid. By default, this is false - @type performance_profile: string - @param performance_profile: options are: LowMemory, MaxPerformance, Balanced, BalancedTemp, HighPerformance - @type reuse_temp_file: bool - @param reuse_temp_file: Indicates that an existing temp file may be used. This should be selected if multiple instances wish to use the same file to prevent high disk usage. - @type concurrency: int - @param concurrency: defaults to the number of cpus in the machine - @type update_on_start : bool - @param update_on_start : When this is set to true the datafile is updated / downloaded immediately on initialisation. This is useful if no initial datafile is present. - @type file_system_watcher: bool - @param file_system_watcher: whether to check the datafile's path for changes and update the connected FlowElement's data - @type polling_interval: int - @param polling_interval: How often to poll for updates to the datafile (minutes) - @type update_time_maximum_randomisation : int - @param update_time_maximum_randomisation : - Maximum randomisation offset in seconds to polling time interval - @type verify_md5 : bool - @type create_temp_copy: bool - @param create_temp_copy: whether to copy datafile to temporary location when updating - @type data_file_update_base_url: string - @param data_file_update_base_url: base url for the datafile update service - """ if settings is None: settings = {} diff --git a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py index 4f40f3c1..a19b7da2 100644 --- a/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/src/fiftyone_devicedetection_cloud/hardwareprofile_cloud.py @@ -22,7 +22,10 @@ from __future__ import absolute_import from fiftyone_pipeline_cloudrequestengine.cloudengine import CloudEngine +from fiftyone_pipeline_cloudrequestengine.clouddata import \ + CloudMissingPropertyService +from fiftyone_pipeline_engines.aspectdata import AspectData from fiftyone_pipeline_engines.aspectdata_dictionary import AspectDataDictionary from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue @@ -32,6 +35,140 @@ # that property has no value, for example "hardwarevendornullreason". NULL_REASON_SUFFIX = "nullreason" +# The key the cloud request engine's data is read under, and the old key, +# which is deprecated. An older cloud request engine gives the old key +# alone. +RESPONSE_DATA_KEY = "cloud-response" +OLD_DATA_KEY = "cloud" + + +def _cloud_response(flowdata): + """! + The raw JSON the cloud request engine gave, read under cloud-response, + or under the old key when there is nothing under cloud-response. + + @rtype string + @return the JSON text of the cloud service's answer + """ + + try: + data = flowdata.get(RESPONSE_DATA_KEY) + except Exception: + data = None + if data is None: + data = flowdata.get(OLD_DATA_KEY) + return data.get("cloud") + + +class HardwareProfileData(dict, AspectData): + """! + One hardware profile from a profile lookup, read in the way the device + data is read. It is Aspect Data, so a property is read as an attribute, + such as profile.hardwarevendor, with get() or with []. + + It is also a dict of property name to AspectPropertyValue. Iteration, + len and keys() give the names the cloud service gave, and get() takes a + default as a dict's get() does. + Attribute access, get(), [] and in compare names ignoring letter case. + A property whose name is also a dict method, such as keys, is read + with get() or []. + """ + + # Attributes set in the constructor, which are never read as properties. + _OWN_ATTRIBUTES = ("flow_element", "missing_property_service") + + def __init__(self, flow_element, contents): + """! + @type flow_element: HardwareProfileCloud + @param flow_element: the engine that made the profile + + @type contents: dict + @param contents: property name to AspectPropertyValue + """ + + dict.__init__(self, contents) + AspectData.__init__(self, flow_element) + + # The device data's service, so a name that is not a property is + # reported in the same way for both. + self.missing_property_service = CloudMissingPropertyService() + + def _matching_key(self, key): + """! + The key held for a property name given in another letter case, or + None when there is none. + """ + + if isinstance(key, str): + lowered = key.lower() + for held in dict.keys(self): + if isinstance(held, str) and held.lower() == lowered: + return held + return None + + def __getitem__(self, key): + try: + return dict.__getitem__(self, key) + except KeyError: + held = self._matching_key(key) + if held is None: + raise + return dict.__getitem__(self, held) + + def __contains__(self, key): + return (dict.__contains__(self, key) + or self._matching_key(key) is not None) + + def get(self, key, default=None): + """! + The value of a property by name in any letter case, or default when + the profile does not hold it, as a dict gives. + """ + + try: + return self[key] + except KeyError: + return default + + def get_internal(self, key): + """! + The value of a property by name in any letter case, or None when the + profile does not hold it, which AspectData.get() reports as missing. + """ + + return self.get(key) + + def as_dictionary(self): + """! + The profile itself, being a dict of property name to value. + """ + + return self + + def __getattr__(self, name): + """! + Read a property as an attribute. Python calls this only for a name + the profile does not otherwise have. A name that is not a property + raises AttributeError with the device data's message, so hasattr() + and getattr() with a default work. + """ + + # Names that start with an underscore, such as the special methods + # copy and pickle look for, are never properties. Reading the + # profile's own attributes as properties would recurse. + if name.startswith("_") or name in self._OWN_ATTRIBUTES: + raise AttributeError(name) + try: + return AspectData.get(self, name) + except Exception as error: + raise AttributeError(str(error)) from error + + def __reduce__(self): + # A copy or a pickle of a profile is a plain dict of its values, + # because the engine a profile refers to holds a lock, which cannot + # be copied or pickled. + return (dict, (dict(self),)) + def _build_aspect_values(hardware): """! @@ -74,9 +211,7 @@ def __init__(self): def process_internal(self, flowdata): - cloud_data = flowdata.get("cloud").get("cloud") - - cloud_data = json.loads(cloud_data) + cloud_data = json.loads(_cloud_response(flowdata)) hardware = cloud_data.get("hardware") or {} @@ -104,7 +239,7 @@ def process_internal(self, flowdata): if property_key not in device: device[property_key] = aspect_value - devices.append(device) + devices.append(HardwareProfileData(self, device)) # The aspect level values are exposed alongside the profiles so a # caller with no matching profiles can still read the reason. diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py index 36bc26ad..2109e4a2 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/live_cloud.py @@ -51,7 +51,7 @@ ACCESSIBLE_PROPERTIES_URL = ( "https://cloud.51degrees.com/api/v4/accessibleproperties?resource=") -CREATE_KEY_URL = "https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" +CREATE_KEY_URL = "https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=live_cloud.py&utm_term=resource-key-required" def _report_old_name(name, old_name): diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_builder_timeout.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_builder_timeout.py new file mode 100644 index 00000000..d923767c --- /dev/null +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_builder_timeout.py @@ -0,0 +1,76 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""! +The Device Detection Cloud Pipeline Builder passes timeout_seconds to the +Cloud Request Engine, as it does the other cloud settings. These tests need +no network connection and no resource key. +""" + +import unittest +import warnings + +import requests + +from fiftyone_pipeline_cloudrequestengine.requestclient import RequestClient + +from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder + +# A timeout other than the default, in seconds. +GIVEN_TIMEOUT_SECONDS = 7 + + +class TimeoutRecorder(RequestClient): + """! + Stands in for the cloud service as it is when it cannot be reached, and + keeps the timeout each request was given. + """ + + def __init__(self): + self.timeouts = [] + + def request(self, type, url, content, originHeader, timeout_seconds=None): + self.timeouts.append(timeout_seconds) + raise requests.exceptions.ConnectionError( + "The stand-in service cannot be reached.") + + +class BuilderTimeoutTests(unittest.TestCase): + + def test_the_builder_passes_the_timeout_to_the_engine(self): + recorder = TimeoutRecorder() + + # The service cannot be reached, which the engine gives as a + # warning and builds regardless. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + DeviceDetectionCloudPipelineBuilder( + resource_key="a-resource-key", + timeout_seconds=GIVEN_TIMEOUT_SECONDS, + http_client=recorder) + + # The two discovery requests made when the engine is built. + self.assertEqual([GIVEN_TIMEOUT_SECONDS] * 2, recorder.timeouts) + + +if __name__ == "__main__": + unittest.main() diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py index 7d769b84..702e5c04 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_cloud.py @@ -30,6 +30,8 @@ from fiftyone_devicedetection_cloud.devicedetection_cloud import DeviceDetectionCloud +from fiftyone_pipeline_engines.missingproperty_service import CONFIGURATOR_URL + from .live_cloud import find_resource_key, no_resource_key pytestmark = pytest.mark.live @@ -156,10 +158,16 @@ def test_missing_property_service_not_found_anywhere(self): fd.device.get("notpresent") result = str(raised.exception) - self.assertIn("notpresent", result) - self.assertIn("device", result) - self.assertIn(', '.join(pipeline.get_element("device").get_properties().keys()), result) - self.assertIn("https://51degrees.com/documentation/_services__cloud__resource_keys.html", result) + properties = pipeline.get_element("device").get_properties() + self.assertIn( + "Property 'notpresent' not found in data for element 'device'. " + "This is because your resource key does not include access to " + "this property. The properties included for your key under " + "'device' are ", result) + self.assertIn( + ",".join(meta.get("name", name) + for name, meta in properties.items()), result) + self.assertIn(CONFIGURATOR_URL, result) def test_cloud_request_origin(self): """! diff --git a/device-detection/fiftyone_devicedetection_cloud/tests/test_hardwareprofile_cloud.py b/device-detection/fiftyone_devicedetection_cloud/tests/test_hardwareprofile_cloud.py index 1ea4e781..643d9bd1 100644 --- a/device-detection/fiftyone_devicedetection_cloud/tests/test_hardwareprofile_cloud.py +++ b/device-detection/fiftyone_devicedetection_cloud/tests/test_hardwareprofile_cloud.py @@ -22,16 +22,20 @@ """! These tests run against fixed cloud responses, so they need no network -connection and no resource key. They cover the case that used to stop the -TAC and native model examples with an AttributeError, which is a resource -key that returns the hardware profiles but is not entitled to the hardware -properties themselves. +connection and no resource key. They cover a resource key that returns the +hardware profiles but is not entitled to the hardware properties themselves, +and how a profile is read, both as Aspect Data in the way the device data is +read and as a dict. """ +import copy import json +import pickle import unittest from fiftyone_devicedetection_cloud.hardwareprofile_cloud import HardwareProfileCloud +from fiftyone_pipeline_cloudrequestengine.clouddata import CloudData +from fiftyone_pipeline_engines.aspectdata import AspectData # A response of the shape the cloud service returns for a TAC lookup when # the resource key has the hardware aspect but is not entitled to the @@ -105,20 +109,55 @@ def set_element_data(self, data): self.element_data = data +class _KeyedFlowData: + """! + Stands in for a FlowData holding a cloud response under each key given. + Asking for any other key raises, as a FlowData does. + """ + + def __init__(self, responses): + self._responses = responses + self.element_data = None + + def get(self, key): + if key not in self._responses: + raise Exception("There is no element data for " + key) + return _FakeCloudElementData(self._responses[key]) + + def set_element_data(self, data): + self.element_data = data + + def _process(response): """! Run a fixed cloud response through the hardware profile engine and return the resulting element data. """ + return _process_flowdata(_FakeFlowData(response)) + + +def _process_flowdata(flowdata): + """! + Run the hardware profile engine on a stand-in FlowData and return the + resulting element data. + """ + engine = HardwareProfileCloud() - flowdata = _FakeFlowData(response) engine.process_internal(flowdata) return flowdata.element_data +def _first_profile(response): + """! + The first hardware profile the engine made from a fixed cloud response. + """ + + return _process(response).get("profiles")[0] + + class HardwareProfileCloudTests(unittest.TestCase): def test_null_reason_is_carried_into_each_profile(self): @@ -171,6 +210,145 @@ def test_aspect_level_values_are_exposed_alongside_the_profiles(self): self.assertIn("paid feature", hardware.get("hardwarevendor").no_value_message()) + def test_the_response_is_read_under_cloud_response(self): + # The cloud request engine's data is read under cloud-response, the + # key the specification gives, rather than the deprecated cloud. + hardware = _process_flowdata(_KeyedFlowData({ + "cloud-response": ENTITLED_RESPONSE, + "cloud": NO_HARDWARE_RESPONSE})) + + self.assertEqual( + "Apple", hardware.get("profiles")[0]["hardwarevendor"].value()) + + def test_the_old_key_is_read_when_there_is_no_cloud_response(self): + # An older cloud request engine gives its data under cloud alone. + hardware = _process_flowdata(_KeyedFlowData({ + "cloud": ENTITLED_RESPONSE})) + + self.assertEqual( + "Apple", hardware.get("profiles")[0]["hardwarevendor"].value()) + + +class HardwareProfileAspectDataTests(unittest.TestCase): + """! + A profile is read in the way the device data is read, because each + device detection engine returns Aspect Data that is interface compatible + with the others. + """ + + def test_a_profile_is_aspect_data(self): + profile = _first_profile(ENTITLED_RESPONSE) + + self.assertIsInstance(profile, AspectData) + + def test_properties_are_read_as_attributes(self): + profile = _first_profile(ENTITLED_RESPONSE) + + self.assertIs(profile["hardwarevendor"], profile.hardwarevendor) + self.assertIs(profile["hardwarename"], profile.hardwarename) + self.assertIs(profile["hardwaremodel"], profile.hardwaremodel) + self.assertEqual("Apple", profile.hardwarevendor.value()) + + def test_attributes_are_read_in_any_letter_case(self): + profile = _first_profile(ENTITLED_RESPONSE) + + self.assertIs(profile["hardwarevendor"], profile.HardwareVendor) + self.assertIs(profile["hardwaremodel"], profile.HARDWAREMODEL) + + def test_a_property_with_no_value_is_read_as_an_attribute(self): + profile = _first_profile(NOT_ENTITLED_RESPONSE) + + self.assertFalse(profile.hardwarevendor.has_value()) + self.assertIn("paid feature", + profile.hardwarevendor.no_value_message()) + + def test_names_are_compared_ignoring_letter_case(self): + profile = _first_profile(ENTITLED_RESPONSE) + value = profile["hardwarevendor"] + + self.assertIs(value, profile["HardwareVendor"]) + self.assertIs(value, profile.get("HardwareVendor")) + self.assertIs(value, profile.get_internal("HardwareVendor")) + self.assertIn("HardwareVendor", profile) + + def test_a_name_that_is_not_a_property_gives_the_device_data_message(self): + profile = _first_profile(ENTITLED_RESPONSE) + + # The device data is CloudData, so CloudData for the same engine + # gives the message the device data gives. + with self.assertRaises(Exception) as expected: + CloudData(HardwareProfileCloud(), {}).get("notaproperty") + + with self.assertRaises(AttributeError) as raised: + profile.notaproperty + + self.assertEqual(str(expected.exception), str(raised.exception)) + + def test_hasattr_and_getattr_with_a_default_work(self): + profile = _first_profile(ENTITLED_RESPONSE) + default = object() + + self.assertTrue(hasattr(profile, "hardwarevendor")) + self.assertFalse(hasattr(profile, "notaproperty")) + self.assertIs(profile["hardwarevendor"], + getattr(profile, "hardwarevendor", default)) + self.assertIs(default, getattr(profile, "notaproperty", default)) + + def test_as_dictionary_gives_every_property(self): + profile = _first_profile(ENTITLED_RESPONSE) + + self.assertEqual( + ["hardwaremodel", "hardwarename", "hardwarevendor"], + sorted(profile.as_dictionary())) + + +class HardwareProfileDictTests(unittest.TestCase): + """! + A profile is also a dict of property name to AspectPropertyValue, and + behaves as one. + """ + + def test_indexing_by_name(self): + profile = _first_profile(ENTITLED_RESPONSE) + + self.assertEqual("Apple", profile["hardwarevendor"].value()) + + with self.assertRaises(KeyError) as raised: + profile["notaproperty"] + + self.assertEqual(("notaproperty",), raised.exception.args) + + def test_get_with_a_default(self): + profile = _first_profile(ENTITLED_RESPONSE) + default = object() + + self.assertEqual("Apple", profile.get("hardwarevendor").value()) + self.assertIsNone(profile.get("notaproperty")) + self.assertIs(default, profile.get("notaproperty", default)) + + def test_in_iteration_len_keys_and_equality(self): + profile = _first_profile(ENTITLED_RESPONSE) + names = ["hardwaremodel", "hardwarename", "hardwarevendor"] + + self.assertIsInstance(profile, dict) + self.assertIn("hardwarevendor", profile) + self.assertNotIn("notaproperty", profile) + self.assertEqual(names, sorted(profile)) + self.assertEqual(names, sorted(profile.keys())) + self.assertEqual(3, len(profile)) + self.assertEqual({name: profile[name] for name in names}, profile) + + def test_copies_and_pickles(self): + profile = _first_profile(ENTITLED_RESPONSE) + + self.assertIs(profile["hardwarevendor"], + copy.copy(profile)["hardwarevendor"]) + self.assertEqual( + "Apple", copy.deepcopy(profile)["hardwarevendor"].value()) + self.assertEqual( + "A1586", + pickle.loads(pickle.dumps(profile))["hardwaremodel"].value()) + if __name__ == "__main__": unittest.main() diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py index 9e6d0b5f..0ef8d9b8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/configurator_console.py @@ -25,7 +25,19 @@ from fiftyone_devicedetection_cloud.devicedetection_cloud_pipelinebuilder import DeviceDetectionCloudPipelineBuilder from fiftyone_devicedetection_examples.example_utils import ExampleUtils -# This example is displayed at the end of the [Configurator](https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + +# This example is displayed at the end of the [configuration page](https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=top) # process, which is used to create resource keys for use with the 51Degrees cloud service. # # It shows how to call the cloud with the newly created key and how to access the values @@ -79,6 +91,8 @@ def run(self, resource_key, logger, output): output("device.ismobile: " + f"{ExampleUtils.get_human_readable(device, 'ismobile')}") + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def main(argv): # Use the command line args to get the resource key if present. # Otherwise, get it from the environment variable. @@ -99,9 +113,10 @@ def main(argv): "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + - "https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-configurator_console.py&utm_term=resource-key-required. " + "Once complete, populate the environment variable " + "mentioned at the start of this message with the key.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py index 8258122c..ae926b41 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/failuretomatch.py @@ -24,7 +24,7 @@ # First create the device detection pipeline with the desired settings. -# You need to create a resource key at https://configure.51degrees.com?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=top +# You need to create a resource key at https://51degrees.com/configure?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=top # and paste it into the code, replacing !!YOUR_RESOURCE_KEY!! below. # Alternatively, set the _51DEGREES_RESOURCE_KEY environment variable. The # key is read through ExampleUtils, as every other example reads it, so @@ -33,6 +33,18 @@ # it has no key. from fiftyone_devicedetection_examples.example_utils import ExampleUtils +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + resource_key = ExampleUtils.get_resource_key() or "!!YOUR_RESOURCE_KEY!!" if resource_key == "!!YOUR_RESOURCE_KEY!!": @@ -40,7 +52,7 @@ # The address stays on one line, because the campaign lint reads a # line at a time and a split address looks to it like a missing tag. print(" To include the properties used in this example, go to " - "https://configure.51degrees.com/bxXqZhLT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=resource-key-required") + "https://51degrees.com/configure/bxXqZhLT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-failuretomatch.py&utm_term=resource-key-required") else: pipeline = DeviceDetectionCloudPipelineBuilder({ @@ -90,3 +102,5 @@ else: # Output why the value isn't meaningful print(flowdata2.device.ismobile.no_value_message()) + +ExampleUtils.print_find_out_more(FIND_OUT_MORE) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.json b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.json index 214a588a..2caf8d14 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.json +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.json @@ -7,7 +7,7 @@ "elementParameters": { "settings": { // Obtain a resource key with the properties required to run this - // example for free: https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.json&utm_term=settings + // example for free: https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.json&utm_term=settings "resource_key": "!!ENTER_YOUR_RESOURCE_KEY_HERE!!", "cloud_request_origin": "51Degrees.example.com" } diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py index 3a92915b..7d525358 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_console.py @@ -45,6 +45,18 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_devicedetection_examples.example_utils import ExampleUtils +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class GettingStartedConsole(): def run(self, config, logger, output): @@ -59,6 +71,8 @@ def run(self, config, logger, output): for values in self.EvidenceValues: self.analyseEvidence(values, pipeline, output) + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def analyseEvidence(self, evidence, pipeline, output): # FlowData is a data structure that is used to convey @@ -176,9 +190,10 @@ def main(argv): "see " + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this example can be " + - "created for free at https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + + "created for free at https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_console.py&utm_term=resource-key-required. " + "Once complete, populate the config file or environment variable " + "mentioned at the start of this message with the key.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) else: GettingStartedConsole().run(config, logger, print) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py index 93f448b3..7d9b1fd8 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/app.py @@ -84,6 +84,19 @@ import os import sys +# Where to go next, printed when the example ends. The page ends with the +# same links, in templates/index.html. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class GettingStartedWeb(): app = Flask(__name__) @@ -204,9 +217,12 @@ def main(argv): "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + - "https://configure.51degrees.com/g3gMZdPY?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + + "https://51degrees.com/configure/g3gMZdPY?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-gettingstarted_web-app.py&utm_term=resource-key-required. " + "Once complete, populated the environment variable " + "mentioned at the start of this message with the key.") + # Printed once the server stops, or straight away when there is no key. + ExampleUtils.print_find_out_more(FIND_OUT_MORE) + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/templates/index.html index f9ff4bd8..0d055c53 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/gettingstarted_web/templates/index.html @@ -153,6 +153,14 @@

Client-side evidence and Apple models

Want to try on-premise? Contact us to discuss requirements.

Contact us + +

Find out more

+ {# diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py index d8b06656..b628ac2d 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/metadata_console.py @@ -53,6 +53,18 @@ from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter from fiftyone_devicedetection_examples.example_utils import ExampleUtils +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + bgRed = "\u001b[41m" fgWhite = "\u001b[37;1m" colReset = "\u001b[0m" @@ -73,6 +85,8 @@ def run(self, resource_key, logger, output): # them to the cloud, so that's the engine we want the keys from. self.outputEvidenceKeyDetails(pipeline.get_element("cloud"), output) + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def outputEvidenceKeyDetails(self, engine, output): if (issubclass(type(engine.get_evidence_key_filter()), BasicListEvidenceKeyFilter)): @@ -123,9 +137,10 @@ def main(argv): "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + - "https://configure.51degrees.com/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + + "https://51degrees.com/configure/1QWJwHxl?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-metadata_console.py&utm_term=resource-key-required. " + "Once complete, populate the environment variable " + "mentioned at the start of this message with the key.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) if __name__ == "__main__": main(sys.argv[1:]) \ No newline at end of file diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py index 98b7212e..4b0b28b5 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/nativemodellookup_console.py @@ -45,6 +45,18 @@ from fiftyone_devicedetection_shared.constants import Constants from fiftyone_devicedetection_cloud.hardwareprofile_cloud import HardwareProfileCloud +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class NativeModelLookupConsole(): def run(self, resource_key, logger, output, cloudEndPoint = ""): @@ -82,6 +94,8 @@ def run(self, resource_key, logger, output, cloudEndPoint = ""): # Repeat for an alternative native model name. self.analyseModel(self._nativeModel2, pipeline, output) + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def analyseModel(self, nativemodel, pipeline, output): # Create the FlowData instance. data = pipeline.create_flowdata() @@ -133,9 +147,10 @@ def main(argv): "you will first need a license key, which can be purchased from our " + "pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. Once this is done, a resource " + "key with the properties required by this example can be created at " + - "https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. You can now populate the " + + "https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-nativemodellookup_console.py&utm_term=resource-key-required. You can now populate the " + "environment variable mentioned at the start of this message with the " + "resource key or pass it as the first argument on the command line.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.json b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.json index 2bb00c22..9a393c31 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.json +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.json @@ -8,7 +8,7 @@ "settings": { // TAC lookup requires a license key, which can be purchased from our pricing page: // https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.json&utm_term=settings. You can then obtain a resource key with the properties - // required to run this example here: https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.json&utm_term=settings + // required to run this example here: https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.json&utm_term=settings "resource_key": "!!ENTER_YOUR_RESOURCE_KEY_HERE!!", // Update this with your domain name. "cloud_request_origin": "51Degrees.example.com" diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py index 63355916..dca57cc6 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/taclookup_console.py @@ -48,6 +48,18 @@ from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder from fiftyone_devicedetection_shared.constants import Constants +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class TacLookupConsole(): def run(self, config, logger, output): @@ -72,6 +84,8 @@ def run(self, config, logger, output): # Repeat for an alternative TAC. self.analyseTac(self._tac2, pipeline, output) + ExampleUtils.print_find_out_more(FIND_OUT_MORE, output) + def analyseTac(self, tac, pipeline, output): # Create the FlowData instance. data = pipeline.create_flowdata() @@ -134,10 +148,11 @@ def main(argv): "that you will first need a license key, which can be purchased " + "from our pricing page: https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. Once this is " + "done, a resource key with the properties required by this example " + - "can be created at https://configure.51degrees.com/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. You " + + "can be created at https://51degrees.com/configure/QKyYH5XT?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-taclookup_console.py&utm_term=resource-key-required. You " + "can now populate the environment variable mentioned at the start " + "of this message with the resource key or pass it as the first " + "argument on the command line.") + ExampleUtils.print_find_out_more(FIND_OUT_MORE) else: TacLookupConsole().run(config, logger, print) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py index 728f0594..c534ffbd 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/app.py @@ -52,6 +52,19 @@ from fiftyone_pipeline_core.logger import Logger from fiftyone_pipeline_core.web import webevidence, set_response_header +# Where to go next, printed when the example ends. The page ends with the +# same links, in templates/index.html. +FIND_OUT_MORE = [ + ("Device detection", + "https://51degrees.com/device-detection?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=find-out-more-device-detection"), + ("Resource keys", + "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=find-out-more-resource-keys"), + ("Pricing", + "https://51degrees.com/pricing?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=find-out-more-pricing"), + ("Device detection for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/device-detection"), +] + class UserAgentClientHintsWeb(): app = Flask(__name__) @@ -139,10 +152,13 @@ def main(argv): "https://51degrees.com/documentation/_services__cloud__resource_keys.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + "A resource key with the properties required by this " + "example can be created for free at " + - "https://configure.51degrees.com/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + + "https://51degrees.com/configure/?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_examples-src-fiftyone_devicedetection_examples-cloud-useragentclienthints_web-app.py&utm_term=resource-key-required. " + "Once complete, populated the environment variable " + "mentioned at the start of this message with the key.") + # Printed once the server stops, or straight away when there is no key. + ExampleUtils.print_find_out_more(FIND_OUT_MORE) + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/templates/index.html b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/templates/index.html index b206b40c..1b84bc94 100644 --- a/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/templates/index.html +++ b/device-detection/fiftyone_devicedetection_examples/src/fiftyone_devicedetection_examples/cloud/useragentclienthints_web/templates/index.html @@ -101,6 +101,14 @@

Detection results

Want to try on-premise? Contact us to discuss requirements.

Contact us
+ +

Find out more

+ " + # Where to go next. + output += find_out_more_html() + # Return the full output to the page return output diff --git a/pipeline/fiftyone_pipeline_core/examples/simple_flow_element.py b/pipeline/fiftyone_pipeline_core/examples/simple_flow_element.py new file mode 100644 index 00000000..e8c5ecd5 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/examples/simple_flow_element.py @@ -0,0 +1,220 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + + +## @example simple_flow_element.py +# +# This example creates a flow element of its own and adds it to a pipeline. +# The element takes a date of birth as evidence and works out the star sign +# from a table held in its code. +# +# You will learn: +# +# 1. How to write a flow element, which names the evidence it reads and the +# properties it gives, and works out their values +# 2. How to add the flow element to a pipeline +# 3. How to pass evidence to the pipeline and read the results +# +# The other star sign examples build on this one. The on-premise engine +# example in fiftyone_pipeline_engines reads the table from a data file, the +# client-side evidence example beside this one gets the date of birth from a +# form in a web page, and the cloud engine example in the cloud request +# engine's examples gets the star sign from a remote service. +# +# This example is available in full on [GitHub](https://github.com/51Degrees/51degrees-python/blob/main/pipeline/fiftyone_pipeline_core/examples/simple_flow_element.py). +# +# It needs no data file, no key and no network. Run it from the pipeline +# folder, once fiftyone_pipeline_core is installed, with +# ``` +# python fiftyone_pipeline_core/examples/simple_flow_element.py +# ``` +# +# Expected output, before the Find out more section: +# ``` +# With a date of birth of 1992-12-18, your star sign is Sagittarius. +# With a date of birth of 1990-12-25, your star sign is Capricorn. +# With a date of birth of 1992-04-15, your star sign is Aries. +# ``` + +from datetime import date + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +# Where to go next, printed when the example ends. +FIND_OUT_MORE = [ + ("Pipeline API examples", + "https://51degrees.com/documentation/_pipeline_api__examples__index.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-examples-simple_flow_element.py&utm_term=find-out-more-examples"), + ("Pipeline API documentation", + "https://51degrees.com/documentation/_pipeline_api__concepts__pipeline_basics.html?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-examples-simple_flow_element.py&utm_term=find-out-more-documentation"), + ("Build your own", + "https://51degrees.com/build-your-own?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-examples-simple_flow_element.py&utm_term=find-out-more-build-your-own"), + ("Contact us", + "https://51degrees.com/contact-us?utm_source=code&utm_medium=example&utm_campaign=51degrees-python&utm_content=fiftyone_pipeline_core-examples-simple_flow_element.py&utm_term=find-out-more-contact"), + ("Pipeline API for Python", + "https://github.com/51Degrees/51degrees-python/tree/main/pipeline"), + ("Pipeline API specification", + "https://github.com/51Degrees/specifications/tree/main/pipeline-specification"), +] + +# The evidence the flow element reads, a date of birth in the form +# YYYY-MM-DD, as a date input in a web form sends it in the query string. +DATE_OF_BIRTH = "query.dateOfBirth" + +# The dates of birth the example looks up. +DATES_OF_BIRTH = ["1992-12-18", "1990-12-25", "1992-04-15"] + +# Each star sign, with the month and day it starts and the month and day it +# ends. Capricorn runs over the end of the year. +STAR_SIGNS = [ + ("Aries", (3, 21), (4, 19)), + ("Taurus", (4, 20), (5, 20)), + ("Gemini", (5, 21), (6, 20)), + ("Cancer", (6, 21), (7, 22)), + ("Leo", (7, 23), (8, 22)), + ("Virgo", (8, 23), (9, 22)), + ("Libra", (9, 23), (10, 22)), + ("Scorpio", (10, 23), (11, 21)), + ("Sagittarius", (11, 22), (12, 21)), + ("Capricorn", (12, 22), (1, 19)), + ("Aquarius", (1, 20), (2, 18)), + ("Pisces", (2, 19), (3, 20)), +] + + +def find_star_sign(month, day): + + """! + The star sign for a month and day, from the table above. + """ + + birthday = (month, day) + for name, start, end in STAR_SIGNS: + if start <= end: + if start <= birthday <= end: + return name + # A star sign that runs over the end of the year. + elif birthday >= start or birthday <= end: + return name + return None + + +class SimpleFlowElement(FlowElement): + + """! + A flow element that takes a date of birth as evidence and gives the star + sign, worked out from the table held in this example. + """ + + def __init__(self): + + super(SimpleFlowElement, self).__init__() + + # The results are kept in the flow data under this key, so they are + # read as flowdata.astrology. + self.datakey = "astrology" + + # The properties the flow element gives, each named in lower case, + # as the 51Degrees flow elements name theirs. + self.properties = { + "starsign": { + "type": "string", + "description": "The star sign for the date of birth." + } + } + + def get_evidence_key_filter(self): + + """! + The evidence the flow element reads. The pipeline sets aside any + evidence that no flow element reads. + """ + + return BasicListEvidenceKeyFilter([DATE_OF_BIRTH]) + + def process_internal(self, flowdata): + + """! + The work of the flow element, which reads the evidence and adds the + results to the flow data. + """ + + result = {} + + date_of_birth = flowdata.evidence.get(DATE_OF_BIRTH) + if date_of_birth: + born = date.fromisoformat(date_of_birth) + result["starsign"] = find_star_sign(born.month, born.day) + + # The results go into element data, which the flow data keeps under + # the flow element's data key. + flowdata.set_element_data(ElementDataDictionary(self, result)) + + +def print_find_out_more(): + + """! + Print the "Find out more" section, a line of "=" and then each label + with its link. + """ + + print("=" * 79) + print("Find out more:") + for label, url in FIND_OUT_MORE: + print(f"\t{label}: {url}") + + +def main(): + + # A console has no page, so leave out the elements that write the + # client-side script and set response headers. + settings = { + "add_javascript_builder": False, + "use_setheader_properties": False, + } + + pipeline = PipelineBuilder(settings).add(SimpleFlowElement()).build() + + # Closing the pipeline closes the flow elements in it. + with pipeline: + + for date_of_birth in DATES_OF_BIRTH: + + # Flow data carries the evidence into the pipeline and the + # results out of it. + flowdata = pipeline.create_flowdata() + flowdata.evidence.add(DATE_OF_BIRTH, date_of_birth) + flowdata.process() + + print( + f"With a date of birth of {date_of_birth}, your star sign " + f"is {flowdata.astrology.starsign}.") + + print_find_out_more() + + +if __name__ == "__main__": + main() diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/aspectproperty_value.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/aspectproperty_value.py index 3a3f1479..d082f992 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/aspectproperty_value.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/aspectproperty_value.py @@ -21,6 +21,15 @@ # ********************************************************************* +# The message value() raises with when there is no value and no reason was +# given, the same as .NET's. +NO_VALUE_MESSAGE = "This instance does not have a set value" + +# Stands for a value that was not given. A private object, so no value a +# caller passes, the text "noValue" included, is taken for it. +_NOT_SET = object() + + class AspectPropertyValue(object): """! @@ -29,13 +38,13 @@ class AspectPropertyValue(object): If not value is set, a specific no value message is returned """ - def __init__(self, no_value_message = None, value = "noValue"): + def __init__(self, no_value_message = None, value = _NOT_SET): """! Constructor for AspectPropertyValue @type value: mixed - @param value: the property value to store + @param value: the property value to store. Leave it out for no value. @type noValueMessage: string @param noValueMessage: if there is no value, the reason for there not being one @@ -44,7 +53,7 @@ def __init__(self, no_value_message = None, value = "noValue"): self.noValueMessage = no_value_message - if(value != "noValue"): + if value is not _NOT_SET: self.__value = value self.__hasValue = True else: @@ -66,18 +75,20 @@ def value(self): """! Get the value out of the AspectPropertyValue wrapper - Note that this will return an error (including no_value_message if set) - if there is no value. So check has_value() first to be sure. + With no value this raises, with no_value_message or, when that is + not set, NO_VALUE_MESSAGE. So check has_value() first to be sure. @rtype mixed @return the stored value - + @raise Exception: where there is no value, with no_value_message or + NO_VALUE_MESSAGE + """ if self.__hasValue: return self.__value else: - raise Exception(self.no_value_message()) + raise Exception(self.no_value_message() or NO_VALUE_MESSAGE) def no_value_message(self): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/basiclist_evidence_keyfilter.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/basiclist_evidence_keyfilter.py index 20c2d5f5..05ceec49 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/basiclist_evidence_keyfilter.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/basiclist_evidence_keyfilter.py @@ -29,6 +29,11 @@ class BasicListEvidenceKeyFilter(EvidenceKeyFilter): An instance of EvidenceKeyFilter that uses a simple array of keys Evidence not using these keys is filtered out + A key is looked up in a set made from list at the first lookup, and + made again whenever list is set to another list. To change the keys + once the filter has been used, set list to a new list rather than + changing the list in place. + """ def __init__(self, keys_list): @@ -56,8 +61,15 @@ def filter_evidence_key(self, key): @return: Is this key in the filter's keys list? """ - - if key.lower() in self.list: - return True - else: - return False + + # The set is kept with the list it was made from, as one value, so + # a lookup on another thread never finds a set beside a list it was + # not made from. Reading list each time finds a list set after the + # filter was made, or set by a subclass that does not call the + # constructor. + listed = self.list + made = getattr(self, "_list_as_set", None) + if made is None or made[0] is not listed: + made = (listed, set(listed)) + self._list_as_set = made + return key.lower() in made[1] diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py index 51fc0bc1..5ee546d3 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/elementdata.py @@ -28,6 +28,13 @@ class ElementData(object): Stores information created by a FlowElement based on FlowData. Stored in FlowData + Only the Flow Element that creates an instance writes it, so an instance + does not need to be safe to write from several threads. A finished + instance, once the Flow Element that writes it has finished, is safe to + read from several threads at once, such as the caller's own threads. An + instance that is written from several threads, or read while another + thread writes it, is not catered for. + """ def __init__(self, flow_element): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py index 914d7596..4f33afee 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/flowdata.py @@ -60,6 +60,10 @@ def process(self): @rtype: FlowData @return: Returns flowdata + @raise Exception: the first error a flow element raised, or the + error saying this FlowData has already been processed, unless the + Pipeline suppresses process exceptions. Every error is in errors + either way. """ @@ -153,6 +157,8 @@ def get(self, flow_element_key): @rtype: ElementData @return: Returns data in the FlowData instance that is under the specified key + @raise Exception: where this FlowData holds no data under the + key, with a message naming the keys it does hold """ @@ -173,6 +179,8 @@ def __getattr__(self, flow_element_key): @param flow_element_key: datakey of the FlowElement that created the data of interest @rtype: ElementData + @raise Exception: where this FlowData holds no data under the + key, with a message naming the keys it does hold """ @@ -188,6 +196,8 @@ def __getitem__(self, flow_element_key): @param flow_element_key: datakey of the FlowElement that created the data of interest @rtype: ElementData + @raise Exception: where this FlowData holds no data under the + key, with a message naming the keys it does hold """ diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py index bcb2fe1f..a7c42d67 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/jsonbundler.py @@ -50,6 +50,9 @@ def __init__(self): self.properties = {"json" : { "type": "dict"} } + # The delayed execution information of each flow element, keyed by + # the element rather than its data key, as two Pipelines sharing + # this element can hold different elements under one data key. self.property_cache = {} def get_evidence_key_filter(self): @@ -116,11 +119,6 @@ def process_internal(self, flowdata): output = {"javascriptProperties": [] } - if len(self.property_cache): - property_cache_set = True - else: - property_cache_set = False - for flow_element in flowdata.pipeline.flow_elements: if (flow_element.datakey == "jsonbundler" or @@ -131,7 +129,9 @@ def process_internal(self, flowdata): properties = flow_element.get_properties() - if not property_cache_set: + property_cache = self.property_cache.get(flow_element) + + if property_cache is None: delay_execution_list = [] delayed_evidence_properties = {} @@ -160,12 +160,17 @@ def process_internal(self, flowdata): flow_element.datakey.lower() + '.' + x, delayed_evidence_properties_list)) - self.property_cache[flow_element.datakey] = { + property_cache = { "delayExecutionList": delay_execution_list, "evidenceProperties": delayed_evidence_properties } - property_cache = self.property_cache[flow_element.datakey] + # Kept once the element has properties. An element can have + # none until its property metadata has been fetched, so an + # answer taken from no properties is read again on the next + # request. + if properties: + self.property_cache[flow_element] = property_cache # Create empty area for FlowElement properties to go diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py index 5823f175..05f2996e 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/messages.py @@ -33,7 +33,7 @@ class Messages(): # as format argument. WRONG_PROPERTY_FORMAT = \ - "Property Name '{}' is not in the expected format i.e. SetHeader[Component][HeaderName]. " + "Property Name '{}' is not in the expected format i.e. SetHeader[Identifier][HeaderName]. " # Element not found in flowData. This takes the element datakey # as format argument. diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py index 4ee68108..c6d93557 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipeline.py @@ -20,9 +20,32 @@ # such notice(s) shall fulfill the requirements of that article. # ********************************************************************* +from types import MappingProxyType + +from .evidence_keyfilter import EvidenceKeyFilter from .flowdata import FlowData from .logger import Logger + +class PipelineEvidenceKeyFilter(EvidenceKeyFilter): + + """! + The Evidence a Pipeline accepts, being every key one of its FlowElements + accepts. Each element's filter is read when a key is checked, as an + element's list can change once its data or the cloud service's answer + arrives. + + """ + + def __init__(self, pipeline): + self.pipeline = pipeline + + def filter_evidence_key(self, key): + return any( + element.get_evidence_key_filter().filter_evidence_key(key) + for element in self.pipeline.flow_elements_list.values()) + + class Pipeline: """! @@ -31,6 +54,12 @@ class Pipeline: collecting ElementData Should be constructed through the PipelineBuilder class + A Pipeline cannot be changed once it is built. flow_elements and + flow_elements_display_list are tuples, and flow_elements_list, the + elements by data key, is a read-only mapping. Each is a property with + no setter. Code can read them, and adding, removing or replacing an + element, or binding one of them again, raises an error. + """ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = False, data_file_update_service = None, auto_close_elements = True): @@ -39,7 +68,9 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = Pipeline constructor. @type flow_elements: list[FlowElement] - @param flow_elements: A list of flowElements + @param flow_elements: The flowElements, in the order they run. The + pipeline keeps a copy, so changing this list afterwards changes + nothing. @type logger: Logger @param logger: A logger to attach to the pipeline @@ -60,13 +91,16 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = """ - self.flow_elements = flow_elements + self._flow_elements = tuple(flow_elements) self.logger = logger - self.flow_elements_list = {} + # Filled below as each element is registered, so an element finds + # those registered before it, and read only through this view. + elements_by_key = {} + self._flow_elements_list = MappingProxyType(elements_by_key) - self.flow_elements_display_list = [] + self._flow_elements_display_list = () self.suppress_process_exceptions = suppress_process_exceptions @@ -76,17 +110,38 @@ def __init__(self, flow_elements, logger=Logger(), suppress_process_exceptions = self.data_file_update_service = data_file_update_service self.data_file_update_service.register_pipeline(self) - for flow_element in flow_elements: + for flow_element in self._flow_elements: # Notify element that it has been registered in the pipeline flow_element.on_registration(self) - self.flow_elements_list[flow_element.datakey] = flow_element + elements_by_key[flow_element.datakey] = flow_element flow_element.pipelines.append(self) if not flow_element.exclude_from_messages: - self.flow_elements_display_list.append(flow_element.datakey) + self._flow_elements_display_list += (flow_element.datakey,) + + @property + def flow_elements(self): + """! + The FlowElements, in the order they run, as a tuple. + """ + return self._flow_elements + + @property + def flow_elements_list(self): + """! + The FlowElements by data key, as a read-only mapping. + """ + return self._flow_elements_list + + @property + def flow_elements_display_list(self): + """! + The data keys named in messages, as a tuple. + """ + return self._flow_elements_display_list def create_flowdata(self): @@ -112,6 +167,9 @@ def close(self): one as well, because the results it holds keep the engine's data in memory until they are released. + @raise Exception: the first error a flow element raised when it was + closed, once every element has been closed + """ if not self.auto_close_elements: @@ -168,6 +226,19 @@ def get_element(self, key): return self.flow_elements_list[key] + def get_evidence_key_filter(self): + + """! + The Evidence the Pipeline accepts, being every key one of its + FlowElements accepts. + + @rtype: EvidenceKeyFilter + @return: Returns an EvidenceKeyFilter + + """ + + return PipelineEvidenceKeyFilter(self) + def element_not_found(self, element, flowdata): """! Trigger error when an element cannot be found in the pipeline @@ -176,12 +247,15 @@ def element_not_found(self, element, flowdata): @param element: name of flowelement @type flowdata: flowdata @param element: flowdata being processsed - @rtype: Exception - @return: Returns exception + @raise Exception: always, naming the keys the FlowData does hold """ - raise Exception("There is no element data for " + element + " against this flow data. Available element data keys are: " + str(flowdata.pipeline.flow_elements_display_list)) + # A list, so the keys are shown in square brackets. + raise Exception( + "There is no element data for " + element + " against this " + "flow data. Available element data keys are: " + + str(list(flowdata.pipeline.flow_elements_display_list))) def get_properties(self): diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py index e208b686..2f0746d1 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/pipelinebuilder.py @@ -28,6 +28,17 @@ from .setheaderelement import SetHeaderElement import importlib +# The data keys of the JSONBundler and the JavaScriptBuilder. +JSON_BUILDER_KEY = "jsonbundler" +JAVASCRIPT_BUILDER_KEY = "javascriptbuilder" + + +def _data_key(element): + + # Flow data keeps each element's results under its data key in lower + # case, so two keys that differ only in letters are the same key. + return str(element.datakey).lower() + class PipelineBuilder(object): """ @@ -40,6 +51,9 @@ class PipelineBuilder(object): def __init__(self, settings= {}): """ Pipeline Builder constructor. + The SequenceElement, JSONBundler, JavaScriptBuilder and + SetHeaderElement switches below each add their element only when the + caller has not added one with the same data key. @type settings: dictionary @param settings : settings for the pipeline builder including: `add_javascript_builder (Bool)` - Whether to add the JavaScriptBuilder, with the @@ -188,14 +202,52 @@ def build(self): """ - flow_elements = ( - self.get_sequence_elements() + - self.flow_elements + - self.get_javascript_elements() + - self.get_setheader_elements()) + flow_elements = self._add_web_elements(self.flow_elements) return Pipeline(flow_elements, logger=self.logger, suppress_process_exceptions=self.suppress_process_exceptions, data_file_update_service=self.data_file_update_service, auto_close_elements=self.auto_close_elements) + def _add_web_elements(self, flow_elements): + + """! + The caller's elements with each web integration element the settings + ask for, unless the caller added one with the same data key. The + SequenceElement goes first and the others after every other element, + except that a JSONBundler goes immediately before a JavaScriptBuilder + the caller added, because the JavaScriptBuilder reads the + JSONBundler's results. + + @type flow_elements: list + @param flow_elements: The FlowElements the caller added + @rtype: list + @return: Returns a new list of FlowElements + """ + + present = set(_data_key(element) for element in flow_elements) + + def absent(element): + return _data_key(element) not in present + + elements = [element for element in self.get_sequence_elements() + if absent(element)] + elements.extend(flow_elements) + + for element in self.get_javascript_elements(): + if not absent(element): + continue + builder = next( + (index for index, added in enumerate(elements) + if _data_key(added) == JAVASCRIPT_BUILDER_KEY), + None) + if _data_key(element) == JSON_BUILDER_KEY and builder is not None: + elements.insert(builder, element) + else: + elements.append(element) + + elements.extend(element for element in self.get_setheader_elements() + if absent(element)) + + return elements + def add_logger(self, logger): """ Add an instance of the logger class to the pipeline. diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py index 083c5617..2845b00b 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/setheaderelement.py @@ -24,7 +24,10 @@ from .flowelement import FlowElement from .elementdata_dictionary import ElementDataDictionary from .messages import Messages -import re + +# The start of the name of every property whose value is for a response +# header. +SET_HEADER_PREFIX = "SetHeader" class SetHeaderElement(FlowElement): @@ -43,7 +46,16 @@ def __init__(self): self.properties = {"responseheaderdictionary" : { "type": "dict"} } + # A copy of the SetHeader properties of the first Pipeline found to + # have any, for code that reads them. Processing does not use it. self.setheader_properties = {} + + # The SetHeader properties of each Pipeline the element processes + # for, keyed by the Pipeline, as each Pipeline holds its own + # elements. Each value is what get_setheader_properties_pipeline + # returns for that Pipeline. The element already holds each Pipeline + # it is added to in pipelines, so the keys keep none alive longer. + self._setheader_properties_by_pipeline = {} self.exclude_from_messages = True @@ -66,10 +78,25 @@ def process_internal(self, flowdata): """ - if not self.setheader_properties: - self.setheader_properties = self.get_setheader_properties_pipeline(flowdata.pipeline) - - response_headers = self.get_response_header_value(flowdata, self.setheader_properties) + # An empty list is built again on the next request, because a cloud + # engine has no properties until the cloud service answers. Requests + # that build a list at the same time build the same one, so either + # may be kept. + pipeline = flowdata.pipeline + setheader_properties = self._setheader_properties_by_pipeline.get( + pipeline) + if not setheader_properties: + setheader_properties = self.get_setheader_properties_pipeline( + pipeline) + self._setheader_properties_by_pipeline[pipeline] = ( + setheader_properties) + if setheader_properties and not self.setheader_properties: + self.setheader_properties = { + datakey: list(names) + for datakey, names in setheader_properties.items()} + + response_headers = self.get_response_header_value( + flowdata, setheader_properties) data = ElementDataDictionary(self, {"responseheaderdictionary": response_headers}) @@ -101,8 +128,9 @@ def get_setheader_properties_pipeline(self, pipeline): # Loop over each flowElement property for propertykey, propertymeta in properties.items(): - # Check if the property starts with SetHeader - if("setheader" in propertykey): + # Property keys are in lower case, and only a name that + # starts with SetHeader is for a response header. + if propertykey.startswith(SET_HEADER_PREFIX.lower()): setHeader_element_list.append(propertymeta["name"]) # Add SetHeader element list in dict against flowElement.datakey as key @@ -192,42 +220,27 @@ def get_property_value(self, flowData, element_key, property_key): def get_response_header_name(self, property_key): """! - Determines which response header the property value will be appended to by - stripping the 'SetHeader' string and the 'Component Name' from the property name. - - @param property_key: Key for SetHeaderAcceptCH property + The response header a SetHeader property's value is for. The + property name is SetHeader[Identifier][HeaderName]. The identifier + may start with either case and has no upper-case letters after its + first character, so the header name starts at the first upper-case + letter after that character. + + @param property_key: A property name, such as SetHeaderBrowserAccept-CH @rtype String - @return Response Header name + @return The response header name, such as Accept-CH """ - actual_property_name = property_key - response_header = "" + if not property_key.startswith(SET_HEADER_PREFIX): + raise Exception( + Messages.PROPERTY_NOT_SET_HEADER.format(property_key)) - # Check if property name starts with SetHeader. - # If Yes, Discard SetHeader from property name. - if ("SetHeader" not in property_key[0:9]): - raise Exception(Messages.PROPERTY_NOT_SET_HEADER.format(actual_property_name)) - else: - # Discard SetHeader from property_key - property_key = property_key.replace("SetHeader", "", 1) + name = property_key[len(SET_HEADER_PREFIX):] - # Check if the first letter of Component name is in Uppercase - # If Yes, Split the propertyKey based on Uppercase letters - if(property_key[0].isupper() is False): - raise Exception(Messages.WRONG_PROPERTY_FORMAT.format(actual_property_name)) - else: - # Get the Component name string to be removed from the key - parts = re.findall(r'([A-Z][^A-Z]*)', property_key) - discard_letter = parts[0] - - # Check if property name contains the header name that starts with upper case - # If Yes, Remove the previously found Component Name to get the Header Name - if len(parts)<=1 : - raise Exception(Messages.WRONG_PROPERTY_FORMAT.format(actual_property_name)) - - else: - # Replace the first letter with the empty string. - response_header = property_key.replace(discard_letter, "") + for index in range(1, len(name)): + if "A" <= name[index] <= "Z": + return name[index:] - return response_header + # There is no identifier, or no header name after it. + raise Exception(Messages.WRONG_PROPERTY_FORMAT.format(property_key)) diff --git a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py index b5d06b0e..05804c10 100644 --- a/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py +++ b/pipeline/fiftyone_pipeline_core/src/fiftyone_pipeline_core/web.py @@ -71,6 +71,21 @@ def request_line_path(request): return raw.split("?", 1)[0].split("#", 1)[0] return request.path +def request_line_query(request): + + """! + The query string of the request line, whole and undecoded, or empty + when it has none. Taken from the request line when the server says what + that was, and otherwise from the query string WSGI gives, which it + leaves undecoded. + """ + + raw = request.environ.get("RAW_URI") or request.environ.get("REQUEST_URI") + if raw: + target = raw.split("#", 1)[0] + return target.split("?", 1)[1] if "?" in target else "" + return request.environ.get("QUERY_STRING", "") + def webevidence(request): """! @@ -129,13 +144,14 @@ def webevidence(request): # The request itself, for the usage sharing element, which accepts no # evidence and shares what it finds. A read-only view of the parts above - # and the method and path of the request line, keyed as evidence is. It - # leaves out the Cookie header, which cookies already cover. Evidence - # keeps it from every other element. + # and the method, path and query string of the request line, keyed as + # evidence is. It leaves out the Cookie header, which cookies already + # cover. Evidence keeps it from every other element. view = {key: value for key, value in webevidence.items() if key not in ("header.cookie", "server.host-ip")} view["server.request-method"] = request.method view["server.request-path"] = request_line_path(request) + view["server.request-query"] = request_line_query(request) webevidence[HTTP_REQUEST_KEY] = MappingProxyType(view) return webevidence diff --git a/pipeline/fiftyone_pipeline_core/tests/test_aspectproperty_value.py b/pipeline/fiftyone_pipeline_core/tests/test_aspectproperty_value.py new file mode 100644 index 00000000..f35249d7 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_aspectproperty_value.py @@ -0,0 +1,68 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue + +NO_VALUE = "This instance does not have a set value" + + +class AspectPropertyValueTests(unittest.TestCase): + + """! + Reading the value of an AspectPropertyValue that has no value raises, + with the reason given or, when none was given, a message saying there + is no value, as the Null values section of properties.md in the + specifications asks. A value that is set, whatever it is, is read back + as it was set. + """ + + def test_reading_no_value_raises_with_a_message(self): + cases = [ + ("no reason", AspectPropertyValue(), NO_VALUE), + ("a None reason", AspectPropertyValue(None), NO_VALUE), + ("an empty reason", AspectPropertyValue(""), NO_VALUE), + ("a reason", AspectPropertyValue("No match"), "No match"), + ] + for name, apv, message in cases: + with self.subTest(name): + self.assertFalse(apv.has_value()) + with self.assertRaises(Exception) as raised: + apv.value() + self.assertEqual(message, str(raised.exception)) + + def test_no_reason_leaves_the_message_unset(self): + self.assertIsNone(AspectPropertyValue().no_value_message()) + + def test_a_value_is_read_as_set(self): + for value in ["Value", None, False, 0, "", ["a", "b"], + "noValue"]: + with self.subTest(value=value): + apv = AspectPropertyValue(value=value) + self.assertTrue(apv.has_value()) + self.assertEqual(value, apv.value()) + + def test_the_text_novalue_given_in_place_is_a_value(self): + apv = AspectPropertyValue(None, "noValue") + self.assertTrue(apv.has_value()) + self.assertEqual("noValue", apv.value()) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_basiclist_evidence_keyfilter.py b/pipeline/fiftyone_pipeline_core/tests/test_basiclist_evidence_keyfilter.py new file mode 100644 index 00000000..4a13f898 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_basiclist_evidence_keyfilter.py @@ -0,0 +1,152 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The basic list filter accepts the keys it was given, in any letters, and +nothing else. Its list attribute is read by other code, so it keeps holding +the keys in lower case, and the filter matches what the list holds when the +list is set after the filter is made. A key is looked up in a set made from +the list, so a lookup does not search the list, and the answers are the ones +a search of the list gives.""" + +import unittest + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter + + +class SetsItsOwnList(BasicListEvidenceKeyFilter): + """A filter that sets its list without calling the base constructor.""" + + def __init__(self): + self.list = ["header.user-agent"] + + +class RefusesSearch(list): + """A list that fails when it is searched for a key.""" + + def __contains__(self, item): + raise AssertionError("The list was searched for " + repr(item)) + + +# Lists of keys, as given to the constructor, and keys to look up in them. +LISTS = [ + [], + ["header.user-agent"], + ["Header.User-Agent", "query.sequence", "header.user-agent"], + ["header.sec-ch-ua", "header.sec-ch-ua-mobile", "cookie.51D_x"], +] +KEYS = [ + "", "header.user-agent", "HEADER.USER-AGENT", "header.user", + "header.sec-ch-ua", "Header.Sec-CH-UA-Mobile", "cookie.51d_x", + "query.sequence", "query.sequence2", +] + + +class BasicListEvidenceKeyFilterTests(unittest.TestCase): + + def test_a_listed_key_is_accepted_in_any_letters(self): + key_filter = BasicListEvidenceKeyFilter( + ["Header.User-Agent", "query.sequence"]) + for key in ("header.user-agent", "HEADER.USER-AGENT", + "Header.User-Agent", "query.sequence", "Query.Sequence"): + self.assertTrue(key_filter.filter_evidence_key(key), key) + + def test_a_key_that_is_not_listed_is_refused(self): + key_filter = BasicListEvidenceKeyFilter(["header.user-agent"]) + for key in ("header.user", "header.user-agent2", "user-agent", + "cookie.user-agent", ""): + self.assertFalse(key_filter.filter_evidence_key(key), key) + + def test_an_empty_list_accepts_nothing(self): + key_filter = BasicListEvidenceKeyFilter([]) + self.assertFalse(key_filter.filter_evidence_key("header.user-agent")) + self.assertFalse(key_filter.filter_evidence_key("")) + + def test_filtering_keeps_listed_keys_as_they_were_given(self): + key_filter = BasicListEvidenceKeyFilter(["header.user-agent"]) + self.assertEqual( + {"Header.User-Agent": "a browser"}, + key_filter.filter_evidence({ + "Header.User-Agent": "a browser", + "header.accept": "text/html"})) + + def test_the_list_holds_the_keys_in_lower_case_in_order(self): + key_filter = BasicListEvidenceKeyFilter( + ["Query.Sequence", "header.user-agent", "Header.Host"]) + self.assertEqual( + ["query.sequence", "header.user-agent", "header.host"], + key_filter.list) + self.assertIsInstance(key_filter.list, list) + + def test_a_list_set_after_the_filter_is_made_is_matched(self): + key_filter = BasicListEvidenceKeyFilter(["header.user-agent"]) + key_filter.list = ["header.accept"] + self.assertTrue(key_filter.filter_evidence_key("Header.Accept")) + self.assertFalse(key_filter.filter_evidence_key("header.user-agent")) + + def test_a_subclass_that_sets_its_own_list_is_matched(self): + key_filter = SetsItsOwnList() + self.assertTrue(key_filter.filter_evidence_key("Header.User-Agent")) + self.assertFalse(key_filter.filter_evidence_key("header.accept")) + self.assertEqual(["header.user-agent"], key_filter.list) + + def test_the_answers_are_those_a_search_of_the_list_gives(self): + for keys in LISTS: + key_filter = BasicListEvidenceKeyFilter(keys) + for key in KEYS: + self.assertEqual( + key.lower() in key_filter.list, + key_filter.filter_evidence_key(key), (keys, key)) + + def test_a_list_set_as_it_is_gives_the_answers_of_a_search(self): + # A list set directly is not lower-cased, so an entry with capitals + # matches no key, as a search of the list finds none. + for keys in LISTS: + key_filter = BasicListEvidenceKeyFilter([]) + key_filter.list = list(keys) + for key in KEYS: + self.assertEqual( + key.lower() in keys, + key_filter.filter_evidence_key(key), (keys, key)) + + def test_a_list_set_after_a_lookup_is_matched(self): + key_filter = BasicListEvidenceKeyFilter(["header.user-agent"]) + self.assertTrue(key_filter.filter_evidence_key("header.user-agent")) + key_filter.list = ["header.accept"] + self.assertTrue(key_filter.filter_evidence_key("Header.Accept")) + self.assertFalse(key_filter.filter_evidence_key("header.user-agent")) + + def test_a_list_changed_before_the_first_lookup_is_matched(self): + key_filter = BasicListEvidenceKeyFilter(["header.user-agent"]) + key_filter.list.append("header.accept") + self.assertTrue(key_filter.filter_evidence_key("Header.Accept")) + self.assertTrue(key_filter.filter_evidence_key("header.user-agent")) + + def test_a_key_is_looked_up_without_searching_the_list(self): + key_filter = BasicListEvidenceKeyFilter([]) + key_filter.list = RefusesSearch(["header.user-agent"]) + self.assertTrue(key_filter.filter_evidence_key("Header.User-Agent")) + self.assertFalse(key_filter.filter_evidence_key("header.accept")) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_client_side_evidence_example.py b/pipeline/fiftyone_pipeline_core/tests/test_client_side_evidence_example.py new file mode 100644 index 00000000..eec8dc29 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_client_side_evidence_example.py @@ -0,0 +1,116 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""Runs the custom flow element example, +examples/client_side_evidence_custom_flow_element.py, through Flask's test +client. The page works out a star sign from a date of birth in the query +string, and a hemisphere from a latitude cookie, and the JSON route answers +with both for the page's script.""" + +import importlib.util +import json +import unittest +from pathlib import Path + +EXAMPLE = (Path(__file__).resolve().parents[1] / "examples" / + "client_side_evidence_custom_flow_element.py") + +# A date of birth in the form a date input submits, and its star sign. +DATE_OF_BIRTH = "1990-03-25" +STAR_SIGN = "aries" + +# The cookie the example's JavaScript sets with the browser's latitude. The +# test client sends only the cookies it holds, and drops a Cookie header +# passed with a request, so each test sets the cookie on the client. +LATITUDE_COOKIE = "latitude" +SOUTHERN_LATITUDE = "-33.9" +NORTHERN_LATITUDE = "51.5" + + +def load_example(): + """Runs the example's file as a module and returns the module.""" + spec = importlib.util.spec_from_file_location( + "client_side_evidence_example", EXAMPLE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class ClientSideEvidenceExampleTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.example = load_example() + + @classmethod + def tearDownClass(cls): + cls.example.myPipeline.close() + super().tearDownClass() + + def setUp(self): + self.client = self.example.app.test_client() + + def get_page(self, **options): + response = self.client.get("/", **options) + page = response.get_data(as_text=True) + self.assertEqual(200, response.status_code, page) + return page + + def test_page_holds_the_form_and_the_script(self): + page = self.get_page() + + self.assertIn("

Starsign

", page) + self.assertIn("name='dateOfBirth'", page) + # The script the JavaScript builder writes posts to the JSON route. + self.assertIn("/json", page) + self.assertIn("const loadHemisphere", page) + self.assertNotIn("Your starsign is", page) + + def test_page_gives_the_star_sign_for_the_date_of_birth(self): + page = self.get_page(query_string={"dateOfBirth": DATE_OF_BIRTH}) + + self.assertIn(f"

Your starsign is {STAR_SIGN}

", page) + + def test_page_gives_the_hemisphere_for_the_latitude_cookie(self): + self.client.set_cookie(LATITUDE_COOKIE, NORTHERN_LATITUDE) + page = self.get_page() + + self.assertIn( + "

Look at the Northern hemisphere stars tonight!

", page) + + def test_json_route_gives_both_for_the_page_script(self): + self.client.set_cookie(LATITUDE_COOKIE, SOUTHERN_LATITUDE) + response = self.client.post( + "/json", data={"dateOfBirth": DATE_OF_BIRTH}) + text = response.get_data(as_text=True) + + self.assertEqual(200, response.status_code, text) + self.assertEqual("application/json", response.headers["Content-Type"]) + astrology = json.loads(text)["astrology"] + self.assertEqual(STAR_SIGN, astrology["starsign"]) + self.assertEqual("Southern", astrology["hemisphere"]) + self.assertIn("navigator.geolocation", astrology["getlatitude"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_documented_errors.py b/pipeline/fiftyone_pipeline_core/tests/test_documented_errors.py new file mode 100644 index 00000000..07dff4a7 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_documented_errors.py @@ -0,0 +1,59 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Each function a user calls that raises says in its docstring what it raises, +with an @raise line naming the exception, as exception-handling.md lines 17 +to 19 ask. That is the Epytext form the docstrings of this repository use. +""" + +import unittest + +from fiftyone_pipeline_core.aspectproperty_value import \ + AspectPropertyValue +from fiftyone_pipeline_core.flowdata import FlowData +from fiftyone_pipeline_core.pipeline import Pipeline + +# Each function, and the exceptions its docstring has to name. +DOCUMENTED = [ + (FlowData.process, ["Exception"]), + (FlowData.get, ["Exception"]), + (FlowData.__getattr__, ["Exception"]), + (FlowData.__getitem__, ["Exception"]), + (Pipeline.close, ["Exception"]), + (Pipeline.element_not_found, ["Exception"]), + (AspectPropertyValue.value, ["Exception"]), +] + + +class DocumentedErrorsTests(unittest.TestCase): + + def test_each_function_names_what_it_raises(self): + for function, names in DOCUMENTED: + for name in names: + with self.subTest(function=function.__qualname__, raises=name): + self.assertIn( + "@raise " + name + ":", function.__doc__ or "") + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py new file mode 100644 index 00000000..58d88e28 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_elementdata_documentation.py @@ -0,0 +1,26 @@ + +""" +The documentation of Element Data says it is safe to read from several threads +once the Flow Element that writes it has finished, and is not catered for when +written from several threads or read while another thread writes, as the +thread safety page of the specification requires. +""" + +import unittest + +from fiftyone_pipeline_core.elementdata import ElementData + + +class ElementDataDocumentationTests(unittest.TestCase): + + def test_the_documentation_says_what_is_safe_with_threads(self): + text = " ".join(ElementData.__doc__.split()) + self.assertIn("safe to read from several threads", text) + self.assertIn("once the Flow Element that writes it has finished", text) + self.assertIn("not catered for", text) + self.assertIn("written from several threads", text) + self.assertIn("read while another thread writes", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_find_out_more.py b/pipeline/fiftyone_pipeline_core/tests/test_find_out_more.py new file mode 100644 index 00000000..919888fe --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_find_out_more.py @@ -0,0 +1,71 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Every example ends with a "Find out more" section, which gives the person +running it links to follow up. The check reads the example's source, so it +needs no network. +""" + +import os +import re +import unittest + +# The product folder. Each link's utm_content is the path of its file below +# this folder. +PRODUCT = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..")) + +# The example of a custom flow element, a web page that ends with the +# section. +EXAMPLE = os.path.join( + PRODUCT, "fiftyone_pipeline_core", "examples", + "client_side_evidence_custom_flow_element.py") + +# The name of the section, or of the list that holds its links. +SECTION_NAME = re.compile(r"(?i)find[ _]out[ _]more") + +# A 51degrees.com link in the section, which carries a utm_term that starts +# with "find-out-more". +SECTION_LINK = re.compile( + r"https://51degrees\.com/[^\s\"'<>)]*utm_term=find-out-more[^\s\"'<>)]*") + +# This product's folder in the repository, which every section links to. +REPOSITORY = "https://github.com/51Degrees/51degrees-python/tree/main/pipeline" + + +class FindOutMoreTests(unittest.TestCase): + + def test_the_example_has_a_find_out_more_section(self): + with open(EXAMPLE, encoding="utf-8") as source: + text = source.read() + self.assertRegex(text, SECTION_NAME) + links = SECTION_LINK.findall(text) + self.assertTrue(links, "The section has no tagged 51degrees.com link") + content = os.path.relpath(EXAMPLE, PRODUCT).replace(os.sep, "-") + for link in links: + self.assertIn(f"&utm_content={content}&", link) + self.assertIn(REPOSITORY, text) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_minify_performance.py b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_minify_performance.py new file mode 100644 index 00000000..85523cbe --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_javascriptbuilder_minify_performance.py @@ -0,0 +1,111 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Measures what minifying the JavaScript adds to each request, as +javascript-builder.md asks. Each figure is the fastest of several rounds, +so a runner that pauses during one round does not fail the test. The +figures are given as a warning, so every build shows them. +""" + +import time +import unittest +import warnings + +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +# The requests in each round, with minifying on and with it off. +REQUESTS = 50 + +# The rounds timed. The fastest round of each is compared. +ROUNDS = 3 + +# The most minifying may add to one request, in milliseconds. About three +# times the most any runner measured, which was with coverage on. +ADDED_LIMIT_MS = 100 + + +class ScriptEngine(FlowElement): + """Gives a JavaScript Property, so the JavaScript Builder has a script + to write.""" + + def __init__(self): + super(ScriptEngine, self).__init__() + self.datakey = "script" + self.properties = {"javascript": {"type": "javascript"}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary( + self, {"javascript": "console.log('hello world')"})) + + +def pipeline(minify): + """A Pipeline whose JavaScript Builder minifies or not.""" + return PipelineBuilder( + {"javascript_builder_settings": {"minify": minify}}) \ + .add(ScriptEngine()).build() + + +def javascript_ms(pipeline, requests): + """The milliseconds one request takes, the JavaScript included, over + the requests, and the JavaScript the last one gave.""" + started = time.perf_counter() + for _ in range(requests): + with pipeline.create_flowdata() as flowdata: + flowdata.process() + javascript = flowdata.javascriptbuilder.javascript + return (time.perf_counter() - started) * 1000 / requests, javascript + + +def fastest_ms(pipeline): + """The fastest round's milliseconds for each request, and the + JavaScript given.""" + rounds = [javascript_ms(pipeline, REQUESTS) for _ in range(ROUNDS)] + return min(ms for ms, _ in rounds), rounds[-1][1] + + +class MinifyPerformanceTests(unittest.TestCase): + + def test_minifying_adds_little_to_each_request(self): + minified, plain = pipeline(True), pipeline(False) + # One request each first, so what happens once is left out. + javascript_ms(minified, 1) + javascript_ms(plain, 1) + + minified_ms, minified_js = fastest_ms(minified) + plain_ms, plain_js = fastest_ms(plain) + added_ms = minified_ms - plain_ms + figures = ( + f"Minifying added {added_ms:.3f} ms to each request, " + f"{minified_ms:.3f} ms minified and {plain_ms:.3f} ms not, " + f"the fastest of {ROUNDS} rounds of {REQUESTS} requests each") + warnings.warn(figures) + + self.assertLess(len(minified_js), len(plain_js)) + self.assertLess(added_ms, ADDED_LIMIT_MS, figures) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_delayexecution.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_delayexecution.py index 73fff908..0f344e25 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_delayexecution.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_delayexecution.py @@ -21,7 +21,13 @@ # ********************************************************************* """The JSON Builder writes delayexecution only for a Property that has a -value, as the JSON Builder specification's meta-Property table says.""" +value, as the JSON Builder specification's meta-Property table says. + +The delayed execution information comes from each element's property +metadata, which an element can lack for a while after the pipeline is +built, for example while that metadata is still being fetched. The JSON +Builder reads the information again while the element has no properties, +and keeps it once the element has them.""" import unittest @@ -52,6 +58,44 @@ def process_internal(self, flowdata): "plainnull": None})) +# The property metadata of DiscoveringElement once it is known. Value can +# be found once the delayed JavaScript property has run. +KNOWN_PROPERTIES = { + "javascript": delayed("JavaScript"), + "value": { + "name": "Value", + "type": "string", + "evidenceproperties": ["javascript"]}} + + +class DiscoveringElement(FlowElement): + """ + An element with no properties until discover is called, as an element + whose property metadata could not be fetched when the pipeline was built + has none until a later fetch succeeds. + """ + + def __init__(self): + super(DiscoveringElement, self).__init__() + self.datakey = "discovering" + self.properties = {} + + def discover(self): + self.properties = dict(KNOWN_PROPERTIES) + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary(self, { + "javascript": AspectPropertyValue(value="script"), + "value": AspectPropertyValue( + no_value_message="Needs the JavaScript to run")})) + + +def process(pipeline): + flowdata = pipeline.create_flowdata() + flowdata.process() + return flowdata.jsonbundler.json["discovering"] + + class JsonBuilderDelayExecutionTests(unittest.TestCase): def test_delayexecution_is_written_only_for_a_property_with_a_value(self): @@ -64,6 +108,49 @@ def test_delayexecution_is_written_only_for_a_property_with_a_value(self): self.assertNotIn("novaluedelayexecution", section) self.assertNotIn("plainnulldelayexecution", section) + def test_information_is_read_once_the_properties_are_known(self): + element = DiscoveringElement() + pipeline = PipelineBuilder({}).add(element).build() + self.addCleanup(pipeline.close) + + # Requests processed before the element's properties are known + # have nothing for the element. + self.assertEqual({}, process(pipeline)) + self.assertEqual({}, process(pipeline)) + + # A later request has the delayed execution information. + element.discover() + section = process(pipeline) + self.assertIs(True, section.get("javascriptdelayexecution"), section) + self.assertEqual( + ["discovering.javascript"], + section.get("valueevidenceproperties"), + section) + + def test_information_is_kept_once_it_is_known(self): + element = DiscoveringElement() + element.discover() + pipeline = PipelineBuilder({}).add(element).build() + self.addCleanup(pipeline.close) + + section = process(pipeline) + self.assertIs(True, section.get("javascriptdelayexecution"), section) + + # Metadata with no delayed execution flags, so the next request + # shows whether the information was read again. + element.properties = { + name: { + key: value for key, value in meta.items() + if key not in ("delayexecution", "evidenceproperties")} + for name, meta in KNOWN_PROPERTIES.items()} + + section = process(pipeline) + self.assertIs(True, section.get("javascriptdelayexecution"), section) + self.assertEqual( + ["discovering.javascript"], + section.get("valueevidenceproperties"), + section) + if __name__ == "__main__": unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_pipelines.py b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_pipelines.py new file mode 100644 index 00000000..1b8b46d8 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_jsonbundler_pipelines.py @@ -0,0 +1,88 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""One JSON Builder in two Pipelines whose elements share a data key. The +delayed execution information of each Pipeline comes from its own +element, as multi-pipeline-elements.md asks of an element that keeps state +for a Pipeline.""" + +import unittest + +from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.jsonbundler import JSONBundlerElement +from fiftyone_pipeline_core.pipeline import Pipeline +from fiftyone_pipeline_core.sequenceelement import SequenceElement + + +class DeviceElement(FlowElement): + """An element under the data key device whose JavaScript property runs + later or not, as its property metadata says.""" + + def __init__(self, delayed): + super(DeviceElement, self).__init__() + self.datakey = "device" + self.properties = { + "javascript": { + "name": "JavaScript", "type": "javascript", + "delayexecution": delayed}, + "value": { + "name": "Value", "type": "string", + "evidenceproperties": ["javascript"]}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary(self, { + "javascript": AspectPropertyValue(value="script"), + "value": AspectPropertyValue(value="value")})) + + +def device_section(pipeline): + with pipeline.create_flowdata() as flowdata: + flowdata.process() + return flowdata.jsonbundler.json["device"] + + +class JsonBuilderPipelinesTests(unittest.TestCase): + + def test_each_pipeline_has_the_information_of_its_own_element(self): + json_builder = JSONBundlerElement() + delayed = Pipeline( + [SequenceElement(), DeviceElement(True), json_builder]) + not_delayed = Pipeline( + [SequenceElement(), DeviceElement(False), json_builder]) + + for _ in range(2): + section = device_section(delayed) + self.assertIs( + True, section.get("javascriptdelayexecution"), section) + self.assertEqual( + ["device.javascript"], + section.get("valueevidenceproperties"), section) + + section = device_section(not_delayed) + self.assertNotIn("javascriptdelayexecution", section) + self.assertNotIn("valueevidenceproperties", section) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_pipeline_configuration.py b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_configuration.py new file mode 100644 index 00000000..f85845f3 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_configuration.py @@ -0,0 +1,151 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Element options set from a configuration file, as the Flow Elements section +of the pipeline configuration feature specification says. +""" + +import json +import os +import tempfile +import unittest + +from fiftyone_pipeline_core.elementdata_dictionary import \ + ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +UPDATE_URL = "http://127.0.0.1:9/update" + + +class Configured(FlowElement): + """! + An element whose options, a string, a number and a boolean, each change + the message it gives, and which keeps the two data update options that + have older names. + """ + + def __init__( + self, greeting="hello", repeat=1, shout=False, + data_update_url=None, data_update_verify_md5=True): + super(Configured, self).__init__() + self.datakey = "configured" + self.properties = {"message": {"type": "string"}} + self.greeting = greeting + self.repeat = repeat + self.shout = shout + self.data_update_url = data_update_url + self.data_update_verify_md5 = data_update_verify_md5 + + def process_internal(self, flow_data): + message = " ".join([self.greeting] * self.repeat) + if self.shout: + message = message.upper() + flow_data.set_element_data( + ElementDataDictionary(self, {"message": message})) + + +class PipelineConfigurationTests(unittest.TestCase): + + def build_from_file(self, parameters): + """! + Write a configuration file that adds a Configured element with the + parameters, read it back and build the pipeline it describes. + """ + configuration = { + "PipelineOptions": { + "Elements": [ + { + "elementName": Configured.__name__, + "elementPath": __name__, + "elementParameters": parameters, + } + ] + } + } + with tempfile.TemporaryDirectory() as folder: + path = os.path.join(folder, "pipeline.json") + with open(path, "w", encoding="utf-8") as file: + json.dump(configuration, file) + with open(path, encoding="utf-8") as file: + configuration = json.load(file) + return PipelineBuilder().build_from_configuration(configuration) + + def message(self, pipeline): + flow_data = pipeline.create_flowdata() + flow_data.process() + return flow_data.get("configured").get("message") + + def test_options_from_a_file_use_the_names_in_code(self): + """! + Options given in a configuration file under the names the element + takes in code reach it under those names and take effect, the same + as when it is configured in code. + """ + + pipeline = self.build_from_file( + {"greeting": "good day", "repeat": 2, "shout": True}) + in_code = PipelineBuilder( + {"add_javascript_builder": False, + "use_setheader_properties": False}) \ + .add(Configured(greeting="good day", repeat=2, shout=True)) \ + .build() + + element = pipeline.get_element("configured") + self.assertEqual("good day", element.greeting) + self.assertEqual(2, element.repeat) + self.assertTrue(element.shout) + self.assertEqual("GOOD DAY GOOD DAY", self.message(pipeline)) + self.assertEqual(self.message(in_code), self.message(pipeline)) + + def test_older_option_names_still_reach_the_element(self): + """! + The older names verify_md5 and data_file_update_base_url, given in + a configuration file, reach the element as data_update_verify_md5 + and data_update_url. The value given differs from the default, so + the test passes only when the name is mapped. + """ + + pipeline = self.build_from_file( + {"verify_md5": False, "data_file_update_base_url": UPDATE_URL}) + + element = pipeline.get_element("configured") + self.assertFalse(element.data_update_verify_md5) + self.assertEqual(UPDATE_URL, element.data_update_url) + + def test_newer_option_names_reach_the_element(self): + """! + The current names of the two data update options reach the element + unchanged. + """ + + pipeline = self.build_from_file( + {"data_update_verify_md5": False, "data_update_url": UPDATE_URL}) + + element = pipeline.get_element("configured") + self.assertFalse(element.data_update_verify_md5) + self.assertEqual(UPDATE_URL, element.data_update_url) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py new file mode 100644 index 00000000..16cb4612 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_evidence_key_filter.py @@ -0,0 +1,93 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + + +class Element(FlowElement): + """An element that accepts the keys it holds when asked.""" + + def __init__(self, datakey, keys): + super(Element, self).__init__() + self.datakey = datakey + self.keys = keys + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(self.keys) + + def process_internal(self, flowdata): + pass + + +class PipelineEvidenceKeyFilterTests(unittest.TestCase): + """! + advertize-accepted-evidence.md line 5: "All Flow Elements and Pipelines + MUST be capable of programmatically describing the Evidence keys that + they can make use of." + """ + + def build(self, *elements): + builder = PipelineBuilder(NO_DEFAULT_ELEMENTS) + for element in elements: + builder.add(element) + pipeline = builder.build() + self.addCleanup(pipeline.close) + return pipeline + + def test_the_pipeline_accepts_what_any_element_accepts(self): + accepted = self.build( + Element("one", ["header.user-agent"]), + Element("two", ["query.mark"])).get_evidence_key_filter() + self.assertTrue(accepted.filter_evidence_key("header.user-agent")) + self.assertTrue(accepted.filter_evidence_key("Header.User-Agent")) + self.assertTrue(accepted.filter_evidence_key("query.mark")) + self.assertFalse(accepted.filter_evidence_key("cookie.other")) + + def test_the_pipeline_reads_its_elements_filters_when_asked(self): + later = Element("later", []) + accepted = self.build(later).get_evidence_key_filter() + self.assertFalse(accepted.filter_evidence_key("header.x-later")) + later.keys = ["header.x-later"] + self.assertTrue(accepted.filter_evidence_key("header.x-later")) + + def test_the_pipeline_filters_evidence_to_what_it_accepts(self): + pipeline = self.build(Element("one", ["header.user-agent"])) + flowdata = pipeline.create_flowdata() + flowdata.evidence.add("header.user-agent", "a browser") + flowdata.evidence.add("cookie.other", "not read") + self.assertEqual( + {"header.user-agent": "a browser"}, + pipeline.get_evidence_key_filter().filter_evidence( + flowdata.evidence.get_all())) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_pipeline_properties.py b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_properties.py new file mode 100644 index 00000000..5f9c6e47 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_properties.py @@ -0,0 +1,119 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +Property metadata at the Pipeline level, as the property metadata section of +the properties feature specification says. +""" + +import unittest + +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +# No JavaScript or set header elements, so the pipeline holds only the +# elements a test adds. +NO_DEFAULT_ELEMENTS = { + "add_javascript_builder": False, + "use_setheader_properties": False, +} + +FIRST_PROPERTIES = { + "IsMobile": {"type": "bool", "category": "Device", + "description": "Whether the device is a mobile device."}, + "ScreenWidth": {"type": "int", "category": "Screen", + "description": "The width of the screen in pixels."}, +} + +SECOND_PROPERTIES = { + "Country": {"type": "string", "category": "Location", + "description": "The country the request came from."}, +} + + +class First(FlowElement): + """! + An element with two properties, whose metadata it holds from the start. + """ + + def __init__(self): + super(First, self).__init__() + self.datakey = "first" + self.properties = FIRST_PROPERTIES + + +class Second(FlowElement): + """! + An element with one property, whose metadata it holds from the start. + """ + + def __init__(self): + super(Second, self).__init__() + self.datakey = "second" + self.properties = SECOND_PROPERTIES + + +class PipelinePropertiesTests(unittest.TestCase): + + def test_metadata_of_every_property_of_both_elements(self): + """! + The pipeline gives the metadata of every property of both of its + elements, keyed by each element's data key and then by the + property's name in lower case, without processing anything. + """ + + pipeline = PipelineBuilder(NO_DEFAULT_ELEMENTS) \ + .add(First()).add(Second()).build() + + self.assertEqual( + { + "first": { + "ismobile": FIRST_PROPERTIES["IsMobile"], + "screenwidth": FIRST_PROPERTIES["ScreenWidth"], + }, + "second": { + "country": SECOND_PROPERTIES["Country"], + }, + }, + pipeline.get_properties()) + + def test_metadata_of_the_elements_the_builder_adds(self): + """! + The elements the builder adds by default are in the pipeline too, so + the pipeline gives their metadata, the same as each element gives. + """ + + pipeline = PipelineBuilder().add(First()).add(Second()).build() + + properties = pipeline.get_properties() + + self.assertEqual( + [element.datakey for element in pipeline.flow_elements], + list(properties)) + for element in pipeline.flow_elements: + with self.subTest(element=element.datakey): + self.assertEqual( + element.get_properties(), properties[element.datakey]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pipeline/fiftyone_pipeline_core/tests/test_pipeline_read_only.py b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_read_only.py new file mode 100644 index 00000000..92cdb1d7 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_pipeline_read_only.py @@ -0,0 +1,281 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import unittest + +from fiftyone_pipeline_core.basiclist_evidence_keyfilter import \ + BasicListEvidenceKeyFilter +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipeline import Pipeline +from fiftyone_pipeline_core.pipelinebuilder import PipelineBuilder + +# The errors Python raises for a change to a collection that cannot be +# changed. +READ_ONLY = (AttributeError, TypeError) + + +class CountingElement(FlowElement): + + """! + An element that counts the times it processes, and accepts the header + named after its data key. + + """ + + def __init__(self, datakey): + super(CountingElement, self).__init__() + self.datakey = datakey + self.calls = 0 + + def get_evidence_key_filter(self): + return BasicListEvidenceKeyFilter(["header." + self.datakey]) + + def process_internal(self, flowdata): + self.calls += 1 + + +class RegistrationElement(CountingElement): + + """! + An element that records what the pipeline holds when the element is + registered with it. + + """ + + def on_registration(self, pipeline): + self.keys_seen = list(pipeline.flow_elements_list) + self.elements_seen = len(pipeline.flow_elements) + + +def builder(*elements): + # Only the elements given, so the pipeline's order and length are known. + result = PipelineBuilder({ + "add_javascript_builder": False, + "use_setheader_properties": False}) + for element in elements: + result.add(element) + return result + + +def build(*elements): + return builder(*elements).build() + + +def process(pipeline): + flowdata = pipeline.create_flowdata() + flowdata.process() + return flowdata + + +def accepts(pipeline, key): + return pipeline.get_evidence_key_filter().filter_evidence_key(key) + + +class PipelineCannotBeChangedTests(unittest.TestCase): + + # Appending an element to a built pipeline raises, and the element does + # not run. + def test_appending_an_element_raises_and_it_does_not_run(self): + first = CountingElement("first") + pipeline = build(first) + added = CountingElement("added") + + with self.assertRaises(READ_ONLY): + pipeline.flow_elements.append(added) + + process(pipeline) + self.assertEqual([1, 0], [first.calls, added.calls]) + self.assertEqual(1, len(pipeline.flow_elements)) + + # Replacing or removing an element raises, and every element the + # pipeline was built with still runs. + def test_replacing_or_removing_an_element_raises(self): + first = CountingElement("first") + second = CountingElement("second") + pipeline = build(first, second) + other = CountingElement("other") + + with self.assertRaises(READ_ONLY): + pipeline.flow_elements[0] = other + with self.assertRaises(READ_ONLY): + pipeline.flow_elements.remove(second) + with self.assertRaises(READ_ONLY): + del pipeline.flow_elements[0] + + process(pipeline) + self.assertEqual( + [1, 1, 0], [first.calls, second.calls, other.calls]) + + # Binding the attributes again raises, and the pipeline still runs + # and finds the elements it was built with. + def test_binding_the_attributes_again_raises_and_changes_nothing(self): + first = CountingElement("first") + pipeline = build(first) + other = CountingElement("other") + + for name, value in ( + ("flow_elements", (other,)), + ("flow_elements_list", {"other": other}), + ("flow_elements_display_list", ("other",))): + with self.subTest(attribute=name): + with self.assertRaises(AttributeError): + setattr(pipeline, name, value) + with self.assertRaises(AttributeError): + delattr(pipeline, name) + + process(pipeline) + self.assertEqual([1, 0], [first.calls, other.calls]) + self.assertIs(first, pipeline.get_element("first")) + self.assertNotIn("other", pipeline.flow_elements_display_list) + + # Adding to the elements by data key raises, and the pipeline neither + # finds the element nor accepts its evidence. + def test_adding_to_the_lookup_raises_and_changes_nothing(self): + first = CountingElement("first") + pipeline = build(first) + added = CountingElement("added") + + with self.assertRaises(READ_ONLY): + pipeline.flow_elements_list["added"] = added + with self.assertRaises(READ_ONLY): + pipeline.flow_elements_list.update({"added": added}) + + self.assertEqual(["first"], list(pipeline.flow_elements_list)) + with self.assertRaises(KeyError): + pipeline.get_element("added") + self.assertFalse(accepts(pipeline, "header.added")) + process(pipeline) + self.assertEqual([1, 0], [first.calls, added.calls]) + + # Removing from the elements by data key raises, and the pipeline still + # finds the element and accepts its evidence. + def test_removing_from_the_lookup_raises_and_changes_nothing(self): + first = CountingElement("first") + pipeline = build(first) + + with self.assertRaises(READ_ONLY): + del pipeline.flow_elements_list["first"] + with self.assertRaises(READ_ONLY): + pipeline.flow_elements_list.pop("first") + + self.assertIs(first, pipeline.get_element("first")) + self.assertTrue(accepts(pipeline, "header.first")) + + # The data keys an error message lists cannot be changed either. + def test_changing_the_data_keys_in_messages_raises(self): + pipeline = build(CountingElement("first")) + + with self.assertRaises(READ_ONLY): + pipeline.flow_elements_display_list.append("added") + + self.assertEqual( + ["first"], list(pipeline.flow_elements_display_list)) + + # The pipeline keeps its own copy of the elements it is given, so + # changing the caller's list afterwards changes nothing. + def test_changing_the_list_given_to_the_pipeline_changes_nothing(self): + first = CountingElement("first") + elements = [first] + pipeline = Pipeline(elements) + added = CountingElement("added") + + elements.append(added) + + process(pipeline) + self.assertEqual([1, 0], [first.calls, added.calls]) + self.assertEqual(1, len(pipeline.flow_elements)) + + +class PipelineReadTests(unittest.TestCase): + + # The elements read in order, by index, by length and by membership. + def test_reading_the_elements(self): + first = CountingElement("first") + second = CountingElement("second") + pipeline = build(first, second) + + self.assertEqual(2, len(pipeline.flow_elements)) + self.assertIs(first, pipeline.flow_elements[0]) + self.assertIs(second, pipeline.flow_elements[-1]) + self.assertEqual([first, second], list(pipeline.flow_elements)) + self.assertEqual( + ["first", "second"], + [element.datakey for element in pipeline.flow_elements]) + self.assertIn(second, pipeline.flow_elements) + self.assertEqual(1, pipeline.flow_elements.index(second)) + + # The elements by data key read by key, in the order they were added, + # and through get_element. + def test_reading_the_lookup(self): + first = CountingElement("first") + second = CountingElement("second") + pipeline = build(first, second) + + self.assertIs(first, pipeline.get_element("first")) + self.assertIs(second, pipeline.flow_elements_list["second"]) + self.assertIn("first", pipeline.flow_elements_list) + self.assertEqual(2, len(pipeline.flow_elements_list)) + self.assertEqual( + ["first", "second"], list(pipeline.flow_elements_list)) + self.assertEqual( + [first, second], list(pipeline.flow_elements_list.values())) + self.assertIsNone(pipeline.flow_elements_list.get("missing")) + with self.assertRaises(KeyError): + pipeline.get_element("missing") + self.assertEqual( + ["first", "second"], list(pipeline.flow_elements_display_list)) + self.assertTrue(accepts(pipeline, "header.second")) + + # Processing runs each element once. + def test_processing_runs_every_element(self): + first = CountingElement("first") + second = CountingElement("second") + + process(build(first, second)) + + self.assertEqual([1, 1], [first.calls, second.calls]) + + # An element being registered finds every element in the pipeline, and + # by data key those registered before it, which is how an element finds + # one it depends on. + def test_an_element_sees_those_before_it_when_registered(self): + first = RegistrationElement("first") + second = RegistrationElement("second") + + build(first, second) + + self.assertEqual([], first.keys_seen) + self.assertEqual(["first"], second.keys_seen) + self.assertEqual([2, 2], [first.elements_seen, second.elements_seen]) + + # Adding to the builder after build leaves the built pipeline as it was. + def test_adding_to_the_builder_after_build_changes_nothing(self): + first = CountingElement("first") + pipeline_builder = builder(first) + pipeline = pipeline_builder.build() + added = CountingElement("added") + + pipeline_builder.add(added) + + process(pipeline) + self.assertEqual([1, 0], [first.calls, added.calls]) + self.assertEqual(1, len(pipeline.flow_elements)) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_request_line.py b/pipeline/fiftyone_pipeline_core/tests/test_request_line.py new file mode 100644 index 00000000..90ec9f04 --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_request_line.py @@ -0,0 +1,76 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +"""The request line the web integration hands over in server.http-request, +being the method, the path and the query string, each exactly as the +request line held them, as web-integration.md lines 100 to 114 ask.""" + +import unittest + +from flask import Flask, request + +from fiftyone_pipeline_core.web import webevidence + + +def view_of(path, **environ): + """The request view webevidence gives for a request for the path, with + the WSGI environment values given.""" + app = Flask(__name__) + with app.test_request_context(path, environ_overrides=environ): + return webevidence(request)["server.http-request"] + + +class RequestLineTests(unittest.TestCase): + + def test_all_three_values_are_given(self): + view = view_of("/page?campaign=a") + + self.assertEqual("GET", view["server.request-method"]) + self.assertEqual("/page", view["server.request-path"]) + self.assertEqual("campaign=a", view["server.request-query"]) + + def test_the_query_is_empty_when_there_is_none(self): + view = view_of("/page") + + self.assertEqual("", view["server.request-query"]) + + def test_the_values_are_as_the_request_line_held_them(self): + view = view_of( + "/a/b/~user", RAW_URI="/a%2Fb/%7Euser?b=2&a=%20one+two&a=3", + QUERY_STRING="b=2&a=%20one+two&a=3") + + self.assertEqual("/a%2Fb/%7Euser", view["server.request-path"]) + self.assertEqual( + "b=2&a=%20one+two&a=3", view["server.request-query"]) + + def test_a_fragment_is_not_part_of_the_query(self): + view = view_of("/page", RAW_URI="/page?a=1#part") + + self.assertEqual("a=1", view["server.request-query"]) + + def test_the_query_string_is_taken_whole_with_no_request_line(self): + # A server that does not say what the request line was gives the + # query string undecoded, as WSGI does. + view = view_of( + "/page?b=2&a=%20one", RAW_URI="", REQUEST_URI="") + + self.assertEqual("b=2&a=%20one", view["server.request-query"]) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py index 240080fb..9f96df6a 100644 --- a/pipeline/fiftyone_pipeline_core/tests/test_setheader.py +++ b/pipeline/fiftyone_pipeline_core/tests/test_setheader.py @@ -28,6 +28,9 @@ from fiftyone_pipeline_core.messages import Messages from fiftyone_pipeline_core.web import * from fiftyone_pipeline_core.aspectproperty_value import AspectPropertyValue +from fiftyone_pipeline_core.elementdata_dictionary import ElementDataDictionary +from fiftyone_pipeline_core.flowelement import FlowElement +from fiftyone_pipeline_core.pipeline import Pipeline from fiftyone_pipeline_core.setheaderelement import SetHeaderElement from .classes.testpipeline import TestPipeline @@ -83,30 +86,154 @@ def test_set_response_header(self, data, expected_value): response = set_response_header(flowdata, response) self.assertEqual(expected_value, Constants.ACCEPTCH_HEADER in response.headers) - # Test get response header function for valid formats. + # Test get response header function for valid formats. The name is + # SetHeader[Identifier][HeaderName], and the identifier has no upper-case + # characters after its first character, which may be either case. @parameterized.expand([ ("SetHeaderBrowserAccept-CH", "Accept-CH"), + ("SetHeaderHardwareAccept-CH", "Accept-CH"), + ("SetHeaderPlatformAccept-CH", "Accept-CH"), ("SetHeaderBrowserCritical-CH", "Critical-CH"), - ("SetHeaderUnknownAccept-CH", "Accept-CH") + ("SetHeaderUnknownAccept-CH", "Accept-CH"), + ("SetHeaderbrowserAccept-CH", "Accept-CH"), + # The header name holds the identifier, which must be kept. + ("SetHeaderAcceptAccept-CH", "Accept-CH") ]) def test_get_response_header_name(self, property_key, expected_value): setHeaderElement = SetHeaderElement() actual_value = setHeaderElement.get_response_header_name(property_key) self.assertEqual(expected_value, actual_value) - # Test get response header function raises exception + # Test get response header function raises exception # for invalid property formats. @parameterized.expand([ ("TestBrowserAccept-CH", Messages.PROPERTY_NOT_SET_HEADER), - ("SetHeaderbrowserAccept-CH", Messages.WRONG_PROPERTY_FORMAT), - ("SetHeaderBrowseraccept-ch", Messages.WRONG_PROPERTY_FORMAT) + ("SetHeaderBrowseraccept-ch", Messages.WRONG_PROPERTY_FORMAT), + ("SetHeaderbrowser", Messages.WRONG_PROPERTY_FORMAT), + ("SetHeaderBrowser", Messages.WRONG_PROPERTY_FORMAT), + ("SetHeader", Messages.WRONG_PROPERTY_FORMAT) ]) - def test_get_response_header_name(self, property_value, expected_message): + def test_get_response_header_name_error( + self, property_value, expected_message): setHeaderElement = SetHeaderElement() - try: + with self.assertRaises(Exception) as caught: setHeaderElement.get_response_header_name(property_value) - except Exception as e: - message = str(e) - self.assertEqual(expected_message.format(property_value), message) + self.assertEqual( + expected_message.format(property_value), str(caught.exception)) + + # Only a property whose name starts with SetHeader is for a response + # header, so a name that holds SetHeader later on is left alone. + def test_get_setheader_properties_pipeline_needs_the_prefix(self): + + class Element(object): + datakey = "device" + + def get_properties(self): + return { + "setheaderbrowseraccept-ch": { + "name": "SetHeaderBrowserAccept-CH"}, + "notsetheaderbrowseraccept-ch": { + "name": "NotSetHeaderBrowserAccept-CH"}} + + class Pipeline(object): + flow_elements = [Element()] + + self.assertEqual( + {"device": ["SetHeaderBrowserAccept-CH"]}, + SetHeaderElement().get_setheader_properties_pipeline(Pipeline())) + + +# An element with one property for a response header, whose value is the +# header's value. +class HeaderSourceElement(FlowElement): + + def __init__(self, datakey, property_name, value): + super(HeaderSourceElement, self).__init__() + self.datakey = datakey + self.property_name = property_name + self.value = value + self.properties = { + property_name: {"name": property_name, "type": "string"}} + + def process_internal(self, flowdata): + flowdata.set_element_data(ElementDataDictionary( + self, + {self.property_name: AspectPropertyValue(value=self.value)})) + + +def response_headers(pipeline): + flowdata = pipeline.create_flowdata() + flowdata.process() + return flowdata["set-headers"]["responseheaderdictionary"] + + +BROWSER_HEADERS = {"Accept-CH": Constants.ACCEPTCH_BROWSER_VALUE} +PLATFORM_HEADERS = {"Critical-CH": Constants.ACCEPTCH_PLATFORM_VALUE} + + +# The multi-pipeline elements specification says an element that keeps +# state for the Pipeline it is in keeps that state for each Pipeline it is +# added to. One Set Headers Element is shared by two Pipelines whose other +# elements give different SetHeader properties. +class SetHeaderMultiPipelineTests(unittest.TestCase): + + def setUp(self): + self.set_headers = set_headers = SetHeaderElement() + # The Pipelines share an element, so neither closes its elements. + self.pipelines = { + "browser": Pipeline( + [HeaderSourceElement( + "browser", "SetHeaderBrowserAccept-CH", + Constants.ACCEPTCH_BROWSER_VALUE), + set_headers], + auto_close_elements=False), + "platform": Pipeline( + [HeaderSourceElement( + "platform", "SetHeaderPlatformCritical-CH", + Constants.ACCEPTCH_PLATFORM_VALUE), + set_headers], + auto_close_elements=False)} + self.expected = { + "browser": BROWSER_HEADERS, "platform": PLATFORM_HEADERS} + + # Each Pipeline gives the headers of its own elements, whichever + # processes first, and keeps doing so after the other has processed. + @parameterized.expand([ + ("browser_first", ["browser", "platform"]), + ("platform_first", ["platform", "browser"])]) + def test_each_pipeline_gives_its_own_headers(self, _, order): + for name in order + order: + self.assertEqual( + self.expected[name], response_headers(self.pipelines[name]), + name) + + # setheader_properties holds the SetHeader properties of the first + # Pipeline that processed, as a dict of data key to property names, for + # code that reads it. + def test_setheader_properties_holds_the_first_pipelines_properties(self): + response_headers(self.pipelines["browser"]) + response_headers(self.pipelines["platform"]) + self.assertEqual( + {"browser": ["SetHeaderBrowserAccept-CH"]}, + self.set_headers.setheader_properties) + + def test_one_pipeline_gives_the_same_headers_on_every_request(self): + for _ in range(3): + self.assertEqual( + BROWSER_HEADERS, response_headers(self.pipelines["browser"])) + + # A cloud engine has no properties until the cloud service has + # answered, so a Pipeline whose SetHeader properties are not known on + # its first request gives the headers once they are. + def test_headers_are_given_once_the_properties_are_known(self): + source = HeaderSourceElement( + "browser", "SetHeaderBrowserAccept-CH", + Constants.ACCEPTCH_BROWSER_VALUE) + properties = source.properties + source.properties = {} + pipeline = Pipeline([source, SetHeaderElement()]) + self.assertEqual({}, response_headers(pipeline)) + source.properties = properties + self.assertEqual(BROWSER_HEADERS, response_headers(pipeline)) diff --git a/pipeline/fiftyone_pipeline_core/tests/test_simple_flow_element_example.py b/pipeline/fiftyone_pipeline_core/tests/test_simple_flow_element_example.py new file mode 100644 index 00000000..a25f141b --- /dev/null +++ b/pipeline/fiftyone_pipeline_core/tests/test_simple_flow_element_example.py @@ -0,0 +1,142 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +The simple flow element example needs no data file, no key and no network, +so it is run here as a person runs it, from the command line, with none of +the _51DEGREES_ environment variables set. What it prints is checked, being +the star signs and the Find out more section it ends with. +""" + +import os +import re +import subprocess +import sys +import unittest + +# The product folder. Each link's utm_content is the path of the example +# below this folder. +PRODUCT = os.path.abspath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "..")) + +EXAMPLE = os.path.join( + PRODUCT, "fiftyone_pipeline_core", "examples", "simple_flow_element.py") + +# How long to wait for the example before failing, so a run that never +# ends cannot hold up the tests. +RUN_TIMEOUT_SECONDS = 120 + +# The start of the name of every 51Degrees environment variable. Each one is +# removed from the example's environment, to show it needs none of them. +SETTING_PREFIX = "_51DEGREES_" + +# What the example prints for each date of birth it looks up. +STAR_SIGNS = [ + "With a date of birth of 1992-12-18, your star sign is Sagittarius.", + "With a date of birth of 1990-12-25, your star sign is Capricorn.", + "With a date of birth of 1992-04-15, your star sign is Aries.", +] + +# The heading of the Find out more section, and the line above it. +HEADING = "Find out more:" +RULE = "=" * 79 + +# A line of the section, which is a tab, a label and a link. +SECTION_LINE = re.compile(r"\t(?P