Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion ci/write-package-versions.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,42 @@ param (
$ErrorActionPreference = "Stop"
. (Join-Path $PSScriptRoot 'packages.ps1')

# The lowest version each package here is required at by another package
# here, read from the install_requires of every setup.py. A package this
# commit would not release keeps its last release, which can be below a
# minimum that names the version this release gives it, and pip then looks
# for that version on a package index, where it does not exist yet.
function Get-RequiredMinimums($RepoName, $Packages) {
$minimums = @{}
foreach ($package in $Packages) {
$setup = Join-Path $RepoName $package.Path 'setup.py'
if (-not (Test-Path $setup)) {
continue
}
$text = Get-Content $setup -Raw
$pattern = '["''](?<name>[A-Za-z0-9_.-]+)\s*>=\s*(?<version>\d+(\.\d+)+)["'']'
foreach ($match in [regex]::Matches($text, $pattern)) {
$name = $match.Groups['name'].Value
$version = [version]$match.Groups['version'].Value
if (-not $minimums.ContainsKey($name) -or
$minimums[$name] -lt $version) {
$minimums[$name] = $version
}
}
}
return $minimums
}

# Writes the version.txt that each published package's setup.py reads. Without
# one a package installed from this checkout is 0.0.0, which is below the
# versions the packages here require of each other, so pip refuses it, or
# replaces it with a release from PyPI. Each package gets the version this
# commit would release it at, as the publish gives it when it builds the
# package, or its last release when this commit would not release it.
# package, or its last release when this commit would not release it, and
# never less than the version another package here requires of it.
$plan = @(Get-ReleasePlan $RepoName)
$published = @(Get-Packages $RepoName | Where-Object { $_.Publish -eq $true })
$minimums = Get-RequiredMinimums $RepoName $published
foreach ($package in $published) {
# A version.txt committed with the package is left as it is.
if (git -C $RepoName ls-files -- "$($package.Path)/version.txt") {
Expand All @@ -23,6 +51,12 @@ foreach ($package in $published) {
} else {
$version = "$((Get-ReleasedVersions $RepoName $package.Name)[-1])"
}
$required = $minimums[$package.Name]
if ($null -ne $required -and [version]$version -lt $required) {
Write-Output ("$($package.Name) $version is below the $required " +
"another package here requires, so it is written as $required")
$version = "$required"
}
$version | Out-File (Join-Path $RepoName $package.Path 'version.txt')
Write-Output "$($package.Name) $version"
}
7 changes: 6 additions & 1 deletion device-detection/fiftyone_devicedetection_cloud/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,12 @@ def read(file_name):
python_requires=">=3.10",
packages=["fiftyone_devicedetection_cloud"],
package_dir={"": "src"},
install_requires=["fiftyone_devicedetection_shared"],
install_requires=[
"fiftyone_devicedetection_shared",
# The first version with CloudMissingPropertyService, which the
# hardware profile results of this package use.
"fiftyone_pipeline_cloudrequestengine>=4.5.45",
],
license="EUPL-1.2",
classifiers=[
"Development Status :: 5 - Production/Stable",
Expand Down
5 changes: 4 additions & 1 deletion device-detection/fiftyone_devicedetection_onpremise/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,9 @@ def stage_native_source():
# The first versions with close(), which releases the native memory
# this package holds.
"fiftyone_pipeline_core>=4.5.42",
"fiftyone_pipeline_engines>=4.5.42",
# The first version with OnPremiseMissingPropertyService, which the
# results of this package give a caller that reads a Property the
# data file in use does not hold.
"fiftyone_pipeline_engines>=4.5.44",
],
)
5 changes: 4 additions & 1 deletion ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,9 @@ def stage_native_source():
# The first versions with close(), which releases the native memory
# this package holds.
"fiftyone_pipeline_core>=4.5.42",
"fiftyone_pipeline_engines>=4.5.42",
# The first version with OnPremiseMissingPropertyService, which the
# results of this package give a caller that reads a Property the
# data file in use does not hold.
"fiftyone_pipeline_engines>=4.5.44",
],
)
4 changes: 3 additions & 1 deletion pipeline/fiftyone_pipeline_cloudrequestengine/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,9 @@ def read(file_name):
packages=["fiftyone_pipeline_cloudrequestengine"],
package_dir={"": "src"},
install_requires=[
'fiftyone_pipeline_engines',
# The first version with MissingPropertyReason, which this
# package gives as the reason a cloud Property has no value.
'fiftyone_pipeline_engines>=4.5.44',
'requests',
],
license="EUPL-1.2",
Expand Down
195 changes: 195 additions & 0 deletions pipeline/fiftyone_pipeline_engines/tests/test_package_requirements.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
# *********************************************************************
# This Original Work is copyright of 51 Degrees Mobile Experts Limited.
# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House,
# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU.
#
# This Original Work is licensed under the European Union Public Licence
# (EUPL) v.1.2 and is subject to its terms as set out below.
#
# If a copy of the EUPL was not distributed with this file, You can obtain
# one at https://opensource.org/licenses/EUPL-1.2.
#
# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be
# amended by the European Commission) shall be deemed incompatible for
# the purposes of the Work and the provisions of the compatibility
# clause in Article 5 of the EUPL shall not apply.
#
# If using the Work as, or as part of, a network application, by
# including the attribution notice(s) required under Article 5 of the EUPL
# in the end user terms of the application under an appropriate heading,
# such notice(s) shall fulfill the requirements of that article.
# *********************************************************************

import ast
import os
import re
import unittest

# Each row is a name one package here imports from another package here,
# the package that holds it and the version of that package it came in at,
# so the importing package has to require at least that version. Without
# the minimum, pip leaves an older copy of the dependency in place when the
# importing package alone is upgraded, and the import fails. Every part of
# a row is read from the checkout, so none of it is trusted.
IMPORTED_NAMES = (
("pipeline/fiftyone_pipeline_cloudrequestengine",
"fiftyone_pipeline_engines",
"4.5.44",
"fiftyone_pipeline_engines.missingproperty_service",
"MissingPropertyReason"),
("device-detection/fiftyone_devicedetection_cloud",
"fiftyone_pipeline_cloudrequestengine",
"4.5.45",
"fiftyone_pipeline_cloudrequestengine.clouddata",
"CloudMissingPropertyService"),
("device-detection/fiftyone_devicedetection_onpremise",
"fiftyone_pipeline_engines",
"4.5.44",
"fiftyone_pipeline_engines.missingproperty_service",
"OnPremiseMissingPropertyService"),
("ip-intelligence/fiftyone_ipintelligence_onpremise",
"fiftyone_pipeline_engines",
"4.5.44",
"fiftyone_pipeline_engines.missingproperty_service",
"OnPremiseMissingPropertyService"),
)

# Where each package that holds one of the names above keeps its source.
PACKAGE_FOLDERS = {
"fiftyone_pipeline_engines": "pipeline/fiftyone_pipeline_engines",
"fiftyone_pipeline_cloudrequestengine":
"pipeline/fiftyone_pipeline_cloudrequestengine",
}

# A requirement that gives a minimum version, as setup.py writes one.
MINIMUM = re.compile(r"^>=\s*([0-9]+(?:\.[0-9]+)*)$")

# The name and the rest of one entry of an install_requires list.
REQUIREMENT = re.compile(r"^([A-Za-z0-9_.-]+)\s*(.*)$")

# The entries of an install_requires list, each in its own quotes.
QUOTED = re.compile("[\"']([^\"']+)[\"']")


def repository_root():
"""The folder holding ci/packages.json, found by walking up from this
file, so the test reads the checkout it runs from."""
folder = os.path.dirname(os.path.abspath(__file__))
while True:
if os.path.isfile(os.path.join(folder, "ci", "packages.json")):
return folder
parent = os.path.dirname(folder)
if parent == folder:
raise AssertionError(
"no ci/packages.json above " + os.path.abspath(__file__))
folder = parent


def requirements(package_folder):
"""The install_requires of a package's setup.py, as the name required
mapped to the rest of the requirement."""
path = os.path.join(repository_root(), package_folder, "setup.py")
with open(path, "rb") as file:
text = file.read().decode("utf-8")
found = re.search(r"install_requires\s*=\s*\[(.*?)\]", text, re.S)
assert found is not None, "no install_requires in " + path
out = {}
for item in QUOTED.findall(found.group(1)):
parts = REQUIREMENT.match(item.strip())
if parts:
out[parts.group(1)] = parts.group(2).strip()
return out


def version_numbers(text):
"""A version as a tuple of its numbers, for comparing two of them."""
return tuple(int(part) for part in text.split("."))


def imports_in(package_folder, module, name):
"""Whether any source file of the package imports the name from the
module."""
source = os.path.join(repository_root(), package_folder, "src")
for folder, _, files in os.walk(source):
if "cxx" in folder.split(os.sep):
continue
for file_name in files:
if not file_name.endswith(".py"):
continue
with open(os.path.join(folder, file_name), "rb") as file:
text = file.read().decode("utf-8", "replace")
try:
tree = ast.parse(text)
except SyntaxError:
continue
for node in ast.walk(tree):
if (isinstance(node, ast.ImportFrom)
and node.module == module
and any(alias.name == name
for alias in node.names)):
return True
return False


def module_names(module):
"""The names a module of this checkout holds at its top level."""
parts = module.split(".")
path = os.path.join(repository_root(), PACKAGE_FOLDERS[parts[0]],
"src", *parts) + ".py"
with open(path, "rb") as file:
text = file.read().decode("utf-8")
out = set()
for node in ast.parse(text).body:
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef,
ast.ClassDef)):
out.add(node.name)
elif isinstance(node, ast.Assign):
out.update(target.id for target in node.targets
if isinstance(target, ast.Name))
return out


class PackageRequirementTests(unittest.TestCase):

"""A package that imports a name from another package here requires
that package at the version the name came in at. Upgrading one package
on its own then brings the dependency with it, where a requirement
with no version leaves an older copy in place and the import fails."""

def test_each_name_is_imported_where_the_row_says(self):
for folder, _, _, module, name in IMPORTED_NAMES:
with self.subTest(package=folder, name=name):
self.assertTrue(
imports_in(folder, module, name),
"{0} does not import {1} from {2}".format(
folder, name, module))

def test_each_name_is_held_by_the_package_the_row_names(self):
for _, package, _, module, name in IMPORTED_NAMES:
with self.subTest(package=package, name=name):
self.assertEqual(package, module.split(".")[0])
self.assertIn(name, module_names(module))

def test_each_package_is_required_at_the_version_of_the_name(self):
for folder, package, version, module, name in IMPORTED_NAMES:
with self.subTest(package=folder, requires=package):
required = requirements(folder)
self.assertIn(
package, required,
"{0} imports {1} from {2} and does not require "
"{3}".format(folder, name, module, package))
found = MINIMUM.match(required[package])
self.assertIsNotNone(
found,
"{0} requires {1} as {2!r}, with no minimum "
"version".format(folder, package, required[package]))
self.assertGreaterEqual(
version_numbers(found.group(1)),
version_numbers(version),
"{0} requires {1} {2}, below the {3} that holds "
"{4}".format(folder, package, required[package],
version, name))


if __name__ == "__main__":
unittest.main()
Loading