diff --git a/ci/write-package-versions.ps1 b/ci/write-package-versions.ps1 index 44b23f25..c522e5ef 100644 --- a/ci/write-package-versions.ps1 +++ b/ci/write-package-versions.ps1 @@ -4,14 +4,42 @@ param ( $ErrorActionPreference = "Stop" . (Join-Path $PSScriptRoot 'packages.ps1') +# The lowest version each package here is required at by another package +# here, read from the install_requires of every setup.py. A package this +# commit would not release keeps its last release, which can be below a +# minimum that names the version this release gives it, and pip then looks +# for that version on a package index, where it does not exist yet. +function Get-RequiredMinimums($RepoName, $Packages) { + $minimums = @{} + foreach ($package in $Packages) { + $setup = Join-Path $RepoName $package.Path 'setup.py' + if (-not (Test-Path $setup)) { + continue + } + $text = Get-Content $setup -Raw + $pattern = '["''](?[A-Za-z0-9_.-]+)\s*>=\s*(?\d+(\.\d+)+)["'']' + foreach ($match in [regex]::Matches($text, $pattern)) { + $name = $match.Groups['name'].Value + $version = [version]$match.Groups['version'].Value + if (-not $minimums.ContainsKey($name) -or + $minimums[$name] -lt $version) { + $minimums[$name] = $version + } + } + } + return $minimums +} + # Writes the version.txt that each published package's setup.py reads. Without # one a package installed from this checkout is 0.0.0, which is below the # versions the packages here require of each other, so pip refuses it, or # replaces it with a release from PyPI. Each package gets the version this # commit would release it at, as the publish gives it when it builds the -# package, or its last release when this commit would not release it. +# package, or its last release when this commit would not release it, and +# never less than the version another package here requires of it. $plan = @(Get-ReleasePlan $RepoName) $published = @(Get-Packages $RepoName | Where-Object { $_.Publish -eq $true }) +$minimums = Get-RequiredMinimums $RepoName $published foreach ($package in $published) { # A version.txt committed with the package is left as it is. if (git -C $RepoName ls-files -- "$($package.Path)/version.txt") { @@ -23,6 +51,12 @@ foreach ($package in $published) { } else { $version = "$((Get-ReleasedVersions $RepoName $package.Name)[-1])" } + $required = $minimums[$package.Name] + if ($null -ne $required -and [version]$version -lt $required) { + Write-Output ("$($package.Name) $version is below the $required " + + "another package here requires, so it is written as $required") + $version = "$required" + } $version | Out-File (Join-Path $RepoName $package.Path 'version.txt') Write-Output "$($package.Name) $version" } diff --git a/device-detection/fiftyone_devicedetection_cloud/setup.py b/device-detection/fiftyone_devicedetection_cloud/setup.py index 7318548d..cbf3f7dc 100644 --- a/device-detection/fiftyone_devicedetection_cloud/setup.py +++ b/device-detection/fiftyone_devicedetection_cloud/setup.py @@ -65,7 +65,12 @@ def read(file_name): python_requires=">=3.10", packages=["fiftyone_devicedetection_cloud"], package_dir={"": "src"}, - install_requires=["fiftyone_devicedetection_shared"], + install_requires=[ + "fiftyone_devicedetection_shared", + # The first version with CloudMissingPropertyService, which the + # hardware profile results of this package use. + "fiftyone_pipeline_cloudrequestengine>=4.5.45", + ], license="EUPL-1.2", classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/device-detection/fiftyone_devicedetection_onpremise/setup.py b/device-detection/fiftyone_devicedetection_onpremise/setup.py index 83b58226..b482bddc 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/setup.py +++ b/device-detection/fiftyone_devicedetection_onpremise/setup.py @@ -267,6 +267,9 @@ def stage_native_source(): # The first versions with close(), which releases the native memory # this package holds. "fiftyone_pipeline_core>=4.5.42", - "fiftyone_pipeline_engines>=4.5.42", + # The first version with OnPremiseMissingPropertyService, which the + # results of this package give a caller that reads a Property the + # data file in use does not hold. + "fiftyone_pipeline_engines>=4.5.44", ], ) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py index 2d850cf7..265e714f 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/setup.py @@ -283,6 +283,9 @@ def stage_native_source(): # The first versions with close(), which releases the native memory # this package holds. "fiftyone_pipeline_core>=4.5.42", - "fiftyone_pipeline_engines>=4.5.42", + # The first version with OnPremiseMissingPropertyService, which the + # results of this package give a caller that reads a Property the + # data file in use does not hold. + "fiftyone_pipeline_engines>=4.5.44", ], ) diff --git a/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py b/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py index 223d23d8..5c51df1a 100644 --- a/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py +++ b/pipeline/fiftyone_pipeline_cloudrequestengine/setup.py @@ -54,7 +54,9 @@ def read(file_name): packages=["fiftyone_pipeline_cloudrequestengine"], package_dir={"": "src"}, install_requires=[ - 'fiftyone_pipeline_engines', + # The first version with MissingPropertyReason, which this + # package gives as the reason a cloud Property has no value. + 'fiftyone_pipeline_engines>=4.5.44', 'requests', ], license="EUPL-1.2", diff --git a/pipeline/fiftyone_pipeline_engines/tests/test_package_requirements.py b/pipeline/fiftyone_pipeline_engines/tests/test_package_requirements.py new file mode 100644 index 00000000..a5985e4b --- /dev/null +++ b/pipeline/fiftyone_pipeline_engines/tests/test_package_requirements.py @@ -0,0 +1,195 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +import ast +import os +import re +import unittest + +# Each row is a name one package here imports from another package here, +# the package that holds it and the version of that package it came in at, +# so the importing package has to require at least that version. Without +# the minimum, pip leaves an older copy of the dependency in place when the +# importing package alone is upgraded, and the import fails. Every part of +# a row is read from the checkout, so none of it is trusted. +IMPORTED_NAMES = ( + ("pipeline/fiftyone_pipeline_cloudrequestengine", + "fiftyone_pipeline_engines", + "4.5.44", + "fiftyone_pipeline_engines.missingproperty_service", + "MissingPropertyReason"), + ("device-detection/fiftyone_devicedetection_cloud", + "fiftyone_pipeline_cloudrequestengine", + "4.5.45", + "fiftyone_pipeline_cloudrequestengine.clouddata", + "CloudMissingPropertyService"), + ("device-detection/fiftyone_devicedetection_onpremise", + "fiftyone_pipeline_engines", + "4.5.44", + "fiftyone_pipeline_engines.missingproperty_service", + "OnPremiseMissingPropertyService"), + ("ip-intelligence/fiftyone_ipintelligence_onpremise", + "fiftyone_pipeline_engines", + "4.5.44", + "fiftyone_pipeline_engines.missingproperty_service", + "OnPremiseMissingPropertyService"), +) + +# Where each package that holds one of the names above keeps its source. +PACKAGE_FOLDERS = { + "fiftyone_pipeline_engines": "pipeline/fiftyone_pipeline_engines", + "fiftyone_pipeline_cloudrequestengine": + "pipeline/fiftyone_pipeline_cloudrequestengine", +} + +# A requirement that gives a minimum version, as setup.py writes one. +MINIMUM = re.compile(r"^>=\s*([0-9]+(?:\.[0-9]+)*)$") + +# The name and the rest of one entry of an install_requires list. +REQUIREMENT = re.compile(r"^([A-Za-z0-9_.-]+)\s*(.*)$") + +# The entries of an install_requires list, each in its own quotes. +QUOTED = re.compile("[\"']([^\"']+)[\"']") + + +def repository_root(): + """The folder holding ci/packages.json, found by walking up from this + file, so the test reads the checkout it runs from.""" + folder = os.path.dirname(os.path.abspath(__file__)) + while True: + if os.path.isfile(os.path.join(folder, "ci", "packages.json")): + return folder + parent = os.path.dirname(folder) + if parent == folder: + raise AssertionError( + "no ci/packages.json above " + os.path.abspath(__file__)) + folder = parent + + +def requirements(package_folder): + """The install_requires of a package's setup.py, as the name required + mapped to the rest of the requirement.""" + path = os.path.join(repository_root(), package_folder, "setup.py") + with open(path, "rb") as file: + text = file.read().decode("utf-8") + found = re.search(r"install_requires\s*=\s*\[(.*?)\]", text, re.S) + assert found is not None, "no install_requires in " + path + out = {} + for item in QUOTED.findall(found.group(1)): + parts = REQUIREMENT.match(item.strip()) + if parts: + out[parts.group(1)] = parts.group(2).strip() + return out + + +def version_numbers(text): + """A version as a tuple of its numbers, for comparing two of them.""" + return tuple(int(part) for part in text.split(".")) + + +def imports_in(package_folder, module, name): + """Whether any source file of the package imports the name from the + module.""" + source = os.path.join(repository_root(), package_folder, "src") + for folder, _, files in os.walk(source): + if "cxx" in folder.split(os.sep): + continue + for file_name in files: + if not file_name.endswith(".py"): + continue + with open(os.path.join(folder, file_name), "rb") as file: + text = file.read().decode("utf-8", "replace") + try: + tree = ast.parse(text) + except SyntaxError: + continue + for node in ast.walk(tree): + if (isinstance(node, ast.ImportFrom) + and node.module == module + and any(alias.name == name + for alias in node.names)): + return True + return False + + +def module_names(module): + """The names a module of this checkout holds at its top level.""" + parts = module.split(".") + path = os.path.join(repository_root(), PACKAGE_FOLDERS[parts[0]], + "src", *parts) + ".py" + with open(path, "rb") as file: + text = file.read().decode("utf-8") + out = set() + for node in ast.parse(text).body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, + ast.ClassDef)): + out.add(node.name) + elif isinstance(node, ast.Assign): + out.update(target.id for target in node.targets + if isinstance(target, ast.Name)) + return out + + +class PackageRequirementTests(unittest.TestCase): + + """A package that imports a name from another package here requires + that package at the version the name came in at. Upgrading one package + on its own then brings the dependency with it, where a requirement + with no version leaves an older copy in place and the import fails.""" + + def test_each_name_is_imported_where_the_row_says(self): + for folder, _, _, module, name in IMPORTED_NAMES: + with self.subTest(package=folder, name=name): + self.assertTrue( + imports_in(folder, module, name), + "{0} does not import {1} from {2}".format( + folder, name, module)) + + def test_each_name_is_held_by_the_package_the_row_names(self): + for _, package, _, module, name in IMPORTED_NAMES: + with self.subTest(package=package, name=name): + self.assertEqual(package, module.split(".")[0]) + self.assertIn(name, module_names(module)) + + def test_each_package_is_required_at_the_version_of_the_name(self): + for folder, package, version, module, name in IMPORTED_NAMES: + with self.subTest(package=folder, requires=package): + required = requirements(folder) + self.assertIn( + package, required, + "{0} imports {1} from {2} and does not require " + "{3}".format(folder, name, module, package)) + found = MINIMUM.match(required[package]) + self.assertIsNotNone( + found, + "{0} requires {1} as {2!r}, with no minimum " + "version".format(folder, package, required[package])) + self.assertGreaterEqual( + version_numbers(found.group(1)), + version_numbers(version), + "{0} requires {1} {2}, below the {3} that holds " + "{4}".format(folder, package, required[package], + version, name)) + + +if __name__ == "__main__": + unittest.main()