diff --git a/.github/workflows/free-threaded-python.yml b/.github/workflows/free-threaded-python.yml new file mode 100644 index 00000000..9c188309 --- /dev/null +++ b/.github/workflows/free-threaded-python.yml @@ -0,0 +1,126 @@ +name: Free-threaded Python + +# Builds the Device Detection and IP Intelligence on-premise packages on the +# free-threaded builds of Python and runs their tests there, with the free +# Lite data files. The native modules do not declare that they run without +# the GIL, so Python turns the GIL back on when it loads them. +# tests/test_free_threaded.py in each package checks that it does, and is +# skipped on every other build of Python. +on: + pull_request: + paths: + - 'device-detection/fiftyone_devicedetection_onpremise/**' + - 'ip-intelligence/fiftyone_ipintelligence_onpremise/**' + - 'pipeline/fiftyone_pipeline_core/**' + - 'pipeline/fiftyone_pipeline_engines/**' + - 'pipeline/fiftyone_pipeline_engines_fiftyone/**' + - 'cxx' + - '.github/workflows/free-threaded-python.yml' + push: + branches: + - main + - rc + paths: + - 'device-detection/fiftyone_devicedetection_onpremise/**' + - 'ip-intelligence/fiftyone_ipintelligence_onpremise/**' + - 'pipeline/fiftyone_pipeline_core/**' + - 'pipeline/fiftyone_pipeline_engines/**' + - 'pipeline/fiftyone_pipeline_engines_fiftyone/**' + - 'cxx' + - '.github/workflows/free-threaded-python.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + free-threaded: + name: Python ${{ matrix.python }} on ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python: '3.13t' + - os: ubuntu-latest + python: '3.14t' + - os: windows-latest + python: '3.14t' + runs-on: ${{ matrix.os }} + defaults: + run: + shell: pwsh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + # The native source is the cxx submodule, and the data files are + # linked from its own data submodule. + submodules: recursive + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Check the build is free-threaded + run: | + python -c "import sys, sysconfig; print(sys.version); assert sysconfig.get_config_var('Py_GIL_DISABLED') == 1, 'not a free-threaded build'; assert not sys._is_gil_enabled(), 'the GIL is on at start'" + + - name: Install the packages the tests use + run: | + python -m pip install --upgrade pip + python -m pip install pytest flask flask-unittest json5 ruamel.yaml Cython setuptools + # The pipeline packages come from this checkout, as the tests run + # with the pipeline code they are released with. + python -m pip install ` + -e pipeline/fiftyone_pipeline_core ` + -e pipeline/fiftyone_pipeline_engines ` + -e pipeline/fiftyone_pipeline_engines_fiftyone ` + -e pipeline/fiftyone_pipeline_cloudrequestengine ` + -e device-detection/fiftyone_devicedetection_shared ` + -e device-detection/fiftyone_devicedetection_cloud ` + -e ip-intelligence/fiftyone_ipintelligence_shared + # These two require the on-premise package by name, which the + # tests read from its source folder instead. + python -m pip install --no-deps ` + -e device-detection/fiftyone_devicedetection ` + -e device-detection/fiftyone_devicedetection_examples + + - name: Fetch the free Lite data files + run: | + ./cxx/data/get-data-files.ps1 51Degrees-LiteV4.1.hash '20000 Evidence Records.yml' '20000 User Agents.csv' 51Degrees-LiteV41.ipi 51Degrees-IPIV4AsnIpiV41.ipi + New-Item -ItemType Directory -Force -Path ip-intelligence/assets | Out-Null + New-Item -ItemType HardLink ` + -Path ip-intelligence/assets/51Degrees-LiteIpiV41.ipi ` + -Target cxx/data/ip-intelligence/51Degrees-LiteV41.ipi | Out-Null + + - name: Build the native modules + run: | + if ($IsWindows) { + # A short temporary path, for the MSVC path length limit. + $env:TEMP = New-Item -ItemType Directory -Force -Path 'C:\tmp' + $env:TMP = $env:TEMP + } + foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') { + Push-Location $package + python setup.py build_clib build_ext --inplace + if ($LASTEXITCODE) { exit $LASTEXITCODE } + Pop-Location + } + + - name: Run the tests + run: | + $failed = $false + foreach ($package in 'device-detection/fiftyone_devicedetection_onpremise', 'ip-intelligence/fiftyone_ipintelligence_onpremise') { + Write-Host "::group::$package" + Push-Location $package + $env:PYTHONPATH = Join-Path $PWD 'src' + # -rs lists any test that skipped, and none should here. + python -m pytest -rs tests + if ($LASTEXITCODE) { $failed = $true } + Pop-Location + Write-Host '::endgroup::' + } + if ($failed) { exit 1 } diff --git a/device-detection/fiftyone_devicedetection_onpremise/readme.md b/device-detection/fiftyone_devicedetection_onpremise/readme.md index eb1b0178..de263c0d 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/readme.md +++ b/device-detection/fiftyone_devicedetection_onpremise/readme.md @@ -78,6 +78,18 @@ To run the tests use: `python -m unittest discover -s tests -p test*.py -b` +## Free-threaded Python + +The package runs on the free-threaded builds of Python 3.13 and 3.14. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on when it loads the module and prints this warning. + +``` +RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_devicedetection_onpremise._DeviceDetectionHashEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. +``` + +The native code then runs with the GIL held, as it does on the standard build of Python, so the package gains nothing from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. + +Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. + ## Find out more - [Device Detection](https://51degrees.com/device-detection?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_devicedetection_onpremise&utm_term=find-out-more-device-detection) diff --git a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py index c4ea2a09..d50c108c 100644 --- a/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py +++ b/device-detection/fiftyone_devicedetection_onpremise/src/fiftyone_devicedetection_onpremise/swig_data.py @@ -47,6 +47,15 @@ class SwigData(AspectData): Flow Data, and so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). + + Free-threaded Python is supported. The native module does not declare + that it can run without the GIL, so Python turns the GIL back on when it + loads the module, with a RuntimeWarning. The native code then runs with + the GIL held, as it does on the standard build of Python and as it is + tested, so the package gains nothing from free-threading. Set + PYTHON_GIL=1, or run Python with -X gil=1, to start with the GIL on and + so without the warning. A results instance is still not safe to use from + several threads at once. """ def __init__(self, flow_element, swig_results): diff --git a/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py new file mode 100644 index 00000000..dd6e3228 --- /dev/null +++ b/device-detection/fiftyone_devicedetection_onpremise/tests/test_free_threaded.py @@ -0,0 +1,83 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +On a free-threaded build of Python, loading the native module turns the GIL +back on, with a RuntimeWarning, because the module does not declare that it +runs without the GIL. A change that loaded it with the GIL still off fails +here. Each check runs in a new interpreter, as this one may have loaded the +module already. +""" + +import os +import subprocess +import sys +import sysconfig +import unittest + +# The SWIG module, which loads the native module when it is imported. +MODULE = "fiftyone_devicedetection_onpremise.DeviceDetectionHashEngineModule" +NATIVE_MODULE = "_DeviceDetectionHashEngineModule" + +FREE_THREADED = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 + +# Prints whether the GIL is on before and after the import, then the +# warnings the import gave, one to a line. +CHECK = "\n".join([ + "import sys, warnings", + "warnings.simplefilter('always')", + "before = sys._is_gil_enabled()", + "with warnings.catch_warnings(record=True) as caught:", + " import " + MODULE, + "print(before, sys._is_gil_enabled())", + "for warning in caught:", + " print(warning.category.__name__, str(warning.message))", +]) + + +@unittest.skipUnless(FREE_THREADED, "Python is not a free-threaded build") +class FreeThreadedTests(unittest.TestCase): + + def import_in_a_new_interpreter(self): + env = dict(os.environ) + # PYTHON_GIL would set the GIL whatever the module declares. + env.pop("PYTHON_GIL", None) + result = subprocess.run( + [sys.executable, "-c", CHECK], env=env, capture_output=True, + text=True, check=True) + return result.stdout.splitlines() + + def test_loading_the_native_module_turns_the_gil_on(self): + lines = self.import_in_a_new_interpreter() + self.assertEqual("False True", lines[0]) + + def test_loading_the_native_module_warns_that_the_gil_is_on(self): + warnings = self.import_in_a_new_interpreter()[1:] + self.assertTrue( + any(line.startswith("RuntimeWarning ") + and "GIL" in line and NATIVE_MODULE in line + for line in warnings), + warnings) + + +if __name__ == "__main__": + unittest.main() diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md index 5cd4011b..ff42b01d 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/readme.md @@ -56,6 +56,18 @@ flowdata.process() print(flowdata.ip.registeredname.value()) ``` +## Free-threaded Python + +The package runs on the free-threaded builds of Python 3.13 and 3.14. Its native module does not declare that it can run without the global interpreter lock (GIL), so Python turns the GIL back on when it loads the module and prints this warning. + +``` +RuntimeWarning: The global interpreter lock (GIL) has been enabled to load module 'fiftyone_ipintelligence_onpremise._IpIntelligenceEngineModule', which has not declared that it can run safely without the GIL. To override this behavior and keep the GIL disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0. +``` + +The native code then runs with the GIL held, as it does on the standard build of Python, so the package gains nothing from free-threading. To start with the GIL on, and so without the warning, set the environment variable `PYTHON_GIL=1` or run Python with `-X gil=1`. The package has not been tested with the GIL kept off by `PYTHON_GIL=0`. + +Whichever build of Python runs it, the results of one request are not safe to use from several threads at once. Give each thread its own Flow Data, or coordinate access with a lock of your own. + ## Find out more - [IP Intelligence](https://51degrees.com/ip-intelligence?utm_source=pypi&utm_medium=package&utm_campaign=51degrees-python&utm_content=fiftyone_ipintelligence_onpremise&utm_term=find-out-more-ip-intelligence) diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py index 5285aca6..c4e8aa55 100644 --- a/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/src/fiftyone_ipintelligence_onpremise/swig_data.py @@ -56,6 +56,15 @@ class SwigData(AspectData): Flow Data, and so its own results instance, or coordinates its own access, for example with a lock of its own around every read and around close(). + + Free-threaded Python is supported. The native module does not declare + that it can run without the GIL, so Python turns the GIL back on when it + loads the module, with a RuntimeWarning. The native code then runs with + the GIL held, as it does on the standard build of Python and as it is + tested, so the package gains nothing from free-threading. Set + PYTHON_GIL=1, or run Python with -X gil=1, to start with the GIL on and + so without the warning. A results instance is still not safe to use from + several threads at once. """ def __init__(self, flow_element, swig_results, echo_ip=None, echo_ipv6=None): diff --git a/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py new file mode 100644 index 00000000..b694dd50 --- /dev/null +++ b/ip-intelligence/fiftyone_ipintelligence_onpremise/tests/test_free_threaded.py @@ -0,0 +1,83 @@ +# ********************************************************************* +# This Original Work is copyright of 51 Degrees Mobile Experts Limited. +# Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, +# Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. +# +# This Original Work is licensed under the European Union Public Licence +# (EUPL) v.1.2 and is subject to its terms as set out below. +# +# If a copy of the EUPL was not distributed with this file, You can obtain +# one at https://opensource.org/licenses/EUPL-1.2. +# +# The 'Compatible Licences' set out in the Appendix to the EUPL (as may be +# amended by the European Commission) shall be deemed incompatible for +# the purposes of the Work and the provisions of the compatibility +# clause in Article 5 of the EUPL shall not apply. +# +# If using the Work as, or as part of, a network application, by +# including the attribution notice(s) required under Article 5 of the EUPL +# in the end user terms of the application under an appropriate heading, +# such notice(s) shall fulfill the requirements of that article. +# ********************************************************************* + +""" +On a free-threaded build of Python, loading the native module turns the GIL +back on, with a RuntimeWarning, because the module does not declare that it +runs without the GIL. A change that loaded it with the GIL still off fails +here. Each check runs in a new interpreter, as this one may have loaded the +module already. +""" + +import os +import subprocess +import sys +import sysconfig +import unittest + +# The SWIG module, which loads the native module when it is imported. +MODULE = "fiftyone_ipintelligence_onpremise.IpIntelligenceEngineModule" +NATIVE_MODULE = "_IpIntelligenceEngineModule" + +FREE_THREADED = sysconfig.get_config_var("Py_GIL_DISABLED") == 1 + +# Prints whether the GIL is on before and after the import, then the +# warnings the import gave, one to a line. +CHECK = "\n".join([ + "import sys, warnings", + "warnings.simplefilter('always')", + "before = sys._is_gil_enabled()", + "with warnings.catch_warnings(record=True) as caught:", + " import " + MODULE, + "print(before, sys._is_gil_enabled())", + "for warning in caught:", + " print(warning.category.__name__, str(warning.message))", +]) + + +@unittest.skipUnless(FREE_THREADED, "Python is not a free-threaded build") +class FreeThreadedTests(unittest.TestCase): + + def import_in_a_new_interpreter(self): + env = dict(os.environ) + # PYTHON_GIL would set the GIL whatever the module declares. + env.pop("PYTHON_GIL", None) + result = subprocess.run( + [sys.executable, "-c", CHECK], env=env, capture_output=True, + text=True, check=True) + return result.stdout.splitlines() + + def test_loading_the_native_module_turns_the_gil_on(self): + lines = self.import_in_a_new_interpreter() + self.assertEqual("False True", lines[0]) + + def test_loading_the_native_module_warns_that_the_gil_is_on(self): + warnings = self.import_in_a_new_interpreter()[1:] + self.assertTrue( + any(line.startswith("RuntimeWarning ") + and "GIL" in line and NATIVE_MODULE in line + for line in warnings), + warnings) + + +if __name__ == "__main__": + unittest.main()