diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 35bebc4c..7dc3c7d5 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -34,6 +34,10 @@ jobs: run: | poetry install .github/scripts/use_cpu_torch.sh "poetry run python" + - name: Show CPU (golden files depend on pinned maths paths; test/regression/cpu_pins.py) + run: | + grep -m1 "model name" /proc/cpuinfo + if grep -qw avx512f /proc/cpuinfo; then echo "AVX-512 yes"; else echo "AVX-512 no"; fi - name: Lint with flake8 (syntax errors and undefined names) run: pipx run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - name: Test with pytest (with coverage) @@ -66,5 +70,9 @@ jobs: run: | poetry install .github/scripts/use_cpu_torch.sh "poetry run python" + - name: Show CPU (golden files depend on pinned maths paths; test/regression/cpu_pins.py) + run: | + grep -m1 "model name" /proc/cpuinfo + if grep -qw avx512f /proc/cpuinfo; then echo "AVX-512 yes"; else echo "AVX-512 no"; fi - name: Test with pytest run: poetry run pytest diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..30aa2031 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,11 @@ +"""Runs before every test module: pin the CPU maths libraries (see +``test/regression/cpu_pins.py``). Must not import torch.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent / "regression")) + +import cpu_pins # noqa: E402 + +cpu_pins.apply() diff --git a/test/regression/README.md b/test/regression/README.md index f4abc9af..01be7ac7 100644 --- a/test/regression/README.md +++ b/test/regression/README.md @@ -36,6 +36,12 @@ dataset, no network access and no GPU, and they run on every pull request. `--check` compares with the file and writes nothing. - `conftest.py`: one thread and deterministic algorithms for every test here. +Float values are compared to within 1e-6, which only holds if every CPU takes the +same code path through the maths libraries. `cpu_pins.py` pins them (PyTorch kernels +at AVX2, MKL in its `COMPATIBLE` mode, oneDNN off); `test/conftest.py` and `regen.py` +apply the pins before torch starts. A golden test that fails only on some machines +points at a library path that is not pinned yet, not at a tolerance to widen. + ## When a golden test fails Your change altered a result. Either that is the point of the change, or it is a bug. diff --git a/test/regression/conftest.py b/test/regression/conftest.py index 4246facb..e71d7138 100644 --- a/test/regression/conftest.py +++ b/test/regression/conftest.py @@ -11,11 +11,14 @@ @pytest.fixture(autouse=True) def deterministic_cpu(): - """One thread and deterministic algorithms for each test, then restore.""" + """``networks.deterministic()`` for each test, then restore.""" + import networks + threads = torch.get_num_threads() det = torch.are_deterministic_algorithms_enabled() - torch.set_num_threads(1) - torch.use_deterministic_algorithms(True) + mkldnn = torch.backends.mkldnn.enabled + networks.deterministic() yield + torch.backends.mkldnn.enabled = mkldnn torch.use_deterministic_algorithms(det) torch.set_num_threads(threads) diff --git a/test/regression/cpu_pins.py b/test/regression/cpu_pins.py new file mode 100644 index 00000000..97f60f22 --- /dev/null +++ b/test/regression/cpu_pins.py @@ -0,0 +1,28 @@ +"""Pin the CPU maths libraries to one code path on every processor. + +The golden files store float32 voltages, traces and weights to within 1e-6. +Without these pins the same code gives different rounding on different CPUs, +because each library picks its fastest code path for the processor it runs on. +On GitHub's runners this moved voltages by up to 1.5e-5 (and an Izhikevich +voltage by 0.23 mV) against files made on an AVX2-only CPU (CI run +40800432540, 2026-10-28). Measured on that AVX2 CPU, changing only MKL's code +path changed 1 to 4 of the 16 goldens. + +- ``ATEN_CPU_CAPABILITY=avx2``: PyTorch's own kernels use AVX2 even on + AVX-512 processors. +- ``MKL_CBWR=COMPATIBLE``: MKL (matrix products) uses its code path that gives + the same results on all x86 processors. +- oneDNN (convolutions) is switched off in ``networks.deterministic()``; its + blocking depends on cache sizes, and it does not follow ``MKL_CBWR``. + +The variables must be set before torch runs its first operation, so +``test/conftest.py`` and ``regen.py`` call ``apply()`` before importing torch. +""" + +import os + +PINS = {"ATEN_CPU_CAPABILITY": "avx2", "MKL_CBWR": "COMPATIBLE"} + + +def apply(): + os.environ.update(PINS) diff --git a/test/regression/golden/conn_postpre_b1.pt.xz b/test/regression/golden/conn_postpre_b1.pt.xz index 9bf06fd5..b82fc33d 100644 Binary files a/test/regression/golden/conn_postpre_b1.pt.xz and b/test/regression/golden/conn_postpre_b1.pt.xz differ diff --git a/test/regression/golden/conn_postpre_b8.pt.xz b/test/regression/golden/conn_postpre_b8.pt.xz index 3fda7fbe..dc3daca7 100644 Binary files a/test/regression/golden/conn_postpre_b8.pt.xz and b/test/regression/golden/conn_postpre_b8.pt.xz differ diff --git a/test/regression/golden/conn_postpre_dt05.pt.xz b/test/regression/golden/conn_postpre_dt05.pt.xz index e80d89b0..e9e6f702 100644 Binary files a/test/regression/golden/conn_postpre_dt05.pt.xz and b/test/regression/golden/conn_postpre_dt05.pt.xz differ diff --git a/test/regression/golden/conv2d_if_izh.pt.xz b/test/regression/golden/conv2d_if_izh.pt.xz index 01554df1..56d5de80 100644 Binary files a/test/regression/golden/conv2d_if_izh.pt.xz and b/test/regression/golden/conv2d_if_izh.pt.xz differ diff --git a/test/regression/golden/csrm_window.pt.xz b/test/regression/golden/csrm_window.pt.xz index 2e3058b3..d29af1cc 100644 Binary files a/test/regression/golden/csrm_window.pt.xz and b/test/regression/golden/csrm_window.pt.xz differ diff --git a/test/regression/golden/dc2015_b1.pt.xz b/test/regression/golden/dc2015_b1.pt.xz index 07bfb41d..9aa6a565 100644 Binary files a/test/regression/golden/dc2015_b1.pt.xz and b/test/regression/golden/dc2015_b1.pt.xz differ diff --git a/test/regression/golden/dc2015_b16.pt.xz b/test/regression/golden/dc2015_b16.pt.xz index 558550bc..ca2a41a7 100644 Binary files a/test/regression/golden/dc2015_b16.pt.xz and b/test/regression/golden/dc2015_b16.pt.xz differ diff --git a/test/regression/golden/encode_clamp.pt.xz b/test/regression/golden/encode_clamp.pt.xz index 5cd4c2fb..b9d9699f 100644 Binary files a/test/regression/golden/encode_clamp.pt.xz and b/test/regression/golden/encode_clamp.pt.xz differ diff --git a/test/regression/golden/encode_clamp_dt05.pt.xz b/test/regression/golden/encode_clamp_dt05.pt.xz index ec7d64ff..45b014a2 100644 Binary files a/test/regression/golden/encode_clamp_dt05.pt.xz and b/test/regression/golden/encode_clamp_dt05.pt.xz differ diff --git a/test/regression/golden/local2d_postpre.pt.xz b/test/regression/golden/local2d_postpre.pt.xz index b797016d..012b1c7b 100644 Binary files a/test/regression/golden/local2d_postpre.pt.xz and b/test/regression/golden/local2d_postpre.pt.xz differ diff --git a/test/regression/golden/meanfield_decay.pt.xz b/test/regression/golden/meanfield_decay.pt.xz index b612ab0f..7e4d7c45 100644 Binary files a/test/regression/golden/meanfield_decay.pt.xz and b/test/regression/golden/meanfield_decay.pt.xz differ diff --git a/test/regression/golden/meanfield_norm.pt.xz b/test/regression/golden/meanfield_norm.pt.xz index 2e98fef6..c2a22b4a 100644 Binary files a/test/regression/golden/meanfield_norm.pt.xz and b/test/regression/golden/meanfield_norm.pt.xz differ diff --git a/test/regression/golden/mstdp_mstdpet.pt.xz b/test/regression/golden/mstdp_mstdpet.pt.xz index 4ad4b132..710b8b33 100644 Binary files a/test/regression/golden/mstdp_mstdpet.pt.xz and b/test/regression/golden/mstdp_mstdpet.pt.xz differ diff --git a/test/regression/golden/mstdp_mstdpet_dt05.pt.xz b/test/regression/golden/mstdp_mstdpet_dt05.pt.xz index 3fcaf0c5..f4be026b 100644 Binary files a/test/regression/golden/mstdp_mstdpet_dt05.pt.xz and b/test/regression/golden/mstdp_mstdpet_dt05.pt.xz differ diff --git a/test/regression/golden/rules_mix.pt.xz b/test/regression/golden/rules_mix.pt.xz index 08aa5039..55f986ff 100644 Binary files a/test/regression/golden/rules_mix.pt.xz and b/test/regression/golden/rules_mix.pt.xz differ diff --git a/test/regression/golden/rules_mix_dt05.pt.xz b/test/regression/golden/rules_mix_dt05.pt.xz index b1563dec..065d47f6 100644 Binary files a/test/regression/golden/rules_mix_dt05.pt.xz and b/test/regression/golden/rules_mix_dt05.pt.xz differ diff --git a/test/regression/networks.py b/test/regression/networks.py index af1408ed..587d18f3 100644 --- a/test/regression/networks.py +++ b/test/regression/networks.py @@ -388,9 +388,17 @@ def rules_mix_dt05(dt): def deterministic(): - """CPU determinism settings required for exact comparisons.""" + """CPU determinism settings required for exact comparisons (see + ``cpu_pins.py``). Fails when the capability pin was applied too late.""" + capability = torch.backends.cpu.get_cpu_capability() + if capability != "AVX2": + raise RuntimeError( + f"PyTorch CPU capability is {capability}, not AVX2: run through pytest " + "(test/conftest.py) or regen.py, which pin it before torch starts" + ) torch.set_num_threads(1) torch.use_deterministic_algorithms(True) + torch.backends.mkldnn.enabled = False def run_kwargs(spec): diff --git a/test/regression/regen.py b/test/regression/regen.py index 1a917775..8191b89f 100644 --- a/test/regression/regen.py +++ b/test/regression/regen.py @@ -23,6 +23,10 @@ HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) +import cpu_pins # noqa: E402 + +cpu_pins.apply() # before torch is imported + import networks as N # noqa: E402 import torch # noqa: E402