diff --git a/AGENTS.md b/AGENTS.md index 35bcd09e..abbfec49 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,8 +56,11 @@ The root `pyproject.toml` declares a `[tool.uv.workspace]`, so one `uv.lock` cov wheels, so it is a standalone uv project with its own `uv.lock`. Run it with `cd packages/bench-third-party && uv sync`, never from the root environment. - `benches/` β€” monoprop's own benchmark suite (`conftest.py`, `bench_*.py`, `results/`). It stays in - the repository and imports the tools package. Benchmark names are Bencher's history key, so they - must not move with a library release; that is why the suite is not in `monoprop-bench-tools`. + the repository and imports the tools package. `bench_bindings.py` contains serial-only boundary + microbenchmarks; run it on a fixed host and Release build. Artifacts record Python, compiled + nanobind frontend, and runtime backend versions. Benchmark names are Bencher's history key, so + they must not move with a library release; that is why the suite is not in + `monoprop-bench-tools`. Dependency groups follow from that split: `test` is monoprop's own suite only (cibuildwheel installs it against a built wheel, so it must not reference a workspace member), `workspace-test` adds diff --git a/README.md b/README.md index 1c6d1d15..f8adacaf 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,9 @@ and against [`MajoranaPropagation.jl`](https://github.com/SparqleSim/MajoranaPro Head to our [benchmarks page](https://docs.monoprop.algorithmiq.tech/benchmarks) for more details. Every commit on `main` also runs the internal benchmark suite, tracked over time -with [Bencher](https://bencher.dev/) to catch performance regressions. +with [Bencher](https://bencher.dev/) to catch propagation and Python/C++ binding +regressions. Reports record the Python, nanobind frontend, and runtime backend +versions for reproducible comparisons. πŸ“– **Full documentation:** diff --git a/benches/README.md b/benches/README.md index 781bf27f..91354b3a 100644 --- a/benches/README.md +++ b/benches/README.md @@ -5,16 +5,27 @@ The `monoprop` repository includes a pytest suite measuring the **time** and The suite is separated from the test suite and can be run with `just bench`. See below for more detailed instructions. -See the the [Benchmarks](../docs/content/docs/benchmarks.mdx) section of the +See the [Benchmarks](../docs/content/docs/benchmarks.mdx) section of the documentation for detailed instructions. ## What lives where This directory holds only monoprop's own benchmarks β€” `conftest.py` (the fixtures -and the results schema), `bench_random.py`, `bench_models.py`, and `results/`. +and the results schema), `bench_random.py`, `bench_models.py`, `bench_bindings.py`, +and `results/`. The binding microbenchmarks are serial-only and isolate Python/C++ +call and return-conversion costs from propagation work. Run them directly with: + +```bash +uv run pytest benches/bench_bindings.py --benchmark-only +``` + Benchmark names are the key [Bencher](https://bencher.dev/) stores history under, so they stay here rather than moving with a library release. +Each run records its Python version, the nanobind version compiled into the +extension, and the installed `nanobind-backend` version. Keep the host, Release +build, and benchmark command fixed when comparing binding revisions. + Everything reusable is in the `monoprop-bench-tools` package ([`../packages/monoprop-bench-tools`](../packages/monoprop-bench-tools)): the memory instrumentation, the model builders, and the two renderers that turn a diff --git a/benches/bench_bindings.py b/benches/bench_bindings.py new file mode 100644 index 00000000..a8c0beb4 --- /dev/null +++ b/benches/bench_bindings.py @@ -0,0 +1,117 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Microbenchmarks isolating Python/C++ binding overhead.""" + +from __future__ import annotations + +from itertools import combinations, islice + +import pytest + +import monoprop +from monoprop import MajoranaPropagator +from monoprop.fermi import MajoranaOperator + +BOUNDARY_ROUNDS = 20 +CHEAP_CALL_ITERATIONS = 10_000 +FUNCTIONAL_ITERATIONS = 1_000 +LARGE_TERM_COUNT = 1_024 +NUM_MODES = 32 +OUTPUT_ATOL = 1e-300 + + +@pytest.fixture(scope="module", autouse=True) +def _serial_only(bench_comm): + if bench_comm is not None and bench_comm.Get_size() > 1: + pytest.skip("binding microbenchmarks are serial-only") + + +@pytest.fixture(scope="module") +def binding_propagators(): + """Build fixed small and large propagators outside timed regions.""" + small = MajoranaPropagator( + MajoranaOperator({(0, 1, 2, 3): 1.0}, NUM_MODES), + [], + cutoff=2 * NUM_MODES, + ) + large_terms = dict.fromkeys( + islice(combinations(range(2 * NUM_MODES), 4), LARGE_TERM_COUNT), 1.0 + ) + large = MajoranaPropagator( + MajoranaOperator(large_terms, NUM_MODES), + [], + cutoff=2 * NUM_MODES, + ) + return {"small": small, "large": large} + + +def test_binding_is_antihermitian_positional(benchmark): + """Benchmark positional free-function dispatch.""" + result = benchmark.pedantic( + monoprop.is_antihermitian, + args=([0, 1],), + rounds=BOUNDARY_ROUNDS, + iterations=CHEAP_CALL_ITERATIONS, + ) + assert result is True + + +def test_binding_is_antihermitian_keyword(benchmark): + """Benchmark keyword free-function dispatch.""" + result = benchmark.pedantic( + monoprop.is_antihermitian, + kwargs={"indices": [0, 1]}, + rounds=BOUNDARY_ROUNDS, + iterations=CHEAP_CALL_ITERATIONS, + ) + assert result is True + + +def test_binding_size(benchmark, binding_propagators): + """Benchmark a bound method with a scalar return value.""" + simulator = binding_propagators["small"]._simulator + result = benchmark.pedantic( + simulator.size, + rounds=BOUNDARY_ROUNDS, + iterations=CHEAP_CALL_ITERATIONS, + ) + assert result == 1 + + +def test_binding_expectation_value_functional(benchmark, binding_propagators): + """Benchmark calls through an already-created functional.""" + functional = binding_propagators["small"].expectation_value_functional() + result = benchmark.pedantic( + functional, + args=([],), + rounds=BOUNDARY_ROUNDS, + iterations=FUNCTIONAL_ITERATIONS, + ) + assert isinstance(result, float) + + +@pytest.mark.parametrize("operator_size", ["small", "large"]) +def test_binding_evolved_operator(benchmark, binding_propagators, operator_size): + """Benchmark raw evolved-operator dictionary conversion.""" + simulator = binding_propagators[operator_size]._simulator + iterations = 100 if operator_size == "small" else 1 + expected_terms = 1 if operator_size == "small" else LARGE_TERM_COUNT + result = benchmark.pedantic( + simulator.evolved_operator, + args=([], OUTPUT_ATOL), + rounds=BOUNDARY_ROUNDS, + iterations=iterations, + ) + assert len(result) == expected_terms diff --git a/benches/conftest.py b/benches/conftest.py index ef22dd80..2204fb33 100644 --- a/benches/conftest.py +++ b/benches/conftest.py @@ -36,8 +36,10 @@ import hashlib import json import os +import platform import socket from dataclasses import asdict, fields +from importlib.metadata import PackageNotFoundError, version from pathlib import Path from typing import TYPE_CHECKING, Any @@ -191,6 +193,11 @@ def _core_md5() -> str: def _meta() -> dict[str, Any]: """Return this run's configuration metadata for the report.""" + try: + nanobind_backend_version = version("nanobind-backend") + except PackageNotFoundError: + nanobind_backend_version = "not installed" + return { "label": os.environ.get("monoprop_BENCH_LABEL", "?"), # noqa: SIM112 "ranks": _size(), @@ -202,6 +209,9 @@ def _meta() -> dict[str, Any]: "monoprop_core_md5": _core_md5(), "monoprop_variant": monoprop.__variant__, "monoprop_compiler_flags": monoprop.__compiler_flags__, + "python_version": platform.python_version(), + "nanobind_version": monoprop.__nanobind_version__, + "nanobind_backend_version": nanobind_backend_version, "monoprop_max_num_modes": monoprop.MAX_NUM_MODES, "malloc_arena_max": os.environ.get("MALLOC_ARENA_MAX", "default"), "omp_num_threads": os.environ.get("OMP_NUM_THREADS", "default"), diff --git a/docs/content/docs/benchmarks.mdx b/docs/content/docs/benchmarks.mdx index 937f1f96..22def8ce 100644 --- a/docs/content/docs/benchmarks.mdx +++ b/docs/content/docs/benchmarks.mdx @@ -446,11 +446,28 @@ aborts if MPI is missing; extra args are forwarded to `mpiexec` (as root, add Override any config field via `---` (e.g. `--pauli-num-layers 30`); `just bench --help` lists all. +**Bindings** (`bench_bindings.py`; fixed, serial-only) β€” low-work calls that +isolate Python/C++ boundary costs: positional and keyword dispatch, a bound +method, an already-created expectation-value functional, and raw evolved-operator +conversion for one and 1,024 terms. These are microbenchmarks of binding overhead, +not propagation throughput. Run only this group with: + +```bash +uv run pytest benches/bench_bindings.py --benchmark-only +``` + +For an upgrade comparison, use Release builds on one pinned host and keep the +interpreter and command fixed. Compare the old dependency, the new dependency +before wrapper edits, and the new dependency after wrapper edits to separate +automatic nanobind improvements from source-level changes. + ### Output Each run writes `results/time-