From a7098bd1f8d155c8284fd78531d0ab7a6efdcdc0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 19:51:38 +0000 Subject: [PATCH 1/3] Run the Python CodSpeed benchmarks on Python 3.13 pytest-codspeed records Python frames only through the CPython perf trampoline, which exists from Python 3.12. The job used 3.11 from `.python-version`, so CodSpeed reported all 40 Python benchmarks as having no flamegraph on every pull request. The extension targets the stable ABI from 3.11, so it runs unchanged on 3.13. Claude-Session: https://claude.ai/code/session_01KF3pMFx7U5ngk8BzQQFS2W Signed-off-by: Claude --- .github/workflows/codspeed.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 943408c841f..3f0fc3a2e0b 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -143,6 +143,11 @@ jobs: # Same settings as the bench profile, which keeps the debug info that CodSpeed uses to # attribute native frames. See `profile.bench_python` in Cargo.toml. MATURIN_PEP517_ARGS: "--profile bench_python" + # pytest-codspeed records Python frames only through the CPython perf trampoline, which + # exists from Python 3.12. On the 3.11 interpreter from `.python-version`, every Python + # benchmark reported "no flamegraph". The extension is built for the stable ABI from 3.11, + # so the same build runs on 3.13. + UV_PYTHON: "3.13" steps: - uses: runs-on/action@v2 if: github.repository == 'vortex-data/vortex' From 065fba3358ee6f3589a8b4d75612fbcf14d38195 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 20:38:11 +0000 Subject: [PATCH 2/3] Stop mimalloc purges inside simulated CodSpeed benchmarks mimalloc returns freed memory to the OS 10 ms after a free. Under the CodSpeed simulation, benchmarks run many times slower, so the purge of one benchmark's memory can land in a later benchmark's single measured iteration. That iteration then spends its time in `madvise` calls and page faults, which the simulation leaves out, and CodSpeed reports it as dominated by system calls or as measuring no time. Under Valgrind, `MIMALLOC_PURGE_DELAY=-1` cut the `madvise` calls of the `filter_powerlaw` benchmarks from 23 to 1. Claude-Session: https://claude.ai/code/session_01KF3pMFx7U5ngk8BzQQFS2W Signed-off-by: Claude --- .github/workflows/codspeed.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 3f0fc3a2e0b..155e1bb2af4 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -122,6 +122,11 @@ jobs: # allocator and cold caches. This adds about 15 µs to each benchmark and hides small # changes in short benchmarks. Run all benchmarks of a target in one process, as before. CODSPEED_PROCESS_ISOLATION: "false" + # mimalloc returns freed memory to the OS (`madvise`) 10 ms after it is freed. The + # simulation runs many times slower than native code, so that delay can end inside a + # later benchmark's measured iteration. That iteration then makes system calls and page + # faults, which the simulation leaves out of the measure. Freed memory stays mapped. + MIMALLOC_PURGE_DELAY: "-1" with: run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} From 83d7d470df480da1f835aaf4790e2a3ce3c3e618 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 21:12:29 +0000 Subject: [PATCH 3/3] Keep system calls out of simulated CodSpeed iterations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tracing the measured regions under callgrind showed where the remaining system calls and the empty measurement came from: - `ilike_contains` and `filter_run_end[len=4096_run=16_density=0.1]` paid for one-time process setup (the first regex compilation and the session id interner, which read the CPU quota from the file system). Both bench binaries now run the operation once in `main()`. - `zstd_compress_string` grew glibc's heap with `brk`. zstd allocates through glibc, which trims its heap after large frees. The simulation job now sets `MALLOC_TRIM_THRESHOLD_` and `MALLOC_TOP_PAD_`. - `filter_powerlaw_by_mostly_false[1000]` did under 1 µs of work, and about half of its measured iteration was mimalloc setting up a fresh page after the previous, larger case. The 1000-row size is gone from `filter_bool.rs`. The benchmarking guide gains a rule against leaving one-time setup to the first measured iteration. Claude-Session: https://claude.ai/code/session_01KF3pMFx7U5ngk8BzQQFS2W Signed-off-by: Claude --- .github/workflows/codspeed.yml | 16 +++++++--- docs/developer-guide/benchmarking.md | 8 ++++- vortex-array/benches/filter_bool.rs | 7 ++++- vortex-array/benches/like.rs | 42 +++++++++++++++++--------- vortex-array/benches/run_end_filter.rs | 12 ++++++++ 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 155e1bb2af4..ae6e9a93069 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -122,11 +122,19 @@ jobs: # allocator and cold caches. This adds about 15 µs to each benchmark and hides small # changes in short benchmarks. Run all benchmarks of a target in one process, as before. CODSPEED_PROCESS_ISOLATION: "false" - # mimalloc returns freed memory to the OS (`madvise`) 10 ms after it is freed. The - # simulation runs many times slower than native code, so that delay can end inside a - # later benchmark's measured iteration. That iteration then makes system calls and page - # faults, which the simulation leaves out of the measure. Freed memory stays mapped. + # Allocators return freed memory to the OS and grow their heaps with system calls. A + # system call inside a measured iteration is left out of the measure, so CodSpeed reports + # that benchmark as dominated by system calls. Keep freed memory mapped instead. + # + # mimalloc, which every benchmark binary uses, returns memory 10 ms after it is freed. + # The simulation runs many times slower than native code, so that delay can end inside a + # later benchmark's measured iteration. MIMALLOC_PURGE_DELAY: "-1" + # glibc `malloc`, which C dependencies such as zstd use, shrinks its heap after large + # frees and grows it again with `brk`. Shrink only when 1 GiB is free at the top of the + # heap, and grow in 64 MiB steps so that a later benchmark rarely needs to grow it. + MALLOC_TRIM_THRESHOLD_: "1073741824" + MALLOC_TOP_PAD_: "67108864" with: run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md index f7140bb9b0f..91dd8a77b00 100644 --- a/docs/developer-guide/benchmarking.md +++ b/docs/developer-guide/benchmarking.md @@ -193,7 +193,8 @@ and memory access costs. As a result: [Known unstable benchmarks](#known-unstable-benchmarks)). - System calls are not measured. A benchmark that depends on I/O or kernel calls **must** use the [walltime instrument](https://codspeed.io/docs/instruments/walltime) or be gated with - `#[cfg(not(codspeed))]`. + `#[cfg(not(codspeed))]`. The simulation job keeps mimalloc and glibc from returning freed + memory to the OS, so that allocator system calls do not land in a measured iteration. **Walltime legs.** A walltime leg is one CI job that measures wall-clock time on a dedicated machine for one CPU feature set (AVX2, AVX-512 or NEON). `.github/workflows/codspeed.yml` defines @@ -255,6 +256,11 @@ unnoticed. When a section above explains a rule, the rule links to that section. [How CodSpeed measures a benchmark](#how-codspeed-measures-a-benchmark). - **Do not measure code that a pull request cannot change.** See [Keep third-party and frozen baselines out of CodSpeed](#keep-third-party-and-frozen-baselines-out-of-codspeed). +- **Do not leave one-time process setup to the first measured iteration.** In simulation, each + benchmark runs one cold iteration. Lazy statics, such as the session id interner or the state + that the first regex compilation creates, then initialize inside whichever benchmark runs + first, often with system calls. Run the operation once in `main()` before `divan::main()`, as + `vortex-array/benches/like.rs` and `vortex-array/benches/run_end_filter.rs` do. - **Do not drop the `mimalloc` global allocator when you rewrite a benchmark file.** See [Use `mimalloc` as the global allocator](#use-mimalloc-as-the-global-allocator). - **Do not read a one-time change in later benchmarks as a regression after you add or remove diff --git a/vortex-array/benches/filter_bool.rs b/vortex-array/benches/filter_bool.rs index 10d4a4db090..d850841aaac 100644 --- a/vortex-array/benches/filter_bool.rs +++ b/vortex-array/benches/filter_bool.rs @@ -36,7 +36,12 @@ fn main() { static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); -const SIZES: &[usize] = &[1_000, 10_000, 100_000, 250_000]; +/// Array lengths for the filter benchmarks. +/// +/// 1000 rows took less than 1 µs, so the allocator work left behind by the previous, larger case +/// was a large part of the single measured iteration in simulation, and CodSpeed reported no +/// execution time for `filter_powerlaw_by_mostly_false[1000]`. +const SIZES: &[usize] = &[10_000, 100_000, 250_000]; const DENSITY_SWEEP_SIZE: usize = 100_000; const ARRAY_SEED: u64 = 42; const MASK_SEED: u64 = 43; diff --git a/vortex-array/benches/like.rs b/vortex-array/benches/like.rs index bce9e8fa5fb..7d7f29ed0bd 100644 --- a/vortex-array/benches/like.rs +++ b/vortex-array/benches/like.rs @@ -34,6 +34,14 @@ static GLOBAL: MiMalloc = MiMalloc; fn main() { LazyLock::force(&SESSION); + + // The first regex that a process compiles reads the CPU quota from the file system and seeds + // a hash. Without this, those system calls land in the single measured iteration of the first + // regex benchmark under the CodSpeed simulation. + like(strings(), "%HELLO%", ILIKE) + .execute::(&mut SESSION.create_execution_ctx()) + .unwrap(); + divan::main(); } @@ -57,18 +65,29 @@ fn strings() -> ArrayRef { .into_array() } +/// Case-insensitive, non-negated matching. +const ILIKE: LikeOptions = LikeOptions { + negated: false, + case_insensitive: true, +}; + +/// `array LIKE pattern`, with a constant pattern. +fn like(array: ArrayRef, pattern: &str, options: LikeOptions) -> ArrayRef { + Like::try_new( + array, + ConstantArray::new(pattern, ARRAY_SIZE).into_array(), + options, + ) + .unwrap() + .into_array() +} + fn bench_like(bencher: Bencher, pattern: &str, options: LikeOptions) { let array = strings(); bencher .with_inputs(|| { ( - Like::try_new( - array.clone(), - ConstantArray::new(pattern, ARRAY_SIZE).into_array(), - options, - ) - .unwrap() - .into_array(), + like(array.clone(), pattern, options), SESSION.create_execution_ctx(), ) }) @@ -120,12 +139,5 @@ fn like_per_row_patterns(bencher: Bencher) { #[divan::bench] fn ilike_contains(bencher: Bencher) { - bench_like( - bencher, - "%HELLO%", - LikeOptions { - negated: false, - case_insensitive: true, - }, - ); + bench_like(bencher, "%HELLO%", ILIKE); } diff --git a/vortex-array/benches/run_end_filter.rs b/vortex-array/benches/run_end_filter.rs index 073da7824a8..f83ed9fb712 100644 --- a/vortex-array/benches/run_end_filter.rs +++ b/vortex-array/benches/run_end_filter.rs @@ -27,6 +27,18 @@ use vortex_buffer::BitBuffer; static GLOBAL: MiMalloc = MiMalloc; fn main() { + // The first call initializes process-wide state, such as the session id interner, which + // reads the CPU quota from the file system. Without this call, those system calls land in the + // single measured iteration of the first benchmark under the CodSpeed simulation. + let length = 4_096; + filter_run_end_primitive::( + &build_run_ends(length, 16), + 0, + length as u64, + &build_mask(length, 0.5), + ) + .expect("filter"); + divan::main(); }