Address recurring CodSpeed warnings

.github/workflows/codspeed.yml

@@ -122,6 +122,19 @@ jobs:
           # allocator and cold caches. This adds about 15 µs to each benchmark and hides small
           # changes in short benchmarks. Run all benchmarks of a target in one process, as before.
           CODSPEED_PROCESS_ISOLATION: "false"
+          # Allocators return freed memory to the OS and grow their heaps with system calls. A
+          # system call inside a measured iteration is left out of the measure, so CodSpeed reports
+          # that benchmark as dominated by system calls. Keep freed memory mapped instead.
+          #
+          # mimalloc, which every benchmark binary uses, returns memory 10 ms after it is freed.
+          # The simulation runs many times slower than native code, so that delay can end inside a
+          # later benchmark's measured iteration.
+          MIMALLOC_PURGE_DELAY: "-1"
+          # glibc `malloc`, which C dependencies such as zstd use, shrinks its heap after large
+          # frees and grows it again with `brk`. Shrink only when 1 GiB is free at the top of the
+          # heap, and grow in 64 MiB steps so that a later benchmark rarely needs to grow it.
+          MALLOC_TRIM_THRESHOLD_: "1073741824"
+          MALLOC_TOP_PAD_: "67108864"
         with:
           run: cargo codspeed run
           token: ${{ secrets.CODSPEED_TOKEN }}
@@ -143,6 +156,11 @@ jobs:
       # Same settings as the bench profile, which keeps the debug info that CodSpeed uses to
       # attribute native frames. See `profile.bench_python` in Cargo.toml.
       MATURIN_PEP517_ARGS: "--profile bench_python"
+      # pytest-codspeed records Python frames only through the CPython perf trampoline, which
+      # exists from Python 3.12. On the 3.11 interpreter from `.python-version`, every Python
+      # benchmark reported "no flamegraph". The extension is built for the stable ABI from 3.11,
+      # so the same build runs on 3.13.
+      UV_PYTHON: "3.13"
     steps:
       - uses: runs-on/action@v2
         if: github.repository == 'vortex-data/vortex'

docs/developer-guide/benchmarking.md

@@ -193,7 +193,8 @@ and memory access costs. As a result:
   [Known unstable benchmarks](#known-unstable-benchmarks)).
 - System calls are not measured. A benchmark that depends on I/O or kernel calls **must** use the
   [walltime instrument](https://codspeed.io/docs/instruments/walltime) or be gated with
-  `#[cfg(not(codspeed))]`.
+  `#[cfg(not(codspeed))]`. The simulation job keeps mimalloc and glibc from returning freed
+  memory to the OS, so that allocator system calls do not land in a measured iteration.
 
 **Walltime legs.** A walltime leg is one CI job that measures wall-clock time on a dedicated
 machine for one CPU feature set (AVX2, AVX-512 or NEON). `.github/workflows/codspeed.yml` defines
@@ -255,6 +256,11 @@ unnoticed. When a section above explains a rule, the rule links to that section.
   [How CodSpeed measures a benchmark](#how-codspeed-measures-a-benchmark).
 - **Do not measure code that a pull request cannot change.** See
   [Keep third-party and frozen baselines out of CodSpeed](#keep-third-party-and-frozen-baselines-out-of-codspeed).
+- **Do not leave one-time process setup to the first measured iteration.** In simulation, each
+  benchmark runs one cold iteration. Lazy statics, such as the session id interner or the state
+  that the first regex compilation creates, then initialize inside whichever benchmark runs
+  first, often with system calls. Run the operation once in `main()` before `divan::main()`, as
+  `vortex-array/benches/like.rs` and `vortex-array/benches/run_end_filter.rs` do.
 - **Do not drop the `mimalloc` global allocator when you rewrite a benchmark file.** See
   [Use `mimalloc` as the global allocator](#use-mimalloc-as-the-global-allocator).
 - **Do not read a one-time change in later benchmarks as a regression after you add or remove

vortex-array/benches/filter_bool.rs

@@ -36,7 +36,12 @@ fn main() {
 static SESSION: LazyLock<VortexSession> =
     LazyLock::new(|| VortexSession::empty().with::<KernelSession>());
 
-const SIZES: &[usize] = &[1_000, 10_000, 100_000, 250_000];
+/// Array lengths for the filter benchmarks.
+///
+/// 1000 rows took less than 1 µs, so the allocator work left behind by the previous, larger case
+/// was a large part of the single measured iteration in simulation, and CodSpeed reported no
+/// execution time for `filter_powerlaw_by_mostly_false[1000]`.
+const SIZES: &[usize] = &[10_000, 100_000, 250_000];
 const DENSITY_SWEEP_SIZE: usize = 100_000;
 const ARRAY_SEED: u64 = 42;
 const MASK_SEED: u64 = 43;

vortex-array/benches/like.rs

@@ -34,6 +34,14 @@ static GLOBAL: MiMalloc = MiMalloc;
 
 fn main() {
     LazyLock::force(&SESSION);
+
+    // The first regex that a process compiles reads the CPU quota from the file system and seeds
+    // a hash. Without this, those system calls land in the single measured iteration of the first
+    // regex benchmark under the CodSpeed simulation.
+    like(strings(), "%HELLO%", ILIKE)
+        .execute::<BoolArray>(&mut SESSION.create_execution_ctx())
+        .unwrap();
+
     divan::main();
 }
 
@@ -57,18 +65,29 @@ fn strings() -> ArrayRef {
     .into_array()
 }
 
+/// Case-insensitive, non-negated matching.
+const ILIKE: LikeOptions = LikeOptions {
+    negated: false,
+    case_insensitive: true,
+};
+
+/// `array LIKE pattern`, with a constant pattern.
+fn like(array: ArrayRef, pattern: &str, options: LikeOptions) -> ArrayRef {
+    Like::try_new(
+        array,
+        ConstantArray::new(pattern, ARRAY_SIZE).into_array(),
+        options,
+    )
+    .unwrap()
+    .into_array()
+}
+
 fn bench_like(bencher: Bencher, pattern: &str, options: LikeOptions) {
     let array = strings();
     bencher
         .with_inputs(|| {
             (
-                Like::try_new(
-                    array.clone(),
-                    ConstantArray::new(pattern, ARRAY_SIZE).into_array(),
-                    options,
-                )
-                .unwrap()
-                .into_array(),
+                like(array.clone(), pattern, options),
                 SESSION.create_execution_ctx(),
             )
         })
@@ -120,12 +139,5 @@ fn like_per_row_patterns(bencher: Bencher) {
 
 #[divan::bench]
 fn ilike_contains(bencher: Bencher) {
-    bench_like(
-        bencher,
-        "%HELLO%",
-        LikeOptions {
-            negated: false,
-            case_insensitive: true,
-        },
-    );
+    bench_like(bencher, "%HELLO%", ILIKE);
 }

vortex-array/benches/run_end_filter.rs

@@ -27,6 +27,18 @@ use vortex_buffer::BitBuffer;
 static GLOBAL: MiMalloc = MiMalloc;
 
 fn main() {
+    // The first call initializes process-wide state, such as the session id interner, which
+    // reads the CPU quota from the file system. Without this call, those system calls land in the
+    // single measured iteration of the first benchmark under the CodSpeed simulation.
+    let length = 4_096;
+    filter_run_end_primitive::<u32>(
+        &build_run_ends(length, 16),
+        0,
+        length as u64,
+        &build_mask(length, 0.5),
+    )
+    .expect("filter");
+
     divan::main();
 }