diff --git a/.gitignore b/.gitignore index 53921ab06c..a54124cca0 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ build*/ # lsp info compile_commands.json + +# local agent scratch files (may contain secrets) +.scratch/**/*.env diff --git a/.scratch/continuous-benchmarking/issues/01-cmake-google-benchmark.md b/.scratch/continuous-benchmarking/issues/01-cmake-google-benchmark.md new file mode 100644 index 0000000000..aeadb24cd1 --- /dev/null +++ b/.scratch/continuous-benchmarking/issues/01-cmake-google-benchmark.md @@ -0,0 +1,12 @@ +# 01: Add `IPPL_ENABLE_BENCHMARK` CMake option and fetch Google Benchmark + +**What to build:** A CMake-controlled path that, when enabled, fetches Google Benchmark v1.9.4 and makes it available to targets. A minimal smoke benchmark executable proves the whole path works: configure with `IPPL_ENABLE_BENCHMARK=ON`, build, run, and see Google Benchmark JSON output. + +**Blocked by:** None (can start immediately) + +**Status:** resolved + +- [ ] `option(IPPL_ENABLE_BENCHMARK ... OFF)` exists in root `CMakeLists.txt`. +- [ ] Google Benchmark v1.9.4 is fetched via `FetchContent` when the option is ON. +- [ ] A minimal `BenchmarkSmoke` executable builds and runs under `ctest`/manually. +- [ ] When the option is OFF, no benchmark code is compiled or linked. diff --git a/.scratch/continuous-benchmarking/issues/02-add-ippl-benchmark-helper.md b/.scratch/continuous-benchmarking/issues/02-add-ippl-benchmark-helper.md new file mode 100644 index 0000000000..299736a5a8 --- /dev/null +++ b/.scratch/continuous-benchmarking/issues/02-add-ippl-benchmark-helper.md @@ -0,0 +1,12 @@ +# 02: Implement `add_ippl_benchmark()` CMake helper + +**What to build:** A reusable CMake function that creates a benchmark target, accepts `REPORT_TIMERS`/`IGNORE_TIMERS`, and generates a compile-time timer-whitelist header. The smoke benchmark from ticket 01 is converted to use this helper. + +**Blocked by:** 01 + +**Status:** resolved + +- [ ] `cmake/AddIpplBenchmark.cmake` defines `add_ippl_benchmark( SOURCES ... [REPORT_TIMERS ...] [IGNORE_TIMERS ...])`. +- [ ] The function generates `_timers.h` in the build tree containing a `constexpr std::array reportedTimers`. +- [ ] Supplying both `REPORT_TIMERS` and `IGNORE_TIMERS` is a CMake configure error. +- [ ] The smoke benchmark uses the helper and compiles with a whitelist. diff --git a/.scratch/continuous-benchmarking/issues/03-bmf-json-writer.md b/.scratch/continuous-benchmarking/issues/03-bmf-json-writer.md new file mode 100644 index 0000000000..4fa8d680bb --- /dev/null +++ b/.scratch/continuous-benchmarking/issues/03-bmf-json-writer.md @@ -0,0 +1,12 @@ +# 03: Implement BMF JSON writer for `IpplTimings` + +**What to build:** A small utility that reads the existing `IpplTimings` measurements, filters them against the generated whitelist, and writes a BMF JSON file. The smoke benchmark is updated to emit BMF JSON alongside Google Benchmark JSON. + +**Blocked by:** 02 + +**Status:** resolved + +- [ ] A header-only (or small library) BMF writer computes `latency` (mean/min/max) and `count` per timer. +- [ ] Only whitelisted timer names appear in the BMF output. +- [ ] The smoke benchmark produces a valid BMF JSON file. +- [ ] No changes to `IpplTimings` internals. diff --git a/.scratch/continuous-benchmarking/issues/04-landau-damping-bench.md b/.scratch/continuous-benchmarking/issues/04-landau-damping-bench.md new file mode 100644 index 0000000000..ea58954b72 --- /dev/null +++ b/.scratch/continuous-benchmarking/issues/04-landau-damping-bench.md @@ -0,0 +1,13 @@ +# 04: Implement `LandauDampingBench` + +**What to build:** The first real benchmark executable for the Alpine module, instrumenting `LandauDampingManager`. It uses `add_ippl_benchmark()` and the BMF writer, runs `pre_run()` once as warmup, then benchmarks `advance()` per Google Benchmark iteration. + +**Blocked by:** 03 + +**Status:** resolved + +- [ ] `LandauDampingBench` builds with `IPPL_ENABLE_BENCHMARK=ON`. +- [ ] Running it produces Google Benchmark JSON and BMF JSON. +- [ ] Only rank 0 emits JSON artifacts. +- [ ] `IpplTimings::resetAllTimers()` is called before the measured loop. +- [ ] The benchmark still prints `IpplTimings` output for local debugging. diff --git a/.scratch/continuous-benchmarking/issues/05-ci-bencher-integration.md b/.scratch/continuous-benchmarking/issues/05-ci-bencher-integration.md new file mode 100644 index 0000000000..9e683e968c --- /dev/null +++ b/.scratch/continuous-benchmarking/issues/05-ci-bencher-integration.md @@ -0,0 +1,12 @@ +# 05: Add CI jobs and bencher.dev submission + +**What to build:** GitLab CI jobs on Eiger OpenMP, Daint GH200 CUDA, and MI300 ROCm that build with `IPPL_ENABLE_BENCHMARK=ON`, run `LandauDampingBench`, and submit both Google Benchmark JSON and BMF JSON to bencher.dev. Jobs remain non-blocking. + +**Blocked by:** 01, 04 + +**Status:** resolved + +- [x] One benchmark job exists per platform under `ci/cscs/`. +- [x] Each job runs `bencher run --adapter cpp_google` and `bencher run --adapter json --file `. +- [x] Jobs are gated by `IPPL_RUN_BENCHMARKS` and do not fail the pipeline on alerts. +- [x] Bencher testbed names match the spec (`eiger-openmp-1node`, `daint-gh200-1node`, `mi300-rocm-1node`). diff --git a/.scratch/continuous-benchmarking/spec.md b/.scratch/continuous-benchmarking/spec.md new file mode 100644 index 0000000000..6c8d5ac328 --- /dev/null +++ b/.scratch/continuous-benchmarking/spec.md @@ -0,0 +1,78 @@ +# Continuous Benchmarking for IPPL/OpalX + +Status: ready-for-agent + +## Problem Statement + +IPPL/OpalX has no systematic, automated way to detect performance regressions in its solver and particle kernels before code merges. Kernel runtimes are currently observed ad-hoc through `IpplTimings` console output and CSV dumps. Without a continuous benchmarking pipeline, regressions are only noticed late, are hard to attribute to a specific change, and require manual reproduction across the three CSCS target platforms (Eiger OpenMP, Daint GH200 CUDA, MI300 ROCm). + +## Solution + +Introduce an optional `IPPL_ENABLE_BENCHMARK` CMake path that fetches Google Benchmark and builds dedicated benchmark executables alongside existing demos/tests. The first benchmark will instrument `LandauDampingManager` via the existing `IpplTimings` framework, emit both Google Benchmark JSON and a BMF JSON file, and submit both to bencher.dev from per-platform GitLab CI jobs. The pipeline starts non-blocking and flips to blocking once thresholds are trustworthy. + +## User Stories + +1. As an IPPL developer, I want benchmark executables to live next to the demos they instrument, so that I can reuse the existing manager classes without modifying the demo binary. +2. As a CI maintainer, I want a single CMake option (`IPPL_ENABLE_BENCHMARK`) to enable the benchmarking build, so that benchmark jobs are opt-in and do not affect normal builds. +3. As a performance engineer, I want Google Benchmark fetched automatically via CMake `FetchContent`, so that no manual dependency installation is required on any platform. +4. As a developer adding a new benchmark, I want an `add_ippl_benchmark()` CMake helper that accepts `REPORT_TIMERS` and `IGNORE_TIMERS`, so that the set of `IpplTimings` phases submitted to bencher.dev is version-controlled and visible next to the target. +5. As a benchmark author, I want the helper to generate a compile-time timer whitelist header, so that the binary only emits the timers I care about with no runtime file I/O or parsing. +6. As a CI runner, I want the benchmark executable to write Google Benchmark JSON and a BMF JSON file, so that bencher.dev can ingest headline latency plus per-phase breakdowns. +7. As a platform maintainer, I want one GitLab CI job per CSCS platform (OpenMP, CUDA, ROCm), so that regressions are caught on every target architecture. +8. As a project lead, I want the CI jobs to be non-blocking initially, so that we can seed baselines and tune statistical thresholds before making them merge-blocking. +9. As a reviewer, I want benchmark results to appear on PRs via bencher.dev, so that performance impact is visible alongside code changes. +10. As a developer debugging a regression, I want the benchmark to still print `IpplTimings` output and CSV dumps, so that I can investigate locally with familiar tooling. +11. As a maintainer of a non-CSCS fork, I want the benchmarking code to be self-contained in IPPL and not require bencher.dev credentials to build or run, so that the feature is portable. +12. As a future benchmark author, I want the first benchmark (`LandauDampingBench`) to be a clear template, so that adding `PenningTrapBench` or `BumponTailInstabilityBench` is mostly copy-paste. +13. As a CI operator, I want only rank 0 to emit JSON output, so that multi-rank runs do not produce corrupt or duplicate benchmark artifacts. +14. As a performance engineer, I want benchmark iterations to call `IpplTimings::resetAllTimers()` before the measured loop, so that cumulative timer state does not leak across Google Benchmark iterations. +15. As a project lead, I want the bencher.dev project name and testbed names documented in the spec, so that CI configuration is reproducible. + +## Implementation Decisions + +- **CMake option**: Add `option(IPPL_ENABLE_BENCHMARK "Enable Google Benchmark-based benchmarks" OFF)` in the root `CMakeLists.txt`. +- **Dependency fetching**: Google Benchmark v1.9.4 is fetched via `FetchContent_Declare` in `cmake/Dependencies.cmake`, mirroring the existing GTest fetch. Benchmark tests and installation are disabled. +- **Benchmark helper**: A new CMake module `cmake/AddIpplBenchmark.cmake` defines `add_ippl_benchmark( SOURCES ... [REPORT_TIMERS ...] [IGNORE_TIMERS ...])`. + - If neither `REPORT_TIMERS` nor `IGNORE_TIMERS` is given, all `IpplTimings` phases are reportable. + - `REPORT_TIMERS` is a whitelist; `IGNORE_TIMERS` is a blacklist; supplying both is an error. + - The helper generates a header `_timers.h` in the build tree containing a `constexpr std::array reportedTimers`. +- **BMF writer**: A small header-only utility (e.g. `src/Utility/BenchmarkMetrics.h`) collects timer statistics from `IpplTimings` and writes BMF JSON. It filters emitted timers against the generated whitelist. +- **First benchmark**: `demos/alpine/LandauDampingBench.cpp` builds a `LandauDampingManager<>` in `SetUp()`, runs `pre_run()` once as warmup, resets timers, then benchmarks `manager->advance()` per Google Benchmark iteration. `TearDown()` prints timers and writes BMF JSON. +- **JSON emission**: Only rank 0 uses the real `benchmark::BenchmarkReporter`; other ranks use `benchmark::NullReporter`. +- **CI integration**: Extend `ci/cscs/` with a benchmark stage/job per platform. Jobs run only when `IPPL_RUN_BENCHMARKS` is `"true"` and execute two `bencher run` submissions: + 1. `--adapter cpp_google` against the Google Benchmark JSON. + 2. `--adapter json --file timings.bmf.json` for per-phase metrics. + - The variable defaults to `"false"` in `ci/cscs/common.yml` so normal build/test pipelines are unaffected. + - Set it to `"true"` via the GitLab pipeline trigger variables or the CSCS CI admin console to enable benchmarks. +- **Bencher project**: Target project is `ippl/OpalX` (name TBD with team); testbeds are `eiger-openmp-1node`, `daint-gh200-1node`, `mi300-rocm-1node`. +- **CI blocking policy**: Jobs set `IPPL_BENCH_ERROR_ON_ALERT=false` (or equivalent non-blocking configuration) until baselines are seeded and thresholds tuned. +- **Warmup semantics**: `SetUp()` calls `pre_run()` once and `IpplTimings::resetAllTimers()` so that initialization and solver warmup are not included in the measured `advance()` loop. +- **Benchmark arguments**: Problem size defaults are configurable via Google Benchmark `->Args(...)` or CLI args; the CI default is left as a spec decision (initially `16 16 16` with `10000000` particles and a small number of steps to keep iteration time reasonable). + +## Testing Decisions + +- **Seam 1 — CMake helper**: Verify that `add_ippl_benchmark(LandauDampingBench ... REPORT_TIMERS solve pushVelocity ...)` configures successfully and that the generated `_timers.h` contains exactly the requested timer names. This is tested by a configure-and-build check, not a runtime unit test. +- **Seam 2 — BMF writer**: If the BMF writer is extracted into a small pure function (string list + timer statistics → JSON string), it can be covered by a lightweight unit test that asserts correct JSON shape and whitelist filtering. Otherwise it is tested indirectly via the benchmark executable. +- **Seam 3 — First benchmark executable**: Run `LandauDampingBench` locally (single-rank and multi-rank) and assert that: + - It exits 0. + - Google Benchmark JSON is produced and contains the expected benchmark name. + - BMF JSON is produced and contains only whitelisted timer names. + - Only rank 0 writes JSON artifacts. +- **Seam 4 — CI integration**: A dry-run on one platform (e.g. OpenMP) confirms that `bencher run` is invoked with the correct adapters and that the job is non-blocking. +- **Regression policy**: No runtime assertions on performance numbers are added to the test suite; bencher.dev owns regression detection. + +## Out of Scope + +- Modifying existing demo/test binaries to act as benchmarks (the plan explicitly creates separate benchmark executables). +- Large-scale (>1 node) benchmarks for this first effort; the testbeds remain single-node. +- OpalX-specific benchmark integration or regression-test coupling; this effort is scoped to IPPL demos only. +- Bencher.dev dashboard configuration beyond project/testbed naming; the skill assumes the project exists or will be created separately. +- Runtime JSON config for the timer whitelist; the generated-header approach is chosen for simplicity. +- Changing `IpplTimings` internals; only the existing public interface is consumed. + +## Further Notes + +- The generated timer whitelist header should be placed in the build tree next to the benchmark binary (e.g. `${CMAKE_CURRENT_BINARY_DIR}/_timers.h`) and included privately by the benchmark source. +- Keep benchmark binaries out of the default `IPPL_ENABLE_TESTS` path so that `ctest` does not try to run Google Benchmark executables as Catch/GTest-style tests. +- The LandauDamping integration test in `demos/alpine/CMakeLists.txt` already passes arguments `"16" "16" "16" "10000000" "25" "FFT" "0.01" "LeapFrog" "--overallocate" "2.0" "--info" "10"`; these are a reasonable starting point for the benchmark, though step count and overallocate factor may be tuned for stable iteration time. +- When `IPPL_ENABLE_BENCHMARK` is OFF, no Google Benchmark code or headers should be compiled or linked. diff --git a/CMakeLists.txt b/CMakeLists.txt index 439514f155..63484cbae8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,6 +58,7 @@ option(IPPL_MARK_FAILING_TESTS "Prefix names of tests that are known to fail with 'known_fail' for filtering with ctest" OFF) option(IPPL_ENABLE_SCRIPTS "Generate job script templates for some benchmarks/tests" OFF) +option(IPPL_ENABLE_BENCHMARK "Enable Google Benchmark-based benchmarks" OFF) option(IPPL_ENABLE_CATALYST "Enable ParaView Catalyst" OFF) @@ -142,3 +143,7 @@ add_subdirectory(demos) if(IPPL_ENABLE_SCRIPTS) add_subdirectory(scripts) endif() + +if(IPPL_ENABLE_BENCHMARK) + add_subdirectory(benchmarks) +endif() diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt new file mode 100644 index 0000000000..407bbf4186 --- /dev/null +++ b/benchmarks/CMakeLists.txt @@ -0,0 +1,10 @@ +# ------------------------------------------------------------------------------ +# benchmarks/CMakeLists.txt +# +# Optional benchmark executables built when IPPL_ENABLE_BENCHMARK is ON. +# ------------------------------------------------------------------------------ +message(STATUS "Configuring benchmarks/") + +include(AddIpplBenchmark) + +add_ippl_benchmark(BenchmarkSmoke SOURCES SmokeBench.cpp REPORT_TIMERS smoke) diff --git a/benchmarks/SmokeBench.cpp b/benchmarks/SmokeBench.cpp new file mode 100644 index 0000000000..8a241797b6 --- /dev/null +++ b/benchmarks/SmokeBench.cpp @@ -0,0 +1,58 @@ +// ------------------------------------------------------------------------------ +// Smoke benchmark for IPPL's Google Benchmark integration. +// +// This is a minimal executable used to verify that the CMake FetchContent setup, +// compile flags, benchmark::benchmark linking, add_ippl_benchmark() timer +// whitelist generation, and BenchmarkMetrics BMF writing all work correctly. +// ------------------------------------------------------------------------------ + +#include "Ippl.h" + +#include + +#include "Utility/BenchmarkMetrics.h" +#include "Utility/IpplTimings.h" + +#include "BenchmarkSmoke_timers.h" + +class SmokeFixture : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State&) override { IpplTimings::resetAllTimers(); } + + void TearDown(const ::benchmark::State&) override { + ippl::benchmark::writeBMF("BenchmarkSmoke.bmf.json", reportedTimers); + } +}; + +BENCHMARK_DEFINE_F(SmokeFixture, BM_Smoke)(benchmark::State& state) { + static IpplTimings::TimerRef smokeTimer = IpplTimings::getTimer("smoke"); + + for (auto _ : state) { + IpplTimings::startTimer(smokeTimer); + + int sum = 0; + for (int i = 0; i < 100; ++i) { + sum += i; + } + benchmark::DoNotOptimize(sum); + + IpplTimings::stopTimer(smokeTimer); + } +} + +BENCHMARK_REGISTER_F(SmokeFixture, BM_Smoke); + +int main(int argc, char** argv) { + ippl::initialize(argc, argv); + + ::benchmark::Initialize(&argc, argv); + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) { + ippl::finalize(); + return 1; + } + ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); + + ippl::finalize(); + return 0; +} diff --git a/ci/cscs/benchmark/benchmark-cuda.yml b/ci/cscs/benchmark/benchmark-cuda.yml new file mode 100644 index 0000000000..ac42da0ea1 --- /dev/null +++ b/ci/cscs/benchmark/benchmark-cuda.yml @@ -0,0 +1,44 @@ +# ----------------------------------------------------------------------------- + +# CUDA benchmark on Daint GH200 +# ----------------------------------------------------------------------------- +ippl-benchmark-cuda: + extends: .uenv-runner-daint-gh200 + stage: benchmark + image: "/capstor/store/cscs/cscs/public/uenvs/opal-x-gh200-mpich-gcc-2026-08-04-cuda-13.1.1.squashfs" + needs: ["ippl-build-cuda13-sm90-release", "ippl-test-cuda13-sm90-release-4-ranks"] + variables: + SLURM_JOB_NUM_NODES: 1 + SLURM_TIMELIMIT: "0:15:00" + BENCHMARK_MIN_TIME: "1.0s" + BENCHER_PROJECT: "ippl-opalx" + CUDA_UENV: "/capstor/store/cscs/cscs/public/uenvs/opal-x-gh200-mpich-gcc-2026-08-04-cuda-13.1.1.squashfs" + WITH_UENV_VIEW: "default" + WRAPPER: "/user-environment/wrapper-mpi.sh" + SCRATCH: "/capstor/scratch/cscs/biddisco" + SRUN_FLAGS: "--uenv=${CUDA_UENV} --view=${WITH_UENV_VIEW} --repo=$SCRATCH/.uenv-images-ci-daint ${WRAPPER}" + BUILD_DIR: "build-$CI_COMMIT_SHORT_SHA-release" + SLURM_NTASKS: 4 + SLURM_CPUS_PER_TASK: 32 + BENCHER_TESTBED: "daint-gh200-1node" + before_script: + - echo "CI_PROJECT_URL=$CI_PROJECT_URL" + - echo "CI_COMMIT_SHORT_SHA=$CI_COMMIT_SHORT_SHA" + - echo "IPPL_RUN_BENCHMARKS=$IPPL_RUN_BENCHMARKS" + - export BUILD_PATH=$(pwd)/$BUILD_DIR + - pwd + - env | sort | grep CI + script: + - ./ci/scripts/install_bencher.sh + - export PATH=$HOME/.cargo/bin:$PATH + - ./ci/scripts/run_benchmark.sh + artifacts: + paths: + - LandauDampingBench.json + - LandauDampingBench.bmf.json + expire_in: 1 week + allow_failure: true + rules: + - if: $IPPL_RUN_BENCHMARKS == "true" + when: always + - when: never diff --git a/ci/cscs/benchmark/benchmark-openmp.yml b/ci/cscs/benchmark/benchmark-openmp.yml new file mode 100644 index 0000000000..0b27c27093 --- /dev/null +++ b/ci/cscs/benchmark/benchmark-openmp.yml @@ -0,0 +1,42 @@ +# ----------------------------------------------------------------------------- +# OpenMP benchmark on Eiger +# ----------------------------------------------------------------------------- +ippl-benchmark-openmp: + extends: .uenv-runner-eiger-zen2 + stage: benchmark + image: "prgenv-gnu/25.6:v2" + needs: ["ippl-build-openmp-release", "ippl-test-openmp-release-4-ranks"] + variables: + SLURM_JOB_NUM_NODES: 1 + SLURM_TIMELIMIT: "0:15:00" + BENCHMARK_MIN_TIME: "1.0s" + BENCHER_PROJECT: "ippl-opalx" + EIGER_UENV: "prgenv-gnu/25.6:v2" + WITH_UENV_VIEW: "default" + SCRATCH: "/capstor/scratch/cscs/biddisco" + SRUN_FLAGS: "--uenv=${EIGER_UENV} --view=${WITH_UENV_VIEW} --repo=$SCRATCH/.uenv-images-ci-eiger" + BUILD_DIR: "build-$CI_COMMIT_SHORT_SHA-release" + SLURM_NTASKS: 4 + SLURM_CPUS_PER_TASK: 32 + BENCHER_TESTBED: "eiger-openmp-1node" + before_script: + - echo "CI_PROJECT_URL=$CI_PROJECT_URL" + - echo "CI_COMMIT_SHORT_SHA=$CI_COMMIT_SHORT_SHA" + - echo "IPPL_RUN_BENCHMARKS=$IPPL_RUN_BENCHMARKS" + - export BUILD_PATH=$(pwd)/$BUILD_DIR + - pwd + - env | sort | grep CI + script: + - ./ci/scripts/install_bencher.sh + - export PATH=$HOME/.cargo/bin:$PATH + - ./ci/scripts/run_benchmark.sh + artifacts: + paths: + - LandauDampingBench.json + - LandauDampingBench.bmf.json + expire_in: 1 week + allow_failure: true + rules: + - if: $IPPL_RUN_BENCHMARKS == "true" + when: always + - when: never diff --git a/ci/cscs/benchmark/benchmark-rocm.yml b/ci/cscs/benchmark/benchmark-rocm.yml new file mode 100644 index 0000000000..c4808c0a40 --- /dev/null +++ b/ci/cscs/benchmark/benchmark-rocm.yml @@ -0,0 +1,42 @@ +# ----------------------------------------------------------------------------- +# ROCm benchmark on Beverin MI300 +# ----------------------------------------------------------------------------- +ippl-benchmark-rocm: + extends: .uenv-runner-beverin-mi300 + stage: benchmark + image: "prgenv-gnu/25.07-6.3.3:v12" + needs: ["ippl-build-rocm6_3-release", "ippl-test-rocm6_3-release-4-ranks"] + variables: + SLURM_JOB_NUM_NODES: 1 + SLURM_TIMELIMIT: "0:15:00" + BENCHMARK_MIN_TIME: "1.0s" + BENCHER_PROJECT: "ippl-opalx" + ROCM6_3_UENV: "prgenv-gnu/25.07-6.3.3:v12" + WITH_UENV_VIEW: "default" + SCRATCH: "/capstor/scratch/cscs/biddisco" + SRUN_FLAGS: "--uenv=${ROCM6_3_UENV} --view=${WITH_UENV_VIEW} --repo=$SCRATCH/.uenv-images-ci-beverin" + BUILD_DIR: "build-$CI_COMMIT_SHORT_SHA-release" + SLURM_NTASKS: 4 + SLURM_CPUS_PER_TASK: 32 + BENCHER_TESTBED: "mi300-rocm-1node" + before_script: + - echo "CI_PROJECT_URL=$CI_PROJECT_URL" + - echo "CI_COMMIT_SHORT_SHA=$CI_COMMIT_SHORT_SHA" + - echo "IPPL_RUN_BENCHMARKS=$IPPL_RUN_BENCHMARKS" + - export BUILD_PATH=$(pwd)/$BUILD_DIR + - pwd + - env | sort | grep CI + script: + - ./ci/scripts/install_bencher.sh + - export PATH=$HOME/.cargo/bin:$PATH + - ./ci/scripts/run_benchmark.sh + artifacts: + paths: + - LandauDampingBench.json + - LandauDampingBench.bmf.json + expire_in: 1 week + allow_failure: true + rules: + - if: $IPPL_RUN_BENCHMARKS == "true" + when: always + - when: never diff --git a/ci/cscs/common.yml b/ci/cscs/common.yml index 50beecc42d..1a6aa5a6d5 100644 --- a/ci/cscs/common.yml +++ b/ci/cscs/common.yml @@ -1,6 +1,11 @@ include: - remote: "https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml" +variables: + IPPL_RUN_BENCHMARKS: + value: "false" + description: "Set to 'true' to run the bencher benchmark jobs after tests." + .pr-number-extractor: before_script: - echo "CI_PROJECT_URL=$CI_PROJECT_URL" @@ -18,3 +23,4 @@ include: stages: - ippl_build - ippl_test + - benchmark diff --git a/ci/cscs/cscs-gh200.yml b/ci/cscs/cscs-gh200.yml index 3c4b6c5037..4ded4632a0 100644 --- a/ci/cscs/cscs-gh200.yml +++ b/ci/cscs/cscs-gh200.yml @@ -2,3 +2,4 @@ include: # NVIDIA/GH200: - local: "ci/cscs/cuda/build_sm90.yml" - local: "ci/cscs/cuda/run_sm90.yml" + - local: "ci/cscs/benchmark/benchmark-cuda.yml" diff --git a/ci/cscs/cscs-mi300.yml b/ci/cscs/cscs-mi300.yml index 161cd1e1b6..dadc6bd0af 100644 --- a/ci/cscs/cscs-mi300.yml +++ b/ci/cscs/cscs-mi300.yml @@ -2,3 +2,4 @@ include: # AMD/MI300: - local: "ci/cscs/rocm/build_rocm-6.3.yml" - local: "ci/cscs/rocm/run_rocm-6.3.yml" + - local: "ci/cscs/benchmark/benchmark-rocm.yml" diff --git a/ci/cscs/cscs-openmp.yml b/ci/cscs/cscs-openmp.yml index be016a84a8..a3a089c932 100644 --- a/ci/cscs/cscs-openmp.yml +++ b/ci/cscs/cscs-openmp.yml @@ -2,3 +2,4 @@ include: # CPU - local: "ci/cscs/openmp/build_openmp.yml" - local: "ci/cscs/openmp/run_openmp.yml" + - local: "ci/cscs/benchmark/benchmark-openmp.yml" diff --git a/ci/cscs/cuda/build_sm90.yml b/ci/cscs/cuda/build_sm90.yml index ff41d53e25..8f352e6b4e 100644 --- a/ci/cscs/cuda/build_sm90.yml +++ b/ci/cscs/cuda/build_sm90.yml @@ -41,6 +41,7 @@ variables: -DMPIEXEC_EXECUTABLE=/usr/bin/srun -DMPIEXEC_PREFLAGS="$SRUN_FLAGS" -DMPIEXEC_MAX_NUMPROCS=4 + -DIPPL_ENABLE_BENCHMARK=ON - echo "Build directory size (before cleanup):" $(du -sh $BUILD_PATH | cut -f1) - find $BUILD_PATH -name \*.o -delete - find $BUILD_PATH/bin -type f -exec strip {} + diff --git a/ci/cscs/dashboard-configure-build.cmake b/ci/cscs/dashboard-configure-build.cmake index 8f9c7f333a..244c1d2113 100644 --- a/ci/cscs/dashboard-configure-build.cmake +++ b/ci/cscs/dashboard-configure-build.cmake @@ -47,6 +47,7 @@ string(APPEND CTEST_CONFIGURE_COMMAND " -DCMAKE_BUILD_TYPE=${BUILD_TYPE}") set(VARS_TO_FORWARD IPPL_PLATFORMS IPPL_OPENMP_THREADS + IPPL_ENABLE_BENCHMARK IPPL_ENABLE_SCRIPTS Heffte_VERSION Kokkos_VERSION @@ -64,7 +65,7 @@ set(VARS_TO_FORWARD LAPACKE_LIBRARIES MKL_DIR MPIEXEC_EXECUTABLE - MPIEXEC_PREFLAGS + MPIEXEC_PREFLAGS MPIEXEC_MAX_NUMPROCS ) diff --git a/ci/cscs/openmp/build_openmp.yml b/ci/cscs/openmp/build_openmp.yml index 76850cf95c..8db64b2249 100644 --- a/ci/cscs/openmp/build_openmp.yml +++ b/ci/cscs/openmp/build_openmp.yml @@ -44,6 +44,7 @@ variables: -DMPIEXEC_EXECUTABLE=/usr/bin/srun -DMPIEXEC_PREFLAGS="$SRUN_FLAGS" -DMPIEXEC_MAX_NUMPROCS=4 + -DIPPL_ENABLE_BENCHMARK=ON - echo "Build directory size (before cleanup):" $(du -sh $BUILD_PATH | cut -f1) - find $BUILD_PATH -name \*.o -delete - find $BUILD_PATH/bin -type f -exec strip {} + diff --git a/ci/cscs/rocm/build_rocm-6.3.yml b/ci/cscs/rocm/build_rocm-6.3.yml index 99e96620cd..7ec49ed1dc 100644 --- a/ci/cscs/rocm/build_rocm-6.3.yml +++ b/ci/cscs/rocm/build_rocm-6.3.yml @@ -45,6 +45,7 @@ variables: -DMPIEXEC_EXECUTABLE=/usr/bin/srun -DMPIEXEC_PREFLAGS="$SRUN_FLAGS;${WRAPPER}" -DMPIEXEC_MAX_NUMPROCS=4 + -DIPPL_ENABLE_BENCHMARK=ON # -DHeffte_ENABLE_GPU_AWARE_MPI=OFF - echo "Build directory size (before cleanup):" $(du -sh $BUILD_PATH | cut -f1) - find $BUILD_PATH -name \*.o -delete diff --git a/ci/scripts/install_bencher.sh b/ci/scripts/install_bencher.sh new file mode 100755 index 0000000000..53feef318f --- /dev/null +++ b/ci/scripts/install_bencher.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# ----------------------------------------------------------------------------- +# Install the bencher.dev CLI. +# +# Uses the official install script. Set BENCHER_VERSION to pin a version; +# otherwise the latest release is installed. +# ----------------------------------------------------------------------------- + +set -euo pipefail + +curl --proto '=https' --tlsv1.2 -sSfL https://bencher.dev/download/install-cli.sh | sh + +bencher --version diff --git a/ci/scripts/run_benchmark.sh b/ci/scripts/run_benchmark.sh new file mode 100755 index 0000000000..8fe43044db --- /dev/null +++ b/ci/scripts/run_benchmark.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# ----------------------------------------------------------------------------- +# Run LandauDampingBench and optionally upload results to bencher.dev. +# +# Environment variables expected: +# BUILD_PATH - path to the CMake build directory +# SRUN_FLAGS - flags for srun (may be empty) +# BENCHMARK_MIN_TIME - minimum benchmark run time (default: 1.0s) +# BENCHER_PROJECT - bencher.dev project slug +# BENCHER_TESTBED - bencher.dev testbed name +# BENCHER_API_KEY - optional bencher.dev API key (preferred) +# BENCHER_API_TOKEN - optional bencher.dev API token (fallback) +# CI_COMMIT_REF_NAME - Git branch/tag name +# ----------------------------------------------------------------------------- + +set -euo pipefail + +BENCHMARK_MIN_TIME="${BENCHMARK_MIN_TIME:-1.0s}" + +# Locate the benchmark binary anywhere under the build tree. +BENCH_BIN=$(find "$BUILD_PATH" -name LandauDampingBench -type f -executable | head -n 1) + +if [ -z "$BENCH_BIN" ]; then + echo "LandauDampingBench binary not found in $BUILD_PATH" + echo "Contents of likely locations:" + ls -la "$BUILD_PATH/bin" 2>/dev/null || true + ls -la "$BUILD_PATH/demos/alpine" 2>/dev/null || true + echo "All executables matching *Bench*:" + find "$BUILD_PATH" -type f -executable -name "*Bench*" 2>/dev/null | head -20 + exit 1 +fi + +echo "Using benchmark binary: $BENCH_BIN" + +# shellcheck disable=SC2086 +# --cpu-bind=none avoids conflicts with the outer SLURM allocation's CPU binding. +srun --cpu-bind=none $SRUN_FLAGS \ + "$BENCH_BIN" \ + 16 16 16 100000 5 FFT 0.01 LeapFrog \ + --overallocate 2.0 --info 10 \ + --benchmark_out=LandauDampingBench.json \ + --benchmark_min_time="$BENCHMARK_MIN_TIME" + +ls -la LandauDampingBench.json LandauDampingBench.bmf.json || true + +# Determine bencher credentials. Prefer BENCHER_API_KEY; fall back to BENCHER_API_TOKEN. +# The flag is chosen by variable name: API keys use --key, tokens use --token. +BENCHER_CRED="" +BENCHER_CRED_FLAG="" +if [ -n "${BENCHER_API_KEY:-}" ]; then + BENCHER_CRED="$BENCHER_API_KEY" + BENCHER_CRED_FLAG="--key" + unset BENCHER_API_TOKEN +elif [ -n "${BENCHER_API_TOKEN:-}" ]; then + BENCHER_CRED="$BENCHER_API_TOKEN" + BENCHER_CRED_FLAG="--token" +fi + +if [ -z "$BENCHER_CRED" ]; then + echo "Neither BENCHER_API_KEY nor BENCHER_API_TOKEN set; skipping bencher upload" +fi + +# Only upload from rank 0 to avoid duplicate submissions in multi-rank runs. +if [ "${SLURM_PROCID:-0}" != "0" ] && [ "${PMI_RANK:-0}" != "0" ] && [ "${OMPI_COMM_WORLD_RANK:-0}" != "0" ]; then + echo "Skipping bencher upload on rank ${SLURM_PROCID:-${PMI_RANK:-${OMPI_COMM_WORLD_RANK:-0}}}" + BENCHER_CRED_FLAG="" +fi + +if [ -n "$BENCHER_CRED_FLAG" ]; then + echo "Uploading to bencher.dev using $BENCHER_CRED_FLAG" + + bencher run \ + --project "$BENCHER_PROJECT" \ + --testbed "$BENCHER_TESTBED" \ + --adapter cpp_google \ + --file LandauDampingBench.json \ + --branch "$CI_COMMIT_REF_NAME" \ + "$BENCHER_CRED_FLAG" "$BENCHER_CRED" + + bencher run \ + --project "$BENCHER_PROJECT" \ + --testbed "$BENCHER_TESTBED" \ + --adapter json \ + --file LandauDampingBench.bmf.json \ + --branch "$CI_COMMIT_REF_NAME" \ + "$BENCHER_CRED_FLAG" "$BENCHER_CRED" +fi diff --git a/cmake/AddIpplBenchmark.cmake b/cmake/AddIpplBenchmark.cmake new file mode 100644 index 0000000000..99d229600d --- /dev/null +++ b/cmake/AddIpplBenchmark.cmake @@ -0,0 +1,80 @@ +# ----------------------------------------------------------------------------- +# AddIpplBenchmark.cmake +# +# Defines a helper function `add_ippl_benchmark()` to create a Google Benchmark executable linked +# against IPPL. It also generates a compile-time whitelist of IpplTimings timer names that the +# benchmark is allowed to emit to bencher.dev. +# ----------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------- +# ~~~ +# add_ippl_benchmark( +# [SOURCES ...] # default: .cpp +# [REPORT_TIMERS ...] # whitelist of IpplTimings names +# [IGNORE_TIMERS ...] # blacklist of IpplTimings names +# [NUM_PROCS ] # default: 1 +# [ARGS ...] # args passed to the benchmark binary +# [LABELS ...] # extra ctest labels +# ) +# ~~~ +# ----------------------------------------------------------------------------- +function(add_ippl_benchmark BENCH_NAME) + set(options) + set(oneValueArgs NUM_PROCS) + set(multiValueArgs SOURCES REPORT_TIMERS IGNORE_TIMERS ARGS LABELS) + cmake_parse_arguments(BENCH "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if(BENCH_REPORT_TIMERS AND BENCH_IGNORE_TIMERS) + message( + FATAL_ERROR + "add_ippl_benchmark(${BENCH_NAME}): REPORT_TIMERS and IGNORE_TIMERS are mutually exclusive") + endif() + + if(BENCH_SOURCES) + set(_sources ${BENCH_SOURCES}) + else() + set(_sources ${BENCH_NAME}.cpp) + endif() + + add_executable(${BENCH_NAME} ${_sources}) + + target_link_libraries(${BENCH_NAME} PRIVATE IPPL::ippl benchmark::benchmark) + + if(TARGET ippl_build_flags) + target_link_libraries(${BENCH_NAME} PRIVATE ippl_build_flags) + endif() + + target_include_directories(${BENCH_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}) + + # Generate the timer whitelist header in the build tree. + set(_timer_header "${CMAKE_CURRENT_BINARY_DIR}/${BENCH_NAME}_timers.h") + + if(BENCH_IGNORE_TIMERS) + # For a blacklist we still emit all known names in the header and let the BMF writer drop + # ignored ones at runtime. Until then, emit an empty list so the helper does not need to + # enumerate every possible IpplTimings name. + set(_timer_names) + else() + set(_timer_names ${BENCH_REPORT_TIMERS}) + endif() + + list(LENGTH _timer_names _num_timers) + + set(_entries) + foreach(_name IN LISTS _timer_names) + string(APPEND _entries " \"${_name}\",\n") + endforeach() + + string( + CONCAT _header_content + "#pragma once\n" + "#include \n" + "#include \n\n" + "// Auto-generated by add_ippl_benchmark for ${BENCH_NAME}.\n" + "// Empty list means all IpplTimings phases are reportable.\n" + "inline constexpr std::array reportedTimers{\n" + "${_entries}};\n") + + file(GENERATE OUTPUT ${_timer_header} CONTENT "${_header_content}") +endfunction() diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index e6d340d798..4c78c3c280 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -619,3 +619,40 @@ if(IPPL_ENABLE_UNIT_TESTS) message(STATUS "✅ GoogleTest built from source (${GTest_VERSION})") endif() endif() + +# ------------------------------------------------------------------------------ +# Google Benchmark +# ------------------------------------------------------------------------------ +if(IPPL_ENABLE_BENCHMARK) + find_package(benchmark CONFIG QUIET) + if(NOT benchmark_FOUND) + FetchContent_Declare(benchmark GIT_REPOSITORY "https://github.com/google/benchmark" + GIT_TAG "v1.9.4" GIT_SHALLOW ON) + + set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable Google Benchmark tests" FORCE) + set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Disable Google Benchmark install" FORCE) + set(BENCHMARK_INSTALL_DOCS OFF CACHE BOOL "Disable Google Benchmark docs install" FORCE) + set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "Disable Google Benchmark gtest tests" FORCE) + + FetchContent_MakeAvailable(benchmark) + message(STATUS "✅ Google Benchmark built from source (v1.9.4)") + else() + message(STATUS "✅ Google Benchmark found externally") + endif() +endif() + +# ------------------------------------------------------------------------------ +# FEL module header-only dependencies (nlohmann/json for config parsing, stb_image_write for the +# Poynting-flux visualization). +# ------------------------------------------------------------------------------ +if(IPPL_ENABLE_FEL) + # Fetch the CMake package instead of downloading the release header directly. CMake's + # file(DOWNLOAD) does not fail by default and can leave a zero-byte json.hpp behind when a + # release-asset host is unavailable, which only surfaces later as a confusing compile error. + set(JSON_BuildTests OFF CACHE BOOL "Disable nlohmann/json tests" FORCE) + FetchContent_Declare(nlohmann_json GIT_REPOSITORY https://github.com/nlohmann/json.git + GIT_TAG v3.11.3 GIT_SHALLOW ON) + FetchContent_MakeAvailable(nlohmann_json) + + message(STATUS "✅ nlohmann/json loaded for the FEL module.") +endif() diff --git a/demos/alpine/CMakeLists.txt b/demos/alpine/CMakeLists.txt index a6265161fa..bd9dee220b 100644 --- a/demos/alpine/CMakeLists.txt +++ b/demos/alpine/CMakeLists.txt @@ -56,3 +56,35 @@ add_alpine_example(PenningTrap) add_alpine_example(BumponTailInstability) add_subdirectory(validation) + +if(IPPL_ENABLE_BENCHMARK) + include(AddIpplBenchmark) + + add_ippl_benchmark( + LandauDampingBench + SOURCES + LandauDampingBench.cpp + REPORT_TIMERS + solve + pushVelocity + pushPosition + update + loadBalance + ARGS + "16" + "16" + "16" + "100000" + "5" + "FFT" + "0.01" + "LeapFrog" + "--overallocate" + "2.0" + "--info" + "10" + NUM_PROCS + 2 + LABELS + alpine) +endif() diff --git a/demos/alpine/LandauDampingBench.cpp b/demos/alpine/LandauDampingBench.cpp new file mode 100644 index 0000000000..fe2e5af3f6 --- /dev/null +++ b/demos/alpine/LandauDampingBench.cpp @@ -0,0 +1,124 @@ +// ------------------------------------------------------------------------------ +// LandauDampingBench +// +// Google Benchmark executable for the LandauDampingManager. It reuses the same +// physics setup as the LandauDamping demo and measures the cost of one +// manager->advance() call per benchmark iteration. +// ------------------------------------------------------------------------------ + +constexpr unsigned Dim = 3; +using T = double; +const char* TestName = "LandauDampingBench"; + +#include "Ippl.h" + +#include +#include +#include +#include + +#include "Manager/datatypes.h" + +#include "Utility/BenchmarkMetrics.h" +#include "Utility/IpplTimings.h" + +#include "LandauDampingBench_timers.h" +#include "LandauDampingManager.h" +#include "Manager/PicManager.h" + +struct BenchConfig { + Vector_t nr; + size_type totalP; + int nt; + std::string solver; + double lbt; + std::string stepMethod; + std::vector preconditionerParams; +}; + +static BenchConfig config_m; + +class LandauDampingFixture : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State&) override { + manager_m = std::make_unique>( + config_m.totalP, config_m.nt, config_m.nr, config_m.lbt, config_m.solver, + config_m.stepMethod, config_m.preconditionerParams); + + manager_m->pre_run(); + IpplTimings::resetAllTimers(); + } + + void TearDown(const ::benchmark::State&) override { + IpplTimings::print(); + ippl::benchmark::writeBMF("LandauDampingBench.bmf.json", reportedTimers); + + // Destroy the manager before ippl::finalize() so Kokkos views are + // deallocated while Kokkos is still initialized. + manager_m.reset(); + } + + std::unique_ptr> manager_m; +}; + +BENCHMARK_DEFINE_F(LandauDampingFixture, BM_Advance)(benchmark::State& state) { + for (auto _ : state) { + manager_m->advance(); + } +} + +BENCHMARK_REGISTER_F(LandauDampingFixture, BM_Advance); + +int main(int argc, char** argv) { + ippl::initialize(argc, argv); + + Inform msg(TestName); + + int arg = 1; + for (unsigned d = 0; d < Dim; d++) { + config_m.nr[d] = std::atoi(argv[arg++]); + } + + config_m.totalP = std::atoll(argv[arg++]); + config_m.nt = std::atoi(argv[arg++]); + config_m.solver = argv[arg++]; + config_m.lbt = std::atof(argv[arg++]); + config_m.stepMethod = argv[arg++]; + + if (config_m.solver == "PCG" || config_m.solver == "FEM_PRECON") { + while (arg < argc) { + const std::string token = argv[arg]; + if (token.rfind("--", 0) == 0) { + break; + } + config_m.preconditionerParams.push_back(token); + ++arg; + } + } + + // Consume demo-specific trailing flags (--overallocate, --info) and pass + // the rest to Google Benchmark. + std::vector benchArgv; + benchArgv.push_back(argv[0]); + for (int i = arg; i < argc;) { + const std::string token = argv[i]; + if (token == "--overallocate" || token == "--info") { + i += 2; + } else { + benchArgv.push_back(argv[i]); + ++i; + } + } + int benchArgc = static_cast(benchArgv.size()); + + ::benchmark::Initialize(&benchArgc, benchArgv.data()); + if (::benchmark::ReportUnrecognizedArguments(benchArgc, benchArgv.data())) { + ippl::finalize(); + return 1; + } + ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); + + ippl::finalize(); + return 0; +} diff --git a/docs/benchmarking-plan.md b/docs/benchmarking-plan.md new file mode 100644 index 0000000000..776e0bf680 --- /dev/null +++ b/docs/benchmarking-plan.md @@ -0,0 +1,158 @@ +# IPPL/OpalX Continuous Benchmarking Plan + +## Status + +Implemented on branch `continuous-benchmarking`. Data is flowing from all three +CSCS platforms to bencher.dev. Jobs remain non-blocking while baselines are +being seeded. + +## Goals + +- Detect performance regressions in IPPL/OpalX kernels **before PRs merge**. +- Track solver and kernel runtimes over time across the three CSCS platforms + (Eiger OpenMP, Daint GH200 CUDA, MI300 ROCm). +- Reuse the existing `IpplTimings` instrumentation so no application code needs + to be re-instrumented. +- Keep benchmark CI jobs non-blocking initially, then flip to blocking once + statistical thresholds on `bencher.dev` are trustworthy. + +## Architecture + +``` +IPPL CMake option : IPPL_ENABLE_BENCHMARK +Dependency : google/benchmark fetched via FetchContent (v1.9.4) +First benchmark : LandauDampingBench (demos/alpine/LandauDampingBench.cpp) +Submission : bencher.dev cloud, project "ippl-opalx" +Testbeds : eiger-openmp-1node, daint-gh200-1node, mi300-rocm-1node +CI gate variable : IPPL_RUN_BENCHMARKS (set to "true" to enable) +``` + +## Benchmark executable structure + +A separate benchmark executable is built per instrumented application. It does +**not** replace the existing example/test binary. + +- `SetUp()` — parse problem-size args, construct manager, call `pre_run()` once + (warmup), then `IpplTimings::resetAllTimers()`. +- Body — `for (auto _ : state) manager->advance();` so Google Benchmark + measures per-iteration wall time. +- `TearDown()` — call `IpplTimings::print()` / `dumpToCSV()` for parity, write + a BMF JSON file for per-phase submission to bencher.dev. +- `main()` — `ippl::initialize`, `benchmark::Initialize`, run benchmarks with + only rank 0 emitting JSON (non-zero ranks use a NullReporter). + +## Bridging IpplTimings to bencher.dev + +Google Benchmark captures the headline per-iteration latency. The existing +`IpplTimings` measurements are bridged into bencher via a BMF JSON file: + +``` +{ + "solve": { + "latency": { "value": , "lower_value": , "upper_value": }, + "count": { "value": } + }, + ... +} +``` + +This is consumed by `bencher run --adapter json --file timings.bmf.json`. + +## CMake-driven timer whitelist/blacklist + +Each `add_ippl_benchmark()` call should accept a list of timer names that the +benchmark is allowed to emit to bencher. This makes it easy to configure which +kernels are tracked without touching C++ source. + +### Proposed CMake API + +```cmake +add_ippl_benchmark( + LandauDampingBench + SOURCES LandauDampingBench.cpp + # Only these IpplTimings phases are emitted to bencher.dev + REPORT_TIMERS solve pushVelocity pushPosition update loadBalance + # Or blacklist specific noisy phases instead of whitelisting + # IGNORE_TIMERS initialize solveWarmup dumpData particlesCreation total +) +``` + +### How the list reaches the binary + +At configure time, CMake generates a small header or JSON file in the build + tree, e.g.: + +```cpp +// auto-generated by CMake: build/demos/alpine/LandauDampingBench_timers.h +#pragma once +#include +#include +inline constexpr std::array reportedTimers{ + "solve", "pushVelocity", "pushPosition", "update", "loadBalance"}; +``` + +The benchmark's BMF writer includes this header and only emits timers whose +name appears in `reportedTimers`. + +### Alternative: runtime JSON config + +Instead of a generated header, CMake could emit `LandauDampingBench_timers.json` +next to the binary and the benchmark reads it at startup: + +```json +{ + "report_timers": ["solve", "pushVelocity", "pushPosition", "update", "loadBalance"] +} +``` + +Pros: can be edited post-build without recompilation. +Cons: adds file I/O to the benchmark startup and complicates CI artifact +handling. + +### Recommendation + +Use the **generated header** approach for simplicity and compile-time safety. +Keep the list in CMake so it is version-controlled and clearly visible next to +the target definition. Switch to a runtime JSON config only if we find we need +to tweak the list frequently without recompiling. + +## CI / bencher.dev integration + +- Bencher CLI installed on each testbed via `ci/scripts/install_bencher.sh` + (static binary download). +- One GitLab CI job per platform, gated by `IPPL_RUN_BENCHMARKS=true`. +- Build jobs pass `-DIPPL_ENABLE_BENCHMARK=ON` to compile `LandauDampingBench`. +- Benchmark jobs run after the 4-rank release tests and execute two + `bencher run` commands: + 1. `--adapter cpp_google` for the headline latency from Google Benchmark JSON. + 2. `--adapter json --file LandauDampingBench.bmf.json` for the per-phase + breakdown. +- Upload happens only from rank 0 to avoid duplicate submissions. +- Jobs are non-blocking until `--error-on-alert` is enabled. + +## Completed steps + +1. ✅ CMake `IPPL_ENABLE_BENCHMARK` option and Google Benchmark v1.9.4 fetch. +2. ✅ `add_ippl_benchmark()` helper with `REPORT_TIMERS`/`IGNORE_TIMERS` and + generated timer whitelist header. +3. ✅ BMF JSON writer (`src/Utility/BenchmarkMetrics.h`). +4. ✅ `LandauDampingBench` as the first benchmark. +5. ✅ One GitLab CI benchmark job per CSCS platform. +6. ✅ Bencher CLI installer and rank-0 upload script. + +## Next steps + +1. Seed `main` baselines on all three testbeds. +2. Collect data for ~2 weeks non-blocking. +3. Tune thresholds per kernel & testbed in the bencher.dev UI. +4. Enable `--error-on-alert` once thresholds are trustworthy. +5. Expand to more demos (PenningTrap, BumponTailInstability, etc.) using the + same pattern. + +## Open questions + +- Should benchmarks be integrated with the OpalX regression-test step, or kept + inside IPPL/OpalX initially? +- Which timer subset should be reported by default for LandauDamping? +- What problem size should be the CI default (small 16³ vs medium 32³)? +- How should large-scale (>1 node) benchmarks be funded and scheduled? diff --git a/docs/benchmarking-slides.md b/docs/benchmarking-slides.md new file mode 100644 index 0000000000..04b70ea21d --- /dev/null +++ b/docs/benchmarking-slides.md @@ -0,0 +1,199 @@ +--- +marp: true +theme: default +paginate: true +title: Continuous Benchmarking for IPPL/OpalX +--- + + + +# Continuous Benchmarking for IPPL/OpalX + +**Catch performance regressions before they merge** + +Run small, representative benchmarks on every PR across the three CSCS +platforms, submit results to bencher.dev, and alert when a kernel regresses +beyond a statistical threshold. + +*Status: implemented on `continuous-benchmarking`; data is flowing from Eiger, +Daint GH200, and Beverin MI300.* + +--- + +## What we built + +1. ✅ **Google Benchmark** integrated via CMake `FetchContent` + (`IPPL_ENABLE_BENCHMARK`, off by default). +2. ✅ **`add_ippl_benchmark()`** helper with `REPORT_TIMERS`/`IGNORE_TIMERS` + and a compile-time timer whitelist header. +3. ✅ **BMF JSON writer** bridging `IpplTimings` to bencher.dev. +4. ✅ **LandauDampingBench** as the first benchmark. +5. ✅ Three GitLab CI benchmark jobs, gated by `IPPL_RUN_BENCHMARKS=true`. +6. ✅ Results submitted to **bencher.dev** project `ippl-opalx`. +7. ✅ **Non-blocking at first** — collecting data before enabling alerts. + +--- + +## How it fits together + +``` + PR opened ─► GitLab CI builds IPPL/OpalX (+Google Benchmark) + │ + ▼ + ctest runs unit/integration tests + │ + ▼ + IPPL_RUN_BENCHMARKS=true? + │ yes + ▼ + srun LandauDampingBench (all ranks) + │ + ▼ + bencher run --adapter cpp_google (headline latency) + bencher run --adapter json --file (per-phase breakdown) + │ + ▼ + bencher.dev ─► plots over time, per testbed & branch + ─► alert on PR if regression > threshold +``` + +- Benchmark jobs run after the 4-rank release tests. +- Only rank 0 uploads to bencher to avoid duplicate submissions. + +--- + +## First target: LandauDamping + +- Already instrumented with `IpplTimings`: + - `solve` (field solve) + - `pushVelocity` / `pushPosition` (particle push) + - `update` (particle redistribution) + - `loadBalance` (ORB repartition) + - `particlesCreation`, `dumpData`, `initialize`, `solveWarmup`, `total` +- Google Benchmark measures **per-iteration wall time** natively + (mean / median / stddev / min / max). +- Existing `IpplTimings` measurements are bridged into bencher so + **each kernel becomes its own plot** over time. + +--- + +## What you'll see on bencher.dev + +Project: `ippl-opalx` + +- **Headline plot** (one per testbed): + `LandauDampingFixture/BM_Advance` — mean latency per time step. +- **Per-kernel plots** (one per IpplTimings phase × testbed): + - `solve`, `pushVelocity`, `pushPosition`, `update`, `loadBalance` +- Testbeds: + - `eiger-openmp-1node` + - `daint-gh200-1node` + - `mi300-rocm-1node` +- PR branches compared against `main` via a statistical threshold + (Student's t-test, tunable upper boundary). + +--- + +## Example: real data from Daint GH200 + +Data is already flowing. Each point is one CI run of `LandauDampingBench` on a +single GH200 node (4 ranks). + +- Headline: ~5–20 s per full `advance()` step for 16³ grid, 100k particles, 5 + steps. +- Per-phase breakdown comes from `IpplTimings` via the BMF JSON adapter. + +![h:360px Example bencher.dev plot](bencher-plot-1.png) + +--- + +## ❓ Discussion: what should we benchmark? + +**Applications** +- **Demos:** LandauDamping, PenningTrap, BumponTailInstability, + UniformPlasma, electrostaticPIF, cosmology, FEL, collisions. +- **Solvers:** FFT, CG, FEM, multigrid, Maxwell. +- **Low-level:** scatter/gather, particle comms, interpolation, RNG. + +**Kernels inside LandauDamping** + +| Phase | What it does | Regression risk | +|---|---|---| +| `solve` | Field solve (FFT / CG / FEM) | ★★★ algorithm | +| `pushVelocity` / `pushPosition` | Particle push kernels | ★★ memory bandwidth | +| `update` | Particle redistribution (MPI) | ★★★ communication | +| `loadBalance` | ORB repartition | ★★ algorithm | +| `par2grid` / `grid2par` | Scatter/gather — *not yet timed* | ★★★ bandwidth | + +👉 *Which applications and kernels should we track first?* +👉 *Are scatter/gather the missing pieces?* + +--- + +## ❓ Discussion: everything, or a curated subset? + +- **Everything:** complete coverage, but long CI runs and noisy data. +- **Curated subset:** fast, clean signal, but blind spots remain. + +**Proposed selection criteria:** +1. Exercises a primary solver + its communication pattern. +2. Has a stable, representative problem size. +3. Runs in < 5 minutes in CI. +4. Covers the backends we ship (OpenMP, CUDA, ROCm). + +👉 *Do these criteria match your priorities?* +👉 *What balance of coverage vs CI cost is right for IPPL/OpalX?* + +--- + +## Current configuration & open questions + +**Current CI problem size:** +- 16³ grid, 100k particles, 5 steps +- `--benchmark_min_time=1.0s` +- Single-node, 4 ranks per platform + +👉 *Is this size representative, or should we increase it for better signal?* + +**Gating policy:** +- Jobs are enabled by `IPPL_RUN_BENCHMARKS=true` in the CSCS CI admin console. +- Non-blocking while we collect baseline data (a few weeks). +- Flip to `--error-on-alert` once thresholds are tuned. + +👉 *When should regressions start blocking PRs?* +👉 *Who decides thresholds — kernel owners or the team collectively?* + +--- + +## Next steps + +1. ✅ **Implemented:** CMake option, helper, BMF writer, `LandauDampingBench`, + three CI jobs, bencher upload. +2. **Seed baselines** on `main` for all three testbeds. +3. **Collect data** for ~2 weeks, non-blocking. +4. **Tune thresholds** per kernel & testbed in the bencher.dev UI. +5. **Flip to blocking** on PRs once thresholds are trustworthy. +6. **Expand** to more applications/kernels (PenningTrap, + BumponTailInstability, solvers, etc.). + +*Note: large-scale benchmarks (>1 node) consume significant node-hours and +should be limited to `main`/nightly/release cycles unless a dedicated budget +is available.* + +👉 *Are we aligned? Any blockers or concerns?* diff --git a/src/Utility/BenchmarkMetrics.h b/src/Utility/BenchmarkMetrics.h new file mode 100644 index 0000000000..a27ed2da32 --- /dev/null +++ b/src/Utility/BenchmarkMetrics.h @@ -0,0 +1,127 @@ +// ----------------------------------------------------------------------------- +// BenchmarkMetrics.h +// +// Header-only utility that bridges IpplTimings measurements to bencher.dev's +// Benchmark Metric Format (BMF). It filters emitted timers against a compile-time +// whitelist generated by add_ippl_benchmark() and writes one JSON file per +// benchmark invocation. +// ----------------------------------------------------------------------------- + +#ifndef IPPL_BENCHMARK_METRICS_H +#define IPPL_BENCHMARK_METRICS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Utility/IpplTimings.h" + +namespace ippl { + namespace benchmark { + + namespace detail { + template + bool isWhitelisted(const std::string& name, + const std::array& whitelist) { + if constexpr (N == 0) { + return true; + } + return std::find(whitelist.begin(), whitelist.end(), name) != whitelist.end(); + } + + inline double computeMean(const std::vector& values) { + if (values.empty()) { + return 0.0; + } + return std::accumulate(values.begin(), values.end(), 0.0) / values.size(); + } + + inline double computeMin(const std::vector& values) { + if (values.empty()) { + return 0.0; + } + return *std::min_element(values.begin(), values.end()); + } + + inline double computeMax(const std::vector& values) { + if (values.empty()) { + return 0.0; + } + return *std::max_element(values.begin(), values.end()); + } + } // namespace detail + + /** + * Write a BMF JSON file containing per-timer latency statistics. + * + * @param filename Path to the output JSON file (written by rank 0 only). + * @param whitelist Compile-time array of timer names to emit. An empty array + * means all timers are emitted. + */ + template + void writeBMF(const std::string& filename, + const std::array& whitelist) { + if (ippl::Comm->rank() != 0) { + return; + } + + std::vector timerNames = IpplTimings::getTimerNames(); + + std::ostringstream json; + json << std::setprecision(17) << std::scientific; + json << "{\n"; + + bool first = true; + for (const auto& name : timerNames) { + if (!detail::isWhitelisted(name, whitelist)) { + continue; + } + + IpplTimings::TimerRef ref = IpplTimings::getTimer(name.c_str()); + std::vector values = IpplTimings::getMeasurements(ref); + std::size_t count = values.size(); + + if (count == 0) { + continue; + } + + // IpplTimings stores values in seconds. Bencher's latency measure + // uses nanoseconds, so scale before emitting. + constexpr double seconds_to_nanoseconds = 1.0e9; + double mean = detail::computeMean(values) * seconds_to_nanoseconds; + double minv = detail::computeMin(values) * seconds_to_nanoseconds; + double maxv = detail::computeMax(values) * seconds_to_nanoseconds; + + if (!first) { + json << ",\n"; + } + first = false; + + json << " \"" << name << "\": {\n"; + json << " \"latency\": {\n"; + json << " \"value\": " << mean << ",\n"; + json << " \"lower_value\": " << minv << ",\n"; + json << " \"upper_value\": " << maxv << "\n"; + json << " },\n"; + json << " \"count\": {\n"; + json << " \"value\": " << count << "\n"; + json << " }\n"; + json << " }"; + } + + json << "\n}\n"; + + std::ofstream out(filename); + out << json.str(); + } + + } // namespace benchmark +} // namespace ippl + +#endif