From 2fed20e3b9d93550a68391b56ba75f3f0dac1bab Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 15:18:50 +0200 Subject: [PATCH] perf: axis reductions split columns across cores and use the SIMD kernels On a Zen 3 (16 cores) the axis reductions of a 1024x1024 matrix lost to NumPy 0.23-0.43x, for two reasons: - An axis-0 reduction ran on one core: it has a single outer slab, and RunAxisP only split outer slabs, on the belief that a column split needs a strided gather. A band of columns is a contiguous run of every row, so the kernels now take a column band [lo, hi) and RunAxisP hands bands of at least 512 columns to the workers when there are fewer slabs than workers. Rows are combined with the elementwise SIMD add/mul from 16 columns on, element by element in axis order, so the result is unchanged. - A row reduction (inner == 1) was a sequential scalar loop, latency-bound. Each row now goes through the SIMD sum/max/min. Max and min are exact; the sum is regrouped lane-parallel, as NumPy's pairwise sum is. Zen 3, 16 cores pinned, min of three interleaved runs: SumAxis(0) 696 -> 186 us (0.97x NumPy 1 thread) SumAxis(1) 531 -> 92 us (2.4x NumPy) MaxAxis(1) 601 -> 93 us (2.0x NumPy) New test: every RunAxisP schedule (serial, slab split, column bands for one slab and for fewer slabs than workers, narrow fallback, row sums) for Sum/Prod/Max/Min against a naive in-order oracle, exact except the regrouped row sum (1e-12). Shifting the band offset makes it fail. Co-Authored-By: Claude Opus 5.5 --- docs/perf.md | 26 +++++++ internal/kernels/kernels.go | 121 ++++++++++++++++++------------ internal/kernels/kernels_test.go | 26 +++---- internal/kernels/parallel.go | 73 ++++++++++-------- internal/kernels/parallel_test.go | 70 ++++++++++++++++- ndarray.go | 2 +- 6 files changed, 222 insertions(+), 96 deletions(-) diff --git a/docs/perf.md b/docs/perf.md index 116e529..04f2d7a 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -411,6 +411,32 @@ amd64 (Zen 3, FMA path), arm64 (cfarm185), ppc64le (cfarm120), riscv64 (cfarm94) and loong64 (cfarm401). The s390x host (LinuxONE) did not answer; s390x stays covered by the qemu lane in CI. +## Axis reductions (2026-10-04) + +On the Zen 3 the axis reductions of a 1024×1024 matrix lost to NumPy 0.23–0.43×. +Two causes, both measured: + +- **An axis-0 reduction ran on one core.** It has a single outer slab, and the + driver only split outer slabs; the comment justifying that said a column split + would need a strided gather. It does not: a band of columns is a contiguous run + of every row. `RunAxisP` now splits the columns into bands of at least 512 + when there are fewer slabs than workers, and each row run is added with the + elementwise SIMD kernel (exact per element, so the sum is still in axis order). +- **A row reduction (axis 1, `inner` = 1) was a sequential scalar loop**, + latency-bound at one add per 4 cycles. Each row now goes through the SIMD + sum/max/min. Max and min are exact; the sum is regrouped lane-parallel, as + NumPy's own pairwise sum is. + +Zen 3, 16 cores pinned, minimum of three interleaved runs: + +| op (1024×1024) | before | after | NumPy 1 thread | after vs NumPy | +|----|--:|--:|--:|:--:| +| SumAxis(0) | 696 µs | **186 µs** | 180 µs | 0.97× | +| SumAxis(1) | 531 µs | **92 µs** | 224 µs | 2.4× | +| MaxAxis(1) | 601 µs | **93 µs** | 189 µs | 2.0× | + +On M4: MaxAxis(1) ×6–8, SumAxis(0) ×1.9. + ## SIMD coverage - **amd64 (SSE2)** ships hand-vectorized `sum` (4-accumulator `ADDPD`), `sqrt` diff --git a/internal/kernels/kernels.go b/internal/kernels/kernels.go index 6c84060..a422ee3 100644 --- a/internal/kernels/kernels.go +++ b/internal/kernels/kernels.go @@ -200,62 +200,83 @@ func Dot1D(a, b []float64) float64 { // The parent package materialises a strided view into a contiguous []float64 // laid out conceptually as [outer][axisLen][inner], i.e. the reduced axis sits // in the middle and `inner` trailing elements are contiguous. Each kernel -// reduces the middle axis, writing outer*inner results into dst (also laid out -// as [outer][inner]). -// -// The innermost loop over `inner` walks contiguous memory with unit stride, -// which is the shape a SIMD kernel wants: Phase 1 can replace each step (the -// `inner`-length combine of dst[base:] with src[off:]) with a vectorised -// fused operation across all six 64-bit targets, leaving this scalar version as -// the reference and fallback. axisLen >= 1 is guaranteed by the caller. - -// SumAxis reduces the middle axis by summation. -func SumAxis(dst, src []float64, outer, axisLen, inner int) { +// reduces the middle axis for the inner columns [lo, hi) of every outer slab, +// writing those columns of dst (laid out as [outer][inner]). A column band is +// contiguous within each row, so RunAxisP can hand disjoint bands to different +// workers when there are too few outer slabs to split (an axis-0 reduction has +// one). The full reduction is the band [0, inner). axisLen >= 1 is guaranteed +// by the caller. + +// rowRun is the band width from which a row combine goes through the SIMD +// elementwise kernel; below it the call costs more than the loop. +const rowRun = 16 + +// SumAxis reduces the middle axis by summation. Each row of the band is added +// into dst with the elementwise SIMD add, element by element in axis order, so +// the result is the sequential sum. With inner == 1 every output is the sum of +// one contiguous row, taken with the lane-parallel SIMD sum (a regrouping, as +// numpy's pairwise sum is): a sequential sum there is latency-bound. +func SumAxis(dst, src []float64, outer, axisLen, inner, lo, hi int) { + if inner == 1 { + for o := 0; o < outer; o++ { + dst[o] = sumSIMD(src[o*axisLen : o*axisLen+axisLen]) + } + return + } for o := 0; o < outer; o++ { - base := o * inner + d := dst[o*inner+lo : o*inner+hi] block := o * axisLen * inner - for i := 0; i < inner; i++ { - dst[base+i] = src[block+i] - } + copy(d, src[block+lo:block+hi]) for k := 1; k < axisLen; k++ { - off := block + k*inner - for i := 0; i < inner; i++ { - dst[base+i] += src[off+i] + r := src[block+k*inner+lo : block+k*inner+hi] + if len(d) >= rowRun { + addBin(d, d, r) + continue + } + for i := range d { + d[i] += r[i] } } } } -// ProdAxis reduces the middle axis by multiplication. -func ProdAxis(dst, src []float64, outer, axisLen, inner int) { +// ProdAxis reduces the middle axis by multiplication, in axis order. +func ProdAxis(dst, src []float64, outer, axisLen, inner, lo, hi int) { for o := 0; o < outer; o++ { - base := o * inner + d := dst[o*inner+lo : o*inner+hi] block := o * axisLen * inner - for i := 0; i < inner; i++ { - dst[base+i] = src[block+i] - } + copy(d, src[block+lo:block+hi]) for k := 1; k < axisLen; k++ { - off := block + k*inner - for i := 0; i < inner; i++ { - dst[base+i] *= src[off+i] + r := src[block+k*inner+lo : block+k*inner+hi] + if len(d) >= rowRun { + mulBin(d, d, r) + continue + } + for i := range d { + d[i] *= r[i] } } } } // MaxAxis reduces the middle axis by taking the maximum. Like Max it is -// NaN-propagating (numpy.max(axis=...)). -func MaxAxis(dst, src []float64, outer, axisLen, inner int) { +// NaN-propagating (numpy.max(axis=...)); max is exact and associative, so the +// inner == 1 row reduction through the SIMD max kernel is the same value. +func MaxAxis(dst, src []float64, outer, axisLen, inner, lo, hi int) { + if inner == 1 { + for o := 0; o < outer; o++ { + dst[o] = maxSIMD(src[o*axisLen : o*axisLen+axisLen]) + } + return + } for o := 0; o < outer; o++ { - base := o * inner + d := dst[o*inner+lo : o*inner+hi] block := o * axisLen * inner - for i := 0; i < inner; i++ { - dst[base+i] = src[block+i] - } + copy(d, src[block+lo:block+hi]) for k := 1; k < axisLen; k++ { - off := block + k*inner - for i := 0; i < inner; i++ { - dst[base+i] = max(dst[base+i], src[off+i]) + r := src[block+k*inner+lo : block+k*inner+hi] + for i := range d { + d[i] = max(d[i], r[i]) } } } @@ -263,17 +284,21 @@ func MaxAxis(dst, src []float64, outer, axisLen, inner int) { // MinAxis reduces the middle axis by taking the minimum, NaN-propagating // (numpy.min(axis=...)). -func MinAxis(dst, src []float64, outer, axisLen, inner int) { +func MinAxis(dst, src []float64, outer, axisLen, inner, lo, hi int) { + if inner == 1 { + for o := 0; o < outer; o++ { + dst[o] = minSIMD(src[o*axisLen : o*axisLen+axisLen]) + } + return + } for o := 0; o < outer; o++ { - base := o * inner + d := dst[o*inner+lo : o*inner+hi] block := o * axisLen * inner - for i := 0; i < inner; i++ { - dst[base+i] = src[block+i] - } + copy(d, src[block+lo:block+hi]) for k := 1; k < axisLen; k++ { - off := block + k*inner - for i := 0; i < inner; i++ { - dst[base+i] = min(dst[base+i], src[off+i]) + r := src[block+k*inner+lo : block+k*inner+hi] + for i := range d { + d[i] = min(d[i], r[i]) } } } @@ -319,11 +344,11 @@ func ArgMin(a []float64) int { // ArgMaxAxis writes into dst the index (along the middle axis) of the first // maximum for each [outer][inner] position, or of the first NaN (see ArgMax). // Layout matches the *Axis kernels. -func ArgMaxAxis(dst []float64, src []float64, outer, axisLen, inner int) { +func ArgMaxAxis(dst []float64, src []float64, outer, axisLen, inner, lo, hi int) { for o := 0; o < outer; o++ { base := o * inner block := o * axisLen * inner - for i := 0; i < inner; i++ { + for i := lo; i < hi; i++ { best := src[block+i] bi := 0 for k := 1; k < axisLen && !math.IsNaN(best); k++ { @@ -338,11 +363,11 @@ func ArgMaxAxis(dst []float64, src []float64, outer, axisLen, inner int) { // ArgMinAxis writes into dst the index (along the middle axis) of the first // minimum for each [outer][inner] position, or of the first NaN. -func ArgMinAxis(dst []float64, src []float64, outer, axisLen, inner int) { +func ArgMinAxis(dst []float64, src []float64, outer, axisLen, inner, lo, hi int) { for o := 0; o < outer; o++ { base := o * inner block := o * axisLen * inner - for i := 0; i < inner; i++ { + for i := lo; i < hi; i++ { best := src[block+i] bi := 0 for k := 1; k < axisLen && !math.IsNaN(best); k++ { diff --git a/internal/kernels/kernels_test.go b/internal/kernels/kernels_test.go index 3195cc1..aa69519 100644 --- a/internal/kernels/kernels_test.go +++ b/internal/kernels/kernels_test.go @@ -68,33 +68,33 @@ func TestAxisKernels(t *testing.T) { src := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} dst := make([]float64, 2*2) - SumAxis(dst, src, 2, 3, 2) + SumAxis(dst, src, 2, 3, 2, 0, 2) eqSlice(t, dst, []float64{9, 12, 27, 30}) - ProdAxis(dst, src, 2, 3, 2) + ProdAxis(dst, src, 2, 3, 2, 0, 2) eqSlice(t, dst, []float64{15, 48, 693, 960}) - MaxAxis(dst, src, 2, 3, 2) + MaxAxis(dst, src, 2, 3, 2, 0, 2) eqSlice(t, dst, []float64{5, 6, 11, 12}) - MinAxis(dst, src, 2, 3, 2) + MinAxis(dst, src, 2, 3, 2, 0, 2) eqSlice(t, dst, []float64{1, 2, 7, 8}) // Descending data so MinAxis takes its "found a smaller value" branch. - MinAxis(dst, []float64{6, 5, 4, 3, 2, 1, 12, 11, 10, 9, 8, 7}, 2, 3, 2) + MinAxis(dst, []float64{6, 5, 4, 3, 2, 1, 12, 11, 10, 9, 8, 7}, 2, 3, 2, 0, 2) eqSlice(t, dst, []float64{2, 1, 8, 7}) // Descending data for MaxAxis exercises its non-update path too. - MaxAxis(dst, []float64{6, 5, 4, 3, 2, 1, 12, 11, 10, 9, 8, 7}, 2, 3, 2) + MaxAxis(dst, []float64{6, 5, 4, 3, 2, 1, 12, 11, 10, 9, 8, 7}, 2, 3, 2, 0, 2) eqSlice(t, dst, []float64{6, 5, 12, 11}) // axisLen == 1: the reduce loop body never runs; result is the input. one := make([]float64, 2) - SumAxis(one, []float64{4, 9}, 2, 1, 1) + SumAxis(one, []float64{4, 9}, 2, 1, 1, 0, 1) eqSlice(t, one, []float64{4, 9}) - ProdAxis(one, []float64{4, 9}, 2, 1, 1) + ProdAxis(one, []float64{4, 9}, 2, 1, 1, 0, 1) eqSlice(t, one, []float64{4, 9}) - MaxAxis(one, []float64{4, 9}, 2, 1, 1) + MaxAxis(one, []float64{4, 9}, 2, 1, 1, 0, 1) eqSlice(t, one, []float64{4, 9}) - MinAxis(one, []float64{4, 9}, 2, 1, 1) + MinAxis(one, []float64{4, 9}, 2, 1, 1, 0, 1) eqSlice(t, one, []float64{4, 9}) } @@ -108,7 +108,7 @@ func BenchmarkSumAxis(b *testing.B) { b.ReportAllocs() b.SetBytes(int64(len(src) * 8)) for i := 0; i < b.N; i++ { - SumAxis(dst, src, outer, axisLen, inner) + SumAxis(dst, src, outer, axisLen, inner, 0, inner) } } @@ -133,9 +133,9 @@ func TestArgAndScan(t *testing.T) { // Axis variants over [outer=1][axisLen=3][inner=2]: columns {3,1},{9,5} etc. src := []float64{3, 1, 9, 5, 0, 7} dst := make([]float64, 2) - ArgMaxAxis(dst, src, 1, 3, 2) + ArgMaxAxis(dst, src, 1, 3, 2, 0, 2) eqSlice(t, dst, []float64{1, 2}) // col0 max at k=1 (9); col1 max at k=2 (7) - ArgMinAxis(dst, src, 1, 3, 2) + ArgMinAxis(dst, src, 1, 3, 2, 0, 2) eqSlice(t, dst, []float64{2, 0}) // col0 min at k=2 (0); col1 min at k=0 (1) cs := make([]float64, 6) diff --git a/internal/kernels/parallel.go b/internal/kernels/parallel.go index b861ec9..5b4991d 100644 --- a/internal/kernels/parallel.go +++ b/internal/kernels/parallel.go @@ -180,45 +180,52 @@ func reduceP(a []float64, red func([]float64) float64) float64 { // axisKernel is the shape of the [outer][axisLen][inner] axis reducers // (SumAxis, MaxAxis, …). -type axisKernel func(dst, src []float64, outer, axisLen, inner int) - -// RunAxisP runs an axis-reduction kernel, fanning the `outer` slabs across cores -// above ParThreshold (measured by the total element count outer*axisLen*inner). -// Each worker owns a disjoint band of outer rows and writes the matching disjoint -// [band*inner] region of dst, reading only its own src slab — so the parallel -// result is identical to the serial kernel. Splitting `outer` keeps every -// worker's inner traversal contiguous (the cache-friendly axis), which is the -// case that matters: an axis-1 reduction of an (R x C) matrix has outer=R, -// inner=1, and the serial kernel leaves all but one core idle. +type axisKernel func(dst, src []float64, outer, axisLen, inner, lo, hi int) + +// RunAxisP runs an axis-reduction kernel across cores above ParThreshold +// (measured by the total element count outer*axisLen*inner). With at least one +// outer slab per worker it splits the slabs; with fewer (an axis-0 reduction +// has one slab) it splits the inner columns into bands of at least axisBand, +// which the old driver ran serially. Either way each worker writes disjoint +// elements of dst, each computed exactly as the serial kernel would, so the +// parallel result is identical to it. func RunAxisP(k axisKernel, dst, src []float64, outer, axisLen, inner int) { total := outer * axisLen * inner - if total < ParThreshold { - k(dst, src, outer, axisLen, inner) - return - } - // Split the outer slabs: each worker owns a disjoint band of outer rows and - // writes the matching disjoint dst rows, reading only its own contiguous src - // slab — identical to the serial kernel. This is the case that matters most: - // an axis-1 reduction of an (R x C) matrix has outer=R, inner=1, so the serial - // kernel leaves all but one core idle, and an inner-split would need a strided - // column gather whose copy cost cancels the parallel gain. A single-outer-slab - // reduction (axis 0: outer=1, inner=C) already streams contiguous inner runs - // per accumulation step and stays on the (fast) serial path. - if outer < 2 { - k(dst, src, outer, axisLen, inner) - return - } w := numWorkers(total) - if w > outer { - w = outer + switch { + case total < ParThreshold || w < 2: + k(dst, src, outer, axisLen, inner, 0, inner) + case outer >= w: + // Enough outer slabs: each worker owns a band of them and reads only its + // own contiguous src slab. An axis-1 reduction of an (R x C) matrix + // (outer=R, inner=1) is this case. + parallelFor(outer, w, func(lo, hi int) { + k(dst[lo*inner:hi*inner], src[lo*axisLen*inner:hi*axisLen*inner], hi-lo, axisLen, inner, 0, inner) + }) + case inner >= 2*axisBand: + // Too few slabs (an axis-0 reduction has one): split the columns. A band + // of columns is a contiguous run of every row, so each worker streams + // its runs and writes its own columns of dst; no gather is needed. + bands := min(w, inner/axisBand) + width := roundUp((inner+bands-1)/bands, 8) + parallelFor(bands, bands, func(b0, b1 int) { + for b := b0; b < b1; b++ { + if lo := b * width; lo < inner { + k(dst, src, outer, axisLen, inner, lo, min(lo+width, inner)) + } + } + }) + default: + parallelFor(outer, min(w, outer), func(lo, hi int) { + k(dst[lo*inner:hi*inner], src[lo*axisLen*inner:hi*axisLen*inner], hi-lo, axisLen, inner, 0, inner) + }) } - parallelFor(outer, w, func(lo, hi int) { - dstSlab := dst[lo*inner : hi*inner] - srcSlab := src[lo*axisLen*inner : hi*axisLen*inner] - k(dstSlab, srcSlab, hi-lo, axisLen, inner) - }) } +// axisBand is the narrowest column band RunAxisP gives a worker (512 doubles, +// 4 KiB per row): narrower bands make each row run too short to stream. +const axisBand = 512 + // MaxP returns the maximum element of a (non-empty), parallelised above // ParThreshold. It runs the SIMD max kernel (packed MAXPD + NaN scan on amd64; // the intrinsic-lowered scalar FMAXD reducer on arm64/others) on each chunk and diff --git a/internal/kernels/parallel_test.go b/internal/kernels/parallel_test.go index e55378a..589d65b 100644 --- a/internal/kernels/parallel_test.go +++ b/internal/kernels/parallel_test.go @@ -148,7 +148,7 @@ func TestRunAxisP(t *testing.T) { gotD := make([]float64, s.outer*s.inner) wantD := make([]float64, s.outer*s.inner) RunAxisP(kk, gotD, src, s.outer, s.axisLen, s.inner) - kk(wantD, src, s.outer, s.axisLen, s.inner) + kk(wantD, src, s.outer, s.axisLen, s.inner, 0, s.inner) for i := range wantD { if gotD[i] != wantD[i] { t.Fatalf("RunAxisP %v par=%d [%d]: %v != %v", s, par, i, gotD[i], wantD[i]) @@ -251,3 +251,71 @@ func TestNumWorkers(t *testing.T) { } }) } + +// naiveAxis is an independent oracle for the axis reductions: a plain triple +// loop combining the middle axis in order with op. +func naiveAxis(src []float64, outer, axisLen, inner int, op func(acc, v float64) float64) []float64 { + out := make([]float64, outer*inner) + for o := 0; o < outer; o++ { + for i := 0; i < inner; i++ { + acc := src[o*axisLen*inner+i] + for k := 1; k < axisLen; k++ { + acc = op(acc, src[(o*axisLen+k)*inner+i]) + } + out[o*inner+i] = acc + } + } + return out +} + +// TestRunAxisPBands drives every RunAxisP schedule — serial, outer split, +// column split (one slab, and fewer slabs than workers) and the narrow +// fallback — through Sum/Prod/Max/Min against the naive in-order oracle. With +// inner > 1 the band kernels combine rows in axis order (through the SIMD +// elementwise kernel from rowRun columns on), so the result must be exact; the +// inner == 1 row sum is regrouped by the SIMD sum and checked to 1e-12. +func TestRunAxisPBands(t *testing.T) { + type op struct { + name string + k axisKernel + f func(a, v float64) float64 + } + ops := []op{ + {"sum", SumAxis, func(a, v float64) float64 { return a + v }}, + {"prod", ProdAxis, func(a, v float64) float64 { return a * v }}, + {"max", MaxAxis, func(a, v float64) float64 { return max(a, v) }}, + {"min", MinAxis, func(a, v float64) float64 { return min(a, v) }}, + } + shapes := []struct{ outer, axisLen, inner int }{ + {1, 6, 3 * axisBand}, // one slab: column bands + {3, 5, 2*axisBand + 37}, // fewer slabs than workers: ragged column bands + {3, 4, 100}, // fewer slabs, too narrow to band: outer fallback + {40, 3, rowRun + 5}, // outer split, SIMD row combine + {2, 7, rowRun - 1}, // scalar row combine + {50, 300, 1}, // row sums (inner == 1) + } + for _, s := range shapes { + src := randVec(s.outer*s.axisLen*s.inner, int64(s.inner)) + for i := range src { + src[i] = 0.5 + src[i]*0.25 // keep products finite + } + for _, par := range []int{1 << 30, 1} { + withMaxProcs(8, func() { + withThresholds(par, 1<<14, func() { + for _, o := range ops { + want := naiveAxis(src, s.outer, s.axisLen, s.inner, o.f) + got := make([]float64, len(want)) + RunAxisP(o.k, got, src, s.outer, s.axisLen, s.inner) + for i := range want { + exact := s.inner > 1 || o.name != "sum" + if exact && got[i] != want[i] || + !exact && math.Abs(got[i]-want[i]) > 1e-12*math.Abs(want[i]) { + t.Fatalf("%s %v par=%d [%d]: %v, want %v", o.name, s, par, i, got[i], want[i]) + } + } + } + }) + }) + } + } +} diff --git a/ndarray.go b/ndarray.go index 7a6123b..1cf47a9 100644 --- a/ndarray.go +++ b/ndarray.go @@ -822,7 +822,7 @@ func (a *Array) reduceLayout(axis int) (outer, axisLen, inner int) { // reductions which error on empty input. func (a *Array) reduceAxis( axis int, keepdims bool, - kernel func(dst, src []float64, outer, axisLen, inner int), + kernel func(dst, src []float64, outer, axisLen, inner, lo, hi int), ) (*Array, error) { axis, err := a.normalizeAxis(axis) if err != nil {