From 353b98c13a20ef114298c85a2c47ecf5ffc97f53 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 10:01:37 +0200 Subject: [PATCH 1/5] perf: float32 pairwise sum blocks on SSE2 and NEON NumPy's pairwise sum keeps eight accumulators so that the inner loop can be vectorised "without changing summation ordering". AddBlocks8F32 is that loop: blocks of eight float32 added lane by lane into r, one IEEE float32 addition per lane per block, in order; SSE2 on amd64 (ADDPS on two XMM), NEON on arm64 (VFADD .S4 on two V), the scalar loop elsewhere. pairwiseSum calls it, so float32 sums are unchanged bit for bit. Tests: kernel vs scalar bit for bit over 1-40 blocks with NaN, the infinities, -0, subnormals and overflow; refusals; a PROT_NONE fence on both operands. Proven to bite: a wrong-register kernel fails the match test, a kernel reading one block too many faults in the fence. Run on arm64 natively and on linux/amd64 (docker). Coverage 100%. Co-Authored-By: Claude Opus 5.5 --- internal/kernels/asmgen/amd64/gen.go | 32 ++++++++++++++ internal/kernels/asmgen/arm64/gen.go | 26 ++++++++++++ internal/kernels/blocks8.go | 34 +++++++++++++++ internal/kernels/blocks8_amd64.go | 12 ++++++ internal/kernels/blocks8_arm64.go | 12 ++++++ internal/kernels/blocks8_generic.go | 7 ++++ internal/kernels/blocks8_test.go | 63 ++++++++++++++++++++++++++++ internal/kernels/fence_test.go | 20 +++++++++ internal/kernels/sum_amd64.s | 18 ++++++++ internal/kernels/sum_arm64.s | 14 +++++++ typed.go | 25 +++++------ 11 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 internal/kernels/blocks8.go create mode 100644 internal/kernels/blocks8_amd64.go create mode 100644 internal/kernels/blocks8_arm64.go create mode 100644 internal/kernels/blocks8_generic.go create mode 100644 internal/kernels/blocks8_test.go diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index 19e7cb1..6a47818 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -49,6 +49,7 @@ func main() { f.Add(dotFMAKernel()) f.Add(gemmSSE2HalfKernel()) f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_amd64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) @@ -107,6 +108,37 @@ func sumKernel() *emit.Function { return b.Func() } +// blocks8Kernel builds blocks8SSE2(r, a *float32, blocks int): for each of +// the blocks of eight float32 at a, in order, r[j] += a[8i+j]. The eight +// accumulators live in X0 (r[0:4]) and X1 (r[4:8]); ADDPS is one IEEE +// float32 addition per lane, so the result is the scalar loop's, bit for bit +// (NumPy's pairwise-sum block, whose order it keeps). blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := amd64.Layout( + []string{"r", "a", "blocks"}, + []amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64}, + nil, nil, + ) + b := amd64.NewFunc("blocks8SSE2", sig, 0) + b.LoadArg("r", "DI"). + LoadArg("a", "SI"). + LoadArg("blocks", "CX"). + Raw("MOVUPS (DI), X0"). + Raw("MOVUPS 16(DI), X1"). + Raw("b8loop:"). + Raw("MOVUPS (SI), X2"). + Raw("MOVUPS 16(SI), X3"). + Raw("ADDPS X2, X0"). + Raw("ADDPS X3, X1"). + Raw("ADDQ $32, SI"). + Raw("DECQ CX"). + Raw("JNZ b8loop"). + Raw("MOVUPS X0, (DI)"). + Raw("MOVUPS X1, 16(DI)"). + Ret() + return b.Func() +} + // sqrtKernel builds sqrtSSE2(dst, src *float64, n int). func sqrtKernel() *emit.Function { sig := amd64.Layout( diff --git a/internal/kernels/asmgen/arm64/gen.go b/internal/kernels/asmgen/arm64/gen.go index a9d74fa..2fe444d 100644 --- a/internal/kernels/asmgen/arm64/gen.go +++ b/internal/kernels/asmgen/arm64/gen.go @@ -50,6 +50,7 @@ func main() { f.Add(extremumKernel("maxNEON", "VFMAX", "FMAXD")) f.Add(extremumKernel("minNEON", "VFMIN", "FMIND")) f.Add(gemmKernel()) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_arm64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -151,6 +152,31 @@ func binKernel(name, vop, sop string) *emit.Function { return b.Func() } +// blocks8Kernel builds blocks8NEON(r, a *float32, blocks int): for each of +// the blocks of eight float32 at a, in order, r[j] += a[8i+j]. The eight +// accumulators live in V0 (r[0:4]) and V1 (r[4:8]); VFADD on .S4 is one IEEE +// float32 addition per lane, so the result is the scalar loop's, bit for bit +// (NumPy's pairwise-sum block, whose order it keeps). blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := arm64.Layout( + []string{"r", "a", "blocks"}, + []arm64.Type{arm64.Ptr, arm64.Ptr, arm64.Int64}, + nil, nil, + ) + b := arm64.NewFunc("blocks8NEON", sig, 0) + b.LoadArg("r", "R0").LoadArg("a", "R1").LoadArg("blocks", "R2"). + Raw("VLD1 (R0), [V0.S4, V1.S4]"). + Raw("b8loop:"). + Raw("VLD1.P 32(R1), [V2.S4, V3.S4]"). + Raw("VFADD V2.S4, V0.S4, V0.S4"). + Raw("VFADD V3.S4, V1.S4, V1.S4"). + Raw("SUB $1, R2"). + Raw("CBNZ R2, b8loop"). + Raw("VST1 [V0.S4, V1.S4], (R0)"). + Ret() + return b.Func() +} + // extremumKernel builds NAME(a *float64, n int) float64, the maximum (or // minimum) of a, for n >= 8: four D2 accumulators seeded with the first eight // elements, folded together, then the two lanes, then the tail one at a time. diff --git a/internal/kernels/blocks8.go b/internal/kernels/blocks8.go new file mode 100644 index 0000000..cb2a52a --- /dev/null +++ b/internal/kernels/blocks8.go @@ -0,0 +1,34 @@ +package kernels + +// AddBlocks8F32 adds a, a whole number of blocks of eight float32, into the +// eight accumulators r lane by lane: for each block i in order, +// r[j] += a[8i+j]. It is the inner loop of NumPy's pairwise sum +// (FLOAT_pairwise_sum, whose eight accumulators exist "to allow vectorization +// without changing summation ordering"), so a vector kernel gives exactly the +// same sums as the scalar loop: one IEEE float32 addition per lane per block, +// in the same order. It panics if len(a) is not a multiple of 8. +func AddBlocks8F32(r *[8]float32, a []float32) { + if len(a)%8 != 0 { + panic("kernels: AddBlocks8F32 needs whole blocks of 8") + } + if len(a) == 0 { + return + } + addBlocks8F32(r, a) +} + +// addBlocks8F32Go is the scalar oracle the kernels are tested against, and +// the implementation on the architectures without one. +func addBlocks8F32Go(r *[8]float32, a []float32) { + for i := 0; i+8 <= len(a); i += 8 { + b := a[i : i+8 : i+8] + r[0] += b[0] + r[1] += b[1] + r[2] += b[2] + r[3] += b[3] + r[4] += b[4] + r[5] += b[5] + r[6] += b[6] + r[7] += b[7] + } +} diff --git a/internal/kernels/blocks8_amd64.go b/internal/kernels/blocks8_amd64.go new file mode 100644 index 0000000..7007b43 --- /dev/null +++ b/internal/kernels/blocks8_amd64.go @@ -0,0 +1,12 @@ +package kernels + +// addBlocks8F32 runs the SSE2 kernel (sum_amd64.s): the eight accumulators in +// two XMM registers, one ADDPS per register per block. len(a) is a positive +// multiple of 8 (AddBlocks8F32 checks). +func addBlocks8F32(r *[8]float32, a []float32) { + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8SSE2(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8SSE2(r, a *float32, blocks int) diff --git a/internal/kernels/blocks8_arm64.go b/internal/kernels/blocks8_arm64.go new file mode 100644 index 0000000..5ca7ab8 --- /dev/null +++ b/internal/kernels/blocks8_arm64.go @@ -0,0 +1,12 @@ +package kernels + +// addBlocks8F32 runs the NEON kernel (sum_arm64.s): the eight accumulators in +// two .S4 registers, one VFADD per register per block. len(a) is a positive +// multiple of 8 (AddBlocks8F32 checks). +func addBlocks8F32(r *[8]float32, a []float32) { + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8NEON(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8NEON(r, a *float32, blocks int) diff --git a/internal/kernels/blocks8_generic.go b/internal/kernels/blocks8_generic.go new file mode 100644 index 0000000..28b2307 --- /dev/null +++ b/internal/kernels/blocks8_generic.go @@ -0,0 +1,7 @@ +//go:build !amd64 && !arm64 + +package kernels + +// addBlocks8F32 is the scalar loop on the architectures without a float32 +// block kernel yet. +func addBlocks8F32(r *[8]float32, a []float32) { addBlocks8F32Go(r, a) } diff --git a/internal/kernels/blocks8_test.go b/internal/kernels/blocks8_test.go new file mode 100644 index 0000000..c413271 --- /dev/null +++ b/internal/kernels/blocks8_test.go @@ -0,0 +1,63 @@ +package kernels + +import ( + "math" + "testing" +) + +// TestAddBlocks8F32MatchesScalar: the kernel must give the scalar loop's +// sums bit for bit (one float32 addition per lane per block, in order), for +// every number of blocks up to 40 and values chosen to round, to overflow, +// to underflow and to carry NaN, the infinities and -0. +func TestAddBlocks8F32MatchesScalar(t *testing.T) { + special := []float32{ + 0, float32(math.Copysign(0, -1)), 1, -1, 0.1, -0.3, 3.4e38, -3.4e38, + 1e-45, -1e-45, 1.17549435e-38, float32(math.Inf(1)), float32(math.Inf(-1)), + float32(math.NaN()), 16777216, 1, 0.5, + } + x := uint32(7) + next := func() float32 { + x = x*1664525 + 1013904223 + if x%5 == 0 { + return special[int(x>>8)%len(special)] + } + return float32(int32(x)) / float32(1<<20) + } + for blocks := 1; blocks <= 40; blocks++ { + a := make([]float32, 8*blocks) + for i := range a { + a[i] = next() + } + var r0 [8]float32 + for j := range r0 { + r0[j] = next() + } + got, want := r0, r0 + AddBlocks8F32(&got, a) + addBlocks8F32Go(&want, a) + for j := range got { + g, w := got[j], want[j] + if g != g && w != w { + continue // both NaN: the payload is not what is tested + } + if math.Float32bits(g) != math.Float32bits(w) { + t.Fatalf("blocks=%d lane %d: kernel %v (%#08x), scalar %v (%#08x)", + blocks, j, g, math.Float32bits(g), w, math.Float32bits(w)) + } + } + } +} + +func TestAddBlocks8F32Refusals(t *testing.T) { + var r [8]float32 + AddBlocks8F32(&r, nil) // no blocks: nothing to add + if r != [8]float32{} { + t.Errorf("an empty a changed r: %v", r) + } + defer func() { + if recover() == nil { + t.Error("a length that is not a multiple of 8 did not panic") + } + }() + AddBlocks8F32(&r, make([]float32, 9)) +} diff --git a/internal/kernels/fence_test.go b/internal/kernels/fence_test.go index 87155c8..3d23895 100644 --- a/internal/kernels/fence_test.go +++ b/internal/kernels/fence_test.go @@ -60,6 +60,26 @@ func fences(t *testing.T, n int) [][]float64 { return [][]float64{f.head(n), f.tail(n)} } +// fences32 is fences for float32 operands: the same mappings, viewed as +// float32, with n elements flush against each guard. +func fences32(t *testing.T, n int) [][]float32 { + f := newFenced(t, (n+1)/2) + d := unsafe.Slice((*float32)(unsafe.Pointer(&f.data[0])), 2*len(f.data)) + return [][]float32{d[:n:n], d[len(d)-n:]} +} + +// TestBlocks8StaysInsideItsOperands fences both the blocks the kernel reads +// and the eight accumulators it reads and writes back. +func TestBlocks8StaysInsideItsOperands(t *testing.T) { + for blocks := 1; blocks <= 20; blocks++ { + for _, a := range fences32(t, 8*blocks) { + for _, r := range fences32(t, 8) { + AddBlocks8F32((*[8]float32)(r), a) + } + } + } +} + func TestKernelsStayInsideTheirSlices(t *testing.T) { for n := 1; n <= 70; n++ { for _, a := range fences(t, n) { diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index 0958389..6026831 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -704,3 +704,21 @@ TEXT ·hasFMA(SB), NOSPLIT, $0-1 hasFMA_unsupported: RET +TEXT ·blocks8SSE2(SB), NOSPLIT, $0-24 + MOVQ r+0(FP), DI + MOVQ a+8(FP), SI + MOVQ blocks+16(FP), CX + MOVUPS (DI), X0 + MOVUPS 16(DI), X1 + b8loop: + MOVUPS (SI), X2 + MOVUPS 16(SI), X3 + ADDPS X2, X0 + ADDPS X3, X1 + ADDQ $32, SI + DECQ CX + JNZ b8loop + MOVUPS X0, (DI) + MOVUPS X1, 16(DI) + RET + diff --git a/internal/kernels/sum_arm64.s b/internal/kernels/sum_arm64.s index a3c1791..dc03f0e 100644 --- a/internal/kernels/sum_arm64.s +++ b/internal/kernels/sum_arm64.s @@ -383,3 +383,17 @@ TEXT ·gemmMicro4x8(SB), NOSPLIT, $0-40 VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5) RET +TEXT ·blocks8NEON(SB), NOSPLIT, $0-24 + MOVD r+0(FP), R0 + MOVD a+8(FP), R1 + MOVD blocks+16(FP), R2 + VLD1 (R0), [V0.S4, V1.S4] + b8loop: + VLD1.P 32(R1), [V2.S4, V3.S4] + VFADD V2.S4, V0.S4, V0.S4 + VFADD V3.S4, V1.S4, V1.S4 + SUB $1, R2 + CBNZ R2, b8loop + VST1 [V0.S4, V1.S4], (R0) + RET + diff --git a/typed.go b/typed.go index 9d91938..cc337b4 100644 --- a/typed.go +++ b/typed.go @@ -3,6 +3,8 @@ package ndarray import ( "math" "math/cmplx" + + "github.com/go-ndarray/ndarray/internal/kernels" ) // Inner loops for the dtypes other than Float64. Each takes contiguous storage @@ -464,30 +466,25 @@ const pwBlock = 128 // pairwiseSum is NumPy's FLOAT_pairwise_sum // (numpy/_core/src/umath/loops_utils.h.src, v2.5.3): rounding error O(lg n) // instead of O(n), in the same order of additions. -func pairwiseSum[F float32](a []F) F { +// +// The eight accumulators are fed by kernels.AddBlocks8F32, a vector kernel on +// amd64 and arm64 that adds the same lanes in the same order, so the sums are +// the scalar loop's bit for bit. +func pairwiseSum(a []float32) float32 { n := len(a) switch { case n < 8: // From -0, so that a sum of only -0 stays -0. - res := F(math.Copysign(0, -1)) + res := float32(math.Copysign(0, -1)) for _, v := range a { res += v } return res case n <= pwBlock: - var r [8]F + var r [8]float32 copy(r[:], a[:8]) - i := 8 - for ; i < n-n%8; i += 8 { - r[0] += a[i] - r[1] += a[i+1] - r[2] += a[i+2] - r[3] += a[i+3] - r[4] += a[i+4] - r[5] += a[i+5] - r[6] += a[i+6] - r[7] += a[i+7] - } + i := n - n%8 + kernels.AddBlocks8F32(&r, a[8:i]) res := ((r[0] + r[1]) + (r[2] + r[3])) + ((r[4] + r[5]) + (r[6] + r[7])) for ; i < n; i++ { res += a[i] From 67e4cc07ebd6e6725fe5c99f83cf4e337914ccf8 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 15:15:24 +0200 Subject: [PATCH 2/5] perf: integer widening folds with four accumulators, one loop per operation On amd64 the single loop reloaded the operation from the stack every element, and the A/B on the hosted EPYC showed int32 Sum 48-56% slower on this branch with byte-identical loop code: the new kernel code moved the loop across a 64-byte line. Four independent accumulators (exact: integer + and * wrap modulo 2^64, so order changes nothing) and one loop per operation make it faster and far less sensitive to where it lands. Co-Authored-By: Claude Opus 5.5 --- typed.go | 47 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/typed.go b/typed.go index cc337b4..4c9282a 100644 --- a/typed.go +++ b/typed.go @@ -445,20 +445,51 @@ func foldWidened(op reduceCode, dst, src any, outer, axisLen int) { func widenRuns[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](op reduceCode, dst []D, src []S, outer, axisLen int) { for o := range outer { run := src[o*axisLen : (o+1)*axisLen] - acc := D(run[0]) if op == rSum { - for _, v := range run[1:] { - acc += D(v) - } + dst[o] = sumWiden[S, D](run) } else { - for _, v := range run[1:] { - acc *= D(v) - } + dst[o] = prodWiden[S, D](run) } - dst[o] = acc } } +// sumWiden and prodWiden fold a run with four accumulators. Integer addition +// and multiplication wrap modulo 2^64, so they are associative and the order +// changes nothing: the result is the in-order fold's, exactly. One loop per +// operation, so the operation is not re-read per element (on amd64 the +// single loop reloaded it from the stack every iteration), and four +// independent chains, so the loop is no longer one dependent add an element: +// it had run 50% slower on a Zen 3 just by being moved across a 64-byte line. +func sumWiden[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](s []S) D { + var a0, a1, a2, a3 D + i := 0 + for ; i+4 <= len(s); i += 4 { + a0 += D(s[i]) + a1 += D(s[i+1]) + a2 += D(s[i+2]) + a3 += D(s[i+3]) + } + for ; i < len(s); i++ { + a0 += D(s[i]) + } + return (a0 + a1) + (a2 + a3) +} + +func prodWiden[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](s []S) D { + var a0, a1, a2, a3 D = 1, 1, 1, 1 + i := 0 + for ; i+4 <= len(s); i += 4 { + a0 *= D(s[i]) + a1 *= D(s[i+1]) + a2 *= D(s[i+2]) + a3 *= D(s[i+3]) + } + for ; i < len(s); i++ { + a0 *= D(s[i]) + } + return (a0 * a1) * (a2 * a3) +} + // pwBlock is NumPy's PW_BLOCKSIZE: runs up to this length are summed with // eight accumulators, longer ones are halved. const pwBlock = 128 From 6af19a50ae9f94d8de59c52c352c8534daa8d76c Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 15:18:17 +0200 Subject: [PATCH 3/5] perf: widening folds consume the slice, so the loop has no bounds check Indexing s[i+1..i+3] left four bounds checks an iteration in the amd64 code (the compiler proved none of them); consuming the slice under len(s) >= 4 proves all four. Disassembly: the unrolled int32 loop is now four MOVSXD loads and no check. Co-Authored-By: Claude Opus 5.5 --- typed.go | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/typed.go b/typed.go index 4c9282a..65682c5 100644 --- a/typed.go +++ b/typed.go @@ -460,32 +460,37 @@ func widenRuns[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint6 // single loop reloaded it from the stack every iteration), and four // independent chains, so the loop is no longer one dependent add an element: // it had run 50% slower on a Zen 3 just by being moved across a 64-byte line. +// +// The loops consume the slice (s = s[4:] under len(s) >= 4) rather than index +// it at i+1..i+3: with indices, the compiler proved none of them in range and +// emitted four bounds checks an iteration (amd64 disassembly); this way it +// proves all four from the loop condition and emits none. func sumWiden[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](s []S) D { var a0, a1, a2, a3 D - i := 0 - for ; i+4 <= len(s); i += 4 { - a0 += D(s[i]) - a1 += D(s[i+1]) - a2 += D(s[i+2]) - a3 += D(s[i+3]) + for len(s) >= 4 { + a0 += D(s[0]) + a1 += D(s[1]) + a2 += D(s[2]) + a3 += D(s[3]) + s = s[4:] } - for ; i < len(s); i++ { - a0 += D(s[i]) + for _, v := range s { + a0 += D(v) } return (a0 + a1) + (a2 + a3) } func prodWiden[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](s []S) D { var a0, a1, a2, a3 D = 1, 1, 1, 1 - i := 0 - for ; i+4 <= len(s); i += 4 { - a0 *= D(s[i]) - a1 *= D(s[i+1]) - a2 *= D(s[i+2]) - a3 *= D(s[i+3]) - } - for ; i < len(s); i++ { - a0 *= D(s[i]) + for len(s) >= 4 { + a0 *= D(s[0]) + a1 *= D(s[1]) + a2 *= D(s[2]) + a3 *= D(s[3]) + s = s[4:] + } + for _, v := range s { + a0 *= D(v) } return (a0 * a1) * (a2 * a3) } From 7aa7433410158896f04284e4fa011ce2b072bed6 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 15:20:13 +0200 Subject: [PATCH 4/5] docs/perf.md: float32 block kernels on SSE2/NEON, and the int32 loop the first A/B exposed Co-Authored-By: Claude Opus 5.5 --- docs/perf.md | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/perf.md b/docs/perf.md index 7203410..e31653c 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -837,7 +837,47 @@ arithmetic, wrap-around included): **int32 `Sum` of 2²⁰ elements 2.872 → 1.548 ms (−46%)**, 2¹⁴ −51%, with float64, float32 and int64 unchanged as controls (POWER9, same protocol, p = 0.002). -Left: float32 has no SIMD kernel yet (still 7× float64 for `Sum`). +### float32 sums on SSE2 and NEON (2026-10-18) + +NumPy keeps eight accumulators in its pairwise sum so that the inner loop can +be vectorised "without changing summation ordering". `kernels.AddBlocks8F32` +is that loop: blocks of eight float32 added lane by lane, one IEEE addition +per lane per block, in order. It is SSE2 on amd64 (`ADDPS` on two XMM +registers) and NEON on arm64 (`VFADD` on two `.S4` registers), and the scalar +loop elsewhere, so float32 sums are unchanged bit for bit. The kernel is +tested bit for bit against the scalar loop (NaN, the infinities, −0, +subnormals, overflow), and fenced with `PROT_NONE` pages on both operands. A +kernel adding from the wrong register fails the first test, and one reading +a block too many faults in the fence. + +No quiet amd64 or arm64 machine was at hand: the cfarm hosts refuse their +host keys since the CA revocation, and the development Mac ran at a load of +97. So these were measured with the `bench` workflow on the hosted runners: +base and head alternated ten times in fresh processes on one thread, +`benchstat`, p ≤ 0.002. + +| vs v0.13.x | amd64 (EPYC 9V45) | arm64 (hosted) | +|---|---:|---:| +| float32 `Sum`, 2¹⁴ | −50% | −33% | +| float32 `Sum`, 2²⁰ | −49% | −43% | +| int32 `Sum`, 2¹⁴ | −12% | −12% | +| int32 `Sum`, 2²⁰ | −15% | −13% | +| float64, int64 `Sum` (controls) | ~ | ~ | + +The integer rows come from a defect the first run exposed. With the kernel +added, int32 `Sum` was **48–56% slower on amd64** although its code had not +changed: the disassembly of both binaries showed the same loop, byte for +byte, moved 224 bytes and now straddling a 64-byte line. The loop also +reloaded the operation from the stack every element. The widening folds now +use one loop per operation and four accumulators (exact: integer addition +and multiplication wrap modulo 2⁶⁴, so the order changes nothing). They also +consume the slice instead of indexing `i+1…i+3`, which had left four bounds +checks per iteration in the amd64 code; now there are none. + +Left: ppc64le, s390x, loong64 and riscv64 run the scalar block loop. ppc64le +needs `xvaddsp`, which Go's assembler lacks: go-asmgen/asmgen#27 adds the +VSX single-precision family to its registry, with references from GNU as +and llvm-mc. ## SIMD coverage From d6c752b04bbbb5aa985e4faa14aa4ccc74a7d099 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 15:27:13 +0200 Subject: [PATCH 5/5] docs/perf.md: the float32 block kernels confirmed on a Neoverse N1 and a Zen 3 Co-Authored-By: Claude Opus 5.5 --- docs/perf.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/perf.md b/docs/perf.md index e31653c..538fafe 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -850,11 +850,16 @@ subnormals, overflow), and fenced with `PROT_NONE` pages on both operands. A kernel adding from the wrong register fails the first test, and one reading a block too many faults in the fence. -No quiet amd64 or arm64 machine was at hand: the cfarm hosts refuse their -host keys since the CA revocation, and the development Mac ran at a load of -97. So these were measured with the `bench` workflow on the hosted runners: -base and head alternated ten times in fresh processes on one thread, -`benchstat`, p ≤ 0.002. +Measured twice. First with the `bench` workflow on the hosted runners +(base and head alternated ten times in fresh processes on one thread, +`benchstat`, p ≤ 0.002), because the development Mac ran at a load of 97 and +most cfarm hosts refuse their keys since the host CA was revoked. Then on +real, idle hardware reached through a known_hosts file whose keys were +confirmed over a second network path: cfarm424 (Neoverse N1, 0.0% busy) and +cfarm420 (Zen 3, EPYC 7773X, 1.7% busy), same protocol, pinned to the idlest +CPU. On those, float32 `Sum` was −39% / −32% (N1) and −43% / −41% (Zen 3) at +2¹⁴ / 2²⁰. int32 was −15% / −11% on the N1, and on the Zen 3 within the +noise (±10% at that host's variance). The controls were unchanged. | vs v0.13.x | amd64 (EPYC 9V45) | arm64 (hosted) | |---|---:|---:|