diff --git a/docs/perf.md b/docs/perf.md index 7203410..538fafe 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -837,7 +837,52 @@ arithmetic, wrap-around included): **int32 `Sum` of 2²⁰ elements 2.872 → 1.548 ms (−46%)**, 2¹⁴ −51%, with float64, float32 and int64 unchanged as controls (POWER9, same protocol, p = 0.002). -Left: float32 has no SIMD kernel yet (still 7× float64 for `Sum`). +### float32 sums on SSE2 and NEON (2026-10-18) + +NumPy keeps eight accumulators in its pairwise sum so that the inner loop can +be vectorised "without changing summation ordering". `kernels.AddBlocks8F32` +is that loop: blocks of eight float32 added lane by lane, one IEEE addition +per lane per block, in order. It is SSE2 on amd64 (`ADDPS` on two XMM +registers) and NEON on arm64 (`VFADD` on two `.S4` registers), and the scalar +loop elsewhere, so float32 sums are unchanged bit for bit. The kernel is +tested bit for bit against the scalar loop (NaN, the infinities, −0, +subnormals, overflow), and fenced with `PROT_NONE` pages on both operands. A +kernel adding from the wrong register fails the first test, and one reading +a block too many faults in the fence. + +Measured twice. First with the `bench` workflow on the hosted runners +(base and head alternated ten times in fresh processes on one thread, +`benchstat`, p ≤ 0.002), because the development Mac ran at a load of 97 and +most cfarm hosts refuse their keys since the host CA was revoked. Then on +real, idle hardware reached through a known_hosts file whose keys were +confirmed over a second network path: cfarm424 (Neoverse N1, 0.0% busy) and +cfarm420 (Zen 3, EPYC 7773X, 1.7% busy), same protocol, pinned to the idlest +CPU. On those, float32 `Sum` was −39% / −32% (N1) and −43% / −41% (Zen 3) at +2¹⁴ / 2²⁰. int32 was −15% / −11% on the N1, and on the Zen 3 within the +noise (±10% at that host's variance). The controls were unchanged. + +| vs v0.13.x | amd64 (EPYC 9V45) | arm64 (hosted) | +|---|---:|---:| +| float32 `Sum`, 2¹⁴ | −50% | −33% | +| float32 `Sum`, 2²⁰ | −49% | −43% | +| int32 `Sum`, 2¹⁴ | −12% | −12% | +| int32 `Sum`, 2²⁰ | −15% | −13% | +| float64, int64 `Sum` (controls) | ~ | ~ | + +The integer rows come from a defect the first run exposed. With the kernel +added, int32 `Sum` was **48–56% slower on amd64** although its code had not +changed: the disassembly of both binaries showed the same loop, byte for +byte, moved 224 bytes and now straddling a 64-byte line. The loop also +reloaded the operation from the stack every element. The widening folds now +use one loop per operation and four accumulators (exact: integer addition +and multiplication wrap modulo 2⁶⁴, so the order changes nothing). They also +consume the slice instead of indexing `i+1…i+3`, which had left four bounds +checks per iteration in the amd64 code; now there are none. + +Left: ppc64le, s390x, loong64 and riscv64 run the scalar block loop. ppc64le +needs `xvaddsp`, which Go's assembler lacks: go-asmgen/asmgen#27 adds the +VSX single-precision family to its registry, with references from GNU as +and llvm-mc. ## SIMD coverage diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index 19e7cb1..6a47818 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -49,6 +49,7 @@ func main() { f.Add(dotFMAKernel()) f.Add(gemmSSE2HalfKernel()) f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_amd64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) @@ -107,6 +108,37 @@ func sumKernel() *emit.Function { return b.Func() } +// blocks8Kernel builds blocks8SSE2(r, a *float32, blocks int): for each of +// the blocks of eight float32 at a, in order, r[j] += a[8i+j]. The eight +// accumulators live in X0 (r[0:4]) and X1 (r[4:8]); ADDPS is one IEEE +// float32 addition per lane, so the result is the scalar loop's, bit for bit +// (NumPy's pairwise-sum block, whose order it keeps). blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := amd64.Layout( + []string{"r", "a", "blocks"}, + []amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64}, + nil, nil, + ) + b := amd64.NewFunc("blocks8SSE2", sig, 0) + b.LoadArg("r", "DI"). + LoadArg("a", "SI"). + LoadArg("blocks", "CX"). + Raw("MOVUPS (DI), X0"). + Raw("MOVUPS 16(DI), X1"). + Raw("b8loop:"). + Raw("MOVUPS (SI), X2"). + Raw("MOVUPS 16(SI), X3"). + Raw("ADDPS X2, X0"). + Raw("ADDPS X3, X1"). + Raw("ADDQ $32, SI"). + Raw("DECQ CX"). + Raw("JNZ b8loop"). + Raw("MOVUPS X0, (DI)"). + Raw("MOVUPS X1, 16(DI)"). + Ret() + return b.Func() +} + // sqrtKernel builds sqrtSSE2(dst, src *float64, n int). func sqrtKernel() *emit.Function { sig := amd64.Layout( diff --git a/internal/kernels/asmgen/arm64/gen.go b/internal/kernels/asmgen/arm64/gen.go index a9d74fa..2fe444d 100644 --- a/internal/kernels/asmgen/arm64/gen.go +++ b/internal/kernels/asmgen/arm64/gen.go @@ -50,6 +50,7 @@ func main() { f.Add(extremumKernel("maxNEON", "VFMAX", "FMAXD")) f.Add(extremumKernel("minNEON", "VFMIN", "FMIND")) f.Add(gemmKernel()) + f.Add(blocks8Kernel()) if err := os.WriteFile("sum_arm64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -151,6 +152,31 @@ func binKernel(name, vop, sop string) *emit.Function { return b.Func() } +// blocks8Kernel builds blocks8NEON(r, a *float32, blocks int): for each of +// the blocks of eight float32 at a, in order, r[j] += a[8i+j]. The eight +// accumulators live in V0 (r[0:4]) and V1 (r[4:8]); VFADD on .S4 is one IEEE +// float32 addition per lane, so the result is the scalar loop's, bit for bit +// (NumPy's pairwise-sum block, whose order it keeps). blocks >= 1. +func blocks8Kernel() *emit.Function { + sig := arm64.Layout( + []string{"r", "a", "blocks"}, + []arm64.Type{arm64.Ptr, arm64.Ptr, arm64.Int64}, + nil, nil, + ) + b := arm64.NewFunc("blocks8NEON", sig, 0) + b.LoadArg("r", "R0").LoadArg("a", "R1").LoadArg("blocks", "R2"). + Raw("VLD1 (R0), [V0.S4, V1.S4]"). + Raw("b8loop:"). + Raw("VLD1.P 32(R1), [V2.S4, V3.S4]"). + Raw("VFADD V2.S4, V0.S4, V0.S4"). + Raw("VFADD V3.S4, V1.S4, V1.S4"). + Raw("SUB $1, R2"). + Raw("CBNZ R2, b8loop"). + Raw("VST1 [V0.S4, V1.S4], (R0)"). + Ret() + return b.Func() +} + // extremumKernel builds NAME(a *float64, n int) float64, the maximum (or // minimum) of a, for n >= 8: four D2 accumulators seeded with the first eight // elements, folded together, then the two lanes, then the tail one at a time. diff --git a/internal/kernels/blocks8.go b/internal/kernels/blocks8.go new file mode 100644 index 0000000..cb2a52a --- /dev/null +++ b/internal/kernels/blocks8.go @@ -0,0 +1,34 @@ +package kernels + +// AddBlocks8F32 adds a, a whole number of blocks of eight float32, into the +// eight accumulators r lane by lane: for each block i in order, +// r[j] += a[8i+j]. It is the inner loop of NumPy's pairwise sum +// (FLOAT_pairwise_sum, whose eight accumulators exist "to allow vectorization +// without changing summation ordering"), so a vector kernel gives exactly the +// same sums as the scalar loop: one IEEE float32 addition per lane per block, +// in the same order. It panics if len(a) is not a multiple of 8. +func AddBlocks8F32(r *[8]float32, a []float32) { + if len(a)%8 != 0 { + panic("kernels: AddBlocks8F32 needs whole blocks of 8") + } + if len(a) == 0 { + return + } + addBlocks8F32(r, a) +} + +// addBlocks8F32Go is the scalar oracle the kernels are tested against, and +// the implementation on the architectures without one. +func addBlocks8F32Go(r *[8]float32, a []float32) { + for i := 0; i+8 <= len(a); i += 8 { + b := a[i : i+8 : i+8] + r[0] += b[0] + r[1] += b[1] + r[2] += b[2] + r[3] += b[3] + r[4] += b[4] + r[5] += b[5] + r[6] += b[6] + r[7] += b[7] + } +} diff --git a/internal/kernels/blocks8_amd64.go b/internal/kernels/blocks8_amd64.go new file mode 100644 index 0000000..7007b43 --- /dev/null +++ b/internal/kernels/blocks8_amd64.go @@ -0,0 +1,12 @@ +package kernels + +// addBlocks8F32 runs the SSE2 kernel (sum_amd64.s): the eight accumulators in +// two XMM registers, one ADDPS per register per block. len(a) is a positive +// multiple of 8 (AddBlocks8F32 checks). +func addBlocks8F32(r *[8]float32, a []float32) { + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8SSE2(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8SSE2(r, a *float32, blocks int) diff --git a/internal/kernels/blocks8_arm64.go b/internal/kernels/blocks8_arm64.go new file mode 100644 index 0000000..5ca7ab8 --- /dev/null +++ b/internal/kernels/blocks8_arm64.go @@ -0,0 +1,12 @@ +package kernels + +// addBlocks8F32 runs the NEON kernel (sum_arm64.s): the eight accumulators in +// two .S4 registers, one VFADD per register per block. len(a) is a positive +// multiple of 8 (AddBlocks8F32 checks). +func addBlocks8F32(r *[8]float32, a []float32) { + _ = a[len(a)-1] // the kernel reads len(a) of a + blocks8NEON(&r[0], &a[0], len(a)/8) +} + +//go:noescape +func blocks8NEON(r, a *float32, blocks int) diff --git a/internal/kernels/blocks8_generic.go b/internal/kernels/blocks8_generic.go new file mode 100644 index 0000000..28b2307 --- /dev/null +++ b/internal/kernels/blocks8_generic.go @@ -0,0 +1,7 @@ +//go:build !amd64 && !arm64 + +package kernels + +// addBlocks8F32 is the scalar loop on the architectures without a float32 +// block kernel yet. +func addBlocks8F32(r *[8]float32, a []float32) { addBlocks8F32Go(r, a) } diff --git a/internal/kernels/blocks8_test.go b/internal/kernels/blocks8_test.go new file mode 100644 index 0000000..c413271 --- /dev/null +++ b/internal/kernels/blocks8_test.go @@ -0,0 +1,63 @@ +package kernels + +import ( + "math" + "testing" +) + +// TestAddBlocks8F32MatchesScalar: the kernel must give the scalar loop's +// sums bit for bit (one float32 addition per lane per block, in order), for +// every number of blocks up to 40 and values chosen to round, to overflow, +// to underflow and to carry NaN, the infinities and -0. +func TestAddBlocks8F32MatchesScalar(t *testing.T) { + special := []float32{ + 0, float32(math.Copysign(0, -1)), 1, -1, 0.1, -0.3, 3.4e38, -3.4e38, + 1e-45, -1e-45, 1.17549435e-38, float32(math.Inf(1)), float32(math.Inf(-1)), + float32(math.NaN()), 16777216, 1, 0.5, + } + x := uint32(7) + next := func() float32 { + x = x*1664525 + 1013904223 + if x%5 == 0 { + return special[int(x>>8)%len(special)] + } + return float32(int32(x)) / float32(1<<20) + } + for blocks := 1; blocks <= 40; blocks++ { + a := make([]float32, 8*blocks) + for i := range a { + a[i] = next() + } + var r0 [8]float32 + for j := range r0 { + r0[j] = next() + } + got, want := r0, r0 + AddBlocks8F32(&got, a) + addBlocks8F32Go(&want, a) + for j := range got { + g, w := got[j], want[j] + if g != g && w != w { + continue // both NaN: the payload is not what is tested + } + if math.Float32bits(g) != math.Float32bits(w) { + t.Fatalf("blocks=%d lane %d: kernel %v (%#08x), scalar %v (%#08x)", + blocks, j, g, math.Float32bits(g), w, math.Float32bits(w)) + } + } + } +} + +func TestAddBlocks8F32Refusals(t *testing.T) { + var r [8]float32 + AddBlocks8F32(&r, nil) // no blocks: nothing to add + if r != [8]float32{} { + t.Errorf("an empty a changed r: %v", r) + } + defer func() { + if recover() == nil { + t.Error("a length that is not a multiple of 8 did not panic") + } + }() + AddBlocks8F32(&r, make([]float32, 9)) +} diff --git a/internal/kernels/fence_test.go b/internal/kernels/fence_test.go index 87155c8..3d23895 100644 --- a/internal/kernels/fence_test.go +++ b/internal/kernels/fence_test.go @@ -60,6 +60,26 @@ func fences(t *testing.T, n int) [][]float64 { return [][]float64{f.head(n), f.tail(n)} } +// fences32 is fences for float32 operands: the same mappings, viewed as +// float32, with n elements flush against each guard. +func fences32(t *testing.T, n int) [][]float32 { + f := newFenced(t, (n+1)/2) + d := unsafe.Slice((*float32)(unsafe.Pointer(&f.data[0])), 2*len(f.data)) + return [][]float32{d[:n:n], d[len(d)-n:]} +} + +// TestBlocks8StaysInsideItsOperands fences both the blocks the kernel reads +// and the eight accumulators it reads and writes back. +func TestBlocks8StaysInsideItsOperands(t *testing.T) { + for blocks := 1; blocks <= 20; blocks++ { + for _, a := range fences32(t, 8*blocks) { + for _, r := range fences32(t, 8) { + AddBlocks8F32((*[8]float32)(r), a) + } + } + } +} + func TestKernelsStayInsideTheirSlices(t *testing.T) { for n := 1; n <= 70; n++ { for _, a := range fences(t, n) { diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index 0958389..6026831 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -704,3 +704,21 @@ TEXT ·hasFMA(SB), NOSPLIT, $0-1 hasFMA_unsupported: RET +TEXT ·blocks8SSE2(SB), NOSPLIT, $0-24 + MOVQ r+0(FP), DI + MOVQ a+8(FP), SI + MOVQ blocks+16(FP), CX + MOVUPS (DI), X0 + MOVUPS 16(DI), X1 + b8loop: + MOVUPS (SI), X2 + MOVUPS 16(SI), X3 + ADDPS X2, X0 + ADDPS X3, X1 + ADDQ $32, SI + DECQ CX + JNZ b8loop + MOVUPS X0, (DI) + MOVUPS X1, 16(DI) + RET + diff --git a/internal/kernels/sum_arm64.s b/internal/kernels/sum_arm64.s index a3c1791..dc03f0e 100644 --- a/internal/kernels/sum_arm64.s +++ b/internal/kernels/sum_arm64.s @@ -383,3 +383,17 @@ TEXT ·gemmMicro4x8(SB), NOSPLIT, $0-40 VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5) RET +TEXT ·blocks8NEON(SB), NOSPLIT, $0-24 + MOVD r+0(FP), R0 + MOVD a+8(FP), R1 + MOVD blocks+16(FP), R2 + VLD1 (R0), [V0.S4, V1.S4] + b8loop: + VLD1.P 32(R1), [V2.S4, V3.S4] + VFADD V2.S4, V0.S4, V0.S4 + VFADD V3.S4, V1.S4, V1.S4 + SUB $1, R2 + CBNZ R2, b8loop + VST1 [V0.S4, V1.S4], (R0) + RET + diff --git a/typed.go b/typed.go index 9d91938..65682c5 100644 --- a/typed.go +++ b/typed.go @@ -3,6 +3,8 @@ package ndarray import ( "math" "math/cmplx" + + "github.com/go-ndarray/ndarray/internal/kernels" ) // Inner loops for the dtypes other than Float64. Each takes contiguous storage @@ -443,20 +445,56 @@ func foldWidened(op reduceCode, dst, src any, outer, axisLen int) { func widenRuns[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](op reduceCode, dst []D, src []S, outer, axisLen int) { for o := range outer { run := src[o*axisLen : (o+1)*axisLen] - acc := D(run[0]) if op == rSum { - for _, v := range run[1:] { - acc += D(v) - } + dst[o] = sumWiden[S, D](run) } else { - for _, v := range run[1:] { - acc *= D(v) - } + dst[o] = prodWiden[S, D](run) } - dst[o] = acc } } +// sumWiden and prodWiden fold a run with four accumulators. Integer addition +// and multiplication wrap modulo 2^64, so they are associative and the order +// changes nothing: the result is the in-order fold's, exactly. One loop per +// operation, so the operation is not re-read per element (on amd64 the +// single loop reloaded it from the stack every iteration), and four +// independent chains, so the loop is no longer one dependent add an element: +// it had run 50% slower on a Zen 3 just by being moved across a 64-byte line. +// +// The loops consume the slice (s = s[4:] under len(s) >= 4) rather than index +// it at i+1..i+3: with indices, the compiler proved none of them in range and +// emitted four bounds checks an iteration (amd64 disassembly); this way it +// proves all four from the loop condition and emits none. +func sumWiden[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](s []S) D { + var a0, a1, a2, a3 D + for len(s) >= 4 { + a0 += D(s[0]) + a1 += D(s[1]) + a2 += D(s[2]) + a3 += D(s[3]) + s = s[4:] + } + for _, v := range s { + a0 += D(v) + } + return (a0 + a1) + (a2 + a3) +} + +func prodWiden[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](s []S) D { + var a0, a1, a2, a3 D = 1, 1, 1, 1 + for len(s) >= 4 { + a0 *= D(s[0]) + a1 *= D(s[1]) + a2 *= D(s[2]) + a3 *= D(s[3]) + s = s[4:] + } + for _, v := range s { + a0 *= D(v) + } + return (a0 * a1) * (a2 * a3) +} + // pwBlock is NumPy's PW_BLOCKSIZE: runs up to this length are summed with // eight accumulators, longer ones are halved. const pwBlock = 128 @@ -464,30 +502,25 @@ const pwBlock = 128 // pairwiseSum is NumPy's FLOAT_pairwise_sum // (numpy/_core/src/umath/loops_utils.h.src, v2.5.3): rounding error O(lg n) // instead of O(n), in the same order of additions. -func pairwiseSum[F float32](a []F) F { +// +// The eight accumulators are fed by kernels.AddBlocks8F32, a vector kernel on +// amd64 and arm64 that adds the same lanes in the same order, so the sums are +// the scalar loop's bit for bit. +func pairwiseSum(a []float32) float32 { n := len(a) switch { case n < 8: // From -0, so that a sum of only -0 stays -0. - res := F(math.Copysign(0, -1)) + res := float32(math.Copysign(0, -1)) for _, v := range a { res += v } return res case n <= pwBlock: - var r [8]F + var r [8]float32 copy(r[:], a[:8]) - i := 8 - for ; i < n-n%8; i += 8 { - r[0] += a[i] - r[1] += a[i+1] - r[2] += a[i+2] - r[3] += a[i+3] - r[4] += a[i+4] - r[5] += a[i+5] - r[6] += a[i+6] - r[7] += a[i+7] - } + i := n - n%8 + kernels.AddBlocks8F32(&r, a[8:i]) res := ((r[0] + r[1]) + (r[2] + r[3])) + ((r[4] + r[5]) + (r[6] + r[7])) for ; i < n; i++ { res += a[i]