From f9755940b5f04d250e9650652d5231e072f74707 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 15:51:15 +0200 Subject: [PATCH] perf: SIMD inner product (AVX2/FMA, NEON) for Dot, MatVec and Dot1DP dotRange, under Dot (1-D), MatVec and Dot1DP, was a four-chain scalar loop (gc does not vectorise it); on one Zen 3 core a 2^20 dot took 464 us, 2x NumPy's single thread. Two go-asmgen kernels replace it: - amd64 dotFMA: four YMM accumulators, b read straight from memory by VFMADD231PD; gated on the FMA probe like the GEMM kernel (the flag is renamed useFMA, since it now covers both). - arm64 dotNEON: eight D2 accumulators for the four FP pipes of an Apple core, folded with FMLA against 1.0. The other targets keep dotRange. Zen 3, interleaved with main: Dot 2^20 1 core 464 -> 229 us (NumPy 231) 16 cores 159 -> 99 us MatVec 1024^2 1 core 482 -> 134 us (NumPy 153) 16 cores 165 -> 89 us TestDotSIMD: exact on integer data for every length 0..70 (main loop, folds, tail), within n*eps*sum|a_i*b_i| on random data, NaN and Inf propagate; on amd64 it also runs with FMA forced off. Passes on a real Zen 3 (FMA path). Co-Authored-By: Claude Opus 5.5 --- docs/perf.md | 21 +++++++++++ internal/kernels/asmgen/amd64/gen.go | 52 ++++++++++++++++++++++++++ internal/kernels/asmgen/arm64/gen.go | 54 +++++++++++++++++++++++++++ internal/kernels/gemm.go | 6 +-- internal/kernels/gemm_amd64.go | 21 +++++++++-- internal/kernels/gemm_amd64_test.go | 7 ++-- internal/kernels/gemm_arm64.go | 11 ++++++ internal/kernels/gemm_generic.go | 3 ++ internal/kernels/gemm_test.go | 39 ++++++++++++++++++++ internal/kernels/sum_amd64.s | 45 +++++++++++++++++++++++ internal/kernels/sum_arm64.s | 55 ++++++++++++++++++++++++++++ 11 files changed, 304 insertions(+), 10 deletions(-) diff --git a/docs/perf.md b/docs/perf.md index fa6bede..b213f8c 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -499,6 +499,27 @@ Speed, kernel only, 16 Ki elements: M4 30 µs (`math.Exp` 65 µs); one Zen 3 cor 62 µs (`math.Exp` 215 µs, NumPy single-threaded 78 µs). Whole `Exp` on Zen 3, 16 cores: 1 Ki ×2.7, 16 Ki ×3, 256 Ki ×1.6, 4 Mi ×1.65; from 256 Ki on it is 2–5× NumPy. A SIMD version (AVX2 gathers for the table) is the next step. +## Dot and mat·vec (2026-10-04) + +`dotRange`, under `Dot` (1-D), `MatVec` and `Dot1DP`, was a four-chain scalar +loop (gc does not vectorise it): on one Zen 3 core 2^20 elements took 464 µs, +2× NumPy's single thread. Two generated kernels replace it: `dotFMA` on amd64 +(four YMM accumulators, b read straight from memory by VFMADD231PD; two loads +per FMA make it load-bound, which four chains cover), gated on the FMA probe, +and `dotNEON` on arm64 (eight D2 accumulators for the four FP pipes of an Apple +core). The other targets keep `dotRange`. The lane-parallel sum is a regrouping, +held to the n·ε·Σ|aᵢbᵢ| bound and exact on integer data. + +| Zen 3 | before | after | NumPy 1 thread | +|----|--:|--:|--:| +| Dot 2^20, 1 core | 464 µs | **229 µs** | 231 µs | +| MatVec 1024², 1 core | 482 µs | **134 µs** | 153 µs | +| Dot 2^20, 16 cores | 159 µs | 99 µs | — | +| MatVec 1024², 16 cores | 165 µs | 89 µs | — | + +NumPy's multi-threaded dot still wins on this part (24 µs): OpenMP pins its +threads, so each one rereads its own chunk from its own L3 slice on every +repetition, while goroutines move between core complexes. ## SIMD coverage diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index b0493a1..19e7cb1 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -46,6 +46,7 @@ func main() { f.Add(binKernel("mulSSE2", "MULPD", "MULSD")) f.Add(binKernel("divSSE2", "DIVPD", "DIVSD")) f.Add(gemmFMAKernel()) + f.Add(dotFMAKernel()) f.Add(gemmSSE2HalfKernel()) f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) @@ -380,6 +381,57 @@ func gemmFMAKernel() *emit.Function { return b.Func() } +// dotFMAKernel builds dotFMA(a, b *float64, n int) float64, the inner product +// with four YMM accumulators (16 doubles per iteration, one VFMADD231PD per 4 +// doubles reading b straight from memory): two loads per FMA make the loop +// load-bound, and four chains cover the FMA latency at that rate. The +// accumulators are folded pairwise, then the four lanes, then the tail is +// fused in one element at a time. Gated on hasFMA like the GEMM kernel. +func dotFMAKernel() *emit.Function { + sig := amd64.Layout( + []string{"a", "b", "n"}, []amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64}, + []string{"ret"}, []amd64.Type{amd64.Float64}, + ) + b := amd64.NewFunc("dotFMA", sig, 0) + b.LoadArg("a", "SI").LoadArg("b", "DI").LoadArg("n", "CX") + for r := 0; r < 4; r++ { + b.Raw(fmt.Sprintf("VXORPD Y%d, Y%d, Y%d", r, r, r)) + } + b.Raw("dloop:") + b.Raw("CMPQ CX, $16") + b.Raw("JL dfold") + for r := 0; r < 4; r++ { + b.Raw(fmt.Sprintf("VMOVUPD %d(SI), Y%d", 32*r, 4+r)) + b.Raw(fmt.Sprintf("VFMADD231PD %d(DI), Y%d, Y%d", 32*r, 4+r, r)) + } + b.Raw("ADDQ $128, SI") + b.Raw("ADDQ $128, DI") + b.Raw("SUBQ $16, CX") + b.Raw("JMP dloop") + b.Raw("dfold:") + b.Raw("VADDPD Y1, Y0, Y0") + b.Raw("VADDPD Y3, Y2, Y2") + b.Raw("VADDPD Y2, Y0, Y0") + b.Raw("VEXTRACTF128 $1, Y0, X1") + b.Raw("VADDPD X1, X0, X0") + b.Raw("VUNPCKHPD X0, X0, X1") + b.Raw("VADDSD X1, X0, X0") + b.Raw("dtail:") + b.Raw("TESTQ CX, CX") + b.Raw("JZ ddone") + b.Raw("VMOVSD (SI), X1") + b.Raw("VFMADD231SD (DI), X1, X0") + b.Raw("ADDQ $8, SI") + b.Raw("ADDQ $8, DI") + b.Raw("DECQ CX") + b.Raw("JMP dtail") + b.Raw("ddone:") + b.Raw("VZEROUPPER") + b.StoreRet("X0", "ret") + b.Ret() + return b.Func() +} + // fmaStep emits one k step of the 6x8 FMA tile, step reading the A and B // values at that many steps past SI and DI. func fmaStep(b *amd64.Builder, step int) { diff --git a/internal/kernels/asmgen/arm64/gen.go b/internal/kernels/asmgen/arm64/gen.go index 2a7b8f8..f2c925d 100644 --- a/internal/kernels/asmgen/arm64/gen.go +++ b/internal/kernels/asmgen/arm64/gen.go @@ -41,6 +41,7 @@ func reduceSig() arm64.Signature { func main() { f := emit.NewFile("arm64") f.Add(sumKernel()) + f.Add(dotKernel()) f.Add(sqrtKernel()) f.Add(addKernel()) f.Add(subKernel()) @@ -394,6 +395,59 @@ func gemmKernel() *emit.Function { // exactly 1.0 is exact, and FMA(src, 1.0, acc) rounds identically to acc+src // (the addend is the only inexact step), so this is a true vector add, not an // approximation — it simply reaches it through the only vector-FP op available. +// dotKernel builds dotNEON(a, b *float64, n int) float64, the inner product +// with eight D2 accumulators V0..V7 (16 doubles per iteration): two loads per +// FMA make the loop load-bound, and eight chains keep the four FP pipes of an +// Apple core fed through the FMA latency. The accumulators are folded with +// FMLA against 1.0 (exact; Go's assembler has no plain vector FP add), the two +// lanes added, and the tail fused in one element at a time with FMADDD, as gc +// fuses the scalar loop. +func dotKernel() *emit.Function { + sig := arm64.Layout( + []string{"a", "b", "n"}, []arm64.Type{arm64.Ptr, arm64.Ptr, arm64.Int64}, + []string{"ret"}, []arm64.Type{arm64.Float64}, + ) + b := arm64.NewFunc("dotNEON", sig, 0) + b.LoadArg("a", "R0").LoadArg("b", "R1").LoadArg("n", "R2") + b.Raw("MOVD $0x3FF0000000000000, R3") + b.Raw("VDUP R3, V24.D2") + for r := 0; r < 8; r++ { + b.Raw(fmt.Sprintf("VEOR V%d.B16, V%d.B16, V%d.B16", r, r, r)) + } + b.Raw("dblock:") + b.Raw("CMP $16, R2") + b.Raw("BLT dfold") + b.Raw("VLD1.P 64(R0), [V8.D2, V9.D2, V10.D2, V11.D2]") + b.Raw("VLD1.P 64(R0), [V12.D2, V13.D2, V14.D2, V15.D2]") + b.Raw("VLD1.P 64(R1), [V16.D2, V17.D2, V18.D2, V19.D2]") + b.Raw("VLD1.P 64(R1), [V20.D2, V21.D2, V22.D2, V23.D2]") + for r := 0; r < 8; r++ { + b.Raw(fmt.Sprintf("VFMLA V%d.D2, V%d.D2, V%d.D2", 16+r, 8+r, r)) // V(r) += a*b + } + b.Raw("SUB $16, R2") + b.Raw("B dblock") + b.Raw("dfold:") + for _, p := range [][2]int{{1, 0}, {3, 2}, {5, 4}, {7, 6}, {2, 0}, {6, 4}, {4, 0}} { + b.Raw(fmt.Sprintf("VFMLA V%d.D2, V24.D2, V%d.D2", p[0], p[1])) // V(p1) += V(p0) + } + b.Raw("VMOV V0.D[0], R3") + b.Raw("VMOV V0.D[1], R4") + b.Raw("FMOVD R3, F0") + b.Raw("FMOVD R4, F9") + b.Raw("FADDD F9, F0, F0") + b.Raw("dtail:") + b.Raw("CBZ R2, ddone") + b.Raw("FMOVD.P 8(R0), F10") + b.Raw("FMOVD.P 8(R1), F11") + b.Raw("FMADDD F11, F0, F10, F0") // F0 += F10*F11 + b.Raw("SUB $1, R2") + b.Raw("B dtail") + b.Raw("ddone:") + b.StoreRet("F0", "ret") + b.Ret() + return b.Func() +} + func sumKernel() *emit.Function { b := arm64.NewFunc("sumNEON", reduceSig(), 0) b.LoadArg("a", "R0"). diff --git a/internal/kernels/gemm.go b/internal/kernels/gemm.go index 7673063..7caad0e 100644 --- a/internal/kernels/gemm.go +++ b/internal/kernels/gemm.go @@ -418,7 +418,7 @@ func dotRange(a, b []float64) float64 { func MatVecP(dst, a, v []float64, m, k int) { body := func(lo, hi int) { for i := lo; i < hi; i++ { - dst[i] = dotRange(a[i*k:i*k+k], v) + dst[i] = dotSIMD(a[i*k:i*k+k], v) } } if m*k < matVecThreshold { @@ -455,7 +455,7 @@ func VecMatP(dst, v, a []float64, k, n int) { func Dot1DP(a, b []float64) float64 { n := len(a) if n < ParThreshold { - return dotRange(a, b) + return dotSIMD(a, b) } w := numWorkers(n) partials := make([]float64, w) @@ -474,7 +474,7 @@ func Dot1DP(a, b []float64) float64 { if e > n { e = n } - partials[idx] = dotRange(a[s:e], b[s:e]) + partials[idx] = dotSIMD(a[s:e], b[s:e]) } }) return Sum(partials) diff --git a/internal/kernels/gemm_amd64.go b/internal/kernels/gemm_amd64.go index 2b1a993..9053cba 100644 --- a/internal/kernels/gemm_amd64.go +++ b/internal/kernels/gemm_amd64.go @@ -23,15 +23,16 @@ const ( NR = 8 ) -// gemmUseFMA selects the FMA kernel. It is a var so tests can force the SSE2 -// path on hardware that has FMA (never the reverse: that would fault). -var gemmUseFMA = hasFMA() +// useFMA selects the FMA kernels (GEMM, dot). It is a var so tests can force +// the SSE2 / pure-Go paths on hardware that has FMA (never the reverse: that +// would fault). +var useFMA = hasFMA() // gemmMicro adds the MR x NR tile sum_p pa[p*MR..]*pb[p*NR..] into the C block at // dst[0:], whose rows are ldc apart. pa, pb hold at least kc*MR and kc*NR packed // (zero-padded) doubles; dst has at least (MR-1)*ldc + NR elements. func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { - if gemmUseFMA { + if useFMA { gemmMicro6x8FMA(kc, &pa[0], &pb[0], &dst[0], ldc) return } @@ -45,5 +46,17 @@ func gemmMicro6x8FMA(kc int, pa, pb, c *float64, ldc int) //go:noescape func gemmMicro6x4SSE2(kc int, pa, pb, c *float64, ldc int) +// dotSIMD is the inner product of two equal-length slices: the AVX2/FMA +// kernel when the CPU has FMA, else dotRange. +func dotSIMD(a, b []float64) float64 { + if !useFMA || len(a) == 0 { + return dotRange(a, b) + } + return dotFMA(&a[0], &b[0], len(a)) +} + +//go:noescape +func dotFMA(a, b *float64, n int) float64 + // hasFMA reports CPU and OS support for the FMA kernel (go-asmgen FeatureProbe). func hasFMA() bool diff --git a/internal/kernels/gemm_amd64_test.go b/internal/kernels/gemm_amd64_test.go index 72001a3..5b00f4e 100644 --- a/internal/kernels/gemm_amd64_test.go +++ b/internal/kernels/gemm_amd64_test.go @@ -7,10 +7,11 @@ import "testing" // (or a hypervisor that masks FMA) takes. The FMA path is the default and is // exercised by the rest of the suite wherever hasFMA is true. func TestGemmSSE2Fallback(t *testing.T) { - saved := gemmUseFMA - gemmUseFMA = false - defer func() { gemmUseFMA = saved }() + saved := useFMA + useFMA = false + defer func() { useFMA = saved }() t.Run("correctness", TestGemmCorrectness) t.Run("bands", TestGemmBandSizing) t.Run("float", TestGemmFloatTolerance) + t.Run("dot", TestDotSIMD) } diff --git a/internal/kernels/gemm_arm64.go b/internal/kernels/gemm_arm64.go index dd1c025..53bd211 100644 --- a/internal/kernels/gemm_arm64.go +++ b/internal/kernels/gemm_arm64.go @@ -26,5 +26,16 @@ func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { gemmMicro4x8(kc, &pa[0], &pb[0], &dst[0], ldc) } +// dotSIMD is the inner product of two equal-length slices (NEON kernel). +func dotSIMD(a, b []float64) float64 { + if len(a) == 0 { + return 0 + } + return dotNEON(&a[0], &b[0], len(a)) +} + +//go:noescape +func dotNEON(a, b *float64, n int) float64 + //go:noescape func gemmMicro4x8(kc int, pa, pb, c *float64, ldc int) diff --git a/internal/kernels/gemm_generic.go b/internal/kernels/gemm_generic.go index e32af1e..cd3a82f 100644 --- a/internal/kernels/gemm_generic.go +++ b/internal/kernels/gemm_generic.go @@ -12,6 +12,9 @@ package kernels // fan-out (MatMulP) keep it well ahead of an unblocked scalar loop and beat // single-threaded numpy on large products. +// dotSIMD is dotRange here: no vector-double kernel on these targets. +func dotSIMD(a, b []float64) float64 { return dotRange(a, b) } + // MR, NR are the register-block dimensions; 4x4 keeps 16 + a few scalars live in // the GP/FP register file on these targets without spilling. const ( diff --git a/internal/kernels/gemm_test.go b/internal/kernels/gemm_test.go index d759aa6..014c9ea 100644 --- a/internal/kernels/gemm_test.go +++ b/internal/kernels/gemm_test.go @@ -330,3 +330,42 @@ func TestDot1DP(t *testing.T) { } } } + +// TestDotSIMD holds the SIMD inner product (AVX2/FMA, NEON, or dotRange on the +// other targets) to a sequential oracle: exactly on integer data for every +// length 0..70 (main loop, folds and tail), within the regrouping bound +// n*eps*sum|a_i*b_i| on random data, and NaN/Inf propagation. +func TestDotSIMD(t *testing.T) { + naive := func(a, b []float64) (s, abs float64) { + for i := range a { + s += a[i] * b[i] + abs += math.Abs(a[i] * b[i]) + } + return s, abs + } + for n := 0; n <= 70; n++ { + a, b := intMat(1, n, int64(n)), intMat(1, n, int64(n+100)) + if want, _ := naive(a, b); dotSIMD(a, b) != want { + t.Fatalf("n=%d: %v != %v", n, dotSIMD(a, b), want) + } + } + for _, n := range []int{17, 1000, 4099} { + a, b := randVec(n, int64(n)), randVec(n, int64(2*n)) + want, abs := naive(a, b) + if got := dotSIMD(a, b); math.Abs(got-want) > float64(n)*0x1p-52*abs { + t.Errorf("n=%d: %v vs %v", n, got, want) + } + } + for _, special := range []float64{math.NaN(), math.Inf(1)} { + for _, at := range []int{0, 20, 37} { + a := intMat(1, 38, 5) + a[at] = special + b := intMat(1, 38, 6) + b[at] = 1 + got := dotSIMD(a, b) + if math.IsNaN(special) != math.IsNaN(got) || !math.IsNaN(special) && !math.IsInf(got, 1) { + t.Errorf("%v at %d: dot = %v", special, at, got) + } + } + } +} diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index 6ee227e..0958389 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -535,6 +535,51 @@ TEXT ·gemmMicro6x8FMA(SB), NOSPLIT, $0-40 VZEROUPPER RET +TEXT ·dotFMA(SB), NOSPLIT, $0-32 + MOVQ a+0(FP), SI + MOVQ b+8(FP), DI + MOVQ n+16(FP), CX + VXORPD Y0, Y0, Y0 + VXORPD Y1, Y1, Y1 + VXORPD Y2, Y2, Y2 + VXORPD Y3, Y3, Y3 + dloop: + CMPQ CX, $16 + JL dfold + VMOVUPD 0(SI), Y4 + VFMADD231PD 0(DI), Y4, Y0 + VMOVUPD 32(SI), Y5 + VFMADD231PD 32(DI), Y5, Y1 + VMOVUPD 64(SI), Y6 + VFMADD231PD 64(DI), Y6, Y2 + VMOVUPD 96(SI), Y7 + VFMADD231PD 96(DI), Y7, Y3 + ADDQ $128, SI + ADDQ $128, DI + SUBQ $16, CX + JMP dloop + dfold: + VADDPD Y1, Y0, Y0 + VADDPD Y3, Y2, Y2 + VADDPD Y2, Y0, Y0 + VEXTRACTF128 $1, Y0, X1 + VADDPD X1, X0, X0 + VUNPCKHPD X0, X0, X1 + VADDSD X1, X0, X0 + dtail: + TESTQ CX, CX + JZ ddone + VMOVSD (SI), X1 + VFMADD231SD (DI), X1, X0 + ADDQ $8, SI + ADDQ $8, DI + DECQ CX + JMP dtail + ddone: + VZEROUPPER + MOVSD X0, ret+24(FP) + RET + TEXT ·gemmMicro6x4SSE2(SB), NOSPLIT, $0-40 MOVQ kc+0(FP), CX MOVQ pa+8(FP), SI diff --git a/internal/kernels/sum_arm64.s b/internal/kernels/sum_arm64.s index 3008e42..49c984a 100644 --- a/internal/kernels/sum_arm64.s +++ b/internal/kernels/sum_arm64.s @@ -42,6 +42,61 @@ TEXT ·sumNEON(SB), NOSPLIT, $0-24 FMOVD F0, ret+16(FP) RET +TEXT ·dotNEON(SB), NOSPLIT, $0-32 + MOVD a+0(FP), R0 + MOVD b+8(FP), R1 + MOVD n+16(FP), R2 + MOVD $0x3FF0000000000000, R3 + VDUP R3, V24.D2 + VEOR V0.B16, V0.B16, V0.B16 + VEOR V1.B16, V1.B16, V1.B16 + VEOR V2.B16, V2.B16, V2.B16 + VEOR V3.B16, V3.B16, V3.B16 + VEOR V4.B16, V4.B16, V4.B16 + VEOR V5.B16, V5.B16, V5.B16 + VEOR V6.B16, V6.B16, V6.B16 + VEOR V7.B16, V7.B16, V7.B16 + dblock: + CMP $16, R2 + BLT dfold + VLD1.P 64(R0), [V8.D2, V9.D2, V10.D2, V11.D2] + VLD1.P 64(R0), [V12.D2, V13.D2, V14.D2, V15.D2] + VLD1.P 64(R1), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P 64(R1), [V20.D2, V21.D2, V22.D2, V23.D2] + VFMLA V16.D2, V8.D2, V0.D2 + VFMLA V17.D2, V9.D2, V1.D2 + VFMLA V18.D2, V10.D2, V2.D2 + VFMLA V19.D2, V11.D2, V3.D2 + VFMLA V20.D2, V12.D2, V4.D2 + VFMLA V21.D2, V13.D2, V5.D2 + VFMLA V22.D2, V14.D2, V6.D2 + VFMLA V23.D2, V15.D2, V7.D2 + SUB $16, R2 + B dblock + dfold: + VFMLA V1.D2, V24.D2, V0.D2 + VFMLA V3.D2, V24.D2, V2.D2 + VFMLA V5.D2, V24.D2, V4.D2 + VFMLA V7.D2, V24.D2, V6.D2 + VFMLA V2.D2, V24.D2, V0.D2 + VFMLA V6.D2, V24.D2, V4.D2 + VFMLA V4.D2, V24.D2, V0.D2 + VMOV V0.D[0], R3 + VMOV V0.D[1], R4 + FMOVD R3, F0 + FMOVD R4, F9 + FADDD F9, F0, F0 + dtail: + CBZ R2, ddone + FMOVD.P 8(R0), F10 + FMOVD.P 8(R1), F11 + FMADDD F11, F0, F10, F0 + SUB $1, R2 + B dtail + ddone: + FMOVD F0, ret+24(FP) + RET + TEXT ·sqrtNEON(SB), NOSPLIT, $0-24 MOVD dst+0(FP), R0 MOVD src+8(FP), R1