Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions docs/perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,27 @@ Speed, kernel only, 16 Ki elements: M4 30 µs (`math.Exp` 65 µs); one Zen 3 cor
62 µs (`math.Exp` 215 µs, NumPy single-threaded 78 µs). Whole `Exp` on Zen 3,
16 cores: 1 Ki ×2.7, 16 Ki ×3, 256 Ki ×1.6, 4 Mi ×1.65; from 256 Ki on it is
2–5× NumPy. A SIMD version (AVX2 gathers for the table) is the next step.
## Dot and mat·vec (2026-10-04)

`dotRange`, under `Dot` (1-D), `MatVec` and `Dot1DP`, was a four-chain scalar
loop (gc does not vectorise it): on one Zen 3 core 2^20 elements took 464 µs,
2× NumPy's single thread. Two generated kernels replace it: `dotFMA` on amd64
(four YMM accumulators, b read straight from memory by VFMADD231PD; two loads
per FMA make it load-bound, which four chains cover), gated on the FMA probe,
and `dotNEON` on arm64 (eight D2 accumulators for the four FP pipes of an Apple
core). The other targets keep `dotRange`. The lane-parallel sum is a regrouping,
held to the n·ε·Σ|aᵢbᵢ| bound and exact on integer data.

| Zen 3 | before | after | NumPy 1 thread |
|----|--:|--:|--:|
| Dot 2^20, 1 core | 464 µs | **229 µs** | 231 µs |
| MatVec 1024², 1 core | 482 µs | **134 µs** | 153 µs |
| Dot 2^20, 16 cores | 159 µs | 99 µs | — |
| MatVec 1024², 16 cores | 165 µs | 89 µs | — |

NumPy's multi-threaded dot still wins on this part (24 µs): OpenMP pins its
threads, so each one rereads its own chunk from its own L3 slice on every
repetition, while goroutines move between core complexes.

## SIMD coverage

Expand Down
52 changes: 52 additions & 0 deletions internal/kernels/asmgen/amd64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ func main() {
f.Add(binKernel("mulSSE2", "MULPD", "MULSD"))
f.Add(binKernel("divSSE2", "DIVPD", "DIVSD"))
f.Add(gemmFMAKernel())
f.Add(dotFMAKernel())
f.Add(gemmSSE2HalfKernel())
f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA))

Expand Down Expand Up @@ -380,6 +381,57 @@ func gemmFMAKernel() *emit.Function {
return b.Func()
}

// dotFMAKernel builds dotFMA(a, b *float64, n int) float64, the inner product
// with four YMM accumulators (16 doubles per iteration, one VFMADD231PD per 4
// doubles reading b straight from memory): two loads per FMA make the loop
// load-bound, and four chains cover the FMA latency at that rate. The
// accumulators are folded pairwise, then the four lanes, then the tail is
// fused in one element at a time. Gated on hasFMA like the GEMM kernel.
func dotFMAKernel() *emit.Function {
sig := amd64.Layout(
[]string{"a", "b", "n"}, []amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64},
[]string{"ret"}, []amd64.Type{amd64.Float64},
)
b := amd64.NewFunc("dotFMA", sig, 0)
b.LoadArg("a", "SI").LoadArg("b", "DI").LoadArg("n", "CX")
for r := 0; r < 4; r++ {
b.Raw(fmt.Sprintf("VXORPD Y%d, Y%d, Y%d", r, r, r))
}
b.Raw("dloop:")
b.Raw("CMPQ CX, $16")
b.Raw("JL dfold")
for r := 0; r < 4; r++ {
b.Raw(fmt.Sprintf("VMOVUPD %d(SI), Y%d", 32*r, 4+r))
b.Raw(fmt.Sprintf("VFMADD231PD %d(DI), Y%d, Y%d", 32*r, 4+r, r))
}
b.Raw("ADDQ $128, SI")
b.Raw("ADDQ $128, DI")
b.Raw("SUBQ $16, CX")
b.Raw("JMP dloop")
b.Raw("dfold:")
b.Raw("VADDPD Y1, Y0, Y0")
b.Raw("VADDPD Y3, Y2, Y2")
b.Raw("VADDPD Y2, Y0, Y0")
b.Raw("VEXTRACTF128 $1, Y0, X1")
b.Raw("VADDPD X1, X0, X0")
b.Raw("VUNPCKHPD X0, X0, X1")
b.Raw("VADDSD X1, X0, X0")
b.Raw("dtail:")
b.Raw("TESTQ CX, CX")
b.Raw("JZ ddone")
b.Raw("VMOVSD (SI), X1")
b.Raw("VFMADD231SD (DI), X1, X0")
b.Raw("ADDQ $8, SI")
b.Raw("ADDQ $8, DI")
b.Raw("DECQ CX")
b.Raw("JMP dtail")
b.Raw("ddone:")
b.Raw("VZEROUPPER")
b.StoreRet("X0", "ret")
b.Ret()
return b.Func()
}

// fmaStep emits one k step of the 6x8 FMA tile, step reading the A and B
// values at that many steps past SI and DI.
func fmaStep(b *amd64.Builder, step int) {
Expand Down
54 changes: 54 additions & 0 deletions internal/kernels/asmgen/arm64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func reduceSig() arm64.Signature {
func main() {
f := emit.NewFile("arm64")
f.Add(sumKernel())
f.Add(dotKernel())
f.Add(sqrtKernel())
f.Add(addKernel())
f.Add(subKernel())
Expand Down Expand Up @@ -394,6 +395,59 @@ func gemmKernel() *emit.Function {
// exactly 1.0 is exact, and FMA(src, 1.0, acc) rounds identically to acc+src
// (the addend is the only inexact step), so this is a true vector add, not an
// approximation — it simply reaches it through the only vector-FP op available.
// dotKernel builds dotNEON(a, b *float64, n int) float64, the inner product
// with eight D2 accumulators V0..V7 (16 doubles per iteration): two loads per
// FMA make the loop load-bound, and eight chains keep the four FP pipes of an
// Apple core fed through the FMA latency. The accumulators are folded with
// FMLA against 1.0 (exact; Go's assembler has no plain vector FP add), the two
// lanes added, and the tail fused in one element at a time with FMADDD, as gc
// fuses the scalar loop.
func dotKernel() *emit.Function {
sig := arm64.Layout(
[]string{"a", "b", "n"}, []arm64.Type{arm64.Ptr, arm64.Ptr, arm64.Int64},
[]string{"ret"}, []arm64.Type{arm64.Float64},
)
b := arm64.NewFunc("dotNEON", sig, 0)
b.LoadArg("a", "R0").LoadArg("b", "R1").LoadArg("n", "R2")
b.Raw("MOVD $0x3FF0000000000000, R3")
b.Raw("VDUP R3, V24.D2")
for r := 0; r < 8; r++ {
b.Raw(fmt.Sprintf("VEOR V%d.B16, V%d.B16, V%d.B16", r, r, r))
}
b.Raw("dblock:")
b.Raw("CMP $16, R2")
b.Raw("BLT dfold")
b.Raw("VLD1.P 64(R0), [V8.D2, V9.D2, V10.D2, V11.D2]")
b.Raw("VLD1.P 64(R0), [V12.D2, V13.D2, V14.D2, V15.D2]")
b.Raw("VLD1.P 64(R1), [V16.D2, V17.D2, V18.D2, V19.D2]")
b.Raw("VLD1.P 64(R1), [V20.D2, V21.D2, V22.D2, V23.D2]")
for r := 0; r < 8; r++ {
b.Raw(fmt.Sprintf("VFMLA V%d.D2, V%d.D2, V%d.D2", 16+r, 8+r, r)) // V(r) += a*b
}
b.Raw("SUB $16, R2")
b.Raw("B dblock")
b.Raw("dfold:")
for _, p := range [][2]int{{1, 0}, {3, 2}, {5, 4}, {7, 6}, {2, 0}, {6, 4}, {4, 0}} {
b.Raw(fmt.Sprintf("VFMLA V%d.D2, V24.D2, V%d.D2", p[0], p[1])) // V(p1) += V(p0)
}
b.Raw("VMOV V0.D[0], R3")
b.Raw("VMOV V0.D[1], R4")
b.Raw("FMOVD R3, F0")
b.Raw("FMOVD R4, F9")
b.Raw("FADDD F9, F0, F0")
b.Raw("dtail:")
b.Raw("CBZ R2, ddone")
b.Raw("FMOVD.P 8(R0), F10")
b.Raw("FMOVD.P 8(R1), F11")
b.Raw("FMADDD F11, F0, F10, F0") // F0 += F10*F11
b.Raw("SUB $1, R2")
b.Raw("B dtail")
b.Raw("ddone:")
b.StoreRet("F0", "ret")
b.Ret()
return b.Func()
}

func sumKernel() *emit.Function {
b := arm64.NewFunc("sumNEON", reduceSig(), 0)
b.LoadArg("a", "R0").
Expand Down
6 changes: 3 additions & 3 deletions internal/kernels/gemm.go
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ func dotRange(a, b []float64) float64 {
func MatVecP(dst, a, v []float64, m, k int) {
body := func(lo, hi int) {
for i := lo; i < hi; i++ {
dst[i] = dotRange(a[i*k:i*k+k], v)
dst[i] = dotSIMD(a[i*k:i*k+k], v)
}
}
if m*k < matVecThreshold {
Expand Down Expand Up @@ -455,7 +455,7 @@ func VecMatP(dst, v, a []float64, k, n int) {
func Dot1DP(a, b []float64) float64 {
n := len(a)
if n < ParThreshold {
return dotRange(a, b)
return dotSIMD(a, b)
}
w := numWorkers(n)
partials := make([]float64, w)
Expand All @@ -474,7 +474,7 @@ func Dot1DP(a, b []float64) float64 {
if e > n {
e = n
}
partials[idx] = dotRange(a[s:e], b[s:e])
partials[idx] = dotSIMD(a[s:e], b[s:e])
}
})
return Sum(partials)
Expand Down
21 changes: 17 additions & 4 deletions internal/kernels/gemm_amd64.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,16 @@ const (
NR = 8
)

// gemmUseFMA selects the FMA kernel. It is a var so tests can force the SSE2
// path on hardware that has FMA (never the reverse: that would fault).
var gemmUseFMA = hasFMA()
// useFMA selects the FMA kernels (GEMM, dot). It is a var so tests can force
// the SSE2 / pure-Go paths on hardware that has FMA (never the reverse: that
// would fault).
var useFMA = hasFMA()

// gemmMicro adds the MR x NR tile sum_p pa[p*MR..]*pb[p*NR..] into the C block at
// dst[0:], whose rows are ldc apart. pa, pb hold at least kc*MR and kc*NR packed
// (zero-padded) doubles; dst has at least (MR-1)*ldc + NR elements.
func gemmMicro(kc int, pa, pb, dst []float64, ldc int) {
if gemmUseFMA {
if useFMA {
gemmMicro6x8FMA(kc, &pa[0], &pb[0], &dst[0], ldc)
return
}
Expand All @@ -45,5 +46,17 @@ func gemmMicro6x8FMA(kc int, pa, pb, c *float64, ldc int)
//go:noescape
func gemmMicro6x4SSE2(kc int, pa, pb, c *float64, ldc int)

// dotSIMD is the inner product of two equal-length slices: the AVX2/FMA
// kernel when the CPU has FMA, else dotRange.
func dotSIMD(a, b []float64) float64 {
if !useFMA || len(a) == 0 {
return dotRange(a, b)
}
return dotFMA(&a[0], &b[0], len(a))
}

//go:noescape
func dotFMA(a, b *float64, n int) float64

// hasFMA reports CPU and OS support for the FMA kernel (go-asmgen FeatureProbe).
func hasFMA() bool
7 changes: 4 additions & 3 deletions internal/kernels/gemm_amd64_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ import "testing"
// (or a hypervisor that masks FMA) takes. The FMA path is the default and is
// exercised by the rest of the suite wherever hasFMA is true.
func TestGemmSSE2Fallback(t *testing.T) {
saved := gemmUseFMA
gemmUseFMA = false
defer func() { gemmUseFMA = saved }()
saved := useFMA
useFMA = false
defer func() { useFMA = saved }()
t.Run("correctness", TestGemmCorrectness)
t.Run("bands", TestGemmBandSizing)
t.Run("float", TestGemmFloatTolerance)
t.Run("dot", TestDotSIMD)
}
11 changes: 11 additions & 0 deletions internal/kernels/gemm_arm64.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,16 @@ func gemmMicro(kc int, pa, pb, dst []float64, ldc int) {
gemmMicro4x8(kc, &pa[0], &pb[0], &dst[0], ldc)
}

// dotSIMD is the inner product of two equal-length slices (NEON kernel).
func dotSIMD(a, b []float64) float64 {
if len(a) == 0 {
return 0
}
return dotNEON(&a[0], &b[0], len(a))
}

//go:noescape
func dotNEON(a, b *float64, n int) float64

//go:noescape
func gemmMicro4x8(kc int, pa, pb, c *float64, ldc int)
3 changes: 3 additions & 0 deletions internal/kernels/gemm_generic.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ package kernels
// fan-out (MatMulP) keep it well ahead of an unblocked scalar loop and beat
// single-threaded numpy on large products.

// dotSIMD is dotRange here: no vector-double kernel on these targets.
func dotSIMD(a, b []float64) float64 { return dotRange(a, b) }

// MR, NR are the register-block dimensions; 4x4 keeps 16 + a few scalars live in
// the GP/FP register file on these targets without spilling.
const (
Expand Down
39 changes: 39 additions & 0 deletions internal/kernels/gemm_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,3 +330,42 @@ func TestDot1DP(t *testing.T) {
}
}
}

// TestDotSIMD holds the SIMD inner product (AVX2/FMA, NEON, or dotRange on the
// other targets) to a sequential oracle: exactly on integer data for every
// length 0..70 (main loop, folds and tail), within the regrouping bound
// n*eps*sum|a_i*b_i| on random data, and NaN/Inf propagation.
func TestDotSIMD(t *testing.T) {
naive := func(a, b []float64) (s, abs float64) {
for i := range a {
s += a[i] * b[i]
abs += math.Abs(a[i] * b[i])
}
return s, abs
}
for n := 0; n <= 70; n++ {
a, b := intMat(1, n, int64(n)), intMat(1, n, int64(n+100))
if want, _ := naive(a, b); dotSIMD(a, b) != want {
t.Fatalf("n=%d: %v != %v", n, dotSIMD(a, b), want)
}
}
for _, n := range []int{17, 1000, 4099} {
a, b := randVec(n, int64(n)), randVec(n, int64(2*n))
want, abs := naive(a, b)
if got := dotSIMD(a, b); math.Abs(got-want) > float64(n)*0x1p-52*abs {
t.Errorf("n=%d: %v vs %v", n, got, want)
}
}
for _, special := range []float64{math.NaN(), math.Inf(1)} {
for _, at := range []int{0, 20, 37} {
a := intMat(1, 38, 5)
a[at] = special
b := intMat(1, 38, 6)
b[at] = 1
got := dotSIMD(a, b)
if math.IsNaN(special) != math.IsNaN(got) || !math.IsNaN(special) && !math.IsInf(got, 1) {
t.Errorf("%v at %d: dot = %v", special, at, got)
}
}
}
}
45 changes: 45 additions & 0 deletions internal/kernels/sum_amd64.s
Original file line number Diff line number Diff line change
Expand Up @@ -535,6 +535,51 @@ TEXT ·gemmMicro6x8FMA(SB), NOSPLIT, $0-40
VZEROUPPER
RET

TEXT ·dotFMA(SB), NOSPLIT, $0-32
MOVQ a+0(FP), SI
MOVQ b+8(FP), DI
MOVQ n+16(FP), CX
VXORPD Y0, Y0, Y0
VXORPD Y1, Y1, Y1
VXORPD Y2, Y2, Y2
VXORPD Y3, Y3, Y3
dloop:
CMPQ CX, $16
JL dfold
VMOVUPD 0(SI), Y4
VFMADD231PD 0(DI), Y4, Y0
VMOVUPD 32(SI), Y5
VFMADD231PD 32(DI), Y5, Y1
VMOVUPD 64(SI), Y6
VFMADD231PD 64(DI), Y6, Y2
VMOVUPD 96(SI), Y7
VFMADD231PD 96(DI), Y7, Y3
ADDQ $128, SI
ADDQ $128, DI
SUBQ $16, CX
JMP dloop
dfold:
VADDPD Y1, Y0, Y0
VADDPD Y3, Y2, Y2
VADDPD Y2, Y0, Y0
VEXTRACTF128 $1, Y0, X1
VADDPD X1, X0, X0
VUNPCKHPD X0, X0, X1
VADDSD X1, X0, X0
dtail:
TESTQ CX, CX
JZ ddone
VMOVSD (SI), X1
VFMADD231SD (DI), X1, X0
ADDQ $8, SI
ADDQ $8, DI
DECQ CX
JMP dtail
ddone:
VZEROUPPER
MOVSD X0, ret+24(FP)
RET

TEXT ·gemmMicro6x4SSE2(SB), NOSPLIT, $0-40
MOVQ kc+0(FP), CX
MOVQ pa+8(FP), SI
Expand Down
Loading
Loading