Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions docs/perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -879,10 +879,28 @@ and multiplication wrap modulo 2⁶⁴, so the order changes nothing). They also
consume the slice instead of indexing `i+1…i+3`, which had left four bounds
checks per iteration in the amd64 code; now there are none.

Left: ppc64le, s390x, loong64 and riscv64 run the scalar block loop. ppc64le
needs `xvaddsp`, which Go's assembler lacks: go-asmgen/asmgen#27 adds the
VSX single-precision family to its registry, with references from GNU as
and llvm-mc.
**riscv64 and loong64 (2026-10-18).** RVV: `VSETIVLI $8, E32, M2` gives
vl = 8 on every VLEN ≥ 128, and `VFADDVV` adds a block to the two-register
group. LSX: the eight accumulators sit in two 128-bit registers, and
`VFADD.S` (`VADDF`) adds a block. Each runs behind the feature bit its float64
kernels use. Both were run on the real hardware: the kernel tests passed
there, and a sabotaged kernel failed them, so the kernel path is the one
taken. Same A/B protocol, idle hosts:

| float32 `Sum`, 2¹⁴ / 2²⁰ | change | controls |
|---|---:|---|
| SpacemiT X60 (cfarm95, RVV, VLEN 256) | −30% / −41% | float64, int32 ~ |
| Loongson 3C5000L (cfarm401, LSX) | −24% / −24% | float64, int32 ~ |

The first loong64 kernel used one 256-bit LASX register for all eight
accumulators, and was **26–47% slower** than the scalar loop. Every block's
addition waited on the previous one, a single dependency chain, where the
scalar loop has eight. Two registers give two chains, as SSE2 does on amd64.

Left: ppc64le and s390x run the scalar block loop. Go's assembler lacks
`xvaddsp` (ppc64) and `vfasb` (s390x). go-asmgen/asmgen#27 adds the VSX
single-precision family to go-asmgen's registry, with references from GNU as
and llvm-mc, and needs a cmd/asm patch of its own.

## SIMD coverage

Expand Down
30 changes: 30 additions & 0 deletions internal/kernels/asmgen/loong64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func main() {
f.Add(binKernel(k.name, k.vec, k.sca))
}
f.Add(gemmKernel())
f.Add(blocks8Kernel())
if err := os.WriteFile("sum_loong64.s", []byte(f.String()), 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
Expand Down Expand Up @@ -206,3 +207,32 @@ func gemmKernel() *emit.Function {
b.Ret()
return b.Func()
}

// blocks8Kernel builds blocks8LSX(r, a *float32, blocks int): for each of
// the blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's
// pairwise-sum block). The eight accumulators are split over two 128-bit LSX
// registers, V0 and V1, and VADDF (vfadd.s) is one IEEE float32 addition per
// lane, so the result is the scalar loop's bit for bit. blocks >= 1.
//
// Not one 256-bit LASX register: that version kept every block's addition
// waiting on the previous one, a single dependency chain, and on a Loongson
// 3C5000L it was 26-47% SLOWER than the scalar loop's eight chains. Two
// registers, two chains (as SSE2 on amd64): 24% faster.
func blocks8Kernel() *emit.Function {
sig := loong64.Layout([]string{"r", "a", "blocks"},
[]loong64.Type{loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil)
b := loong64.NewFunc("blocks8LSX", sig, 0)
b.LoadArg("r", "R4").LoadArg("a", "R5").LoadArg("blocks", "R6")
b.Raw("VMOVQ 0(R4), V0").Raw("VMOVQ 16(R4), V1")
b.Label("b8loop").
Raw("VMOVQ 0(R5), V2").
Raw("VMOVQ 16(R5), V3").
Raw("VADDF V2, V0, V0").
Raw("VADDF V3, V1, V1").
Raw("ADDV $32, R5").
Raw("ADDV $-1, R6").
Raw("BNE R6, b8loop")
b.Raw("VMOVQ V0, 0(R4)").Raw("VMOVQ V1, 16(R4)")
b.Ret()
return b.Func()
}
24 changes: 24 additions & 0 deletions internal/kernels/asmgen/riscv64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ func main() {
f.Add(binKernel(k.name, k.op))
}
f.Add(gemmKernel())
f.Add(blocks8Kernel())
if err := os.WriteFile("sum_riscv64.s", []byte(f.String()), 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
Expand Down Expand Up @@ -168,3 +169,26 @@ func gemmKernel() *emit.Function {
}
return b.Ret().Func()
}

// blocks8Kernel builds blocks8RVV(r, a *float32, blocks int): for each of the
// blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's
// pairwise-sum block). VSETIVLI $8 with E32 and M2 gives vl = 8 on every
// VLEN >= 128, the V extension's minimum; the eight accumulators live in the
// register group V2-V3 and VFADDVV is one IEEE float32 addition per lane, so
// the result is the scalar loop's bit for bit. blocks >= 1.
func blocks8Kernel() *emit.Function {
sig := riscv64.Layout([]string{"r", "a", "blocks"},
[]riscv64.Type{riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil)
b := riscv64.NewFunc("blocks8RVV", sig, 0)
b.LoadArg("r", "X10").LoadArg("a", "X11").LoadArg("blocks", "X12")
b.Raw("VSETIVLI $8, E32, M2, TA, MA, X15").
Raw("VLE32V (X10), V2").
Raw("b8loop:").
Raw("VLE32V (X11), V4").
Raw("VFADDVV V4, V2, V2"). // V2 = V2 + V4
Raw("ADD $32, X11").
Raw("ADD $-1, X12").
Raw("BNEZ X12, b8loop").
Raw("VSE32V V2, (X10)")
return b.Ret().Func()
}
2 changes: 1 addition & 1 deletion internal/kernels/blocks8_generic.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//go:build !amd64 && !arm64
//go:build !amd64 && !arm64 && !loong64 && !riscv64

package kernels

Expand Down
17 changes: 17 additions & 0 deletions internal/kernels/blocks8_loong64.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package kernels

// addBlocks8F32 runs the LSX kernel (sum_loong64.s), the eight accumulators
// in two 128-bit registers, when the CPU has LASX (which every LASX CPU's LSX
// comes with); the scalar loop otherwise. len(a) is a positive multiple of 8
// (AddBlocks8F32 checks).
func addBlocks8F32(r *[8]float32, a []float32) {
if !useLASX {
addBlocks8F32Go(r, a)
return
}
_ = a[len(a)-1] // the kernel reads len(a) of a
blocks8LSX(&r[0], &a[0], len(a)/8)
}

//go:noescape
func blocks8LSX(r, a *float32, blocks int)
17 changes: 17 additions & 0 deletions internal/kernels/blocks8_riscv64.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package kernels

// addBlocks8F32 runs the RVV kernel (sum_riscv64.s), the eight accumulators
// in a two-register group at vl = 8, when the CPU has the V extension; the
// scalar loop otherwise. len(a) is a positive multiple of 8 (AddBlocks8F32
// checks).
func addBlocks8F32(r *[8]float32, a []float32) {
if !useRVV {
addBlocks8F32Go(r, a)
return
}
_ = a[len(a)-1] // the kernel reads len(a) of a
blocks8RVV(&r[0], &a[0], len(a)/8)
}

//go:noescape
func blocks8RVV(r, a *float32, blocks int)
18 changes: 18 additions & 0 deletions internal/kernels/sum_loong64.s
Original file line number Diff line number Diff line change
Expand Up @@ -426,3 +426,21 @@ gstore:
XVMOVQ X17, 32(R8)
RET

TEXT ·blocks8LSX(SB), NOSPLIT, $0-24
MOVV r+0(FP), R4
MOVV a+8(FP), R5
MOVV blocks+16(FP), R6
VMOVQ 0(R4), V0
VMOVQ 16(R4), V1
b8loop:
VMOVQ 0(R5), V2
VMOVQ 16(R5), V3
VADDF V2, V0, V0
VADDF V3, V1, V1
ADDV $32, R5
ADDV $-1, R6
BNE R6, b8loop
VMOVQ V0, 0(R4)
VMOVQ V1, 16(R4)
RET

15 changes: 15 additions & 0 deletions internal/kernels/sum_riscv64.s
Original file line number Diff line number Diff line change
Expand Up @@ -198,3 +198,18 @@ TEXT ·gemmMicro6x8RVV(SB), NOSPLIT, $0-40
VSE64V V28, (X13)
RET

TEXT ·blocks8RVV(SB), NOSPLIT, $0-24
MOV r+0(FP), X10
MOV a+8(FP), X11
MOV blocks+16(FP), X12
VSETIVLI $8, E32, M2, TA, MA, X15
VLE32V (X10), V2
b8loop:
VLE32V (X11), V4
VFADDVV V4, V2, V2
ADD $32, X11
ADD $-1, X12
BNEZ X12, b8loop
VSE32V V2, (X10)
RET

Loading