From 05627bc64deafb307c8f6d532ae9079061ec3b2e Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 27 Oct 2026 15:18:37 +0100 Subject: [PATCH 1/2] perf(gemm): native float32 GEMM on ppc64le (VSX 8x16, rotated A) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gemmMicro8x16VSX follows OpenBLAS's POWER9 sgemm (sgemm_macros_power9.S, LOAD8x16): no splats. Each 4-float A vector is rotated three ways with XXSLDWI, every rotation multiplies every B vector (32 XVMADDASP per k step against 6 loads and 6 rotations), and the store untangles the rotated 4x4 blocks with three XXSELs per row. OpenBLAS rotates B; this kernel rotates A, which has half as many vectors per step. POWER9 (cfarm29), abbench 8 rounds, one core, against v0.27.0, which widened float32 to the float64 GEMM: -59/-42/-37% at 64²/256²/1024². Co-Authored-By: Claude Opus 5.5 --- internal/kernels/asmgen/ppc64le/gen.go | 87 +++++++ internal/kernels/gemm32_other.go | 2 +- internal/kernels/gemm32_ppc64le.go | 16 ++ internal/kernels/sum_ppc64le.s | 336 +++++++++++++++++++++++++ 4 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 internal/kernels/gemm32_ppc64le.go diff --git a/internal/kernels/asmgen/ppc64le/gen.go b/internal/kernels/asmgen/ppc64le/gen.go index b8cc686..1bb47a7 100644 --- a/internal/kernels/asmgen/ppc64le/gen.go +++ b/internal/kernels/asmgen/ppc64le/gen.go @@ -51,6 +51,7 @@ func main() { f.Add(extremumKernel("minVSX", 64, false, abs64, inf64)) f.Add(extremumKernel("max32VSX", 32, true, abs32, inf32)) f.Add(extremumKernel("min32VSX", 32, false, abs32, inf32)) + f.Add(gemm32Kernel(f.Data("oddw", words(1, 3)), f.Data("highw", words(2, 3)))) if err := os.WriteFile("sum_ppc64le.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -442,6 +443,92 @@ func gemmKernel() *emit.Function { return b.Func() } +// gemm32Kernel builds gemmMicro8x16VSX(kc int, pa, pb, c *float32, ldc int): +// +// for p in [0,kc): C[r][col] += pa[p*8+r] * pb[p*16+col] (r<8, col<16) +// +// with the scheme of OpenBLAS's POWER9 sgemm (kernel/power/sgemm_macros_power9.S, +// LOAD8x16): no splats. Each 4-float A vector is rotated three ways, and the +// rotations multiply each B vector, so the tile is accumulated in a rotated +// layout that the store untangles once. OpenBLAS rotates B with xxperm and +// xxpermdi; this kernel rotates A, which has two vectors per k step against +// B's four, with the cyclic word shift XXSLDWI a, a, $v: rotation v holds +// a[(i+v)%4] in lane i (lanes numbered as LXVW4X loads them, memory order). +// +// Accumulator VS(16p+4k+v) lane i is then C[4p+(i+v)%4][4k+i]: A vector p +// (rows 4p..4p+3), B vector k (columns 4k..4k+3), rotation v. Row 4p+r of +// that 4x4 block takes lane i from rotation (r-i) mod 4, so it is three +// selects: x = lanes 0,1 from rotations r, r-1; y = lanes 2,3 from r-2, r-3; +// row = x's low half and y's high half (XXSEL against the odd-lanes and +// high-lanes masks, VS48 and VS49). +// +// Per k step: 2 A loads, 6 rotations, 4 B loads, 32 XVMADDASP. +func gemm32Kernel(odd, high string) *emit.Function { + name := "gemmMicro8x16VSX" + sig := ppc64.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, + []ppc64.Type{ppc64.Int64, ppc64.Ptr, ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil) + b := ppc64.NewFunc(name, sig, 0) + b.LoadArg("kc", "R3").LoadArg("pa", "R4").LoadArg("pb", "R5").LoadArg("c", "R6").LoadArg("ldc", "R7") + b.Raw("SLD $2, R7") + offsets(b, false) + b.Raw("MOVD $%s+0(SB), R8", odd).Raw("LXVW4X (R8)(R0), VS48"). + Raw("MOVD $%s+0(SB), R8", high).Raw("LXVW4X (R8)(R0), VS49") + for r := 0; r < 32; r++ { + b.Raw("XXLXOR VS%d, VS%d, VS%d", r, r, r) + } + b.Raw("CMP R3, $0").Raw("BEQ %sstore", name) + b.Label(name+"loop"). + Raw("LXVW4X (R4)(R0), VS32").Raw("LXVW4X (R4)(R14), VS36") + for k := 0; k < 4; k++ { + b.Raw("LXVW4X (R5)(%s), VS%d", vidx(k), 40+k) + } + for p := 0; p < 2; p++ { + for v := 1; v < 4; v++ { + b.Raw("XXSLDWI VS%d, VS%d, $%d, VS%d", 32+4*p, 32+4*p, v, 32+4*p+v) + } + } + for p := 0; p < 2; p++ { + for k := 0; k < 4; k++ { + for v := 0; v < 4; v++ { + b.XVMADDASP(16*p+4*k+v, 32+4*p+v, 40+k) + } + } + } + b.Raw("ADD $32, R4").Raw("ADD $64, R5").Raw("ADD $-1, R3"). + Raw("CMP R3, $0").Raw("BNE %sloop", name) + b.Label(name + "store") + acc := func(p, k, v int) int { return 16*p + 4*k + (v+4)%4 } + for p := 0; p < 2; p++ { + for r := 0; r < 4; r++ { + for k := 0; k < 4; k++ { + b.Raw("XXSEL VS%d, VS%d, VS48, VS44", acc(p, k, r), acc(p, k, r-1)). + Raw("XXSEL VS%d, VS%d, VS48, VS45", acc(p, k, r-2), acc(p, k, r-3)). + Raw("XXSEL VS44, VS45, VS49, VS46"). + Raw("LXVW4X (R6)(%s), VS47", vidx(k)) + b.XVADDSP(47, 47, 46) + b.Raw("STXVW4X VS47, (R6)(%s)", vidx(k)) + } + if !(p == 1 && r == 3) { + b.Raw("ADD R7, R6") + } + } + } + b.Ret() + return b.Func() +} + +// words is a 16-byte mask with all-ones in the given 4-byte lanes (memory +// order, as LXVW4X loads them) and zeros elsewhere. +func words(on ...int) []byte { + m := make([]byte, 16) + for _, w := range on { + for i := 0; i < 4; i++ { + m[4*w+i] = 0xff + } + } + return m +} + // blocks8Kernel builds blocks8VSX(r, a *float32, blocks int): for each of the // blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's pairwise-sum // block). The eight accumulators are split over two VSX registers, VS0 and VS1 diff --git a/internal/kernels/gemm32_other.go b/internal/kernels/gemm32_other.go index 3c45cdf..bfdd10e 100644 --- a/internal/kernels/gemm32_other.go +++ b/internal/kernels/gemm32_other.go @@ -1,4 +1,4 @@ -//go:build !amd64 && !arm64 +//go:build !amd64 && !arm64 && !ppc64le package kernels diff --git a/internal/kernels/gemm32_ppc64le.go b/internal/kernels/gemm32_ppc64le.go new file mode 100644 index 0000000..e8d5e9e --- /dev/null +++ b/internal/kernels/gemm32_ppc64le.go @@ -0,0 +1,16 @@ +package kernels + +// gemm32 is the float32 GEMM on the VSX 8x16 kernel (gemmMicro8x16VSX in +// sum_ppc64le.s, generated by go-asmgen; see asmgen/ppc64le/gen.go). +var gemm32 = &gemmKernel[float32]{mr: 8, nr: 16, micro: gemmMicro32} + +// gemmMicro32 adds the 8 x 16 tile sum_p pa[p*8..]*pb[p*16..] into the C block +// at dst[0:], whose rows are ldc apart. pa, pb hold at least kc*8 and kc*16 +// packed (zero-padded) floats; dst has at least 7*ldc + 16 elements. +func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { + _, _, _ = pa[kc*8-1], pb[kc*16-1], dst[7*ldc+15] // the kernel's whole reach + gemmMicro8x16VSX(kc, &pa[0], &pb[0], &dst[0], ldc) +} + +//go:noescape +func gemmMicro8x16VSX(kc int, pa, pb, c *float32, ldc int) diff --git a/internal/kernels/sum_ppc64le.s b/internal/kernels/sum_ppc64le.s index fbdff56..b7c60b6 100644 --- a/internal/kernels/sum_ppc64le.s +++ b/internal/kernels/sum_ppc64le.s @@ -1185,6 +1185,306 @@ min32VSXfold: MOVD R6, ret+24(FP) RET +TEXT ·gemmMicro8x16VSX(SB), NOSPLIT, $0-40 + MOVD kc+0(FP), R3 + MOVD pa+8(FP), R4 + MOVD pb+16(FP), R5 + MOVD c+24(FP), R6 + MOVD ldc+32(FP), R7 + SLD $2, R7 + MOVD $16, R14 + MOVD $32, R15 + MOVD $48, R16 + MOVD $oddw<>+0(SB), R8 + LXVW4X (R8)(R0), VS48 + MOVD $highw<>+0(SB), R8 + LXVW4X (R8)(R0), VS49 + XXLXOR VS0, VS0, VS0 + XXLXOR VS1, VS1, VS1 + XXLXOR VS2, VS2, VS2 + XXLXOR VS3, VS3, VS3 + XXLXOR VS4, VS4, VS4 + XXLXOR VS5, VS5, VS5 + XXLXOR VS6, VS6, VS6 + XXLXOR VS7, VS7, VS7 + XXLXOR VS8, VS8, VS8 + XXLXOR VS9, VS9, VS9 + XXLXOR VS10, VS10, VS10 + XXLXOR VS11, VS11, VS11 + XXLXOR VS12, VS12, VS12 + XXLXOR VS13, VS13, VS13 + XXLXOR VS14, VS14, VS14 + XXLXOR VS15, VS15, VS15 + XXLXOR VS16, VS16, VS16 + XXLXOR VS17, VS17, VS17 + XXLXOR VS18, VS18, VS18 + XXLXOR VS19, VS19, VS19 + XXLXOR VS20, VS20, VS20 + XXLXOR VS21, VS21, VS21 + XXLXOR VS22, VS22, VS22 + XXLXOR VS23, VS23, VS23 + XXLXOR VS24, VS24, VS24 + XXLXOR VS25, VS25, VS25 + XXLXOR VS26, VS26, VS26 + XXLXOR VS27, VS27, VS27 + XXLXOR VS28, VS28, VS28 + XXLXOR VS29, VS29, VS29 + XXLXOR VS30, VS30, VS30 + XXLXOR VS31, VS31, VS31 + CMP R3, $0 + BEQ gemmMicro8x16VSXstore +gemmMicro8x16VSXloop: + LXVW4X (R4)(R0), VS32 + LXVW4X (R4)(R14), VS36 + LXVW4X (R5)(R0), VS40 + LXVW4X (R5)(R14), VS41 + LXVW4X (R5)(R15), VS42 + LXVW4X (R5)(R16), VS43 + XXSLDWI VS32, VS32, $1, VS33 + XXSLDWI VS32, VS32, $2, VS34 + XXSLDWI VS32, VS32, $3, VS35 + XXSLDWI VS36, VS36, $1, VS37 + XXSLDWI VS36, VS36, $2, VS38 + XXSLDWI VS36, VS36, $3, VS39 + WORD $0xf000420e // XVMADDASP VS32, VS40, VS0 + WORD $0xf021420e // XVMADDASP VS33, VS40, VS1 + WORD $0xf042420e // XVMADDASP VS34, VS40, VS2 + WORD $0xf063420e // XVMADDASP VS35, VS40, VS3 + WORD $0xf0804a0e // XVMADDASP VS32, VS41, VS4 + WORD $0xf0a14a0e // XVMADDASP VS33, VS41, VS5 + WORD $0xf0c24a0e // XVMADDASP VS34, VS41, VS6 + WORD $0xf0e34a0e // XVMADDASP VS35, VS41, VS7 + WORD $0xf100520e // XVMADDASP VS32, VS42, VS8 + WORD $0xf121520e // XVMADDASP VS33, VS42, VS9 + WORD $0xf142520e // XVMADDASP VS34, VS42, VS10 + WORD $0xf163520e // XVMADDASP VS35, VS42, VS11 + WORD $0xf1805a0e // XVMADDASP VS32, VS43, VS12 + WORD $0xf1a15a0e // XVMADDASP VS33, VS43, VS13 + WORD $0xf1c25a0e // XVMADDASP VS34, VS43, VS14 + WORD $0xf1e35a0e // XVMADDASP VS35, VS43, VS15 + WORD $0xf204420e // XVMADDASP VS36, VS40, VS16 + WORD $0xf225420e // XVMADDASP VS37, VS40, VS17 + WORD $0xf246420e // XVMADDASP VS38, VS40, VS18 + WORD $0xf267420e // XVMADDASP VS39, VS40, VS19 + WORD $0xf2844a0e // XVMADDASP VS36, VS41, VS20 + WORD $0xf2a54a0e // XVMADDASP VS37, VS41, VS21 + WORD $0xf2c64a0e // XVMADDASP VS38, VS41, VS22 + WORD $0xf2e74a0e // XVMADDASP VS39, VS41, VS23 + WORD $0xf304520e // XVMADDASP VS36, VS42, VS24 + WORD $0xf325520e // XVMADDASP VS37, VS42, VS25 + WORD $0xf346520e // XVMADDASP VS38, VS42, VS26 + WORD $0xf367520e // XVMADDASP VS39, VS42, VS27 + WORD $0xf3845a0e // XVMADDASP VS36, VS43, VS28 + WORD $0xf3a55a0e // XVMADDASP VS37, VS43, VS29 + WORD $0xf3c65a0e // XVMADDASP VS38, VS43, VS30 + WORD $0xf3e75a0e // XVMADDASP VS39, VS43, VS31 + ADD $32, R4 + ADD $64, R5 + ADD $-1, R3 + CMP R3, $0 + BNE gemmMicro8x16VSXloop +gemmMicro8x16VSXstore: + XXSEL VS0, VS3, VS48, VS44 + XXSEL VS2, VS1, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R0), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R0) + XXSEL VS4, VS7, VS48, VS44 + XXSEL VS6, VS5, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R14), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R14) + XXSEL VS8, VS11, VS48, VS44 + XXSEL VS10, VS9, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R15), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R15) + XXSEL VS12, VS15, VS48, VS44 + XXSEL VS14, VS13, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R16), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R16) + ADD R7, R6 + XXSEL VS1, VS0, VS48, VS44 + XXSEL VS3, VS2, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R0), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R0) + XXSEL VS5, VS4, VS48, VS44 + XXSEL VS7, VS6, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R14), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R14) + XXSEL VS9, VS8, VS48, VS44 + XXSEL VS11, VS10, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R15), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R15) + XXSEL VS13, VS12, VS48, VS44 + XXSEL VS15, VS14, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R16), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R16) + ADD R7, R6 + XXSEL VS2, VS1, VS48, VS44 + XXSEL VS0, VS3, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R0), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R0) + XXSEL VS6, VS5, VS48, VS44 + XXSEL VS4, VS7, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R14), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R14) + XXSEL VS10, VS9, VS48, VS44 + XXSEL VS8, VS11, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R15), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R15) + XXSEL VS14, VS13, VS48, VS44 + XXSEL VS12, VS15, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R16), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R16) + ADD R7, R6 + XXSEL VS3, VS2, VS48, VS44 + XXSEL VS1, VS0, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R0), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R0) + XXSEL VS7, VS6, VS48, VS44 + XXSEL VS5, VS4, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R14), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R14) + XXSEL VS11, VS10, VS48, VS44 + XXSEL VS9, VS8, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R15), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R15) + XXSEL VS15, VS14, VS48, VS44 + XXSEL VS13, VS12, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R16), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R16) + ADD R7, R6 + XXSEL VS16, VS19, VS48, VS44 + XXSEL VS18, VS17, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R0), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R0) + XXSEL VS20, VS23, VS48, VS44 + XXSEL VS22, VS21, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R14), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R14) + XXSEL VS24, VS27, VS48, VS44 + XXSEL VS26, VS25, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R15), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R15) + XXSEL VS28, VS31, VS48, VS44 + XXSEL VS30, VS29, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R16), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R16) + ADD R7, R6 + XXSEL VS17, VS16, VS48, VS44 + XXSEL VS19, VS18, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R0), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R0) + XXSEL VS21, VS20, VS48, VS44 + XXSEL VS23, VS22, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R14), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R14) + XXSEL VS25, VS24, VS48, VS44 + XXSEL VS27, VS26, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R15), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R15) + XXSEL VS29, VS28, VS48, VS44 + XXSEL VS31, VS30, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R16), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R16) + ADD R7, R6 + XXSEL VS18, VS17, VS48, VS44 + XXSEL VS16, VS19, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R0), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R0) + XXSEL VS22, VS21, VS48, VS44 + XXSEL VS20, VS23, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R14), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R14) + XXSEL VS26, VS25, VS48, VS44 + XXSEL VS24, VS27, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R15), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R15) + XXSEL VS30, VS29, VS48, VS44 + XXSEL VS28, VS31, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R16), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R16) + ADD R7, R6 + XXSEL VS19, VS18, VS48, VS44 + XXSEL VS17, VS16, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R0), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R0) + XXSEL VS23, VS22, VS48, VS44 + XXSEL VS21, VS20, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R14), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R14) + XXSEL VS27, VS26, VS48, VS44 + XXSEL VS25, VS24, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R15), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R15) + XXSEL VS31, VS30, VS48, VS44 + XXSEL VS29, VS28, VS48, VS45 + XXSEL VS44, VS45, VS49, VS46 + LXVW4X (R6)(R16), VS47 + WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 + STXVW4X VS47, (R6)(R16) + RET + DATA abs64<>+0(SB)/1, $0xff DATA abs64<>+1(SB)/1, $0xff DATA abs64<>+2(SB)/1, $0xff @@ -1257,3 +1557,39 @@ DATA inf32<>+14(SB)/1, $0x80 DATA inf32<>+15(SB)/1, $0x7f GLOBL inf32<>(SB), RODATA|NOPTR, $16 +DATA oddw<>+0(SB)/1, $0x00 +DATA oddw<>+1(SB)/1, $0x00 +DATA oddw<>+2(SB)/1, $0x00 +DATA oddw<>+3(SB)/1, $0x00 +DATA oddw<>+4(SB)/1, $0xff +DATA oddw<>+5(SB)/1, $0xff +DATA oddw<>+6(SB)/1, $0xff +DATA oddw<>+7(SB)/1, $0xff +DATA oddw<>+8(SB)/1, $0x00 +DATA oddw<>+9(SB)/1, $0x00 +DATA oddw<>+10(SB)/1, $0x00 +DATA oddw<>+11(SB)/1, $0x00 +DATA oddw<>+12(SB)/1, $0xff +DATA oddw<>+13(SB)/1, $0xff +DATA oddw<>+14(SB)/1, $0xff +DATA oddw<>+15(SB)/1, $0xff +GLOBL oddw<>(SB), RODATA|NOPTR, $16 + +DATA highw<>+0(SB)/1, $0x00 +DATA highw<>+1(SB)/1, $0x00 +DATA highw<>+2(SB)/1, $0x00 +DATA highw<>+3(SB)/1, $0x00 +DATA highw<>+4(SB)/1, $0x00 +DATA highw<>+5(SB)/1, $0x00 +DATA highw<>+6(SB)/1, $0x00 +DATA highw<>+7(SB)/1, $0x00 +DATA highw<>+8(SB)/1, $0xff +DATA highw<>+9(SB)/1, $0xff +DATA highw<>+10(SB)/1, $0xff +DATA highw<>+11(SB)/1, $0xff +DATA highw<>+12(SB)/1, $0xff +DATA highw<>+13(SB)/1, $0xff +DATA highw<>+14(SB)/1, $0xff +DATA highw<>+15(SB)/1, $0xff +GLOBL highw<>(SB), RODATA|NOPTR, $16 + From b62b0ada1b41832528bae6fc34b3ff9fc5af3061 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 27 Oct 2026 21:19:53 +0100 Subject: [PATCH 2/2] docs: ppc64le float32 GEMM measurements and audit entry Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +++--- SECURITY.md | 6 ++++++ docs/perf.md | 22 +++++++++++++++++++++- docs/plan-ndarray.md | 5 +++-- linalg.go | 2 +- 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e8c1bb0..cc7c009 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ scalars with `Scalar`). Float64 is the default and the most SIMD-accelerated one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every 64-bit target, and `Max`/`Min` on every 64-bit target, bit-identical to its - scalar loops. Its `MatMul` runs on a native float32 GEMM on arm64 (NEON) - and amd64 with FMA (AVX2), accumulating in float32 like NumPy's sgemm, and + scalar loops. Its `MatMul` runs on a native float32 GEMM on arm64 (NEON), + amd64 with FMA (AVX2) and ppc64le (VSX), accumulating in float32 like NumPy's sgemm, and elsewhere on the float64 GEMM (accumulated in float64, rounded once) ([docs/design-dtypes.md](docs/design-dtypes.md)). - **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/ @@ -155,7 +155,7 @@ and `Max` by 92–96% ([docs/perf.md](docs/perf.md), *SIMD coverage*). It is a > pair Go supports; v0.1.0 and v0.2.0 were also run on real amd64, arm64, > ppc64le, riscv64 and loong64 hardware, v0.3.0 on real amd64, arm64, ppc64le > and loong64. See **[docs/plan-ndarray.md](docs/plan-ndarray.md)** -> for the roadmap (a native float32 GEMM on the other four 64-bit targets; +> for the roadmap (a native float32 GEMM on s390x, riscv64 and loong64; > linear algebra decompositions). ## Why this module? diff --git a/SECURITY.md b/SECURITY.md index 22bc605..e2aa53a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,12 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-27 (v0.28.0).** The ppc64le float32 GEMM kernel +(`gemmMicro8x16VSX`) reads `kc·8` and `kc·16` packed floats and one 8×16 tile +of C, which its wrapper bounds-checks before the call. It also reads two +16-byte read-only mask constants. It is fenced and tested against the float32 +oracle on a real POWER9, and a sabotaged kernel failed CI's ppc64le lane. + **2026-10-27 (v0.27.0).** The unpacked-A GEMM kernels (`gemmUA*` on amd64 and arm64) read A in place: `mr` rows `lda` elements apart, `kc` elements each. Their Go wrapper bounds-checks the whole reach, diff --git a/docs/perf.md b/docs/perf.md index 3841545..ad33a05 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1275,9 +1275,29 @@ Against NumPy 2.5.4 (OpenBLAS, one core), the ratio ours / NumPy: | N1 float32 | 1.20 | 1.06 | 1.09 | | N1 float64 | 1.18 | 1.11 | 1.18 | +### ppc64le float32 GEMM (2026-10-27) + +`gemmMicro8x16VSX` follows OpenBLAS's POWER9 sgemm +(`kernel/power/sgemm_macros_power9.S`, `LOAD8x16`) and uses no splats. Each +4-float A vector is rotated three ways (`XXSLDWI a, a, $v`), and every +rotation multiplies every B vector: 32 `XVMADDASP` per k step against 6 loads +and 6 rotations. The tile therefore accumulates in a rotated layout, and the +store untangles each 4×4 block with three `XXSEL`s per row. OpenBLAS rotates +B; rotating A costs half as many permutes, because A has two vectors per k +step and B four. + +| float32 `MatMul` vs v0.27.0 (`abbench`, 8 rounds, POWER9) | 64² | 256² | 1024² | +|---|---:|---:|---:| +| widened to the float64 GEMM → native | **−59%** | **−42%** | **−37%** | + +The NumPy 2.5.3 wheel on that host is no reference for products: x @ x takes +0.48 ms at 64² and 2.7 s at 1024², for float32 and float64 alike. That is +14–31× slower than us, consistent with a BLAS that has no POWER9 kernels. A +comparison with a tuned POWER9 OpenBLAS remains to be made. + **Open defect: products are still up to 1.2× slower than OpenBLAS on one core.** The exception is float64 at 1024² on Zen 3, where we are 3% faster. -The kernels for ppc64le, s390x, riscv64 and loong64 are still to write. +The float32 kernels for s390x, riscv64 and loong64 are still to write. **Open: the allocating operations.** `Add` is still 1.5–3.2× slower than NumPy's `a+b`, for float64 as well. The kernel is no longer the difference, diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index f422d71..a001072 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -161,8 +161,9 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). vectorised them on all six 64-bit targets, for float32 and float64), and the transcendental ufuncs. `MatMul` runs on the float64 GEMM since v0.21.0, 77–90% faster than the triple loop, and since v0.26.0 on a native float32 GEMM on arm64 (NEON 8×8) and amd64 with FMA (6×16), - a further 53–61% faster. The float32 kernels for ppc64le, s390x, riscv64 - and loong64 are still to write. + a further 53–61% faster, and since v0.28.0 on ppc64le (VSX 8×16, 37–59% + faster on a POWER9). The float32 kernels for s390x, riscv64 and loong64 + are still to write. - **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic, comparisons returning Bool masks, `Maximum`/`Minimum`, `Logical*`, `Power`, `Where` and the per-axis reductions, in every dtype. Not done: a public diff --git a/linalg.go b/linalg.go index 261e254..677a3c2 100644 --- a/linalg.go +++ b/linalg.go @@ -53,7 +53,7 @@ func (a *Array) MatMul(b *Array) (*Array, error) { // matmulTyped is MatMul for operands of other dtypes, in their common dtype. // // Float32 runs on the packed float32 GEMM where there is a float32 kernel -// (arm64, and amd64 with FMA): it accumulates in float32, as NumPy's BLAS +// (arm64, ppc64le, and amd64 with FMA): it accumulates in float32, as NumPy's BLAS // sgemm does, at twice the SIMD lanes of float64. Elsewhere it runs on the // float64 GEMM: the operands are widened (exact, O(n²)), multiplied and // accumulated in float64, and each element is rounded once to float32, which