From b18e97de2b7bc81051aa47cd53eba6c3e4297e61 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 26 Oct 2026 11:39:24 +0100 Subject: [PATCH] perf: float32 MatMul on the float64 GEMM float32 MatMul ran a generic triple loop, 9x slower than our float64 GEMM. It now widens the operands (exact, O(n^2)), runs the packed, cache-blocked SIMD float64 GEMM, and rounds each element once to float32: at least as accurate as a BLAS sgemm accumulating in float32, on every architecture at once. abbench, 5 rounds, pinned core, float32 MatMul 64^2 / 256^2: POWER9 -77% / -87%, Zen 3 -77% / -90%; float64 flat. TestMatMulFloat32: exact on integer-valued operands, float32 precision against the triple loop on random ones, float32 dtype kept. The float32 case leaves the generated matmulStore (dead now). Co-Authored-By: Claude Opus 5.5 --- README.md | 3 ++- docs/perf.md | 25 ++++++++++++++++++++++++- docs/plan-ndarray.md | 5 +++-- gen_dtypes.go | 3 ++- linalg.go | 11 +++++++++-- linalg_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ zz_dtypes_gen.go | 2 -- 7 files changed, 82 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 04e7c18..534495d 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ scalars with `Scalar`). Float64 is the default and the most SIMD-accelerated one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every 64-bit target, and `Max`/`Min` on amd64 and arm64, bit-identical to its - scalar loops + scalar loops; its `MatMul` runs on the float64 GEMM (accumulated in float64, + rounded once) ([docs/design-dtypes.md](docs/design-dtypes.md)). - **Creation** — `New`/`Zeros`/`Ones`/`Full`/`FromData`/`Arange`/`Linspace`/ `Eye`/`Identity`. diff --git a/docs/perf.md b/docs/perf.md index 3055348..69e71a4 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1049,7 +1049,30 @@ Still to do: vector float max, min or compare at all. - s390x (`VFMAXSB` mode 1) waits for a z machine to test on. - riscv64 (`vfmax.vv` with a `vmfne.vv` NaN mask) is next. -- float32 GEMM. + +### float32 `MatMul` on the float64 GEMM (2026-10-26) + +float32 `MatMul` ran a generic triple loop, 9× slower than our float64 GEMM. +It now runs on the float64 GEMM: + +- the operands are widened, which is exact and costs O(n²); +- the products are accumulated in float64; +- each element is rounded once to float32. + +That is at least as accurate as a BLAS sgemm, which accumulates in float32, +and every architecture gets the packed, cache-blocked SIMD kernels at once. +With integer-valued operands the result is exact; with random ones it agrees +with the float32 triple loop to float32 precision (`TestMatMulFloat32`). + +| float32 `MatMul` (`abbench`, 5 rounds) | 64² | 256² | float64 control | +|---|---:|---:|---| +| POWER9 | −77% | **−87%** | ~ | +| Zen 3 | −77% | **−90%** | ~ | + +float32 is now within the widening of float64: 86.7 against 54.0 µs at 64², +2.56 against 2.25 ms at 256² on the POWER9. A native float32 GEMM, twice the +lanes per register and half the bytes, could go further; it is the next step +for products. **Open defect: the allocating operations.** `Add` is still 1.5–3.2× slower than NumPy's `a+b`, for float64 as well (21.9 against 8.48 µs at 2¹⁴). The diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index d93c8f6..e528c7d 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -159,8 +159,9 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). `Add`/`Sub`/`Mul`/`Div` (v0.18.0) and `Sqrt` (v0.19.0). Still pure Go for float32: `Max`/`Min` on ppc64le, loong64, s390x and riscv64 (v0.20.0 vectorised them on amd64 and arm64 and stopped converting to float64 - everywhere), `Dot`/`MatMul` (9× slower than our float64 GEMM) and the - transcendental ufuncs. + everywhere), and the transcendental ufuncs. `MatMul` runs on the float64 + GEMM since v0.21.0, 77–90% faster than the triple loop; a native float32 + GEMM is still to write. - **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic, comparisons returning Bool masks, `Maximum`/`Minimum`, `Logical*`, `Power`, `Where` and the per-axis reductions, in every dtype. Not done: a public diff --git a/gen_dtypes.go b/gen_dtypes.go index 9324afe..ade494f 100644 --- a/gen_dtypes.go +++ b/gen_dtypes.go @@ -339,7 +339,8 @@ func main() { p("// matmulStore multiplies (m x k) by (k x n) storage slices of one dtype into d.") p("func matmulStore(d, x, y any, m, k, n int) {") p("switch d := d.(type) {") - each("iufc-float64", "case []$T: matmulNum(d, x.([]$T), y.([]$T), m, k, n)") + // float32 and float64 run on the float64 GEMM (matmulTyped). + each("iufc-float64,float32", "case []$T: matmulNum(d, x.([]$T), y.([]$T), m, k, n)") each("b", "case []$T: matmulBool(d, x.([]$T), y.([]$T), m, k, n)") p("}") p("}") diff --git a/linalg.go b/linalg.go index d9ebe60..13cee0d 100644 --- a/linalg.go +++ b/linalg.go @@ -51,11 +51,18 @@ func (a *Array) MatMul(b *Array) (*Array, error) { } // matmulTyped is MatMul for operands of other dtypes, in their common dtype. -// It is a plain triple loop; only Float64 has the blocked SIMD GEMM so far. +// Float32 runs on the blocked SIMD float64 GEMM: the operands are widened +// (exact, O(n²)), multiplied and accumulated in float64, and each element is +// rounded once to float32. That is at least as accurate as accumulating in +// float32, as a BLAS sgemm does, and on a POWER9 it was 9x faster at 256² than +// the triple loop below, which the other dtypes keep. func matmulTyped(a, b *Array) *Array { dt := ResultType(a.dtype, b.dtype) - if dt == Float64 { + switch dt { + case Float64: return a.toFloat64().matmulFloat64(b.toFloat64()) + case Float32: + return a.toFloat64().matmulFloat64(b.toFloat64()).AsType(Float32) } m, k, n := a.shape[0], a.shape[1], b.shape[1] d := makeStore(dt, m*n) diff --git a/linalg_test.go b/linalg_test.go index f31afe0..eb4c42f 100644 --- a/linalg_test.go +++ b/linalg_test.go @@ -22,6 +22,48 @@ func TestMatMul(t *testing.T) { []float64{27, 30, 33, 61, 68, 75, 95, 106, 117}) } +// TestMatMulFloat32 runs the float32 product on the float64 GEMM. With +// integer-valued operands every product and sum is exact in both widths, so +// the result is exact. With random operands it agrees with a float32 +// triple loop to float32 precision. Big enough (40x50x30) to take the blocked +// GEMM path, and the result keeps the Float32 dtype. +func TestMatMulFloat32(t *testing.T) { + m, k, n := 40, 50, 30 + ints := func(seed int) []float32 { + d := make([]float32, 0, m*k) + for i := range cap(d) { + d = append(d, float32((i*seed)%17-8)) + } + return d + } + for _, exact := range []bool{true, false} { + x, y := ints(7), ints(11)[:k*n] + if !exact { + for i := range x { + x[i] = x[i]/3 + 0.1 + } + for i := range y { + y[i] = y[i]/7 - 0.05 + } + } + a := mustArr(t, ok(FromSlice(x, m, k))) + b := mustArr(t, ok(FromSlice(y, k, n))) + r := mustArr(t, ok(a.MatMul(b))) + if r.DType() != Float32 { + t.Fatalf("dtype %v, want float32", r.DType()) + } + got := data[float32](t, r) + want := make([]float32, m*n) + matmulNum(want, x, y, m, k, n) + for i := range want { + d := float64(got[i]) - float64(want[i]) + if exact && d != 0 || !exact && d*d > 1e-10*(1+float64(want[i])*float64(want[i])) { + t.Fatalf("exact=%v [%d] = %v, float32 loop %v", exact, i, got[i], want[i]) + } + } + } +} + func TestMatMulTransposedView(t *testing.T) { // A.T @ A where A is (3x2): A.T is a (2x3) strided view, exercising // materialize inside matmul2D. Result (2x2) == [[35,44],[44,56]] (numpy). diff --git a/zz_dtypes_gen.go b/zz_dtypes_gen.go index b3b3f91..65b9e0c 100644 --- a/zz_dtypes_gen.go +++ b/zz_dtypes_gen.go @@ -1537,8 +1537,6 @@ func matmulStore(d, x, y any, m, k, n int) { matmulNum(d, x.([]uint32), y.([]uint32), m, k, n) case []uint64: matmulNum(d, x.([]uint64), y.([]uint64), m, k, n) - case []float32: - matmulNum(d, x.([]float32), y.([]float32), m, k, n) case []complex64: matmulNum(d, x.([]complex64), y.([]complex64), m, k, n) case []complex128: