From d70670259b0404e0635317846db2f2b144d9ac72 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 26 Oct 2026 21:48:53 +0100 Subject: [PATCH] perf(gemm): pack full B panels with whole-row array moves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit packB copied each NR-element row run of a full panel element by element. A pprof of a 64² float32 product put it at 21% of the time. A full panel (nr == NR, unit column stride) now moves each run with one fixed-size array assignment, *(*[NR]T)(dst) = *(*[NR]T)(src), for the two widths in use (8 and 16). The compiler lowers it to vector moves with no runtime call. The conversions are bounds-checked like any slice-to-array conversion. abbench, 10 rounds, one core, against v0.26.0: Zen 3 float32 -11.6/-7.1/-1.7% (64²/256²/1024²), float64 -4.4/-2.3/~ N1 float32 -16.4/-8.9/~, float64 -9.5/~/~ Co-Authored-By: Claude Opus 5.5 --- docs/perf.md | 11 +++++++++++ internal/kernels/gemm.go | 24 ++++++++++++++++++++++++ internal/kernels/gemm_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/docs/perf.md b/docs/perf.md index d637414..cadd90d 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1232,6 +1232,17 @@ Against NumPy 2.5.4 (OpenBLAS, `OPENBLAS_NUM_THREADS=1`, the same core): | N1 float32 | 30.4 / 16.2 µs | 1.19 / 0.94 ms | 68.2 / 60.6 ms | | N1 float64 | 48.2 / 31.3 µs | 2.46 / 1.95 ms | 154 / 123 ms | +v0.26.1 packs full B panels with one fixed-size array move per row (8 or 16 +elements, lowered to vector moves), where `packB` had copied element by +element. Profiling had put it at 21% of a 64² float32 product. + +| `MatMul` vs v0.26.0 (`abbench`, 10 rounds) | 64² | 256² | 1024² | +|---|---:|---:|---:| +| Zen 3, float32 | −11.6% | −7.1% | −1.7% | +| Zen 3, float64 | −4.4% | −2.3% | ~ | +| Neoverse N1, float32 | −16.4% | −8.9% | ~ | +| Neoverse N1, float64 | −9.5% | ~ | ~ | + **Open defect: products are still slower than OpenBLAS on one core**, by 1.1–1.3× at 1024² and up to 1.9× at 64². The gap is widest where the work around the kernel weighs most: packing, edge tiles, and the result's diff --git a/internal/kernels/gemm.go b/internal/kernels/gemm.go index c3142d8..efe62e4 100644 --- a/internal/kernels/gemm.go +++ b/internal/kernels/gemm.go @@ -203,6 +203,9 @@ func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) { nr := min(NR, nc-jr) dst := pb[jr*kc:] if b.CS == 1 { + if nr == NR && packRowsWhole(dst, b.Data, b.Off+pc*b.RS+jc+jr, b.RS, kc, NR) { + continue + } // Element loops, not copy and clear: on runs this short (NR // elements) each of those is a runtime call. for p := 0; p < kc; p++ { @@ -232,6 +235,27 @@ func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) { } } +// packRowsWhole copies kc runs of nr elements, the p-th starting at +// src[s+p*rs], to dst[p*nr:], each as one fixed-size array assignment, which +// the compiler lowers to a few vector moves instead of a loop or a runtime +// call. It reports false, copying nothing, for a width it has no array type +// for; the caller then copies element by element. +func packRowsWhole[T gemmFloat](dst, src []T, s, rs, kc, nr int) bool { + switch nr { + case 8: + for p := 0; p < kc; p++ { + *(*[8]T)(dst[p*8:]) = *(*[8]T)(src[s+p*rs:]) + } + case 16: + for p := 0; p < kc; p++ { + *(*[16]T)(dst[p*16:]) = *(*[16]T)(src[s+p*rs:]) + } + default: + return false + } + return true +} + // packA copies A[ic:ic+mc][pc:pc+kc] into MR-tall row panels: for each panel ir // the layout is pa[ir*kc + p*MR + r] = A(ic+ir+r, pc+p), with the r >= mr rows // of an edge panel zero-filled so the micro-kernel reads a full MR. diff --git a/internal/kernels/gemm_test.go b/internal/kernels/gemm_test.go index be421fd..bdde8ce 100644 --- a/internal/kernels/gemm_test.go +++ b/internal/kernels/gemm_test.go @@ -187,6 +187,33 @@ func TestGemmFloatTolerance(t *testing.T) { } } +// TestPackRowsWhole: the whole-run copy moves exactly kc runs of 8 or 16 +// strided elements, and declines any other width without writing. +func TestPackRowsWhole(t *testing.T) { + src := randVec(200, 3) + for _, nr := range []int{8, 16} { + const s, rs, kc = 5, 21, 7 + dst := make([]float64, kc*nr+1) + if !packRowsWhole(dst, src, s, rs, kc, nr) { + t.Fatalf("nr=%d declined", nr) + } + for p := 0; p < kc; p++ { + for c := 0; c < nr; c++ { + if dst[p*nr+c] != src[s+p*rs+c] { + t.Fatalf("nr=%d [%d][%d] = %v, want %v", nr, p, c, dst[p*nr+c], src[s+p*rs+c]) + } + } + } + if dst[kc*nr] != 0 { + t.Fatalf("nr=%d wrote past its %d runs", nr, kc) + } + } + dst := make([]float64, 40) + if packRowsWhole(dst, src, 0, 10, 4, 5) || dst[0] != 0 { + t.Fatal("nr=5 copied") + } +} + // TestGemmBufferRealloc forces block sizes larger than the pooled buffers were // first sized for, exercising the reallocation branch in getPackBuf, then a // normal-sized call to confirm the pool still serves valid scratch.