Skip to content

Commit 679c055

Browse files
tannevaledclaude
andauthored
perf(gemm): pack full B panels with whole-row array moves (#75)
packB copied each NR-element row run of a full panel element by element. A pprof of a 64² float32 product put it at 21% of the time. A full panel (nr == NR, unit column stride) now moves each run with one fixed-size array assignment, *(*[NR]T)(dst) = *(*[NR]T)(src), for the two widths in use (8 and 16). The compiler lowers it to vector moves with no runtime call. The conversions are bounds-checked like any slice-to-array conversion. abbench, 10 rounds, one core, against v0.26.0: Zen 3 float32 -11.6/-7.1/-1.7% (64²/256²/1024²), float64 -4.4/-2.3/~ N1 float32 -16.4/-8.9/~, float64 -9.5/~/~ Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 5e51541 commit 679c055

3 files changed

Lines changed: 62 additions & 0 deletions

File tree

‎docs/perf.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1232,6 +1232,17 @@ Against NumPy 2.5.4 (OpenBLAS, `OPENBLAS_NUM_THREADS=1`, the same core):
12321232
| N1 float32 | 30.4 / 16.2 µs | 1.19 / 0.94 ms | 68.2 / 60.6 ms |
12331233
| N1 float64 | 48.2 / 31.3 µs | 2.46 / 1.95 ms | 154 / 123 ms |
12341234

1235+
v0.26.1 packs full B panels with one fixed-size array move per row (8 or 16
1236+
elements, lowered to vector moves), where `packB` had copied element by
1237+
element. Profiling had put it at 21% of a 64² float32 product.
1238+
1239+
| `MatMul` vs v0.26.0 (`abbench`, 10 rounds) | 64² | 256² | 1024² |
1240+
|---|---:|---:|---:|
1241+
| Zen 3, float32 | −11.6% | −7.1% | −1.7% |
1242+
| Zen 3, float64 | −4.4% | −2.3% | ~ |
1243+
| Neoverse N1, float32 | −16.4% | −8.9% | ~ |
1244+
| Neoverse N1, float64 | −9.5% | ~ | ~ |
1245+
12351246
**Open defect: products are still slower than OpenBLAS on one core**, by
12361247
1.1–1.3× at 1024² and up to 1.9× at 64². The gap is widest where the work
12371248
around the kernel weighs most: packing, edge tiles, and the result's

‎internal/kernels/gemm.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,9 @@ func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) {
203203
nr := min(NR, nc-jr)
204204
dst := pb[jr*kc:]
205205
if b.CS == 1 {
206+
if nr == NR && packRowsWhole(dst, b.Data, b.Off+pc*b.RS+jc+jr, b.RS, kc, NR) {
207+
continue
208+
}
206209
// Element loops, not copy and clear: on runs this short (NR
207210
// elements) each of those is a runtime call.
208211
for p := 0; p < kc; p++ {
@@ -232,6 +235,27 @@ func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) {
232235
}
233236
}
234237

238+
// packRowsWhole copies kc runs of nr elements, the p-th starting at
239+
// src[s+p*rs], to dst[p*nr:], each as one fixed-size array assignment, which
240+
// the compiler lowers to a few vector moves instead of a loop or a runtime
241+
// call. It reports false, copying nothing, for a width it has no array type
242+
// for; the caller then copies element by element.
243+
func packRowsWhole[T gemmFloat](dst, src []T, s, rs, kc, nr int) bool {
244+
switch nr {
245+
case 8:
246+
for p := 0; p < kc; p++ {
247+
*(*[8]T)(dst[p*8:]) = *(*[8]T)(src[s+p*rs:])
248+
}
249+
case 16:
250+
for p := 0; p < kc; p++ {
251+
*(*[16]T)(dst[p*16:]) = *(*[16]T)(src[s+p*rs:])
252+
}
253+
default:
254+
return false
255+
}
256+
return true
257+
}
258+
235259
// packA copies A[ic:ic+mc][pc:pc+kc] into MR-tall row panels: for each panel ir
236260
// the layout is pa[ir*kc + p*MR + r] = A(ic+ir+r, pc+p), with the r >= mr rows
237261
// of an edge panel zero-filled so the micro-kernel reads a full MR.

‎internal/kernels/gemm_test.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,33 @@ func TestGemmFloatTolerance(t *testing.T) {
187187
}
188188
}
189189

190+
// TestPackRowsWhole: the whole-run copy moves exactly kc runs of 8 or 16
191+
// strided elements, and declines any other width without writing.
192+
func TestPackRowsWhole(t *testing.T) {
193+
src := randVec(200, 3)
194+
for _, nr := range []int{8, 16} {
195+
const s, rs, kc = 5, 21, 7
196+
dst := make([]float64, kc*nr+1)
197+
if !packRowsWhole(dst, src, s, rs, kc, nr) {
198+
t.Fatalf("nr=%d declined", nr)
199+
}
200+
for p := 0; p < kc; p++ {
201+
for c := 0; c < nr; c++ {
202+
if dst[p*nr+c] != src[s+p*rs+c] {
203+
t.Fatalf("nr=%d [%d][%d] = %v, want %v", nr, p, c, dst[p*nr+c], src[s+p*rs+c])
204+
}
205+
}
206+
}
207+
if dst[kc*nr] != 0 {
208+
t.Fatalf("nr=%d wrote past its %d runs", nr, kc)
209+
}
210+
}
211+
dst := make([]float64, 40)
212+
if packRowsWhole(dst, src, 0, 10, 4, 5) || dst[0] != 0 {
213+
t.Fatal("nr=5 copied")
214+
}
215+
}
216+
190217
// TestGemmBufferRealloc forces block sizes larger than the pooled buffers were
191218
// first sized for, exercising the reallocation branch in getPackBuf, then a
192219
// normal-sized call to confirm the pool still serves valid scratch.

0 commit comments

Comments
 (0)