Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions SECURITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,15 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none.

## Audit log

**2026-10-31 (v0.29.1).** The A-in-place GEMM kernels now take B's row
stride (`ldb`) and may read a row-major B operand in place: `kc` rows `ldb`
elements apart, `NR` elements each. The wrappers bounds-check
`b[(kc-1)*ldb+NR-1]` before the call. The path is taken only for a unit
column stride and a positive row stride, and only for full panels; a partial
panel is still packed, so no column past B's last is read. The fence tests
place a strided B flush against a guard page. A sabotaged kernel (ignoring
`ldb`) failed the GEMM tests on arm64 and on amd64.

**2026-10-30 (v0.29.0).** The s390x float32 GEMM kernel
(`gemmMicro4x16VXE`) reads `kc·4` and `kc·16` packed floats and one 4×16
tile of C. Its wrapper bounds-checks all three before the call, behind the
Expand Down
24 changes: 23 additions & 1 deletion docs/perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -1309,8 +1309,30 @@ the float64 GEMM.
|---|---:|---:|---:|
| widened to the float64 GEMM → native | **−58%** | **−54%** | **−50%** |

### Reading B in place too, for small products (2026-10-31)

BLIS's sup kernels pack neither operand: the "rv" kernel advances B by its
row stride (`b += rs_b`) as it advances A by its column stride. v0.29.1 does
the same on the small-product path (below `GemmThreshold`, one worker). When
both operands are row-major, the A-in-place kernels read B's full panels in
place too, advancing B by `ldb` instead of the packed panel's width. Only a
partial last panel is still packed, for its zero padding. The parallel path,
used for larger products, still packs B once and shares it across workers.

| `MatMul` vs v0.29.0 (`abbench`, 10 rounds) | 64² | 256² | 1024² |
|---|---:|---:|---:|
| Zen 3, float32 | **−16%** | ~ | ~ |
| Zen 3, float64 | **−10%** | ~ | ~ |
| Neoverse N1, float32 | −7.3% | ~ | ~ |
| Neoverse N1, float64 | ~ | ~ | ~ |

The N1 host was contended (load 31, rows up to ±39%), so its "~" rows show
no regression but are not a precise measurement. On Zen 3, float32 64² now
takes 8.02 µs against NumPy's 8.14: **parity**. float64 takes 16.3 against
15.8 µs (1.03×).

**Open defect: products are still up to 1.2× slower than OpenBLAS on one
core.** The exception is float64 at 1024² on Zen 3, where we are 3% faster.
core** at 256² and 1024² on N1, and float32 1024² on Zen 3 is at 1.04×. The exception is float64 at 1024² on Zen 3, where we are 3% faster.
The float32 kernels for riscv64 and loong64 are still to write.

**Open: the allocating operations.** `Add` is still 1.5–3.2× slower than
Expand Down
26 changes: 16 additions & 10 deletions internal/kernels/asmgen/amd64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -563,18 +563,22 @@ func gemm32FMAKernel() *emit.Function {
// each A value is broadcast straight from its row, r*lda bytes past the
// k-step's column, through a scaled index (lda, 2lda, 3lda in R11, 4lda,
// 5lda in R12), and the column advances 4 bytes per step. A small product
// then pays no copy of A at all. B stays packed, so a partial last panel is
// still zero-padded; the kernels for mr < 6 cover the bottom rows of a block
// without reading past A's last row.
// then pays no copy of A at all. The kernels for mr < 6 cover the bottom rows
// of a block without reading past A's last row.
//
// B's rows are ldb elements apart (R13 in bytes), as BLIS's b += rs_b: a
// packed panel passes its own width (16 or 8), and a full panel of a
// row-major B is read in place with ldb its row stride. A partial last panel
// is still packed, for its zero padding.
//
// gemmUAKernel(mr, false) is the float64 counterpart over the 6x8 packing,
// gemmUA<mr>x8FMA: the same loop in PD instructions, 8-byte steps.
func gemm32UAKernel(mr int) *emit.Function { return gemmUAKernel(mr, true) }

func gemmUAKernel(mr int, single bool) *emit.Function {
sig := amd64.Layout(
[]string{"kc", "a", "lda", "pb", "c", "ldc"},
[]amd64.Type{amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Ptr, amd64.Int64},
[]string{"kc", "a", "lda", "pb", "ldb", "c", "ldc"},
[]amd64.Type{amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Int64},
nil, nil,
)
name, size, shift, sfx, bcast := fmt.Sprintf("gemmUA%dx8FMA", mr), 8, 3, "PD", "VBROADCASTSD"
Expand All @@ -586,12 +590,14 @@ func gemmUAKernel(mr int, single bool) *emit.Function {
b.LoadArg("a", "SI")
b.LoadArg("lda", "R8")
b.LoadArg("pb", "DI")
b.LoadArg("ldb", "R13")
b.LoadArg("c", "DX")
b.LoadArg("ldc", "R9")
b.Raw(fmt.Sprintf("SHLQ $%d, R8", shift)) // lda -> bytes
b.Raw(fmt.Sprintf("SHLQ $%d, R9", shift)) // ldc -> bytes
b.Raw("LEAQ (R8)(R8*2), R11") // 3*lda
b.Raw("LEAQ (R8)(R8*4), R12") // 5*lda
b.Raw(fmt.Sprintf("SHLQ $%d, R8", shift)) // lda -> bytes
b.Raw(fmt.Sprintf("SHLQ $%d, R13", shift)) // ldb -> bytes
b.Raw(fmt.Sprintf("SHLQ $%d, R9", shift)) // ldc -> bytes
b.Raw("LEAQ (R8)(R8*2), R11") // 3*lda
b.Raw("LEAQ (R8)(R8*4), R12") // 5*lda
b.Raw("MOVQ DX, R10")
for r := 0; r < mr; r++ {
b.Raw("PREFETCHT0 (R10)")
Expand All @@ -614,7 +620,7 @@ func gemmUAKernel(mr int, single bool) *emit.Function {
b.Raw(fmt.Sprintf("VFMADD231%s Y13, Y14, Y%d", sfx, 2*r+1))
}
b.Raw(fmt.Sprintf("ADDQ $%d, SI", size))
b.Raw("ADDQ $64, DI") // one packed B row: 16 floats or 8 doubles
b.Raw("ADDQ R13, DI") // next B row: ldb bytes on
b.Raw("DECQ CX")
b.Raw("JNZ uakloop")
b.Raw("uastore:")
Expand Down
8 changes: 5 additions & 3 deletions internal/kernels/asmgen/arm64/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -498,8 +498,8 @@ func gemmKernel() *emit.Function {
// Accumulators: float32 row r in V(2r), V(2r+1); float64 row r in V(4r)..V(4r+3).
func gemmUAKernel(mr int, single bool) *emit.Function {
sig := arm64.Layout(
[]string{"kc", "a", "lda", "pb", "c", "ldc"},
[]arm64.Type{arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Ptr, arm64.Int64},
[]string{"kc", "a", "lda", "pb", "ldb", "c", "ldc"},
[]arm64.Type{arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Int64},
nil, nil,
)
name, shift, arr, vecs := fmt.Sprintf("gemmUA%dx8D", mr), 3, "D2", 4
Expand All @@ -517,9 +517,11 @@ func gemmUAKernel(mr int, single bool) *emit.Function {
LoadArg("a", "R1").
LoadArg("lda", "R2").
LoadArg("pb", "R13").
LoadArg("ldb", "R16").
LoadArg("c", "R3").
LoadArg("ldc", "R4").
Raw(fmt.Sprintf("LSL $%d, R2, R2", shift)).
Raw(fmt.Sprintf("LSL $%d, R16, R16", shift)). // B rows ldb elements apart (BLIS's b += rs_b)
Raw(fmt.Sprintf("LSL $%d, R4, R4", shift))
for r := 0; r < mr; r++ { // row pointers R5.. = a + r*lda
if r == 0 {
Expand All @@ -537,7 +539,7 @@ func gemmUAKernel(mr int, single bool) *emit.Function {
fmla, scalar = fmlaElemS, "FMOVS"
}
step := func(lane int) {
b.Raw(fmt.Sprintf("VLD1.P %d(R13), %s", 8*size, blist))
b.Raw(fmt.Sprintf("VLD1.P (R13)(R16), %s", blist)) // post-increment by ldb
for r := 0; r < mr; r++ {
for v := 0; v < vecs; v++ {
b.Raw(fmla(vecs*r+v, 16+v, 20+r, lane))
Expand Down
20 changes: 13 additions & 7 deletions internal/kernels/fence_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,12 +139,14 @@ func TestGemmMicroStaysInsideItsOperands(t *testing.T) {
}
for kc := 1; kc <= 9; kc++ {
for _, ldc := range []int{NR, NR + 3} {
pb := fences(t, kc*NR)[1]
for mr := 1; mr <= MR; mr++ {
for _, lda := range []int{kc, kc + 5} {
for _, a := range fences(t, (mr-1)*lda+kc) {
for _, dst := range fences(t, (mr-1)*ldc+NR) {
gemm64.microUA(kc, a, lda, pb, dst, ldc, mr)
for _, ldb := range []int{NR, NR + 5} { // a packed panel, a row-major B in place
b := fences(t, (kc-1)*ldb+NR)[1]
for mr := 1; mr <= MR; mr++ {
for _, lda := range []int{kc, kc + 5} {
for _, a := range fences(t, (mr-1)*lda+kc) {
for _, dst := range fences(t, (mr-1)*ldc+NR) {
gemm64.microUA(kc, a, lda, b, ldb, dst, ldc, mr)
}
}
}
}
Expand All @@ -161,6 +163,9 @@ func TestGemmMicro32StaysInsideItsOperands(t *testing.T) {
t.Skip("no float32 GEMM kernel here")
}
for kc := 1; kc <= 9; kc++ {
// a row-major B read in place by microUA, rows g.nr+5 apart, ending
// flush against the guard
bs := fences32(t, (kc-1)*(g.nr+5)+g.nr)[1]
for _, ldc := range []int{g.nr, g.nr + 3, 64} {
for _, pa := range fences32(t, kc*g.mr) {
for _, pb := range fences32(t, kc*g.nr) {
Expand All @@ -174,7 +179,8 @@ func TestGemmMicro32StaysInsideItsOperands(t *testing.T) {
for _, lda := range []int{kc, kc + 5} {
for _, a := range fences32(t, (mr-1)*lda+kc) {
for _, dst := range fences32(t, (mr-1)*ldc+g.nr) {
g.microUA(kc, a, lda, pb, dst, ldc, mr)
g.microUA(kc, a, lda, pb, g.nr, dst, ldc, mr)
g.microUA(kc, a, lda, bs, g.nr+5, dst, ldc, mr)
}
}
}
Expand Down
53 changes: 38 additions & 15 deletions internal/kernels/gemm.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,17 +84,20 @@ func rowMajor[T gemmFloat](data []T, ld int) MatOf[T] {
//
// microUA, where there is one, is the same tile with A read in place: rows
// lda > 0 elements apart from a[0], k steps along each row, for the top mr
// (1..mr) rows only. gemmBand then never packs a row-major A.
// (1..mr) rows only, against B rows ldb elements apart (nr for a packed panel).
// gemmBand then never packs a row-major A, and the serial path reads the full
// panels of a row-major B in place too (see packGemmRows).
type gemmKernel[T gemmFloat] struct {
mr, nr int
micro func(kc int, pa, pb, dst []T, ldc int)
microUA func(kc int, a []T, lda int, pb, dst []T, ldc, mr int)
microUA func(kc int, a []T, lda int, b []T, ldb int, dst []T, ldc, mr int)
paPool, pbPool sync.Pool
}

// unpackA lets the GEMM read a row-major A in place where the kernel can. It
// is a var so tests can drive the packed path on the same inputs.
var unpackA = true
// unpackA lets the GEMM read a row-major A in place where the kernel can, and
// unpackB a row-major B on the serial path. They are vars so tests can drive
// the packed paths on the same inputs.
var unpackA, unpackB = true, true

// gemm64 is the float64 GEMM, on the per-architecture gemmMicro.
var gemm64 = &gemmKernel[float64]{mr: MR, nr: NR, micro: gemmMicro, microUA: gemmMicroUA}
Expand Down Expand Up @@ -159,14 +162,28 @@ func (g *gemmKernel[T]) putPackBuf(b *packBuf[T]) {
// r0 (the parallel splitter guarantees this) so packed A panels never straddle a
// band boundary; r1 may be m. This is the single-worker entry: it packs its own
// B panel. The multi-worker path (gemmBand) shares one packed B across workers.
//
// When both operands are row-major and the kernel reads A in place, B's full
// NR-wide panels are read in place as well (BLIS's sup kernels pack neither
// operand) and only the partial last panel is packed, for its zero padding.
// This is the small-product path (below GemmThreshold), where packing B was a
// measurable share of the time.
func (g *gemmKernel[T]) packGemmRows(r0, r1 int, dst []T, a, b MatOf[T], k, n int, buf *packBuf[T]) {
pa, pb := buf.pa, buf.pb
var bIn *MatOf[T]
if g.microUA != nil && unpackA && unpackB && a.CS == 1 && a.RS > 0 && b.CS == 1 && b.RS > 0 {
bIn = &b
}
for jc := 0; jc < n; jc += blockNC { // L3 column block of B
nc := min(blockNC, n-jc)
for pc := 0; pc < k; pc += blockKC { // L2 contraction block
kc := min(blockKC, k-pc)
g.packB(b, pc, kc, jc, nc, pb) // pack B(kc x nc) -> NR-wide panels
g.gemmBand(r0, r1, jc, nc, pc, kc, dst, a, n, pa, pb)
if bIn == nil {
g.packB(b, pc, kc, jc, nc, pb) // pack B(kc x nc) -> NR-wide panels
} else if full := nc - nc%g.nr; full < nc {
g.packB(b, pc, kc, jc+full, nc-full, pb[full*kc:]) // the partial last panel only
}
g.gemmBand(r0, r1, jc, nc, pc, kc, dst, a, n, pa, pb, bIn)
}
}
}
Expand All @@ -180,11 +197,14 @@ func (g *gemmKernel[T]) packGemmRows(r0, r1 int, dst []T, a, b MatOf[T], k, n in
// is what lets several workers share one packed B: the redundant full-B repack
// each worker used to do — 1 MiB copied GOMAXPROCS times per kc block — was the
// dominant non-scaling cost of the old row-split parallelism.
func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pa, pb []T) {
//
// bIn, when not nil, is a row-major B whose full panels the A-in-place kernel
// reads directly; pb then holds only the partial last panel (packGemmRows).
func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pa, pb []T, bIn *MatOf[T]) {
mr0, nr0 := g.mr, g.nr
tile := pa[len(pa)-mr0*nr0:] // see paCap
if g.microUA != nil && unpackA && a.CS == 1 && a.RS > 0 {
g.gemmBandUA(r0, r1, jc, nc, pc, kc, dst, a, n, pb, tile)
g.gemmBandUA(r0, r1, jc, nc, pc, kc, dst, a, n, pb, tile, bIn)
return
}
for ic := r0; ic < r1; ic += blockMC { // L1 row block of A
Expand All @@ -210,22 +230,25 @@ func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T]
// pc, a.RS apart. A tile narrower than NR (the last B panel, zero-padded) runs
// into the scratch tile and only its nr columns are added into dst; a tile
// shorter than MR uses the kernel for its own height, so no row past A's last
// is read.
func (g *gemmKernel[T]) gemmBandUA(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pb, tile []T) {
// is read. With bIn, a full panel's B is read in place, bIn.RS apart.
func (g *gemmKernel[T]) gemmBandUA(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pb, tile []T, bIn *MatOf[T]) {
mr0, nr0 := g.mr, g.nr
for jr := 0; jr < nc; jr += nr0 {
nr := min(nr0, nc-jr)
bp := pb[jr*kc:]
bp, ldb := pb[jr*kc:], nr0
if bIn != nil && nr == nr0 {
bp, ldb = bIn.Data[bIn.Off+pc*bIn.RS+jc+jr:], bIn.RS
}
for i := r0; i < r1; i += mr0 {
mr := min(mr0, r1-i)
ar := a.Data[a.Off+i*a.RS+pc:]
c0 := i*n + jc + jr
if nr == nr0 {
g.microUA(kc, ar, a.RS, bp, dst[c0:], n, mr)
g.microUA(kc, ar, a.RS, bp, ldb, dst[c0:], n, mr)
continue
}
clear(tile)
g.microUA(kc, ar, a.RS, bp, tile, nr0, mr)
g.microUA(kc, ar, a.RS, bp, ldb, tile, nr0, mr)
for r := 0; r < mr; r++ {
row := dst[c0+r*n : c0+r*n+nr]
for c := range row {
Expand Down Expand Up @@ -524,7 +547,7 @@ func (g *gemmKernel[T]) matMul(dst []T, a, b MatOf[T], m, k, n int) {
break
}
r0 := bi * bandRows
g.gemmBand(r0, min(r0+bandRows, m), bl.jc, bl.nc, bl.pc, bl.kc, dst, a, n, pa, pb)
g.gemmBand(r0, min(r0+bandRows, m), bl.jc, bl.nc, bl.pc, bl.kc, dst, a, n, pa, pb, nil)
pr.bandDone.Add(1)
}
}
Expand Down
25 changes: 13 additions & 12 deletions internal/kernels/gemm32_amd64.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,17 @@ func newGemm32(fma bool) *gemmKernel[float32] {
}

// gemmUA32 holds the unpacked-A kernels by tile height.
var gemmUA32 = [...]func(kc int, a *float32, lda int, pb, c *float32, ldc int){
var gemmUA32 = [...]func(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int){
1: gemmUA1x16FMA, 2: gemmUA2x16FMA, 3: gemmUA3x16FMA,
4: gemmUA4x16FMA, 5: gemmUA5x16FMA, 6: gemmUA6x16FMA,
}

// gemmMicroUA32 adds the mr x 16 tile sum_p a[r*lda+p]*pb[p*16..] into dst,
// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0).
func gemmMicroUA32(kc int, a []float32, lda int, pb, dst []float32, ldc, mr int) {
_, _, _ = a[(mr-1)*lda+kc-1], pb[kc*16-1], dst[(mr-1)*ldc+15] // the kernel's whole reach
gemmUA32[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc)
// gemmMicroUA32 adds the mr x 16 tile sum_p a[r*lda+p]*b[p*ldb..] into dst,
// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0) and B
// rows ldb > 0 apart (16 for a packed panel).
func gemmMicroUA32(kc int, a []float32, lda int, b []float32, ldb int, dst []float32, ldc, mr int) {
_, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+15], dst[(mr-1)*ldc+15] // the kernel's whole reach
gemmUA32[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc)
}

// gemmMicro32 adds the 6 x 16 tile sum_p pa[p*6..]*pb[p*16..] into the C block
Expand All @@ -37,19 +38,19 @@ func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) {
func gemmMicro6x16FMA(kc int, pa, pb, c *float32, ldc int)

//go:noescape
func gemmUA1x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA1x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA2x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA2x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA3x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA3x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA4x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA4x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA5x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA5x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA6x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA6x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)
29 changes: 15 additions & 14 deletions internal/kernels/gemm32_arm64.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,41 +5,42 @@ package kernels
var gemm32 = &gemmKernel[float32]{mr: 8, nr: 8, micro: gemmMicro32, microUA: gemmMicroUA32}

// gemmUA32 holds the unpacked-A kernels by tile height.
var gemmUA32 = [...]func(kc int, a *float32, lda int, pb, c *float32, ldc int){
var gemmUA32 = [...]func(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int){
1: gemmUA1x8S, 2: gemmUA2x8S, 3: gemmUA3x8S, 4: gemmUA4x8S,
5: gemmUA5x8S, 6: gemmUA6x8S, 7: gemmUA7x8S, 8: gemmUA8x8S,
}

// gemmMicroUA32 adds the mr x 8 tile sum_p a[r*lda+p]*pb[p*8..] into dst,
// whose rows are ldc apart, reading A in place (mr in 1..8, lda > 0).
func gemmMicroUA32(kc int, a []float32, lda int, pb, dst []float32, ldc, mr int) {
_, _, _ = a[(mr-1)*lda+kc-1], pb[kc*8-1], dst[(mr-1)*ldc+7] // the kernel's whole reach
gemmUA32[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc)
// gemmMicroUA32 adds the mr x 8 tile sum_p a[r*lda+p]*b[p*ldb..] into dst,
// whose rows are ldc apart, reading A in place (mr in 1..8, lda > 0) and B
// rows ldb > 0 apart (8 for a packed panel).
func gemmMicroUA32(kc int, a []float32, lda int, b []float32, ldb int, dst []float32, ldc, mr int) {
_, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+7], dst[(mr-1)*ldc+7] // the kernel's whole reach
gemmUA32[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc)
}

//go:noescape
func gemmUA1x8S(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA1x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA2x8S(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA2x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA3x8S(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA3x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA4x8S(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA4x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA5x8S(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA5x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA6x8S(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA6x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA7x8S(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA7x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

//go:noescape
func gemmUA8x8S(kc int, a *float32, lda int, pb, c *float32, ldc int)
func gemmUA8x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int)

// gemmMicro32 adds the 8 x 8 tile sum_p pa[p*8..]*pb[p*8..] into the C block at
// dst[0:], whose rows are ldc apart. pa, pb hold at least kc*8 packed
Expand Down
Loading
Loading