diff --git a/SECURITY.md b/SECURITY.md index c43c2b0..6d7c7b3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,15 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-31 (v0.29.1).** The A-in-place GEMM kernels now take B's row +stride (`ldb`) and may read a row-major B operand in place: `kc` rows `ldb` +elements apart, `NR` elements each. The wrappers bounds-check +`b[(kc-1)*ldb+NR-1]` before the call. The path is taken only for a unit +column stride and a positive row stride, and only for full panels; a partial +panel is still packed, so no column past B's last is read. The fence tests +place a strided B flush against a guard page. A sabotaged kernel (ignoring +`ldb`) failed the GEMM tests on arm64 and on amd64. + **2026-10-30 (v0.29.0).** The s390x float32 GEMM kernel (`gemmMicro4x16VXE`) reads `kc·4` and `kc·16` packed floats and one 4×16 tile of C. Its wrapper bounds-checks all three before the call, behind the diff --git a/docs/perf.md b/docs/perf.md index 71c3012..59085aa 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1309,8 +1309,30 @@ the float64 GEMM. |---|---:|---:|---:| | widened to the float64 GEMM → native | **−58%** | **−54%** | **−50%** | +### Reading B in place too, for small products (2026-10-31) + +BLIS's sup kernels pack neither operand: the "rv" kernel advances B by its +row stride (`b += rs_b`) as it advances A by its column stride. v0.29.1 does +the same on the small-product path (below `GemmThreshold`, one worker). When +both operands are row-major, the A-in-place kernels read B's full panels in +place too, advancing B by `ldb` instead of the packed panel's width. Only a +partial last panel is still packed, for its zero padding. The parallel path, +used for larger products, still packs B once and shares it across workers. + +| `MatMul` vs v0.29.0 (`abbench`, 10 rounds) | 64² | 256² | 1024² | +|---|---:|---:|---:| +| Zen 3, float32 | **−16%** | ~ | ~ | +| Zen 3, float64 | **−10%** | ~ | ~ | +| Neoverse N1, float32 | −7.3% | ~ | ~ | +| Neoverse N1, float64 | ~ | ~ | ~ | + +The N1 host was contended (load 31, rows up to ±39%), so its "~" rows show +no regression but are not a precise measurement. On Zen 3, float32 64² now +takes 8.02 µs against NumPy's 8.14: **parity**. float64 takes 16.3 against +15.8 µs (1.03×). + **Open defect: products are still up to 1.2× slower than OpenBLAS on one -core.** The exception is float64 at 1024² on Zen 3, where we are 3% faster. +core** at 256² and 1024² on N1, and float32 1024² on Zen 3 is at 1.04×. The exception is float64 at 1024² on Zen 3, where we are 3% faster. The float32 kernels for riscv64 and loong64 are still to write. **Open: the allocating operations.** `Add` is still 1.5–3.2× slower than diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index 9c46f24..e1edbe4 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -563,9 +563,13 @@ func gemm32FMAKernel() *emit.Function { // each A value is broadcast straight from its row, r*lda bytes past the // k-step's column, through a scaled index (lda, 2lda, 3lda in R11, 4lda, // 5lda in R12), and the column advances 4 bytes per step. A small product -// then pays no copy of A at all. B stays packed, so a partial last panel is -// still zero-padded; the kernels for mr < 6 cover the bottom rows of a block -// without reading past A's last row. +// then pays no copy of A at all. The kernels for mr < 6 cover the bottom rows +// of a block without reading past A's last row. +// +// B's rows are ldb elements apart (R13 in bytes), as BLIS's b += rs_b: a +// packed panel passes its own width (16 or 8), and a full panel of a +// row-major B is read in place with ldb its row stride. A partial last panel +// is still packed, for its zero padding. // // gemmUAKernel(mr, false) is the float64 counterpart over the 6x8 packing, // gemmUAx8FMA: the same loop in PD instructions, 8-byte steps. @@ -573,8 +577,8 @@ func gemm32UAKernel(mr int) *emit.Function { return gemmUAKernel(mr, true) } func gemmUAKernel(mr int, single bool) *emit.Function { sig := amd64.Layout( - []string{"kc", "a", "lda", "pb", "c", "ldc"}, - []amd64.Type{amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Ptr, amd64.Int64}, + []string{"kc", "a", "lda", "pb", "ldb", "c", "ldc"}, + []amd64.Type{amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Int64}, nil, nil, ) name, size, shift, sfx, bcast := fmt.Sprintf("gemmUA%dx8FMA", mr), 8, 3, "PD", "VBROADCASTSD" @@ -586,12 +590,14 @@ func gemmUAKernel(mr int, single bool) *emit.Function { b.LoadArg("a", "SI") b.LoadArg("lda", "R8") b.LoadArg("pb", "DI") + b.LoadArg("ldb", "R13") b.LoadArg("c", "DX") b.LoadArg("ldc", "R9") - b.Raw(fmt.Sprintf("SHLQ $%d, R8", shift)) // lda -> bytes - b.Raw(fmt.Sprintf("SHLQ $%d, R9", shift)) // ldc -> bytes - b.Raw("LEAQ (R8)(R8*2), R11") // 3*lda - b.Raw("LEAQ (R8)(R8*4), R12") // 5*lda + b.Raw(fmt.Sprintf("SHLQ $%d, R8", shift)) // lda -> bytes + b.Raw(fmt.Sprintf("SHLQ $%d, R13", shift)) // ldb -> bytes + b.Raw(fmt.Sprintf("SHLQ $%d, R9", shift)) // ldc -> bytes + b.Raw("LEAQ (R8)(R8*2), R11") // 3*lda + b.Raw("LEAQ (R8)(R8*4), R12") // 5*lda b.Raw("MOVQ DX, R10") for r := 0; r < mr; r++ { b.Raw("PREFETCHT0 (R10)") @@ -614,7 +620,7 @@ func gemmUAKernel(mr int, single bool) *emit.Function { b.Raw(fmt.Sprintf("VFMADD231%s Y13, Y14, Y%d", sfx, 2*r+1)) } b.Raw(fmt.Sprintf("ADDQ $%d, SI", size)) - b.Raw("ADDQ $64, DI") // one packed B row: 16 floats or 8 doubles + b.Raw("ADDQ R13, DI") // next B row: ldb bytes on b.Raw("DECQ CX") b.Raw("JNZ uakloop") b.Raw("uastore:") diff --git a/internal/kernels/asmgen/arm64/gen.go b/internal/kernels/asmgen/arm64/gen.go index 479765b..d07eb3f 100644 --- a/internal/kernels/asmgen/arm64/gen.go +++ b/internal/kernels/asmgen/arm64/gen.go @@ -498,8 +498,8 @@ func gemmKernel() *emit.Function { // Accumulators: float32 row r in V(2r), V(2r+1); float64 row r in V(4r)..V(4r+3). func gemmUAKernel(mr int, single bool) *emit.Function { sig := arm64.Layout( - []string{"kc", "a", "lda", "pb", "c", "ldc"}, - []arm64.Type{arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Ptr, arm64.Int64}, + []string{"kc", "a", "lda", "pb", "ldb", "c", "ldc"}, + []arm64.Type{arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Int64}, nil, nil, ) name, shift, arr, vecs := fmt.Sprintf("gemmUA%dx8D", mr), 3, "D2", 4 @@ -517,9 +517,11 @@ func gemmUAKernel(mr int, single bool) *emit.Function { LoadArg("a", "R1"). LoadArg("lda", "R2"). LoadArg("pb", "R13"). + LoadArg("ldb", "R16"). LoadArg("c", "R3"). LoadArg("ldc", "R4"). Raw(fmt.Sprintf("LSL $%d, R2, R2", shift)). + Raw(fmt.Sprintf("LSL $%d, R16, R16", shift)). // B rows ldb elements apart (BLIS's b += rs_b) Raw(fmt.Sprintf("LSL $%d, R4, R4", shift)) for r := 0; r < mr; r++ { // row pointers R5.. = a + r*lda if r == 0 { @@ -537,7 +539,7 @@ func gemmUAKernel(mr int, single bool) *emit.Function { fmla, scalar = fmlaElemS, "FMOVS" } step := func(lane int) { - b.Raw(fmt.Sprintf("VLD1.P %d(R13), %s", 8*size, blist)) + b.Raw(fmt.Sprintf("VLD1.P (R13)(R16), %s", blist)) // post-increment by ldb for r := 0; r < mr; r++ { for v := 0; v < vecs; v++ { b.Raw(fmla(vecs*r+v, 16+v, 20+r, lane)) diff --git a/internal/kernels/fence_test.go b/internal/kernels/fence_test.go index 2bbcbff..579391d 100644 --- a/internal/kernels/fence_test.go +++ b/internal/kernels/fence_test.go @@ -139,12 +139,14 @@ func TestGemmMicroStaysInsideItsOperands(t *testing.T) { } for kc := 1; kc <= 9; kc++ { for _, ldc := range []int{NR, NR + 3} { - pb := fences(t, kc*NR)[1] - for mr := 1; mr <= MR; mr++ { - for _, lda := range []int{kc, kc + 5} { - for _, a := range fences(t, (mr-1)*lda+kc) { - for _, dst := range fences(t, (mr-1)*ldc+NR) { - gemm64.microUA(kc, a, lda, pb, dst, ldc, mr) + for _, ldb := range []int{NR, NR + 5} { // a packed panel, a row-major B in place + b := fences(t, (kc-1)*ldb+NR)[1] + for mr := 1; mr <= MR; mr++ { + for _, lda := range []int{kc, kc + 5} { + for _, a := range fences(t, (mr-1)*lda+kc) { + for _, dst := range fences(t, (mr-1)*ldc+NR) { + gemm64.microUA(kc, a, lda, b, ldb, dst, ldc, mr) + } } } } @@ -161,6 +163,9 @@ func TestGemmMicro32StaysInsideItsOperands(t *testing.T) { t.Skip("no float32 GEMM kernel here") } for kc := 1; kc <= 9; kc++ { + // a row-major B read in place by microUA, rows g.nr+5 apart, ending + // flush against the guard + bs := fences32(t, (kc-1)*(g.nr+5)+g.nr)[1] for _, ldc := range []int{g.nr, g.nr + 3, 64} { for _, pa := range fences32(t, kc*g.mr) { for _, pb := range fences32(t, kc*g.nr) { @@ -174,7 +179,8 @@ func TestGemmMicro32StaysInsideItsOperands(t *testing.T) { for _, lda := range []int{kc, kc + 5} { for _, a := range fences32(t, (mr-1)*lda+kc) { for _, dst := range fences32(t, (mr-1)*ldc+g.nr) { - g.microUA(kc, a, lda, pb, dst, ldc, mr) + g.microUA(kc, a, lda, pb, g.nr, dst, ldc, mr) + g.microUA(kc, a, lda, bs, g.nr+5, dst, ldc, mr) } } } diff --git a/internal/kernels/gemm.go b/internal/kernels/gemm.go index 6ee79d4..90f8833 100644 --- a/internal/kernels/gemm.go +++ b/internal/kernels/gemm.go @@ -84,17 +84,20 @@ func rowMajor[T gemmFloat](data []T, ld int) MatOf[T] { // // microUA, where there is one, is the same tile with A read in place: rows // lda > 0 elements apart from a[0], k steps along each row, for the top mr -// (1..mr) rows only. gemmBand then never packs a row-major A. +// (1..mr) rows only, against B rows ldb elements apart (nr for a packed panel). +// gemmBand then never packs a row-major A, and the serial path reads the full +// panels of a row-major B in place too (see packGemmRows). type gemmKernel[T gemmFloat] struct { mr, nr int micro func(kc int, pa, pb, dst []T, ldc int) - microUA func(kc int, a []T, lda int, pb, dst []T, ldc, mr int) + microUA func(kc int, a []T, lda int, b []T, ldb int, dst []T, ldc, mr int) paPool, pbPool sync.Pool } -// unpackA lets the GEMM read a row-major A in place where the kernel can. It -// is a var so tests can drive the packed path on the same inputs. -var unpackA = true +// unpackA lets the GEMM read a row-major A in place where the kernel can, and +// unpackB a row-major B on the serial path. They are vars so tests can drive +// the packed paths on the same inputs. +var unpackA, unpackB = true, true // gemm64 is the float64 GEMM, on the per-architecture gemmMicro. var gemm64 = &gemmKernel[float64]{mr: MR, nr: NR, micro: gemmMicro, microUA: gemmMicroUA} @@ -159,14 +162,28 @@ func (g *gemmKernel[T]) putPackBuf(b *packBuf[T]) { // r0 (the parallel splitter guarantees this) so packed A panels never straddle a // band boundary; r1 may be m. This is the single-worker entry: it packs its own // B panel. The multi-worker path (gemmBand) shares one packed B across workers. +// +// When both operands are row-major and the kernel reads A in place, B's full +// NR-wide panels are read in place as well (BLIS's sup kernels pack neither +// operand) and only the partial last panel is packed, for its zero padding. +// This is the small-product path (below GemmThreshold), where packing B was a +// measurable share of the time. func (g *gemmKernel[T]) packGemmRows(r0, r1 int, dst []T, a, b MatOf[T], k, n int, buf *packBuf[T]) { pa, pb := buf.pa, buf.pb + var bIn *MatOf[T] + if g.microUA != nil && unpackA && unpackB && a.CS == 1 && a.RS > 0 && b.CS == 1 && b.RS > 0 { + bIn = &b + } for jc := 0; jc < n; jc += blockNC { // L3 column block of B nc := min(blockNC, n-jc) for pc := 0; pc < k; pc += blockKC { // L2 contraction block kc := min(blockKC, k-pc) - g.packB(b, pc, kc, jc, nc, pb) // pack B(kc x nc) -> NR-wide panels - g.gemmBand(r0, r1, jc, nc, pc, kc, dst, a, n, pa, pb) + if bIn == nil { + g.packB(b, pc, kc, jc, nc, pb) // pack B(kc x nc) -> NR-wide panels + } else if full := nc - nc%g.nr; full < nc { + g.packB(b, pc, kc, jc+full, nc-full, pb[full*kc:]) // the partial last panel only + } + g.gemmBand(r0, r1, jc, nc, pc, kc, dst, a, n, pa, pb, bIn) } } } @@ -180,11 +197,14 @@ func (g *gemmKernel[T]) packGemmRows(r0, r1 int, dst []T, a, b MatOf[T], k, n in // is what lets several workers share one packed B: the redundant full-B repack // each worker used to do — 1 MiB copied GOMAXPROCS times per kc block — was the // dominant non-scaling cost of the old row-split parallelism. -func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pa, pb []T) { +// +// bIn, when not nil, is a row-major B whose full panels the A-in-place kernel +// reads directly; pb then holds only the partial last panel (packGemmRows). +func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pa, pb []T, bIn *MatOf[T]) { mr0, nr0 := g.mr, g.nr tile := pa[len(pa)-mr0*nr0:] // see paCap if g.microUA != nil && unpackA && a.CS == 1 && a.RS > 0 { - g.gemmBandUA(r0, r1, jc, nc, pc, kc, dst, a, n, pb, tile) + g.gemmBandUA(r0, r1, jc, nc, pc, kc, dst, a, n, pb, tile, bIn) return } for ic := r0; ic < r1; ic += blockMC { // L1 row block of A @@ -210,22 +230,25 @@ func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T] // pc, a.RS apart. A tile narrower than NR (the last B panel, zero-padded) runs // into the scratch tile and only its nr columns are added into dst; a tile // shorter than MR uses the kernel for its own height, so no row past A's last -// is read. -func (g *gemmKernel[T]) gemmBandUA(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pb, tile []T) { +// is read. With bIn, a full panel's B is read in place, bIn.RS apart. +func (g *gemmKernel[T]) gemmBandUA(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pb, tile []T, bIn *MatOf[T]) { mr0, nr0 := g.mr, g.nr for jr := 0; jr < nc; jr += nr0 { nr := min(nr0, nc-jr) - bp := pb[jr*kc:] + bp, ldb := pb[jr*kc:], nr0 + if bIn != nil && nr == nr0 { + bp, ldb = bIn.Data[bIn.Off+pc*bIn.RS+jc+jr:], bIn.RS + } for i := r0; i < r1; i += mr0 { mr := min(mr0, r1-i) ar := a.Data[a.Off+i*a.RS+pc:] c0 := i*n + jc + jr if nr == nr0 { - g.microUA(kc, ar, a.RS, bp, dst[c0:], n, mr) + g.microUA(kc, ar, a.RS, bp, ldb, dst[c0:], n, mr) continue } clear(tile) - g.microUA(kc, ar, a.RS, bp, tile, nr0, mr) + g.microUA(kc, ar, a.RS, bp, ldb, tile, nr0, mr) for r := 0; r < mr; r++ { row := dst[c0+r*n : c0+r*n+nr] for c := range row { @@ -524,7 +547,7 @@ func (g *gemmKernel[T]) matMul(dst []T, a, b MatOf[T], m, k, n int) { break } r0 := bi * bandRows - g.gemmBand(r0, min(r0+bandRows, m), bl.jc, bl.nc, bl.pc, bl.kc, dst, a, n, pa, pb) + g.gemmBand(r0, min(r0+bandRows, m), bl.jc, bl.nc, bl.pc, bl.kc, dst, a, n, pa, pb, nil) pr.bandDone.Add(1) } } diff --git a/internal/kernels/gemm32_amd64.go b/internal/kernels/gemm32_amd64.go index daac694..cadad56 100644 --- a/internal/kernels/gemm32_amd64.go +++ b/internal/kernels/gemm32_amd64.go @@ -13,16 +13,17 @@ func newGemm32(fma bool) *gemmKernel[float32] { } // gemmUA32 holds the unpacked-A kernels by tile height. -var gemmUA32 = [...]func(kc int, a *float32, lda int, pb, c *float32, ldc int){ +var gemmUA32 = [...]func(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int){ 1: gemmUA1x16FMA, 2: gemmUA2x16FMA, 3: gemmUA3x16FMA, 4: gemmUA4x16FMA, 5: gemmUA5x16FMA, 6: gemmUA6x16FMA, } -// gemmMicroUA32 adds the mr x 16 tile sum_p a[r*lda+p]*pb[p*16..] into dst, -// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0). -func gemmMicroUA32(kc int, a []float32, lda int, pb, dst []float32, ldc, mr int) { - _, _, _ = a[(mr-1)*lda+kc-1], pb[kc*16-1], dst[(mr-1)*ldc+15] // the kernel's whole reach - gemmUA32[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc) +// gemmMicroUA32 adds the mr x 16 tile sum_p a[r*lda+p]*b[p*ldb..] into dst, +// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0) and B +// rows ldb > 0 apart (16 for a packed panel). +func gemmMicroUA32(kc int, a []float32, lda int, b []float32, ldb int, dst []float32, ldc, mr int) { + _, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+15], dst[(mr-1)*ldc+15] // the kernel's whole reach + gemmUA32[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc) } // gemmMicro32 adds the 6 x 16 tile sum_p pa[p*6..]*pb[p*16..] into the C block @@ -37,19 +38,19 @@ func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { func gemmMicro6x16FMA(kc int, pa, pb, c *float32, ldc int) //go:noescape -func gemmUA1x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA1x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA2x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA2x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA3x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA3x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA4x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA4x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA5x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA5x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA6x16FMA(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA6x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) diff --git a/internal/kernels/gemm32_arm64.go b/internal/kernels/gemm32_arm64.go index b1bcde5..155ff42 100644 --- a/internal/kernels/gemm32_arm64.go +++ b/internal/kernels/gemm32_arm64.go @@ -5,41 +5,42 @@ package kernels var gemm32 = &gemmKernel[float32]{mr: 8, nr: 8, micro: gemmMicro32, microUA: gemmMicroUA32} // gemmUA32 holds the unpacked-A kernels by tile height. -var gemmUA32 = [...]func(kc int, a *float32, lda int, pb, c *float32, ldc int){ +var gemmUA32 = [...]func(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int){ 1: gemmUA1x8S, 2: gemmUA2x8S, 3: gemmUA3x8S, 4: gemmUA4x8S, 5: gemmUA5x8S, 6: gemmUA6x8S, 7: gemmUA7x8S, 8: gemmUA8x8S, } -// gemmMicroUA32 adds the mr x 8 tile sum_p a[r*lda+p]*pb[p*8..] into dst, -// whose rows are ldc apart, reading A in place (mr in 1..8, lda > 0). -func gemmMicroUA32(kc int, a []float32, lda int, pb, dst []float32, ldc, mr int) { - _, _, _ = a[(mr-1)*lda+kc-1], pb[kc*8-1], dst[(mr-1)*ldc+7] // the kernel's whole reach - gemmUA32[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc) +// gemmMicroUA32 adds the mr x 8 tile sum_p a[r*lda+p]*b[p*ldb..] into dst, +// whose rows are ldc apart, reading A in place (mr in 1..8, lda > 0) and B +// rows ldb > 0 apart (8 for a packed panel). +func gemmMicroUA32(kc int, a []float32, lda int, b []float32, ldb int, dst []float32, ldc, mr int) { + _, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+7], dst[(mr-1)*ldc+7] // the kernel's whole reach + gemmUA32[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc) } //go:noescape -func gemmUA1x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA1x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA2x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA2x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA3x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA3x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA4x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA4x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA5x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA5x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA6x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA6x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA7x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA7x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) //go:noescape -func gemmUA8x8S(kc int, a *float32, lda int, pb, c *float32, ldc int) +func gemmUA8x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) // gemmMicro32 adds the 8 x 8 tile sum_p pa[p*8..]*pb[p*8..] into the C block at // dst[0:], whose rows are ldc apart. pa, pb hold at least kc*8 packed diff --git a/internal/kernels/gemm32_test.go b/internal/kernels/gemm32_test.go index a2202dd..48e1f56 100644 --- a/internal/kernels/gemm32_test.go +++ b/internal/kernels/gemm32_test.go @@ -35,18 +35,18 @@ func assertGemm32(t *testing.T, m, k, n int, a, b Mat32) { t.Helper() want := make([]float32, m*n) gemmOracle32(want, a, b, m, k, n) - saved := unpackA - defer func() { unpackA = saved }() - for _, unpacked := range []bool{true, false} { - unpackA = unpacked + savedA, savedB := unpackA, unpackB + defer func() { unpackA, unpackB = savedA, savedB }() + for _, p := range [][2]bool{{true, true}, {true, false}, {false, false}} { + unpackA, unpackB = p[0], p[1] for _, gemm := range []int{1 << 30, 1} { withThresholds(1<<20, gemm, func() { got := make([]float32, m*n) MatMul32StridedP(got, a, b, m, k, n) for i := range want { if got[i] != want[i] { - t.Fatalf("m=%d k=%d n=%d gemm=%d unpackA=%v [%d]: %v != %v", - m, k, n, gemm, unpacked, i, got[i], want[i]) + t.Fatalf("m=%d k=%d n=%d gemm=%d unpackA,B=%v [%d]: %v != %v", + m, k, n, gemm, p, i, got[i], want[i]) } } }) diff --git a/internal/kernels/gemm_amd64.go b/internal/kernels/gemm_amd64.go index 62e91d9..0315ca9 100644 --- a/internal/kernels/gemm_amd64.go +++ b/internal/kernels/gemm_amd64.go @@ -45,7 +45,7 @@ func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { // FMA CPUs only; without FMA, A is packed for the SSE2 kernel. var gemmMicroUA = microUAIf(useFMA) -func microUAIf(fma bool) func(kc int, a []float64, lda int, pb, dst []float64, ldc, mr int) { +func microUAIf(fma bool) func(kc int, a []float64, lda int, b []float64, ldb int, dst []float64, ldc, mr int) { if !fma { return nil } @@ -53,35 +53,36 @@ func microUAIf(fma bool) func(kc int, a []float64, lda int, pb, dst []float64, l } // gemmUA64 holds the unpacked-A kernels by tile height. -var gemmUA64 = [...]func(kc int, a *float64, lda int, pb, c *float64, ldc int){ +var gemmUA64 = [...]func(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int){ 1: gemmUA1x8FMA, 2: gemmUA2x8FMA, 3: gemmUA3x8FMA, 4: gemmUA4x8FMA, 5: gemmUA5x8FMA, 6: gemmUA6x8FMA, } -// gemmMicroUA64 adds the mr x 8 tile sum_p a[r*lda+p]*pb[p*8..] into dst, -// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0). -func gemmMicroUA64(kc int, a []float64, lda int, pb, dst []float64, ldc, mr int) { - _, _, _ = a[(mr-1)*lda+kc-1], pb[kc*NR-1], dst[(mr-1)*ldc+NR-1] // the kernel's whole reach - gemmUA64[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc) +// gemmMicroUA64 adds the mr x 8 tile sum_p a[r*lda+p]*b[p*ldb..] into dst, +// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0) and B +// rows ldb > 0 apart (NR for a packed panel). +func gemmMicroUA64(kc int, a []float64, lda int, b []float64, ldb int, dst []float64, ldc, mr int) { + _, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+NR-1], dst[(mr-1)*ldc+NR-1] // the kernel's whole reach + gemmUA64[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc) } //go:noescape -func gemmUA1x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA1x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape -func gemmUA2x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA2x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape -func gemmUA3x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA3x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape -func gemmUA4x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA4x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape -func gemmUA5x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA5x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape -func gemmUA6x8FMA(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA6x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape func gemmMicro6x8FMA(kc int, pa, pb, c *float64, ldc int) diff --git a/internal/kernels/gemm_arm64.go b/internal/kernels/gemm_arm64.go index 406e82b..ea9f71d 100644 --- a/internal/kernels/gemm_arm64.go +++ b/internal/kernels/gemm_arm64.go @@ -46,25 +46,26 @@ func gemmMicro4x8(kc int, pa, pb, c *float64, ldc int) var gemmMicroUA = gemmMicroUA64 // gemmUA64 holds the unpacked-A kernels by tile height. -var gemmUA64 = [...]func(kc int, a *float64, lda int, pb, c *float64, ldc int){ +var gemmUA64 = [...]func(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int){ 1: gemmUA1x8D, 2: gemmUA2x8D, 3: gemmUA3x8D, 4: gemmUA4x8D, } -// gemmMicroUA64 adds the mr x 8 tile sum_p a[r*lda+p]*pb[p*8..] into dst, -// whose rows are ldc apart, reading A in place (mr in 1..4, lda > 0). -func gemmMicroUA64(kc int, a []float64, lda int, pb, dst []float64, ldc, mr int) { - _, _, _ = a[(mr-1)*lda+kc-1], pb[kc*NR-1], dst[(mr-1)*ldc+NR-1] // the kernel's whole reach - gemmUA64[mr](kc, &a[0], lda, &pb[0], &dst[0], ldc) +// gemmMicroUA64 adds the mr x 8 tile sum_p a[r*lda+p]*b[p*ldb..] into dst, +// whose rows are ldc apart, reading A in place (mr in 1..4, lda > 0) and B +// rows ldb > 0 apart (NR for a packed panel). +func gemmMicroUA64(kc int, a []float64, lda int, b []float64, ldb int, dst []float64, ldc, mr int) { + _, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+NR-1], dst[(mr-1)*ldc+NR-1] // the kernel's whole reach + gemmUA64[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc) } //go:noescape -func gemmUA1x8D(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA1x8D(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape -func gemmUA2x8D(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA2x8D(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape -func gemmUA3x8D(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA3x8D(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) //go:noescape -func gemmUA4x8D(kc int, a *float64, lda int, pb, c *float64, ldc int) +func gemmUA4x8D(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) diff --git a/internal/kernels/gemm_test.go b/internal/kernels/gemm_test.go index 1ec642c..d8ec9fe 100644 --- a/internal/kernels/gemm_test.go +++ b/internal/kernels/gemm_test.go @@ -42,10 +42,10 @@ func withBandTiles(bt int, f func()) { // that make the ikj order reproduce the oracle exactly. func assertGemm(t *testing.T, m, k, n int, a, b []float64) { t.Helper() - saved := unpackA - defer func() { unpackA = saved }() - for _, unpacked := range []bool{saved, false} { - unpackA = unpacked + savedA, savedB := unpackA, unpackB + defer func() { unpackA, unpackB = savedA, savedB }() + for _, p := range [][2]bool{{savedA, savedB}, {savedA, false}, {false, false}} { + unpackA, unpackB = p[0], p[1] assertGemmPath(t, m, k, n, a, b) } // A in place behind an offset, rows k+3 apart (lda != k). diff --git a/internal/kernels/gemmua_other.go b/internal/kernels/gemmua_other.go index 190943d..5b53a81 100644 --- a/internal/kernels/gemmua_other.go +++ b/internal/kernels/gemmua_other.go @@ -3,4 +3,4 @@ package kernels // gemmMicroUA is nil here: no unpacked-A float64 kernel, so A is packed. -var gemmMicroUA func(kc int, a []float64, lda int, pb, dst []float64, ldc, mr int) +var gemmMicroUA func(kc int, a []float64, lda int, b []float64, ldb int, dst []float64, ldc, mr int) diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index fa71a7e..7c1bd88 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -677,14 +677,16 @@ TEXT ·gemmMicro6x16FMA(SB), NOSPLIT, $0-40 VZEROUPPER RET -TEXT ·gemmUA1x16FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA1x16FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $2, R8 + SHLQ $2, R13 SHLQ $2, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -702,7 +704,7 @@ TEXT ·gemmUA1x16FMA(SB), NOSPLIT, $0-48 VFMADD231PS Y12, Y14, Y0 VFMADD231PS Y13, Y14, Y1 ADDQ $4, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -713,14 +715,16 @@ TEXT ·gemmUA1x16FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA1x8FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA1x8FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $3, R8 + SHLQ $3, R13 SHLQ $3, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -738,7 +742,7 @@ TEXT ·gemmUA1x8FMA(SB), NOSPLIT, $0-48 VFMADD231PD Y12, Y14, Y0 VFMADD231PD Y13, Y14, Y1 ADDQ $8, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -749,14 +753,16 @@ TEXT ·gemmUA1x8FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA2x16FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA2x16FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $2, R8 + SHLQ $2, R13 SHLQ $2, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -782,7 +788,7 @@ TEXT ·gemmUA2x16FMA(SB), NOSPLIT, $0-48 VFMADD231PS Y12, Y14, Y2 VFMADD231PS Y13, Y14, Y3 ADDQ $4, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -798,14 +804,16 @@ TEXT ·gemmUA2x16FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA2x8FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA2x8FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $3, R8 + SHLQ $3, R13 SHLQ $3, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -831,7 +839,7 @@ TEXT ·gemmUA2x8FMA(SB), NOSPLIT, $0-48 VFMADD231PD Y12, Y14, Y2 VFMADD231PD Y13, Y14, Y3 ADDQ $8, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -847,14 +855,16 @@ TEXT ·gemmUA2x8FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA3x16FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA3x16FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $2, R8 + SHLQ $2, R13 SHLQ $2, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -888,7 +898,7 @@ TEXT ·gemmUA3x16FMA(SB), NOSPLIT, $0-48 VFMADD231PS Y12, Y14, Y4 VFMADD231PS Y13, Y14, Y5 ADDQ $4, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -909,14 +919,16 @@ TEXT ·gemmUA3x16FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA3x8FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA3x8FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $3, R8 + SHLQ $3, R13 SHLQ $3, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -950,7 +962,7 @@ TEXT ·gemmUA3x8FMA(SB), NOSPLIT, $0-48 VFMADD231PD Y12, Y14, Y4 VFMADD231PD Y13, Y14, Y5 ADDQ $8, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -971,14 +983,16 @@ TEXT ·gemmUA3x8FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA4x16FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA4x16FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $2, R8 + SHLQ $2, R13 SHLQ $2, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -1020,7 +1034,7 @@ TEXT ·gemmUA4x16FMA(SB), NOSPLIT, $0-48 VFMADD231PS Y12, Y14, Y6 VFMADD231PS Y13, Y14, Y7 ADDQ $4, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -1046,14 +1060,16 @@ TEXT ·gemmUA4x16FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA4x8FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA4x8FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $3, R8 + SHLQ $3, R13 SHLQ $3, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -1095,7 +1111,7 @@ TEXT ·gemmUA4x8FMA(SB), NOSPLIT, $0-48 VFMADD231PD Y12, Y14, Y6 VFMADD231PD Y13, Y14, Y7 ADDQ $8, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -1121,14 +1137,16 @@ TEXT ·gemmUA4x8FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA5x16FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA5x16FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $2, R8 + SHLQ $2, R13 SHLQ $2, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -1178,7 +1196,7 @@ TEXT ·gemmUA5x16FMA(SB), NOSPLIT, $0-48 VFMADD231PS Y12, Y14, Y8 VFMADD231PS Y13, Y14, Y9 ADDQ $4, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -1209,14 +1227,16 @@ TEXT ·gemmUA5x16FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA5x8FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA5x8FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $3, R8 + SHLQ $3, R13 SHLQ $3, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -1266,7 +1286,7 @@ TEXT ·gemmUA5x8FMA(SB), NOSPLIT, $0-48 VFMADD231PD Y12, Y14, Y8 VFMADD231PD Y13, Y14, Y9 ADDQ $8, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -1297,14 +1317,16 @@ TEXT ·gemmUA5x8FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA6x16FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA6x16FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $2, R8 + SHLQ $2, R13 SHLQ $2, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -1362,7 +1384,7 @@ TEXT ·gemmUA6x16FMA(SB), NOSPLIT, $0-48 VFMADD231PS Y12, Y14, Y10 VFMADD231PS Y13, Y14, Y11 ADDQ $4, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: @@ -1398,14 +1420,16 @@ TEXT ·gemmUA6x16FMA(SB), NOSPLIT, $0-48 VZEROUPPER RET -TEXT ·gemmUA6x8FMA(SB), NOSPLIT, $0-48 +TEXT ·gemmUA6x8FMA(SB), NOSPLIT, $0-56 MOVQ kc+0(FP), CX MOVQ a+8(FP), SI MOVQ lda+16(FP), R8 MOVQ pb+24(FP), DI - MOVQ c+32(FP), DX - MOVQ ldc+40(FP), R9 + MOVQ ldb+32(FP), R13 + MOVQ c+40(FP), DX + MOVQ ldc+48(FP), R9 SHLQ $3, R8 + SHLQ $3, R13 SHLQ $3, R9 LEAQ (R8)(R8*2), R11 LEAQ (R8)(R8*4), R12 @@ -1463,7 +1487,7 @@ TEXT ·gemmUA6x8FMA(SB), NOSPLIT, $0-48 VFMADD231PD Y12, Y14, Y10 VFMADD231PD Y13, Y14, Y11 ADDQ $8, SI - ADDQ $64, DI + ADDQ R13, DI DECQ CX JNZ uakloop uastore: diff --git a/internal/kernels/sum_arm64.s b/internal/kernels/sum_arm64.s index 8588dc0..1075603 100644 --- a/internal/kernels/sum_arm64.s +++ b/internal/kernels/sum_arm64.s @@ -687,14 +687,16 @@ TEXT ·gemmMicro8x8S(SB), NOSPLIT, $0-40 VST1 [V16.S4, V17.S4], (R3) RET -TEXT ·gemmUA1x8S(SB), NOSPLIT, $0-48 +TEXT ·gemmUA1x8S(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $2, R2, R2 + LSL $2, R16, R16 LSL $2, R4, R4 MOVD R1, R5 VEOR V0.B16, V0.B16, V0.B16 @@ -704,16 +706,16 @@ TEXT ·gemmUA1x8S(SB), NOSPLIT, $0-48 CBZ R14, uatail uakloop: VLD1.P 16(R5), [V20.S4] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] SUB $1, R14 @@ -722,7 +724,7 @@ TEXT ·gemmUA1x8S(SB), NOSPLIT, $0-48 CBZ R15, uastore uatloop: FMOVS.P 4(R5), F20 - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] SUB $1, R15 @@ -734,14 +736,16 @@ TEXT ·gemmUA1x8S(SB), NOSPLIT, $0-48 VST1 [V16.S4, V17.S4], (R3) RET -TEXT ·gemmUA1x8D(SB), NOSPLIT, $0-48 +TEXT ·gemmUA1x8D(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $3, R2, R2 + LSL $3, R16, R16 LSL $3, R4, R4 MOVD R1, R5 VEOR V0.B16, V0.B16, V0.B16 @@ -753,12 +757,12 @@ TEXT ·gemmUA1x8D(SB), NOSPLIT, $0-48 CBZ R14, uatail uakloop: VLD1.P 16(R5), [V20.D2] - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] @@ -769,7 +773,7 @@ TEXT ·gemmUA1x8D(SB), NOSPLIT, $0-48 CBZ R15, uastore uatloop: FMOVD.P 8(R5), F20 - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] @@ -785,14 +789,16 @@ TEXT ·gemmUA1x8D(SB), NOSPLIT, $0-48 VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) RET -TEXT ·gemmUA2x8S(SB), NOSPLIT, $0-48 +TEXT ·gemmUA2x8S(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $2, R2, R2 + LSL $2, R16, R16 LSL $2, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -806,22 +812,22 @@ TEXT ·gemmUA2x8S(SB), NOSPLIT, $0-48 uakloop: VLD1.P 16(R5), [V20.S4] VLD1.P 16(R6), [V21.S4] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] @@ -833,7 +839,7 @@ TEXT ·gemmUA2x8S(SB), NOSPLIT, $0-48 uatloop: FMOVS.P 4(R5), F20 FMOVS.P 4(R6), F21 - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -852,14 +858,16 @@ TEXT ·gemmUA2x8S(SB), NOSPLIT, $0-48 VST1 [V16.S4, V17.S4], (R3) RET -TEXT ·gemmUA2x8D(SB), NOSPLIT, $0-48 +TEXT ·gemmUA2x8D(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $3, R2, R2 + LSL $3, R16, R16 LSL $3, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -877,7 +885,7 @@ TEXT ·gemmUA2x8D(SB), NOSPLIT, $0-48 uakloop: VLD1.P 16(R5), [V20.D2] VLD1.P 16(R6), [V21.D2] - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] @@ -886,7 +894,7 @@ TEXT ·gemmUA2x8D(SB), NOSPLIT, $0-48 WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] @@ -902,7 +910,7 @@ TEXT ·gemmUA2x8D(SB), NOSPLIT, $0-48 uatloop: FMOVD.P 8(R5), F20 FMOVD.P 8(R6), F21 - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] @@ -929,14 +937,16 @@ TEXT ·gemmUA2x8D(SB), NOSPLIT, $0-48 VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) RET -TEXT ·gemmUA3x8S(SB), NOSPLIT, $0-48 +TEXT ·gemmUA3x8S(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $2, R2, R2 + LSL $2, R16, R16 LSL $2, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -954,28 +964,28 @@ TEXT ·gemmUA3x8S(SB), NOSPLIT, $0-48 VLD1.P 16(R5), [V20.S4] VLD1.P 16(R6), [V21.S4] VLD1.P 16(R7), [V22.S4] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] @@ -990,7 +1000,7 @@ TEXT ·gemmUA3x8S(SB), NOSPLIT, $0-48 FMOVS.P 4(R5), F20 FMOVS.P 4(R6), F21 FMOVS.P 4(R7), F22 - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1016,14 +1026,16 @@ TEXT ·gemmUA3x8S(SB), NOSPLIT, $0-48 VST1 [V16.S4, V17.S4], (R3) RET -TEXT ·gemmUA3x8D(SB), NOSPLIT, $0-48 +TEXT ·gemmUA3x8D(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $3, R2, R2 + LSL $3, R16, R16 LSL $3, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -1047,7 +1059,7 @@ TEXT ·gemmUA3x8D(SB), NOSPLIT, $0-48 VLD1.P 16(R5), [V20.D2] VLD1.P 16(R6), [V21.D2] VLD1.P 16(R7), [V22.D2] - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] @@ -1060,7 +1072,7 @@ TEXT ·gemmUA3x8D(SB), NOSPLIT, $0-48 WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] @@ -1081,7 +1093,7 @@ TEXT ·gemmUA3x8D(SB), NOSPLIT, $0-48 FMOVD.P 8(R5), F20 FMOVD.P 8(R6), F21 FMOVD.P 8(R7), F22 - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] @@ -1119,14 +1131,16 @@ TEXT ·gemmUA3x8D(SB), NOSPLIT, $0-48 VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) RET -TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-48 +TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $2, R2, R2 + LSL $2, R16, R16 LSL $2, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -1148,7 +1162,7 @@ TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-48 VLD1.P 16(R6), [V21.S4] VLD1.P 16(R7), [V22.S4] VLD1.P 16(R8), [V23.S4] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1157,7 +1171,7 @@ TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-48 WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] @@ -1166,7 +1180,7 @@ TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-48 WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] @@ -1175,7 +1189,7 @@ TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-48 WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] @@ -1193,7 +1207,7 @@ TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-48 FMOVS.P 4(R6), F21 FMOVS.P 4(R7), F22 FMOVS.P 4(R8), F23 - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1226,14 +1240,16 @@ TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-48 VST1 [V16.S4, V17.S4], (R3) RET -TEXT ·gemmUA4x8D(SB), NOSPLIT, $0-48 +TEXT ·gemmUA4x8D(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $3, R2, R2 + LSL $3, R16, R16 LSL $3, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -1263,7 +1279,7 @@ TEXT ·gemmUA4x8D(SB), NOSPLIT, $0-48 VLD1.P 16(R6), [V21.D2] VLD1.P 16(R7), [V22.D2] VLD1.P 16(R8), [V23.D2] - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] @@ -1280,7 +1296,7 @@ TEXT ·gemmUA4x8D(SB), NOSPLIT, $0-48 WORD $0x4FD7122D // FMLA V13.2D, V17.2D, V23.D[0] WORD $0x4FD7124E // FMLA V14.2D, V18.2D, V23.D[0] WORD $0x4FD7126F // FMLA V15.2D, V19.2D, V23.D[0] - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] @@ -1306,7 +1322,7 @@ TEXT ·gemmUA4x8D(SB), NOSPLIT, $0-48 FMOVD.P 8(R6), F21 FMOVD.P 8(R7), F22 FMOVD.P 8(R8), F23 - VLD1.P 64(R13), [V16.D2, V17.D2, V18.D2, V19.D2] + VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] @@ -1355,14 +1371,16 @@ TEXT ·gemmUA4x8D(SB), NOSPLIT, $0-48 VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) RET -TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-48 +TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $2, R2, R2 + LSL $2, R16, R16 LSL $2, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -1388,7 +1406,7 @@ TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-48 VLD1.P 16(R7), [V22.S4] VLD1.P 16(R8), [V23.S4] VLD1.P 16(R9), [V24.S4] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1399,7 +1417,7 @@ TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-48 WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] @@ -1410,7 +1428,7 @@ TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-48 WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] @@ -1421,7 +1439,7 @@ TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-48 WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] @@ -1442,7 +1460,7 @@ TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-48 FMOVS.P 4(R7), F22 FMOVS.P 4(R8), F23 FMOVS.P 4(R9), F24 - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1482,14 +1500,16 @@ TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-48 VST1 [V16.S4, V17.S4], (R3) RET -TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-48 +TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $2, R2, R2 + LSL $2, R16, R16 LSL $2, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -1519,7 +1539,7 @@ TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-48 VLD1.P 16(R8), [V23.S4] VLD1.P 16(R9), [V24.S4] VLD1.P 16(R10), [V25.S4] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1532,7 +1552,7 @@ TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-48 WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] @@ -1545,7 +1565,7 @@ TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-48 WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] WORD $0x4FB9120A // FMLA V10.4S, V16.4S, V25.S[1] WORD $0x4FB9122B // FMLA V11.4S, V17.4S, V25.S[1] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] @@ -1558,7 +1578,7 @@ TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-48 WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] WORD $0x4F991A0A // FMLA V10.4S, V16.4S, V25.S[2] WORD $0x4F991A2B // FMLA V11.4S, V17.4S, V25.S[2] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] @@ -1582,7 +1602,7 @@ TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-48 FMOVS.P 4(R8), F23 FMOVS.P 4(R9), F24 FMOVS.P 4(R10), F25 - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1629,14 +1649,16 @@ TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-48 VST1 [V16.S4, V17.S4], (R3) RET -TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-48 +TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $2, R2, R2 + LSL $2, R16, R16 LSL $2, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -1670,7 +1692,7 @@ TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-48 VLD1.P 16(R9), [V24.S4] VLD1.P 16(R10), [V25.S4] VLD1.P 16(R11), [V26.S4] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1685,7 +1707,7 @@ TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-48 WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] @@ -1700,7 +1722,7 @@ TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-48 WORD $0x4FB9122B // FMLA V11.4S, V17.4S, V25.S[1] WORD $0x4FBA120C // FMLA V12.4S, V16.4S, V26.S[1] WORD $0x4FBA122D // FMLA V13.4S, V17.4S, V26.S[1] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] @@ -1715,7 +1737,7 @@ TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-48 WORD $0x4F991A2B // FMLA V11.4S, V17.4S, V25.S[2] WORD $0x4F9A1A0C // FMLA V12.4S, V16.4S, V26.S[2] WORD $0x4F9A1A2D // FMLA V13.4S, V17.4S, V26.S[2] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] @@ -1742,7 +1764,7 @@ TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-48 FMOVS.P 4(R9), F24 FMOVS.P 4(R10), F25 FMOVS.P 4(R11), F26 - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1796,14 +1818,16 @@ TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-48 VST1 [V16.S4, V17.S4], (R3) RET -TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-48 +TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-56 MOVD kc+0(FP), R0 MOVD a+8(FP), R1 MOVD lda+16(FP), R2 MOVD pb+24(FP), R13 - MOVD c+32(FP), R3 - MOVD ldc+40(FP), R4 + MOVD ldb+32(FP), R16 + MOVD c+40(FP), R3 + MOVD ldc+48(FP), R4 LSL $2, R2, R2 + LSL $2, R16, R16 LSL $2, R4, R4 MOVD R1, R5 ADD R2, R5, R6 @@ -1841,7 +1865,7 @@ TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-48 VLD1.P 16(R10), [V25.S4] VLD1.P 16(R11), [V26.S4] VLD1.P 16(R12), [V27.S4] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] @@ -1858,7 +1882,7 @@ TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-48 WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] WORD $0x4F9B120E // FMLA V14.4S, V16.4S, V27.S[0] WORD $0x4F9B122F // FMLA V15.4S, V17.4S, V27.S[0] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] @@ -1875,7 +1899,7 @@ TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-48 WORD $0x4FBA122D // FMLA V13.4S, V17.4S, V26.S[1] WORD $0x4FBB120E // FMLA V14.4S, V16.4S, V27.S[1] WORD $0x4FBB122F // FMLA V15.4S, V17.4S, V27.S[1] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] @@ -1892,7 +1916,7 @@ TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-48 WORD $0x4F9A1A2D // FMLA V13.4S, V17.4S, V26.S[2] WORD $0x4F9B1A0E // FMLA V14.4S, V16.4S, V27.S[2] WORD $0x4F9B1A2F // FMLA V15.4S, V17.4S, V27.S[2] - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] @@ -1922,7 +1946,7 @@ TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-48 FMOVS.P 4(R10), F25 FMOVS.P 4(R11), F26 FMOVS.P 4(R12), F27 - VLD1.P 32(R13), [V16.S4, V17.S4] + VLD1.P (R13)(R16), [V16.S4, V17.S4] WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0]