From dfd1452d1c4fc98a7ab6e7217f37030f4826c14b Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 15 Nov 2026 15:14:56 +0100 Subject: [PATCH 1/4] refactor: MatMul, MatMul32 and the linalg products run on go-blas/blas The packed GEMM (gemm.go, gemm_.go, gemm32_*, gemmua_other, the GEMM micro-kernels of sum_.s and their generators) moved to github.com/go-blas/blas in go-blas/blas#1, byte for byte. ndarray now calls DgemmStrided / SgemmStrided through internal/blasmm, which reads an operand through its strides (a transposed or a[:, ::2] view is not copied; a broadcast or reversed one is) and picks the float32 path with HasSgemmKernel. Kept here: the dot kernels (moved from gemm_.go to dot_.go, unchanged), mat-vec and Dot1D (matvec.go), blocks8, the float32 elementwise and sqrt kernels, max/min, the thread pool and hwcap. sum_.s is regenerated without the GEMM blocks: every other block is byte-identical. Co-Authored-By: Claude Sonnet 5.5 --- README.md | 12 +- SECURITY.md | 28 +- bench/go.mod | 1 + bench/go.sum | 2 + bench_test.go | 28 + docs/plan-ndarray.md | 3 +- go.mod | 1 + go.sum | 2 + internal/blasmm/blasmm.go | 76 + internal/blasmm/blasmm_test.go | 122 ++ internal/kernels/asmgen/amd64/gen.go | 305 +--- internal/kernels/asmgen/arm64/gen.go | 351 +--- internal/kernels/asmgen/loong64/gen.go | 41 - internal/kernels/asmgen/ppc64le/gen.go | 178 -- internal/kernels/asmgen/riscv64/gen.go | 153 -- internal/kernels/asmgen/s390x/gen.go | 82 - internal/kernels/barrier_test.go | 17 - internal/kernels/dot_amd64.go | 21 + internal/kernels/dot_arm64.go | 13 + internal/kernels/dot_loong64.go | 13 + internal/kernels/dot_ppc64le.go | 14 + internal/kernels/fence_amd64_test.go | 9 - internal/kernels/fence_fallback_test.go | 7 - internal/kernels/fence_loong64_test.go | 9 - internal/kernels/fence_riscv64_test.go | 9 - internal/kernels/fence_s390x_test.go | 9 - internal/kernels/fence_test.go | 73 - internal/kernels/gemm.go | 707 -------- internal/kernels/gemm32_amd64.go | 56 - internal/kernels/gemm32_arm64.go | 54 - internal/kernels/gemm32_other.go | 7 - internal/kernels/gemm32_parts_riscv64_test.go | 62 - internal/kernels/gemm32_ppc64le.go | 16 - internal/kernels/gemm32_riscv64.go | 55 - internal/kernels/gemm32_riscv64_test.go | 48 - internal/kernels/gemm32_s390x.go | 25 - internal/kernels/gemm32_test.go | 162 -- internal/kernels/gemm_amd64.go | 107 -- internal/kernels/gemm_amd64_test.go | 31 - internal/kernels/gemm_arm64.go | 71 - internal/kernels/gemm_bench_test.go | 44 - internal/kernels/gemm_generic.go | 75 - internal/kernels/gemm_loong64.go | 59 - internal/kernels/gemm_loong64_test.go | 19 - internal/kernels/gemm_oneworker_test.go | 63 - internal/kernels/gemm_ppc64le.go | 37 - internal/kernels/gemm_riscv64.go | 47 - internal/kernels/gemm_riscv64_test.go | 17 - internal/kernels/gemm_s390x.go | 47 - internal/kernels/gemm_test.go | 437 ----- internal/kernels/gemmua_other.go | 6 - internal/kernels/generate.go | 7 +- internal/kernels/guard_test.go | 4 - internal/kernels/matvec.go | 114 ++ internal/kernels/matvec_test.go | 175 ++ internal/kernels/packrows8_test.go | 44 - internal/kernels/parallel.go | 2 + internal/kernels/parallel_test.go | 38 +- internal/kernels/stress_test.go | 6 +- internal/kernels/sum_amd64.s | 1238 -------------- internal/kernels/sum_arm64.s | 1488 ----------------- internal/kernels/sum_loong64.s | 113 -- internal/kernels/sum_ppc64le.s | 662 -------- internal/kernels/sum_riscv64.s | 236 --- internal/kernels/sum_s390x.s | 158 -- internal/kernels/vx_s390x_test.go | 13 - linalg.go | 21 +- linalg/blas.go | 21 +- matmul_workers_test.go | 65 + 69 files changed, 717 insertions(+), 7519 deletions(-) create mode 100644 internal/blasmm/blasmm.go create mode 100644 internal/blasmm/blasmm_test.go create mode 100644 internal/kernels/dot_amd64.go create mode 100644 internal/kernels/dot_arm64.go create mode 100644 internal/kernels/dot_loong64.go create mode 100644 internal/kernels/dot_ppc64le.go delete mode 100644 internal/kernels/fence_amd64_test.go delete mode 100644 internal/kernels/fence_fallback_test.go delete mode 100644 internal/kernels/fence_loong64_test.go delete mode 100644 internal/kernels/fence_riscv64_test.go delete mode 100644 internal/kernels/fence_s390x_test.go delete mode 100644 internal/kernels/gemm.go delete mode 100644 internal/kernels/gemm32_amd64.go delete mode 100644 internal/kernels/gemm32_arm64.go delete mode 100644 internal/kernels/gemm32_other.go delete mode 100644 internal/kernels/gemm32_parts_riscv64_test.go delete mode 100644 internal/kernels/gemm32_ppc64le.go delete mode 100644 internal/kernels/gemm32_riscv64.go delete mode 100644 internal/kernels/gemm32_riscv64_test.go delete mode 100644 internal/kernels/gemm32_s390x.go delete mode 100644 internal/kernels/gemm32_test.go delete mode 100644 internal/kernels/gemm_amd64.go delete mode 100644 internal/kernels/gemm_amd64_test.go delete mode 100644 internal/kernels/gemm_arm64.go delete mode 100644 internal/kernels/gemm_bench_test.go delete mode 100644 internal/kernels/gemm_generic.go delete mode 100644 internal/kernels/gemm_loong64.go delete mode 100644 internal/kernels/gemm_loong64_test.go delete mode 100644 internal/kernels/gemm_oneworker_test.go delete mode 100644 internal/kernels/gemm_ppc64le.go delete mode 100644 internal/kernels/gemm_riscv64.go delete mode 100644 internal/kernels/gemm_riscv64_test.go delete mode 100644 internal/kernels/gemm_s390x.go delete mode 100644 internal/kernels/gemm_test.go delete mode 100644 internal/kernels/gemmua_other.go create mode 100644 internal/kernels/matvec.go create mode 100644 internal/kernels/matvec_test.go delete mode 100644 internal/kernels/packrows8_test.go create mode 100644 matmul_workers_test.go diff --git a/README.md b/README.md index 3286c1a..769ea82 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ scalars with `Scalar`). Float64 is the default and the most SIMD-accelerated one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every 64-bit target, and `Max`/`Min` on every 64-bit target, bit-identical to its - scalar loops. Its `MatMul` runs on a native float32 GEMM on arm64 (NEON), + scalar loops. Its `MatMul` runs on go-blas/blas's native float32 GEMM on arm64 (NEON), amd64 with FMA (AVX2), ppc64le (VSX), s390x (z14 vector) and riscv64 (RVV 1.0), accumulating in float32 like NumPy's sgemm, and elsewhere on the float64 GEMM (accumulated in float64, rounded once). Its `Exp` and `Log` are **correctly rounded**, proven by an exhaustive test @@ -115,9 +115,13 @@ loops are [go-asmgen](https://github.com/go-asmgen)-generated SIMD kernels: sum, add/sub/mul/div, sqrt and the dot product (SSE2 or AVX2/FMA on amd64, NEON on arm64, VSX on ppc64le, LASX on loong64, RVV on riscv64 and the vector facility on s390x, the last three when the CPU has them), max/min too on -amd64, arm64 and s390x (z14 and later), and a **panel-packed, cache-blocked GEMM** with an SIMD-FMA -micro-kernel (NEON 4×8; AVX2/FMA 6×8 chosen at run time, SSE2 fallback; VSX -8×8; LASX 8×8; RVV 6×8; s390x 4×8), the OpenBLAS/BLIS structure. The 32-bit +amd64, arm64 and s390x (z14 and later). The **panel-packed, cache-blocked GEMM** +with its SIMD-FMA micro-kernels (NEON 4×8; AVX2/FMA 6×8 chosen at run time, SSE2 +fallback; VSX 8×8; LASX 8×8; RVV 6×8; s390x 4×8; the OpenBLAS/BLIS structure) +no longer lives here: `MatMul`, `Dot` and the linalg products call +[go-blas/blas](https://github.com/go-blas/blas) (`DgemmStrided`/`SgemmStrided`, +which read a transposed or sliced view through its strides without copying), the +one implementation of that kernel. The 32-bit targets run the same pure-Go code those kernels are tested against. `Exp` and `Log` are ports of Arm's optimized-routines (≤ 0.51 ULP measured, against up to 0.88 and 0.72 for Go's `math.Exp`/`math.Log` on arm64), diff --git a/SECURITY.md b/SECURITY.md index 7a18b6c..0c37e14 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,15 +15,19 @@ bounds checks do not cover, so they are held to the following: - **Every wrapper checks the reach of its kernel.** Before an assembly call, the Go wrapper indexes the last element of every operand the kernel reads or - writes (`internal/kernels/{reduce,gemm}_.go`). A caller's mistake is an - index-out-of-range panic in Go, never an access outside a slice. + writes (`internal/kernels/{reduce,dot}_.go`). A caller's mistake is an + index-out-of-range panic in Go, never an access outside a slice. The GEMM + micro-kernels, their wrappers and their fence tests live in + [go-blas/blas](https://github.com/go-blas/blas), which owns that audit; the + entries below about GEMM kernels (v0.26.0 to v0.38.0) are history of code now + there. `TestWrappersRefuseShortOperands` passes each wrapper an operand one element short and requires the panic. - **The kernels stay inside their slices.** `TestKernelsStayInsideTheirSlices` - and `TestGemmMicroStaysInsideItsOperands` (`internal/kernels/fence_test.go`, - Linux and macOS) map each operand flush against a `PROT_NONE` page, once - ending at it and once starting after one, for every length from 1 to 70 and - every GEMM depth from 1 to 9. A read or write one element past either end is a + (`internal/kernels/fence_test.go`, Linux and macOS) maps each operand flush + against a `PROT_NONE` page, once ending at it and once starting after one, for + every length from 1 to 70 (go-blas/blas fences its GEMM micro-kernels at every + depth from 1 to 9 the same way). A read or write one element past either end is a fault. The fence itself was checked by a kernel deliberately reading one element too far, which faults on amd64 (4 KiB pages), arm64 (16 KiB), loong64 (16 KiB) and ppc64le (64 KiB). Prefetch instructions may point past the end: @@ -66,6 +70,18 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-11-15 (unreleased).** The GEMM moved to +[go-blas/blas](https://github.com/go-blas/blas) v0.2.0 (the micro-kernels, their +wrappers, the packing and their fence tests, byte for byte; the audit of that +code is that repository's). What ndarray adds is `internal/blasmm`, which hands +an operand's row and column strides to `DgemmStrided`/`SgemmStrided`. go-blas +checks every stride and slice length (overflow-safe, `*ArgError` on refusal), so +a view ndarray builds wrongly is an error there, which `blasmm` turns into a +panic naming the package, not a read outside a slice. A zero or negative stride +on an axis longer than one (a broadcast or reversed view) is copied to a +contiguous matrix first. ndarray keeps the dot kernels (`dot_.go`), the +mat-vec and the thread pool, with their wrappers and fences. + **2026-11-12 (unreleased, #103).** The gather of `random`'s `Choice` (`takeAxis`, random/arrays.go) now copies bytes through `bytesOf`, an `unsafe.Slice` view of a slice's backing array of exactly `Len()·size` bytes. diff --git a/bench/go.mod b/bench/go.mod index 305b4d0..6b02d4b 100644 --- a/bench/go.mod +++ b/bench/go.mod @@ -8,6 +8,7 @@ require ( ) require ( + github.com/go-blas/blas v0.2.0 // indirect github.com/go-fft/fft v0.29.0 // indirect github.com/go-fleettools/fleettools v0.81.0 // indirect github.com/go-simd/floats v0.17.0 // indirect diff --git a/bench/go.sum b/bench/go.sum index a2f89c3..0d66bd2 100644 --- a/bench/go.sum +++ b/bench/go.sum @@ -1,3 +1,5 @@ +github.com/go-blas/blas v0.2.0 h1:kLWNy4ZKhUEHz8vXWIC7n6CP0ytNuEWTYBuASdF+Sxo= +github.com/go-blas/blas v0.2.0/go.mod h1:WjhlqzQ+RA8tS473f18QtcJl5Q0GonPs3RJIxT5G7Bk= github.com/go-fft/fft v0.29.0 h1:WtJb5K3dmbLWREvvZ50od0fxtjIBiXrg7mVybz/nPoA= github.com/go-fft/fft v0.29.0/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s= github.com/go-fleettools/fleettools v0.81.0 h1:UnMflmNw5b82Z+7Ums7UVaW5RrGuACCVmSwytwT1hxw= diff --git a/bench_test.go b/bench_test.go index 30da0d3..00beb2a 100644 --- a/bench_test.go +++ b/bench_test.go @@ -129,6 +129,34 @@ func benchMat(n int) *Array { return a } +// BenchmarkMatMulVariants covers what BenchmarkMatMul does not: float32 +// operands, a transposed operand (read in place by the GEMM) and a strided one +// (a[:, ::2], which the GEMM entry copies first). +func BenchmarkMatMulVariants(b *testing.B) { + for _, n := range []int{64, 256, 1024} { + x, y := benchMat(n), benchMat(n) + x32, y32 := x.AsType(Float32), y.AsType(Float32) + wide, _ := FromData(make([]float64, 2*n*n), n, 2*n) + half, _ := wide.Slice(All(), Step(2)) + xt := x.Transpose() + b.Run(fmt.Sprintf("f32/n=%d", n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = x32.MatMul(y32) + } + }) + b.Run(fmt.Sprintf("transA/n=%d", n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = xt.MatMul(y) + } + }) + b.Run(fmt.Sprintf("strided/n=%d", n), func(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = half.MatMul(y) + } + }) + } +} + func BenchmarkMatMul(b *testing.B) { for _, n := range []int{64, 128, 256, 512, 1024} { x, y := benchMat(n), benchMat(n) diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index 2f6e0f1..7ca5cc2 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -34,7 +34,8 @@ ndarray/ internal/kernels/ portable scalar inner loops over flat []float64 (Add/Sub/Mul/Div, Map, comparisons, Maximum/Minimum, Sum/Prod/Max/Min, Abs, {Sum,Prod,Max,Min}Axis over - [outer][axisLen][inner], MatMul GEMM, Dot1D) + [outer][axisLen][inner], mat-vec, Dot1D; the GEMM is + github.com/go-blas/blas, through internal/blasmm) docs/plan-ndarray.md this roadmap ``` diff --git a/go.mod b/go.mod index 6db75a0..5028998 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/go-ndarray/ndarray go 1.27.2 require ( + github.com/go-blas/blas v0.2.0 github.com/go-fft/fft v0.29.0 github.com/go-simd/floats v0.17.0 ) diff --git a/go.sum b/go.sum index 66ee611..a2c19a6 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/go-blas/blas v0.2.0 h1:kLWNy4ZKhUEHz8vXWIC7n6CP0ytNuEWTYBuASdF+Sxo= +github.com/go-blas/blas v0.2.0/go.mod h1:WjhlqzQ+RA8tS473f18QtcJl5Q0GonPs3RJIxT5G7Bk= github.com/go-fft/fft v0.29.0 h1:WtJb5K3dmbLWREvvZ50od0fxtjIBiXrg7mVybz/nPoA= github.com/go-fft/fft v0.29.0/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s= github.com/go-simd/floats v0.17.0 h1:Dv0ZmZ3k0IavvHwQgstpUz+TeIf/gGJ9HEGQ/UART8Q= diff --git a/internal/blasmm/blasmm.go b/internal/blasmm/blasmm.go new file mode 100644 index 0000000..3c84844 --- /dev/null +++ b/internal/blasmm/blasmm.go @@ -0,0 +1,76 @@ +// Package blasmm is the row-major strided matrix product of ndarray on top of +// the general-stride GEMM of github.com/go-blas/blas (DgemmStrided and +// SgemmStrided), the one implementation of the packed, cache-blocked, multicore +// kernel. +// +// An operand is read through its row and column strides, as go-blas packs it, +// so a transposed or sliced view (a.T, a[:, ::2]) is never copied. go-blas +// takes strides of at least 1, and ndarray's views can hold others, which are +// normalised here: a stride of an axis of extent 1 is never used, so it is set +// to 1; an operand with a zero (broadcast) or negative (reversed) stride on an +// axis longer than 1 is copied to a contiguous matrix first, an O(rows*cols) +// pass beside the O(m*n*k) product. The result is written row-major, contiguous. +package blasmm + +import "github.com/go-blas/blas" + +// Mat is a strided matrix operand: element (i, j) is Data[Off+i*RS+j*CS]. +type Mat[T float32 | float64] struct { + Data []T + Off int + RS, CS int +} + +// HasGemm32 reports whether SgemmStrided runs on a float32 SIMD kernel here; +// without one the caller multiplies in float64 instead. +func HasGemm32() bool { return blas.HasSgemmKernel() } + +// MatMul writes the m x k by k x n product a*b into dst (m*n elements, row +// major, overwritten). +func MatMul(dst []float64, a, b Mat[float64], m, k, n int) { + matMul(blas.DgemmStrided, dst, a, b, m, k, n) +} + +// MatMul32 is MatMul in float32. It requires HasGemm32. +func MatMul32(dst []float32, a, b Mat[float32], m, k, n int) { + matMul(blas.SgemmStrided, dst, a, b, m, k, n) +} + +type stridedGemm[T float32 | float64] func(m, n, k int, alpha T, a []T, rsa, csa int, b []T, rsb, csb int, beta T, c []T, rsc, csc int) error + +func matMul[T float32 | float64](gemm stridedGemm[T], dst []T, a, b Mat[T], m, k, n int) { + if m == 0 || n == 0 { + return + } + ad, rsa, csa := operand(a, m, k) + bd, rsb, csb := operand(b, k, n) + // The operands are valid views (the callers checked their shapes), so a + // refusal here is a defect in this package, not an input error. + if err := gemm(m, n, k, 1, ad, rsa, csa, bd, rsb, csb, 0, dst[:m*n], n, 1); err != nil { + panic("ndarray: blasmm: " + err.Error()) + } +} + +// operand returns x (rows x cols) as a slice and strides go-blas accepts. +func operand[T float32 | float64](x Mat[T], rows, cols int) (d []T, rs, cs int) { + if rows == 0 || cols == 0 { + return nil, 1, 1 + } + rs, cs = x.RS, x.CS + if rows == 1 { + rs = 1 // never used + } + if cols == 1 { + cs = 1 + } + if rs >= 1 && cs >= 1 { + return x.Data[x.Off:], rs, cs + } + d = make([]T, rows*cols) + for i := 0; i < rows; i++ { + for j := 0; j < cols; j++ { + d[i*cols+j] = x.Data[x.Off+i*x.RS+j*x.CS] + } + } + return d, cols, 1 +} diff --git a/internal/blasmm/blasmm_test.go b/internal/blasmm/blasmm_test.go new file mode 100644 index 0000000..e711c39 --- /dev/null +++ b/internal/blasmm/blasmm_test.go @@ -0,0 +1,122 @@ +package blasmm + +import "testing" + +// view lays a rows x cols matrix of small integers out with the given strides +// in a buffer padded with a sentinel, from offset off; the products below are +// exact, so every comparison is equality. +func view[T float32 | float64](rows, cols, off, rs, cs int, seed int) (Mat[T], [][]T) { + hi := off // highest element index + for _, ext := range [][2]int{{rows - 1, rs}, {cols - 1, cs}} { + if ext[1] > 0 { + hi += ext[0] * ext[1] + } + } + buf := make([]T, hi+1) + for i := range buf { + buf[i] = 9999 // never an element, or a product would be off + } + want := make([][]T, rows) + for i := range want { + want[i] = make([]T, cols) + for j := range want[i] { + v := T((i*7+j*3+seed)%5 - 2) + want[i][j] = v + buf[off+i*rs+j*cs] = v // an aliased view (rs = 0) keeps the last write + } + } + for i := range want { // re-read: aliasing makes the matrix what the buffer says + for j := range want[i] { + want[i][j] = buf[off+i*rs+j*cs] + } + } + return Mat[T]{Data: buf, Off: off, RS: rs, CS: cs}, want +} + +func product[T float32 | float64](a, b [][]T, m, k, n int) []T { + c := make([]T, m*n) + for i := 0; i < m; i++ { + for j := 0; j < n; j++ { + for p := 0; p < k; p++ { + c[i*n+j] += a[i][p] * b[p][j] + } + } + } + return c +} + +func testMatMul[T float32 | float64](t *testing.T, mul func(dst []T, a, b Mat[T], m, k, n int)) { + type lay struct { + name string + off, rs, cs func(rows, cols int) int + } + layouts := []lay{ + {"row-major", func(r, c int) int { return 0 }, func(r, c int) int { return c }, func(r, c int) int { return 1 }}, + {"padded", func(r, c int) int { return 5 }, func(r, c int) int { return c + 3 }, func(r, c int) int { return 1 }}, + {"transposed", func(r, c int) int { return 0 }, func(r, c int) int { return 1 }, func(r, c int) int { return r }}, + {"every other column", func(r, c int) int { return 0 }, func(r, c int) int { return 2 * c }, func(r, c int) int { return 2 }}, + {"every other row", func(r, c int) int { return 0 }, func(r, c int) int { return 2 * c }, func(r, c int) int { return 1 }}, + {"reversed rows", func(r, c int) int { return (r - 1) * c }, func(r, c int) int { return -c }, func(r, c int) int { return 1 }}, + {"reversed columns", func(r, c int) int { return c - 1 }, func(r, c int) int { return c }, func(r, c int) int { return -1 }}, + {"broadcast rows", func(r, c int) int { return 0 }, func(r, c int) int { return 0 }, func(r, c int) int { return 1 }}, + } + shapes := [][3]int{{1, 1, 1}, {1, 5, 4}, {4, 5, 1}, {3, 1, 6}, {7, 5, 3}, {13, 17, 19}, {40, 33, 50}} + for _, sh := range shapes { + m, k, n := sh[0], sh[1], sh[2] + for _, la := range layouts { + for _, lb := range layouts { + a, aw := view[T](m, k, la.off(m, k), la.rs(m, k), la.cs(m, k), 1) + b, bw := view[T](k, n, lb.off(k, n), lb.rs(k, n), lb.cs(k, n), 2) + got := make([]T, m*n) + for i := range got { + got[i] = 7777 // overwritten, not accumulated + } + mul(got, a, b, m, k, n) + want := product(aw, bw, m, k, n) + for i := range want { + if got[i] != want[i] { + t.Fatalf("%v a=%s b=%s: [%d] = %v, want %v", sh, la.name, lb.name, i, got[i], want[i]) + } + } + } + } + } + // A stride of an axis of extent 1 is never used, whatever it is. + a := Mat[T]{Data: []T{2, 3}, RS: 0, CS: 1} // 1 x 2, rs unused + b := Mat[T]{Data: []T{4, 5}, RS: 1, CS: -3} // 2 x 1, cs unused + got := make([]T, 1) + mul(got, a, b, 1, 2, 1) + if got[0] != 23 { + t.Fatalf("extent-1 strides: %v, want 23", got[0]) + } + // Empty products write nothing; k = 0 is the zero matrix. + mul(nil, Mat[T]{}, Mat[T]{}, 0, 3, 4) + mul(nil, Mat[T]{}, Mat[T]{}, 3, 3, 0) + z := []T{5, 5, 5, 5} + mul(z, Mat[T]{}, Mat[T]{}, 2, 0, 2) + for _, v := range z { + if v != 0 { + t.Fatalf("k = 0: %v, want zeros", z) + } + } +} + +func TestMatMul(t *testing.T) { testMatMul(t, MatMul) } + +func TestMatMul32(t *testing.T) { + t.Logf("HasGemm32() = %v; the float32 product is tested where there is a kernel", HasGemm32()) + if HasGemm32() { + testMatMul(t, MatMul32) + } +} + +// A refusal by go-blas is a defect of this package and panics with its name. +func TestMatMulPanicsOnADefect(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("a destination too short was accepted") + } + }() + one := Mat[float64]{Data: []float64{1}, RS: 1, CS: 1} + MatMul(make([]float64, 2), one, one, 1, 1, 2) // B is 1 x 2 but holds one element +} diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index 44b84ec..a10fd5e 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -42,14 +42,8 @@ func main() { // go-scikit performance campaign): sumKernel, sqrtKernel, extremeKernel, // binKernel, exp32CRKernel, log32CRKernel and logKernel below are no longer // emitted. - f.Add(gemmFMAKernel()) - f.Add(gemm32FMAKernel()) - for mr := 1; mr <= gemmMR; mr++ { - f.Add(gemm32UAKernel(mr)) - f.Add(gemmUAKernel(mr, false)) - } + // The GEMM kernels moved to github.com/go-blas/blas, which generates them. f.Add(dotFMAKernel()) - f.Add(gemmSSE2HalfKernel()) f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) f.Add(blocks8Kernel()) f.Add(bin32Kernel("add32SSE2", "ADDPS", "ADDSS")) @@ -681,251 +675,9 @@ const ( gemmNR = 8 ) -func gemmSig() amd64.Signature { - return amd64.Layout( - []string{"kc", "pa", "pb", "c", "ldc"}, - []amd64.Type{amd64.Int64, amd64.Ptr, amd64.Ptr, amd64.Ptr, amd64.Int64}, - nil, nil, - ) -} - -// gemmFMAKernel builds gemmMicro6x8FMA(kc int, pa, pb, c *float64, ldc int): -// -// for p in [0,kc): C[r][col] += pa[p*6+r] * pb[p*8+col] (r<6, col<8) -// -// the BLIS Haswell dgemm shape (6x8, row-major here). Each k step loads the -// 8-double B row into two YMM, broadcasts each of the 6 A values with -// VBROADCASTSD straight from the panel, and issues 12 VFMADD231PD into the 12 -// accumulators Y0..Y11 (row r in Y(2r), Y(2r+1)). Twelve independent chains -// cover the FMA latency on both ports (Haswell: 5 cycles x 2 ports needs >= 10), -// and 8 loads per 12 FMAs stay under the 2-loads-per-cycle budget, so the loop -// is FMA-throughput bound — the property that makes it a real dgemm kernel. -// -// Fused: each step rounds a*b+c once, as tuned BLAS (and gc's own scalar code at -// GOAMD64=v3, and arm64's) do. The SSE2 fallback below does not fuse, so on -// non-integer data the two can differ in the last place; both are the ikj sum. -// The caller gates this kernel on hasFMA (CPUID FMA+AVX+OSXSAVE and XCR0 YMM). -func gemmFMAKernel() *emit.Function { - b := amd64.NewFunc("gemmMicro6x8FMA", gemmSig(), 0) - b.LoadArg("kc", "CX") - b.LoadArg("pa", "SI") // A panel - b.LoadArg("pb", "DI") // B panel - b.LoadArg("c", "DX") // C base - b.LoadArg("ldc", "R8") - b.Raw("SHLQ $3, R8") // ldc -> bytes - // Prefetch the C tile now: its rows are ldc apart, so they are 6 different - // cache lines (12 when a row straddles two), usually not in L1, and the - // store at the end would otherwise stall on each of them in turn. Issued - // first, they arrive while the k loop runs. The last byte of each row is - // fetched too, for a row that does not start on a line boundary. - b.Raw("MOVQ DX, R9") - for r := 0; r < gemmMR; r++ { - b.Raw("PREFETCHT0 (R9)") - b.Raw("PREFETCHT0 63(R9)") - if r < gemmMR-1 { - b.Raw("ADDQ R8, R9") - } - } - for r := 0; r < 12; r++ { - b.Raw(fmt.Sprintf("VXORPD Y%d, Y%d, Y%d", r, r, r)) - } - // Unrolled by two k steps, with a one-step tail. The A panel streams from - // L2 (the B micro-panel stays in L1 across the whole A block), so each pair - // of steps prefetches the A line 8 steps ahead. - b.Raw("MOVQ CX, R10") - b.Raw("SHRQ $1, R10") - b.Raw("JZ ftail") - b.Raw("fkloop:") - b.Raw(fmt.Sprintf("PREFETCHT0 %d(SI)", 8*gemmMR*8)) - fmaStep(b, 0) - fmaStep(b, 1) - b.Raw(fmt.Sprintf("ADDQ $%d, SI", 2*8*gemmMR)) - b.Raw(fmt.Sprintf("ADDQ $%d, DI", 2*8*gemmNR)) - b.Raw("DECQ R10") - b.Raw("JNZ fkloop") - b.Raw("ftail:") - b.Raw("TESTQ $1, CX") - b.Raw("JZ fstore") - fmaStep(b, 0) - b.Raw("fstore:") - for r := 0; r < gemmMR; r++ { - b.Raw(fmt.Sprintf("VADDPD (DX), Y%d, Y15", 2*r)) - b.Raw("VMOVUPD Y15, (DX)") - b.Raw(fmt.Sprintf("VADDPD 32(DX), Y%d, Y15", 2*r+1)) - b.Raw("VMOVUPD Y15, 32(DX)") - if r < gemmMR-1 { - b.Raw("ADDQ R8, DX") - } - } - b.Raw("VZEROUPPER") // no AVX-SSE transition penalty in the Go code after - b.Ret() - return b.Func() -} - // gemm32NR is the float32 tile width on amd64: two YMM of eight floats. const gemm32NR = 16 -// gemm32FMAKernel builds gemmMicro6x16FMA(kc int, pa, pb, c *float32, ldc int), -// the float32 micro-kernel of the packed GEMM: -// -// for p in [0,kc): C[r][col] += pa[p*6+r] * pb[p*16+col] (r<6, col<16) -// -// gemmFMAKernel at twice the lanes, the BLIS Haswell sgemm shape: the 16-float -// B row in Y12,Y13, each of the six A values broadcast with VBROADCASTSS, and -// twelve VFMADD231PS into Y0..Y11 (row r in Y(2r), Y(2r+1)). Gated on hasFMA. -func gemm32FMAKernel() *emit.Function { - b := amd64.NewFunc("gemmMicro6x16FMA", gemmSig(), 0) - b.LoadArg("kc", "CX") - b.LoadArg("pa", "SI") - b.LoadArg("pb", "DI") - b.LoadArg("c", "DX") - b.LoadArg("ldc", "R8") - b.Raw("SHLQ $2, R8") // ldc -> bytes - // Prefetch the C tile: six rows of 64 bytes, each on one or two lines. - b.Raw("MOVQ DX, R9") - for r := 0; r < gemmMR; r++ { - b.Raw("PREFETCHT0 (R9)") - b.Raw("PREFETCHT0 63(R9)") - if r < gemmMR-1 { - b.Raw("ADDQ R8, R9") - } - } - for r := 0; r < 12; r++ { - b.Raw(fmt.Sprintf("VXORPS Y%d, Y%d, Y%d", r, r, r)) - } - b.Raw("MOVQ CX, R10") - b.Raw("SHRQ $1, R10") - b.Raw("JZ stail") - b.Raw("skloop:") - b.Raw(fmt.Sprintf("PREFETCHT0 %d(SI)", 16*gemmMR*4)) - fma32Step(b, 0) - fma32Step(b, 1) - b.Raw(fmt.Sprintf("ADDQ $%d, SI", 2*4*gemmMR)) - b.Raw(fmt.Sprintf("ADDQ $%d, DI", 2*4*gemm32NR)) - b.Raw("DECQ R10") - b.Raw("JNZ skloop") - b.Raw("stail:") - b.Raw("TESTQ $1, CX") - b.Raw("JZ sstore") - fma32Step(b, 0) - b.Raw("sstore:") - for r := 0; r < gemmMR; r++ { - b.Raw(fmt.Sprintf("VADDPS (DX), Y%d, Y15", 2*r)) - b.Raw("VMOVUPS Y15, (DX)") - b.Raw(fmt.Sprintf("VADDPS 32(DX), Y%d, Y15", 2*r+1)) - b.Raw("VMOVUPS Y15, 32(DX)") - if r < gemmMR-1 { - b.Raw("ADDQ R8, DX") - } - } - b.Raw("VZEROUPPER") - b.Ret() - return b.Func() -} - -// gemm32UAKernel builds gemmUAx16FMA(kc int, a *float32, lda int, pb, c -// *float32, ldc int): the 6x16 float32 tile with A read in place instead of -// packed, for mr rows (1..6): -// -// for p in [0,kc): C[r][col] += a[r*lda+p] * pb[p*16+col] (rx8FMA: the same loop in PD instructions, 8-byte steps. -func gemm32UAKernel(mr int) *emit.Function { return gemmUAKernel(mr, true) } - -func gemmUAKernel(mr int, single bool) *emit.Function { - sig := amd64.Layout( - []string{"kc", "a", "lda", "pb", "ldb", "c", "ldc"}, - []amd64.Type{amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Int64, amd64.Ptr, amd64.Int64}, - nil, nil, - ) - name, size, shift, sfx, bcast := fmt.Sprintf("gemmUA%dx8FMA", mr), 8, 3, "PD", "VBROADCASTSD" - if single { - name, size, shift, sfx, bcast = fmt.Sprintf("gemmUA%dx16FMA", mr), 4, 2, "PS", "VBROADCASTSS" - } - b := amd64.NewFunc(name, sig, 0) - b.LoadArg("kc", "CX") - b.LoadArg("a", "SI") - b.LoadArg("lda", "R8") - b.LoadArg("pb", "DI") - b.LoadArg("ldb", "R13") - b.LoadArg("c", "DX") - b.LoadArg("ldc", "R9") - b.Raw(fmt.Sprintf("SHLQ $%d, R8", shift)) // lda -> bytes - b.Raw(fmt.Sprintf("SHLQ $%d, R13", shift)) // ldb -> bytes - b.Raw(fmt.Sprintf("SHLQ $%d, R9", shift)) // ldc -> bytes - b.Raw("LEAQ (R8)(R8*2), R11") // 3*lda - b.Raw("LEAQ (R8)(R8*4), R12") // 5*lda - b.Raw("MOVQ DX, R10") - for r := 0; r < mr; r++ { - b.Raw("PREFETCHT0 (R10)") - b.Raw("PREFETCHT0 63(R10)") - if r < mr-1 { - b.Raw("ADDQ R9, R10") - } - } - for r := 0; r < 2*mr; r++ { - b.Raw(fmt.Sprintf("VXOR%s Y%d, Y%d, Y%d", sfx, r, r, r)) - } - b.Raw("TESTQ CX, CX") - b.Raw("JZ uastore") - b.Raw("uakloop:") - b.Raw(fmt.Sprintf("VMOVU%s (DI), Y12", sfx)) - b.Raw(fmt.Sprintf("VMOVU%s 32(DI), Y13", sfx)) - for r := 0; r < mr; r++ { - b.Raw(fmt.Sprintf("%s %s, Y14", bcast, uaRow(r))) - b.Raw(fmt.Sprintf("VFMADD231%s Y12, Y14, Y%d", sfx, 2*r)) - b.Raw(fmt.Sprintf("VFMADD231%s Y13, Y14, Y%d", sfx, 2*r+1)) - } - b.Raw(fmt.Sprintf("ADDQ $%d, SI", size)) - b.Raw("ADDQ R13, DI") // next B row: ldb bytes on - b.Raw("DECQ CX") - b.Raw("JNZ uakloop") - b.Raw("uastore:") - for r := 0; r < mr; r++ { - b.Raw(fmt.Sprintf("VADD%s (DX), Y%d, Y15", sfx, 2*r)) - b.Raw(fmt.Sprintf("VMOVU%s Y15, (DX)", sfx)) - b.Raw(fmt.Sprintf("VADD%s 32(DX), Y%d, Y15", sfx, 2*r+1)) - b.Raw(fmt.Sprintf("VMOVU%s Y15, 32(DX)", sfx)) - if r < mr-1 { - b.Raw("ADDQ R9, DX") - } - } - b.Raw("VZEROUPPER") - b.Ret() - return b.Func() -} - -// uaRow is the address of row r of the unpacked A column at SI, rows lda -// bytes apart (R8), 3*lda in R11 and 5*lda in R12. -func uaRow(r int) string { - return [...]string{"(SI)", "(SI)(R8*1)", "(SI)(R8*2)", "(SI)(R11*1)", "(SI)(R8*4)", "(SI)(R12*1)"}[r] -} - -// fma32Step emits one k step of the 6x16 float32 FMA tile. -func fma32Step(b *amd64.Builder, step int) { - ao, bo := 4*gemmMR*step, 4*gemm32NR*step - b.Raw(fmt.Sprintf("VMOVUPS %d(DI), Y12", bo)) // B cols 0..7 - b.Raw(fmt.Sprintf("VMOVUPS %d(DI), Y13", bo+32)) // B cols 8..15 - for r := 0; r < gemmMR; r++ { - b.Raw(fmt.Sprintf("VBROADCASTSS %d(SI), Y14", ao+4*r)) - b.Raw(fmt.Sprintf("VFMADD231PS Y12, Y14, Y%d", 2*r)) - b.Raw(fmt.Sprintf("VFMADD231PS Y13, Y14, Y%d", 2*r+1)) - } -} - // dotFMAKernel builds dotFMA(a, b *float64, n int) float64, the inner product // with four YMM accumulators (16 doubles per iteration, one VFMADD231PD per 4 // doubles reading b straight from memory): two loads per FMA make the loop @@ -990,61 +742,6 @@ func fmaStep(b *amd64.Builder, step int) { } } -// gemmSSE2HalfKernel builds gemmMicro6x4SSE2(kc int, pa, pb, c *float64, ldc -// int), the baseline (GOAMD64=v1, no FMA) fallback over the same 6x8 packing: -// it computes the 6-row x 4-col half of the tile whose B columns start at pb, -// so the Go side calls it twice (pb, c) and (pb+4, c+4). A whole 6x8 tile in -// XMM would need 24 accumulators; the half needs 12 (X0..X11, row r in X(2r), -// X(2r+1)) plus the B half-row X12,X13, the broadcast A X14 and a product X15. -// The B stride is still the full NR=8 doubles per k step. -// -// SSE2 has no fused multiply-add, so each step is MULPD then ADDPD: the -// running sum is the plain dst += a*b in ikj order, like the scalar oracle at -// GOAMD64=v1. -func gemmSSE2HalfKernel() *emit.Function { - b := amd64.NewFunc("gemmMicro6x4SSE2", gemmSig(), 0) - b.LoadArg("kc", "CX") - b.LoadArg("pa", "SI") - b.LoadArg("pb", "DI") - b.LoadArg("c", "DX") - b.LoadArg("ldc", "R8") - b.Raw("SHLQ $3, R8") - for r := 0; r < 12; r++ { - b.Raw(fmt.Sprintf("XORPS X%d, X%d", r, r)) - } - b.Raw("TESTQ CX, CX") - b.Raw("JZ hstore") - b.Raw("hkloop:") - b.Raw("MOVUPD (DI), X12") - b.Raw("MOVUPD 16(DI), X13") - for r := 0; r < gemmMR; r++ { - b.Raw(fmt.Sprintf("MOVDDUP %d(SI), X14", 8*r)) // X14 = [a,a] - b.Raw("MOVAPS X12, X15") - b.Raw("MULPD X14, X15") - b.Raw(fmt.Sprintf("ADDPD X15, X%d", 2*r)) - b.Raw("MULPD X13, X14") // the broadcast is spent: reuse it for the product - b.Raw(fmt.Sprintf("ADDPD X14, X%d", 2*r+1)) - } - b.Raw(fmt.Sprintf("ADDQ $%d, SI", 8*gemmMR)) - b.Raw(fmt.Sprintf("ADDQ $%d, DI", 8*gemmNR)) - b.Raw("DECQ CX") - b.Raw("JNZ hkloop") - b.Raw("hstore:") - for r := 0; r < gemmMR; r++ { - b.Raw("MOVUPD (DX), X14") - b.Raw(fmt.Sprintf("ADDPD X%d, X14", 2*r)) - b.Raw("MOVUPD X14, (DX)") - b.Raw("MOVUPD 16(DX), X15") - b.Raw(fmt.Sprintf("ADDPD X%d, X15", 2*r+1)) - b.Raw("MOVUPD X15, 16(DX)") - if r < gemmMR-1 { - b.Raw("ADDQ R8, DX") - } - } - b.Ret() - return b.Func() -} - func nanScan(b *amd64.Builder, reg string) { b.Raw("MOVAPS " + reg + ", X9") b.Raw("CMPPD X9, X9, $3") diff --git a/internal/kernels/asmgen/arm64/gen.go b/internal/kernels/asmgen/arm64/gen.go index c1f5254..29c6bd3 100644 --- a/internal/kernels/asmgen/arm64/gen.go +++ b/internal/kernels/asmgen/arm64/gen.go @@ -25,7 +25,6 @@ package main import ( "fmt" "os" - "strings" "github.com/go-asmgen/asmgen/arm64" "github.com/go-asmgen/asmgen/emit" @@ -51,14 +50,7 @@ func main() { f.Add(sqrt32Kernel()) // max32NEON/min32NEON moved to go-simd/floats (Float32Max, Float32Min; // go-ndarray/ndarray#98). - f.Add(gemmKernel()) - f.Add(gemm32Kernel()) - for mr := 1; mr <= 8; mr++ { - f.Add(gemmUAKernel(mr, true)) - if mr <= 4 { - f.Add(gemmUAKernel(mr, false)) - } - } + // The GEMM kernels moved to github.com/go-blas/blas, which generates them. f.Add(blocks8Kernel()) if err := os.WriteFile("sum_arm64.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) @@ -272,347 +264,6 @@ func extremumKernel(name, vop, sop string) *emit.Function { return b.Func() } -// gemmSig is the micro-kernel signature -// -// gemmMicro4x8(kc int, pa, pb, c *float64, ldc int) -func gemmSig() arm64.Signature { - return arm64.Layout( - []string{"kc", "pa", "pb", "c", "ldc"}, - []arm64.Type{arm64.Int64, arm64.Ptr, arm64.Ptr, arm64.Ptr, arm64.Int64}, - nil, nil, - ) -} - -// fmlaElem returns the Plan 9 line for the by-element (indexed) double FMLA -// -// FMLA V{acc}.2D, V{breg}.2D, V{areg}.D[idx] => V{acc} += V{breg} * V{areg}[idx] -// -// which Go's arm64 assembler cannot name, emitted as its raw 32-bit encoding via -// WORD. The encoding (Advanced SIMD, FMLA by element, FP64, ARM C7-2: Q=1, sz=1, -// L=0) is -// -// 0|1|0|01111|1|1|0|M|Rm[3:0]|0001|H|0|Rn|Rd -// -// with the element index = H (sz=1 so L is unused) and Rm = M:Rm[3:0]. The result -// includes a trailing comment naming the human-readable instruction so the -// generated .s stays auditable; the encoding is cross-checked against objdump in -// the asmgen tests. areg/breg/acc are V-register numbers, idx in {0,1}. -func fmlaElem(acc, breg, areg, idx int) string { - const base = 0x4FC01000 // Q=1, sz=1, opcode 0001, base for FMLA-by-elem .2D - M := (areg >> 4) & 1 - Rm := areg & 0xF - H := idx & 1 - word := base | (M << 20) | (Rm << 16) | (H << 11) | (breg << 5) | acc - return fmt.Sprintf("WORD $0x%08X // FMLA V%d.2D, V%d.2D, V%d.D[%d]", - word, acc, breg, areg, idx) -} - -// gemmKernel builds gemmMicro4x8(kc int, pa, pb, c *float64, ldc int): the NEON -// 4x8 register-blocked GEMM micro-kernel at the heart of the packed GEMM. -// -// It computes a 4-row x 8-col tile of C from packed panels and ADDS it into C: -// -// for p in [0,kc): C[r][col] += pa[p*4+r] * pb[p*8+col] (r<4, col<8) -// -// pa is the packed A panel (MR=4 contiguous A values per k step), pb the packed -// B panel (NR=8 contiguous B values per k step) — both unit-stride, so the loads -// are conflict-free regardless of the source matrices' strides. This contiguity -// is the whole point of packing: it removes the power-of-two L1 set-conflicts -// that defeated the earlier un-packed register-blocked attempt. -// -// Register map: the 4x8 C tile lives in V0..V15 (4 rows x 4 D2 = 16 accumulators); -// the per-step B row in V16..V19 (8 doubles), the four A values in V20,V21 (two -// D2 vectors, two A doubles each), and a vector of 1.0 in V31 for the final -// C += acc. -// -// Two arm64-assembler constraints shape the kernel (both noted at point of use): -// 1. THE BY-LANE FMLA. Every hand-tuned arm64 dgemm broadcasts one A lane -// straight into the FMA with FMLA Vd.2d, Vn.2d, Vm.d[i]. Go's arm64 assembler -// cannot *name* this indexed-element form ("illegal combination ... ELEM"), -// so it is emitted by its raw WORD encoding (fmlaElem below: cross-checked -// against objdump). The four packed A doubles of a k-step are loaded as two -// D2 vectors (V20=[a0,a1], V21=[a2,a3]) and each row's FMA reads its A scalar -// by lane — eliminating the MOVD-to-GP + VDUP round-trip the prior kernel -// paid per A value every k-step. Measured on this Apple-silicon core this is -// ~1.28x faster on the L1-resident micro-kernel (44 -> 58 GFLOP/s/core); -// the earlier note claiming the indexed form is "slower" was wrong (it was -// never actually measured against a correct encoding). See docs/perf.md. -// 2. The closing C += acc is done as C = C + acc*1.0 via VFMLA against -// V31=[1.0,1.0], written when Go's assembler had no VFADD (it has since 1.27). Multiplying by -// exactly 1.0 is exact and FMA(acc,1.0,C) rounds identically to C+acc (C is -// the only inexact addend), so this is a true add — the tile result is -// bit-identical to the scalar oracle's ikj accumulation order. -func gemmKernel() *emit.Function { - b := arm64.NewFunc("gemmMicro4x8", gemmSig(), 0) - b.LoadArg("kc", "R0"). - LoadArg("pa", "R1"). - LoadArg("pb", "R2"). - LoadArg("c", "R3"). - LoadArg("ldc", "R4"). - Raw("LSL $3, R4, R4"). // ldc -> bytes - // Zero the sixteen C accumulators V0..V15. - Raw("VEOR V0.B16, V0.B16, V0.B16"). - Raw("VEOR V1.B16, V1.B16, V1.B16"). - Raw("VEOR V2.B16, V2.B16, V2.B16"). - Raw("VEOR V3.B16, V3.B16, V3.B16"). - Raw("VEOR V4.B16, V4.B16, V4.B16"). - Raw("VEOR V5.B16, V5.B16, V5.B16"). - Raw("VEOR V6.B16, V6.B16, V6.B16"). - Raw("VEOR V7.B16, V7.B16, V7.B16"). - Raw("VEOR V8.B16, V8.B16, V8.B16"). - Raw("VEOR V9.B16, V9.B16, V9.B16"). - Raw("VEOR V10.B16, V10.B16, V10.B16"). - Raw("VEOR V11.B16, V11.B16, V11.B16"). - Raw("VEOR V12.B16, V12.B16, V12.B16"). - Raw("VEOR V13.B16, V13.B16, V13.B16"). - Raw("VEOR V14.B16, V14.B16, V14.B16"). - Raw("VEOR V15.B16, V15.B16, V15.B16"). - Raw("CBZ R0, gstore"). - Raw("gkloop:"). - // B row: 8 contiguous doubles into V16..V19. - Raw("VLD1.P 64(R2), [V16.D2, V17.D2, V18.D2, V19.D2]"). - // A col: 4 contiguous doubles into two D2 vectors V20=[a0,a1], V21=[a2,a3]. - // Each row's FMA reads its A scalar by lane (header note 1) — no VDUP. - Raw("VLD1.P 32(R1), [V20.D2, V21.D2]"). - // FMLA Vacc.2D, Vb.2D, Va.D[i] => Vacc += Vb * Va[i]. row r uses A lane - // (r&1) of V20 (r<2) or V21 (r>=2). - Raw(fmlaElem(0, 16, 20, 0)). // row0 = a0 - Raw(fmlaElem(1, 17, 20, 0)). - Raw(fmlaElem(2, 18, 20, 0)). - Raw(fmlaElem(3, 19, 20, 0)). - Raw(fmlaElem(4, 16, 20, 1)). // row1 = a1 - Raw(fmlaElem(5, 17, 20, 1)). - Raw(fmlaElem(6, 18, 20, 1)). - Raw(fmlaElem(7, 19, 20, 1)). - Raw(fmlaElem(8, 16, 21, 0)). // row2 = a2 - Raw(fmlaElem(9, 17, 21, 0)). - Raw(fmlaElem(10, 18, 21, 0)). - Raw(fmlaElem(11, 19, 21, 0)). - Raw(fmlaElem(12, 16, 21, 1)). // row3 = a3 - Raw(fmlaElem(13, 17, 21, 1)). - Raw(fmlaElem(14, 18, 21, 1)). - Raw(fmlaElem(15, 19, 21, 1)). - Raw("SUB $1, R0"). - Raw("CBNZ R0, gkloop"). - // C += acc, row by row, via VFMLA against V31=[1.0,1.0]: exact, so - // bit-identical to C+acc (written before Go 1.27 named VFADD). - Raw("gstore:"). - Raw("MOVD $0x3FF0000000000000, R6"). - Raw("VDUP R6, V31.D2"). - Raw("MOVD R3, R5"). // row0 - Raw("VLD1 (R5), [V16.D2, V17.D2, V18.D2, V19.D2]"). - Raw("VFMLA V0.D2, V31.D2, V16.D2"). - Raw("VFMLA V1.D2, V31.D2, V17.D2"). - Raw("VFMLA V2.D2, V31.D2, V18.D2"). - Raw("VFMLA V3.D2, V31.D2, V19.D2"). - Raw("VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5)"). - Raw("ADD R4, R3, R3"). // row1 - Raw("MOVD R3, R5"). - Raw("VLD1 (R5), [V16.D2, V17.D2, V18.D2, V19.D2]"). - Raw("VFMLA V4.D2, V31.D2, V16.D2"). - Raw("VFMLA V5.D2, V31.D2, V17.D2"). - Raw("VFMLA V6.D2, V31.D2, V18.D2"). - Raw("VFMLA V7.D2, V31.D2, V19.D2"). - Raw("VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5)"). - Raw("ADD R4, R3, R3"). // row2 - Raw("MOVD R3, R5"). - Raw("VLD1 (R5), [V16.D2, V17.D2, V18.D2, V19.D2]"). - Raw("VFMLA V8.D2, V31.D2, V16.D2"). - Raw("VFMLA V9.D2, V31.D2, V17.D2"). - Raw("VFMLA V10.D2, V31.D2, V18.D2"). - Raw("VFMLA V11.D2, V31.D2, V19.D2"). - Raw("VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5)"). - Raw("ADD R4, R3, R3"). // row3 - Raw("MOVD R3, R5"). - Raw("VLD1 (R5), [V16.D2, V17.D2, V18.D2, V19.D2]"). - Raw("VFMLA V12.D2, V31.D2, V16.D2"). - Raw("VFMLA V13.D2, V31.D2, V17.D2"). - Raw("VFMLA V14.D2, V31.D2, V18.D2"). - Raw("VFMLA V15.D2, V31.D2, V19.D2"). - Raw("VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5)"). - Ret() - return b.Func() -} - -// gemmUAKernel builds the unpacked-A tiles (the "rv" sup kernels of BLIS, -// bli_gemmsup_rv_*): A is read in place, mr rows lda elements apart, instead -// of from a packed panel; B stays packed. -// -// single: gemmUAx8S(kc int, a *float32, lda int, pb, c *float32, ldc int), mr <= 8 -// double: gemmUAx8D(kc int, a *float64, lda int, pb, c *float64, ldc int), mr <= 4 -// -// computing C[r][col] += a[r*lda+p] * pb[p*8+col] for r < mr. Each row has its -// own pointer (R5..R12). A row is contiguous along k, so one plain vector load -// brings U consecutive k values of it (U = 4 floats or 2 doubles) into V20+r, -// and the U k steps that follow take their A value from lane s of that vector -// with the by-element FMLA (fmlaElem/fmlaElemS), as the packed kernels do. -// The kc%U remaining steps load each row's value as a scalar into lane 0. -// A first version loaded every A value with a load-and-replicate (VLD1R) and -// used the vector FMLA: on a Neoverse N1 it was 26-78% slower than packing A, -// apparently because the replicating loads compete with the FMAs for the -// vector pipes. -// Accumulators: float32 row r in V(2r), V(2r+1); float64 row r in V(4r)..V(4r+3). -func gemmUAKernel(mr int, single bool) *emit.Function { - sig := arm64.Layout( - []string{"kc", "a", "lda", "pb", "ldb", "c", "ldc"}, - []arm64.Type{arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Int64, arm64.Ptr, arm64.Int64}, - nil, nil, - ) - name, shift, arr, vecs := fmt.Sprintf("gemmUA%dx8D", mr), 3, "D2", 4 - if single { - name, shift, arr, vecs = fmt.Sprintf("gemmUA%dx8S", mr), 2, "S4", 2 - } - size := 1 << shift - bregs := make([]string, vecs) - for v := range bregs { - bregs[v] = fmt.Sprintf("V%d.%s", 16+v, arr) - } - blist := "[" + strings.Join(bregs, ", ") + "]" - b := arm64.NewFunc(name, sig, 0) - b.LoadArg("kc", "R0"). - LoadArg("a", "R1"). - LoadArg("lda", "R2"). - LoadArg("pb", "R13"). - LoadArg("ldb", "R16"). - LoadArg("c", "R3"). - LoadArg("ldc", "R4"). - Raw(fmt.Sprintf("LSL $%d, R2, R2", shift)). - Raw(fmt.Sprintf("LSL $%d, R16, R16", shift)). // B rows ldb elements apart (BLIS's b += rs_b) - Raw(fmt.Sprintf("LSL $%d, R4, R4", shift)) - for r := 0; r < mr; r++ { // row pointers R5.. = a + r*lda - if r == 0 { - b.Raw("MOVD R1, R5") - } else { - b.Raw(fmt.Sprintf("ADD R2, R%d, R%d", 4+r, 5+r)) - } - } - for v := 0; v < mr*vecs; v++ { - b.Raw(fmt.Sprintf("VEOR V%d.B16, V%d.B16, V%d.B16", v, v, v)) - } - u := 16 / size // k steps per A vector - fmla, scalar := fmlaElem, "FMOVD" - if single { - fmla, scalar = fmlaElemS, "FMOVS" - } - step := func(lane int) { - b.Raw(fmt.Sprintf("VLD1.P (R13)(R16), %s", blist)) // post-increment by ldb - for r := 0; r < mr; r++ { - for v := 0; v < vecs; v++ { - b.Raw(fmla(vecs*r+v, 16+v, 20+r, lane)) - } - } - } - b.Raw(fmt.Sprintf("LSR $%d, R0, R14", shift4(u))). // whole groups of u steps - Raw(fmt.Sprintf("AND $%d, R0, R15", u-1)). // remaining steps - Raw("CBZ R14, uatail"). - Raw("uakloop:") - for r := 0; r < mr; r++ { - b.Raw(fmt.Sprintf("VLD1.P 16(R%d), [V%d.%s]", 5+r, 20+r, arr)) - } - for s := 0; s < u; s++ { - step(s) - } - b.Raw("SUB $1, R14"). - Raw("CBNZ R14, uakloop"). - Raw("uatail:"). - Raw("CBZ R15, uastore"). - Raw("uatloop:") - for r := 0; r < mr; r++ { - b.Raw(fmt.Sprintf("%s.P %d(R%d), F%d", scalar, size, 5+r, 20+r)) - } - step(0) - b.Raw("SUB $1, R15"). - Raw("CBNZ R15, uatloop"). - Raw("uastore:") - for r := 0; r < mr; r++ { - b.Raw(fmt.Sprintf("VLD1 (R3), %s", blist)) - for v := 0; v < vecs; v++ { - b.Raw(fmt.Sprintf("VFADD V%d.%s, V%d.%s, V%d.%s", vecs*r+v, arr, 16+v, arr, 16+v, arr)) - } - b.Raw(fmt.Sprintf("VST1 %s, (R3)", blist)) - if r < mr-1 { - b.Raw("ADD R4, R3, R3") - } - } - b.Ret() - return b.Func() -} - -// shift4 is log2 of u, 2 or 4. -func shift4(u int) int { - if u == 4 { - return 2 - } - return 1 -} - -// fmlaElemS is fmlaElem for single precision: -// -// FMLA V{acc}.4S, V{breg}.4S, V{areg}.S[idx] => V{acc} += V{breg} * V{areg}[idx] -// -// The same Advanced SIMD "FMLA (by element)" encoding with sz=0 (ARM C7-2): -// -// 0|1|0|01111|1|0|L|M|Rm[3:0]|0001|H|0|Rn|Rd -// -// where the element index is H:L and, for sz=0, the register is M:Rm, so -// areg may be any of V0..V31. idx in {0,1,2,3}. -func fmlaElemS(acc, breg, areg, idx int) string { - const base = 0x4F801000 // Q=1, sz=0, opcode 0001: FMLA by element .4S - M := (areg >> 4) & 1 - Rm := areg & 0xF - H, L := (idx>>1)&1, idx&1 - word := base | (L << 21) | (M << 20) | (Rm << 16) | (H << 11) | (breg << 5) | acc - return fmt.Sprintf("WORD $0x%08X // FMLA V%d.4S, V%d.4S, V%d.S[%d]", - word, acc, breg, areg, idx) -} - -// gemm32Kernel builds gemmMicro8x8S(kc int, pa, pb, c *float32, ldc int), the -// float32 micro-kernel of the packed GEMM: an 8-row x 8-col tile, -// -// for p in [0,kc): C[r][col] += pa[p*8+r] * pb[p*8+col] (r<8, col<8) -// -// the float64 kernel's structure at twice the lanes. The tile lives in V0..V15 -// (row r in V(2r), V(2r+1), four floats each), the B row in V16,V17 and the -// eight A values in V18=[a0..a3], V19=[a4..a7], which the by-element FMLA -// (fmlaElemS) reads lane by lane. Sixteen independent accumulator chains cover -// the FMA latency of two 4-cycle pipes (Neoverse N1) with room to spare; each -// k step is two loads for sixteen FMAs. The closing C += acc is a VFADD. -func gemm32Kernel() *emit.Function { - b := arm64.NewFunc("gemmMicro8x8S", gemmSig(), 0) - b.LoadArg("kc", "R0"). - LoadArg("pa", "R1"). - LoadArg("pb", "R2"). - LoadArg("c", "R3"). - LoadArg("ldc", "R4"). - Raw("LSL $2, R4, R4") // ldc -> bytes - for r := 0; r < 16; r++ { - b.Raw(fmt.Sprintf("VEOR V%d.B16, V%d.B16, V%d.B16", r, r, r)) - } - b.Raw("CBZ R0, sstore"). - Raw("skloop:"). - Raw("VLD1.P 32(R2), [V16.S4, V17.S4]"). // B row: 8 floats - Raw("VLD1.P 32(R1), [V18.S4, V19.S4]") // A column: 8 floats - for r := 0; r < 8; r++ { - areg, lane := 18+r/4, r%4 - b.Raw(fmlaElemS(2*r, 16, areg, lane)) - b.Raw(fmlaElemS(2*r+1, 17, areg, lane)) - } - b.Raw("SUB $1, R0"). - Raw("CBNZ R0, skloop"). - Raw("sstore:") - for r := 0; r < 8; r++ { - b.Raw("VLD1 (R3), [V16.S4, V17.S4]") - b.Raw(fmt.Sprintf("VFADD V%d.S4, V16.S4, V16.S4", 2*r)) - b.Raw(fmt.Sprintf("VFADD V%d.S4, V17.S4, V17.S4", 2*r+1)) - b.Raw("VST1 [V16.S4, V17.S4], (R3)") - if r < 7 { - b.Raw("ADD R4, R3, R3") - } - } - b.Ret() - return b.Func() -} - // sumKernel builds sumNEON(a *float64, n int) float64. // // The lane-parallel accumulation is VFMLA against a vector of 1.0: acc += src * diff --git a/internal/kernels/asmgen/loong64/gen.go b/internal/kernels/asmgen/loong64/gen.go index a4035da..704a2b5 100644 --- a/internal/kernels/asmgen/loong64/gen.go +++ b/internal/kernels/asmgen/loong64/gen.go @@ -39,7 +39,6 @@ func main() { //"XVADDD" -> "VADDF", "ADDD" -> "ADDF": the LSX single-precision op. f.Add(bin32Kernel(k.name[:3]+"32LSX", k.vec[1:len(k.vec)-1]+"F", k.sca[:len(k.sca)-1]+"F")) } - f.Add(gemmKernel()) f.Add(blocks8Kernel()) f.Add(sqrt32Kernel()) abs64 := f.Data("abs64", lanes(8, 0x7fffffffffffffff)) @@ -242,46 +241,6 @@ func bin32Kernel(name, vec, sca string) *emit.Function { return b.Func() } -// gemmKernel builds gemmMicro8x8LASX(kc int, pa, pb, c *float64, ldc int): -// -// for p in [0,kc): C[r][col] += pa[p*8+r] * pb[p*8+col] (r<8, col<8) -// -// 16 accumulators X0..X15 (row r in X(2r), X(2r+1), four columns each), the B -// row in X16, X17, and each A value broadcast (XVMOVQ .V4, xvldrepl.d) into X18..X25, -// then xvfmadd.d: 16 fused multiply-adds per k step against 10 loads. Fused, -// like the other FMA kernels; on integer-valued data it equals the scalar -// ikj oracle exactly. -func gemmKernel() *emit.Function { - sig := loong64.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, - []loong64.Type{loong64.Int64, loong64.Ptr, loong64.Ptr, loong64.Ptr, loong64.Int64}, nil, nil) - b := loong64.NewFunc("gemmMicro8x8LASX", sig, 0) - b.LoadArg("kc", "R5").LoadArg("pa", "R6").LoadArg("pb", "R7").LoadArg("c", "R8").LoadArg("ldc", "R9") - b.Raw("SLLV $3, R9") // ldc -> bytes - zero(b, 16) - b.Raw("BEQ R5, gstore") - b.Label("gloop") - b.Raw("XVMOVQ 0(R7), X16").Raw("XVMOVQ 32(R7), X17") - for r := 0; r < gemmMR; r++ { - b.Raw("XVMOVQ %d(R6), X%d.V4", 8*r, 18+r) // xvldrepl.d: pa[r] in all four lanes - } - for r := 0; r < gemmMR; r++ { - b.XVFMADDD(2*r, 18+r, 16, 2*r) - b.XVFMADDD(2*r+1, 18+r, 17, 2*r+1) - } - b.Raw("ADDV $%d, R6", 8*gemmMR).Raw("ADDV $%d, R7", 8*gemmNR).Raw("ADDV $-1, R5").Raw("BNE R5, gloop") - b.Label("gstore") - for r := 0; r < gemmMR; r++ { - b.Raw("XVMOVQ 0(R8), X16").Raw("XVMOVQ 32(R8), X17") - b.Raw("XVADDD X%d, X16, X16", 2*r).Raw("XVADDD X%d, X17, X17", 2*r+1) - b.Raw("XVMOVQ X16, 0(R8)").Raw("XVMOVQ X17, 32(R8)") - if r < gemmMR-1 { - b.Raw("ADDV R9, R8") - } - } - b.Ret() - return b.Func() -} - // blocks8Kernel builds blocks8LSX(r, a *float32, blocks int): for each of // the blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's // pairwise-sum block). The eight accumulators are split over two 128-bit LSX diff --git a/internal/kernels/asmgen/ppc64le/gen.go b/internal/kernels/asmgen/ppc64le/gen.go index db3a544..748175e 100644 --- a/internal/kernels/asmgen/ppc64le/gen.go +++ b/internal/kernels/asmgen/ppc64le/gen.go @@ -39,7 +39,6 @@ func main() { }{{"addVSX", "add", "FADD"}, {"subVSX", "sub", "FSUB"}, {"mulVSX", "mul", "FMUL"}, {"divVSX", "div", "FDIV"}} { f.Add(bin32Kernel(k.name[:3]+"32VSX", k.vec, k.sca+"S")) } - f.Add(gemmKernel()) f.Add(blocks8Kernel()) f.Add(sqrt32Kernel()) abs64 := f.Data("abs64", lanes(8, 0x7fffffffffffffff)) @@ -48,7 +47,6 @@ func main() { // go-ndarray/ndarray#98). f.Add(extremumKernel("maxVSX", 64, true, abs64, inf64)) f.Add(extremumKernel("minVSX", 64, false, abs64, inf64)) - f.Add(gemm32Kernel(f.Data("oddw", words(1, 3)), f.Data("highw", words(2, 3)))) if err := os.WriteFile("sum_ppc64le.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -259,182 +257,6 @@ func bin32Kernel(name, op, sca string) *emit.Function { return b.Func() } -// gemmKernel builds gemmMicro8x8VSX(kc int, pa, pb, c *float64, ldc int): -// -// for p in [0,kc): C[r][col] += pa[p*8+r] * pb[p*8+col] (r<8, col<8) -// -// with the scheme of OpenBLAS's POWER9 dgemm kernel: no splats. Each A pair -// (a_2p, a_2p+1) and its swap (one XXPERMDI) multiply each B column pair, so -// accumulator VS(8p+2k) holds the diagonal (C[2p][2k], C[2p+1][2k+1]) and -// VS(8p+2k+1) the anti-diagonal (C[2p+1][2k], C[2p][2k+1]); the store rebuilds -// the rows with XXPERMDI (DM=1). 32 fused multiply-adds per k step against 8 -// loads and 4 swaps; two k steps per iteration, the second step's loads issued -// between the first step's FMAs. Measured on a POWER9 core (cfarm29): 16.9 -// GFLOP/s, against 13.3 with LXVDSX splats, 14.8 with XXPERMDI splats and 15.4 -// with those unrolled; OpenBLAS's own kernel reaches 23 there. -func gemmKernel() *emit.Function { - name := "gemmMicro8x8VSX" - sig := ppc64.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, - []ppc64.Type{ppc64.Int64, ppc64.Ptr, ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil) - b := ppc64.NewFunc(name, sig, 0) - b.LoadArg("kc", "R3").LoadArg("pa", "R4").LoadArg("pb", "R5").LoadArg("c", "R6").LoadArg("ldc", "R7") - b.Raw("SLD $3, R7") - offsets(b, false) - for r := 0; r < 32; r++ { - b.Raw("XXLXOR VS%d, VS%d, VS%d", r, r, r) - } - b.Raw("CMP R3, $0").Raw("BEQ %sstore", name) - // step uses B vectors bv..bv+3, A pairs av..av+3, swapped pairs sw..sw+3. - loadsOf := func(bv, av int) []string { - var l []string - for k := 0; k < 4; k++ { - l = append(l, fmt.Sprintf("LXVD2X (R5)(%s), VS%d", vidx(k), bv+k), fmt.Sprintf("LXVD2X (R4)(%s), VS%d", vidx(k), av+k)) - } - return l - } - step := func(bv, av, sw int, pending []string) { - for p := 0; p < 4; p++ { - b.Raw("XXPERMDI VS%d, VS%d, $2, VS%d", av+p, av+p, sw+p) - } - i := 0 - for p := 0; p < 4; p++ { - for k := 0; k < 4; k++ { - b.XVMADDADP(8*p+2*k, av+p, bv+k) - b.XVMADDADP(8*p+2*k+1, sw+p, bv+k) - if i%2 == 1 && len(pending) > 0 { - b.Raw("%s", pending[0]) - pending = pending[1:] - } - i++ - } - } - for _, l := range pending { // never leave a load out - b.Raw("%s", l) - } - } - b.Label(name+"loop2").Raw("CMP R3, $2").Raw("BLT %sone", name) - for _, l := range loadsOf(32, 36) { - b.Raw("%s", l) - } - b.Raw("ADD $64, R4").Raw("ADD $64, R5") - step(32, 36, 40, loadsOf(44, 48)) - b.Raw("ADD $64, R4").Raw("ADD $64, R5") - step(44, 48, 52, nil) - b.Raw("ADD $-2, R3").Raw("BR %sloop2", name) - b.Label(name+"one").Raw("CMP R3, $0").Raw("BEQ %sstore", name) - for _, l := range loadsOf(32, 36) { - b.Raw("%s", l) - } - step(32, 36, 40, nil) - b.Label(name + "store") - for p := 0; p < 4; p++ { - for half := 0; half < 2; half++ { // row 2p, then 2p+1 - for k := 0; k < 4; k++ { - d, a := 8*p+2*k, 8*p+2*k+1 - if half == 0 { - b.Raw("XXPERMDI VS%d, VS%d, $1, VS%d", d, a, 56) // [diag.dw0, anti.dw1] - } else { - b.Raw("XXPERMDI VS%d, VS%d, $1, VS%d", a, d, 56) // [anti.dw0, diag.dw1] - } - b.Raw("LXVD2X (R6)(%s), VS%d", vidx(k), 57) - b.XVADDDP(57, 57, 56) - b.Raw("STXVD2X VS%d, (R6)(%s)", 57, vidx(k)) - } - if !(p == 3 && half == 1) { - b.Raw("ADD R7, R6") - } - } - } - b.Ret() - return b.Func() -} - -// gemm32Kernel builds gemmMicro8x16VSX(kc int, pa, pb, c *float32, ldc int): -// -// for p in [0,kc): C[r][col] += pa[p*8+r] * pb[p*16+col] (r<8, col<16) -// -// with the scheme of OpenBLAS's POWER9 sgemm (kernel/power/sgemm_macros_power9.S, -// LOAD8x16): no splats. Each 4-float A vector is rotated three ways, and the -// rotations multiply each B vector, so the tile is accumulated in a rotated -// layout that the store untangles once. OpenBLAS rotates B with xxperm and -// xxpermdi; this kernel rotates A, which has two vectors per k step against -// B's four, with the cyclic word shift XXSLDWI a, a, $v: rotation v holds -// a[(i+v)%4] in lane i (lanes numbered as LXVW4X loads them, memory order). -// -// Accumulator VS(16p+4k+v) lane i is then C[4p+(i+v)%4][4k+i]: A vector p -// (rows 4p..4p+3), B vector k (columns 4k..4k+3), rotation v. Row 4p+r of -// that 4x4 block takes lane i from rotation (r-i) mod 4, so it is three -// selects: x = lanes 0,1 from rotations r, r-1; y = lanes 2,3 from r-2, r-3; -// row = x's low half and y's high half (XXSEL against the odd-lanes and -// high-lanes masks, VS48 and VS49). -// -// Per k step: 2 A loads, 6 rotations, 4 B loads, 32 XVMADDASP. -func gemm32Kernel(odd, high string) *emit.Function { - name := "gemmMicro8x16VSX" - sig := ppc64.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, - []ppc64.Type{ppc64.Int64, ppc64.Ptr, ppc64.Ptr, ppc64.Ptr, ppc64.Int64}, nil, nil) - b := ppc64.NewFunc(name, sig, 0) - b.LoadArg("kc", "R3").LoadArg("pa", "R4").LoadArg("pb", "R5").LoadArg("c", "R6").LoadArg("ldc", "R7") - b.Raw("SLD $2, R7") - offsets(b, false) - b.Raw("MOVD $%s+0(SB), R8", odd).Raw("LXVW4X (R8)(R0), VS48"). - Raw("MOVD $%s+0(SB), R8", high).Raw("LXVW4X (R8)(R0), VS49") - for r := 0; r < 32; r++ { - b.Raw("XXLXOR VS%d, VS%d, VS%d", r, r, r) - } - b.Raw("CMP R3, $0").Raw("BEQ %sstore", name) - b.Label(name + "loop"). - Raw("LXVW4X (R4)(R0), VS32").Raw("LXVW4X (R4)(R14), VS36") - for k := 0; k < 4; k++ { - b.Raw("LXVW4X (R5)(%s), VS%d", vidx(k), 40+k) - } - for p := 0; p < 2; p++ { - for v := 1; v < 4; v++ { - b.Raw("XXSLDWI VS%d, VS%d, $%d, VS%d", 32+4*p, 32+4*p, v, 32+4*p+v) - } - } - for p := 0; p < 2; p++ { - for k := 0; k < 4; k++ { - for v := 0; v < 4; v++ { - b.XVMADDASP(16*p+4*k+v, 32+4*p+v, 40+k) - } - } - } - b.Raw("ADD $32, R4").Raw("ADD $64, R5").Raw("ADD $-1, R3"). - Raw("CMP R3, $0").Raw("BNE %sloop", name) - b.Label(name + "store") - acc := func(p, k, v int) int { return 16*p + 4*k + (v+4)%4 } - for p := 0; p < 2; p++ { - for r := 0; r < 4; r++ { - for k := 0; k < 4; k++ { - b.Raw("XXSEL VS%d, VS%d, VS48, VS44", acc(p, k, r), acc(p, k, r-1)). - Raw("XXSEL VS%d, VS%d, VS48, VS45", acc(p, k, r-2), acc(p, k, r-3)). - Raw("XXSEL VS44, VS45, VS49, VS46"). - Raw("LXVW4X (R6)(%s), VS47", vidx(k)) - b.XVADDSP(47, 47, 46) - b.Raw("STXVW4X VS47, (R6)(%s)", vidx(k)) - } - if !(p == 1 && r == 3) { - b.Raw("ADD R7, R6") - } - } - } - b.Ret() - return b.Func() -} - -// words is a 16-byte mask with all-ones in the given 4-byte lanes (memory -// order, as LXVW4X loads them) and zeros elsewhere. -func words(on ...int) []byte { - m := make([]byte, 16) - for _, w := range on { - for i := 0; i < 4; i++ { - m[4*w+i] = 0xff - } - } - return m -} - // blocks8Kernel builds blocks8VSX(r, a *float32, blocks int): for each of the // blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's pairwise-sum // block). The eight accumulators are split over two VSX registers, VS0 and VS1 diff --git a/internal/kernels/asmgen/riscv64/gen.go b/internal/kernels/asmgen/riscv64/gen.go index 9fa69f9..3f9014f 100644 --- a/internal/kernels/asmgen/riscv64/gen.go +++ b/internal/kernels/asmgen/riscv64/gen.go @@ -38,10 +38,6 @@ func main() { } { f.Add(bin32Kernel(k.name[:3]+"32RVV", k.op)) } - f.Add(gemmKernel()) - f.Add(gemm32Kernel()) - f.Add(gemm32Kernel8x16(gemm8x16Unroll)) - f.Add(gemmVLENBKernel()) f.Add(blocks8Kernel()) f.Add(sqrt32Kernel()) for _, k := range []struct { @@ -206,155 +202,6 @@ func bin32Kernel(name, op string) *emit.Function { return b.Func() } -// gemmKernel builds gemmMicro6x8RVV(kc int, pa, pb, c *float64, ldc int) for -// kc >= 1: C[r][0:8] += sum_p pa[p*6+r] * pb[p*8:p*8+8] for r < 6, the rows of C -// ldc float64s apart. -// -// NR is fixed by the packing, while VLEN varies between RISC-V cores. The V -// extension guarantees VLEN >= 128, so a group of four registers (LMUL=4) holds -// at least eight float64: vl = 8 on every core. That leaves eight groups, six -// of them accumulators, one for the B row and one for C. Each step loads the B -// row once and fuses each A value in with vfmacc.vf (Vacc += f*Vb). -func gemmKernel() *emit.Function { - sig := riscv64.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, - []riscv64.Type{riscv64.Int64, riscv64.Ptr, riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil) - b := riscv64.NewFunc("gemmMicro6x8RVV", sig, 0) - b.LoadArg("kc", "X10").LoadArg("pa", "X11").LoadArg("pb", "X12"). - LoadArg("c", "X13").LoadArg("ldc", "X14") - b.Raw("VSETIVLI $8, E64, M4, TA, MA, X15") - for r := 0; r < 6; r++ { - b.Raw("VMVVI $0, V%d", 4*r) - } - b.Raw("loop:").Raw("VLE64V (X12), V24") - for r := 0; r < 6; r++ { - b.Raw("MOVD %d(X11), F%d", 8*r, r) - } - for r := 0; r < 6; r++ { - b.Raw("VFMACCVF V24, F%d, V%d", r, 4*r) // acc_r += a_r * b - } - b.Raw("ADD $48, X11, X11"). - Raw("ADD $64, X12, X12"). - Raw("ADD $-1, X10, X10"). - Raw("BNEZ X10, loop"). - Raw("SLLI $3, X14, X14") // row stride in bytes - for r := 0; r < 6; r++ { - b.Raw("VLE64V (X13), V28"). - Raw("VFADDVV V%d, V28, V28", 4*r). // C row += acc_r - Raw("VSE64V V28, (X13)") - if r < 5 { - b.Raw("ADD X14, X13, X13") - } - } - return b.Ret().Func() -} - -// gemm32Kernel builds gemmMicro6x16RVV(kc int, pa, pb, c *float32, ldc int) for -// kc >= 1: C[r][0:16] += sum_p pa[p*6+r] * pb[p*16:p*16+16] for r < 6, the rows -// of C ldc float32s apart. gemmKernel in float32: with E32 a group of four -// registers (LMUL=4) holds at least sixteen floats on any VLEN >= 128, so -// vl = 16 everywhere, and the same six accumulator groups, B group and C group -// fit. -func gemm32Kernel() *emit.Function { - sig := riscv64.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, - []riscv64.Type{riscv64.Int64, riscv64.Ptr, riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil) - b := riscv64.NewFunc("gemmMicro6x16RVV", sig, 0) - b.LoadArg("kc", "X10").LoadArg("pa", "X11").LoadArg("pb", "X12"). - LoadArg("c", "X13").LoadArg("ldc", "X14") - b.Raw("VSETIVLI $16, E32, M4, TA, MA, X15") - for r := 0; r < 6; r++ { - b.Raw("VMVVI $0, V%d", 4*r) - } - b.Raw("loop:").Raw("VLE32V (X12), V24") - for r := 0; r < 6; r++ { - b.Raw("MOVF %d(X11), F%d", 4*r, r) - } - for r := 0; r < 6; r++ { - b.Raw("VFMACCVF V24, F%d, V%d", r, 4*r) // acc_r += a_r * b - } - b.Raw("ADD $24, X11, X11"). - Raw("ADD $64, X12, X12"). - Raw("ADD $-1, X10, X10"). - Raw("BNEZ X10, loop"). - Raw("SLLI $2, X14, X14") // row stride in bytes - for r := 0; r < 6; r++ { - b.Raw("VLE32V (X13), V28"). - Raw("VFADDVV V%d, V28, V28", 4*r). // C row += acc_r - Raw("VSE32V V28, (X13)") - if r < 5 { - b.Raw("ADD X14, X13, X13") - } - } - return b.Ret().Func() -} - -// gemm32Kernel8x16 builds gemmMicro8x16RVV(kc int, pa, pb, c *float32, ldc int) -// for kc >= 1: C[r][0:16] += sum_p pa[p*8+r] * pb[p*16:p*16+16] for r < 8, the -// rows of C ldc float32s apart. It needs VLEN >= 256: with E32 and LMUL=2 a -// group of two registers holds sixteen floats only then (vl would be 8 on a -// 128-bit core), so the caller picks it by gemmVLENB. LMUL=4 with vl=16, as in -// gemm32Kernel, fills half of each group on a 256-bit core. Eight accumulator -// groups (V0..V15), `unroll` B groups from V16 and one C group (V28). Each pass -// loads the B rows and A scalars of `unroll` steps ahead of their FMAs, so the -// in-order core waits on loads once per pass; kc % unroll steps run first. -func gemm32Kernel8x16(unroll int) *emit.Function { - sig := riscv64.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, - []riscv64.Type{riscv64.Int64, riscv64.Ptr, riscv64.Ptr, riscv64.Ptr, riscv64.Int64}, nil, nil) - b := riscv64.NewFunc("gemmMicro8x16RVV", sig, 0) - b.LoadArg("kc", "X10").LoadArg("pa", "X11").LoadArg("pb", "X12"). - LoadArg("c", "X13").LoadArg("ldc", "X14") - b.Raw("VSETIVLI $16, E32, M2, TA, MA, X15") - for r := 0; r < 8; r++ { - b.Raw("VMVVI $0, V%d", 2*r) - } - // steps emits n steps: all loads first (B group V16+2s, A scalars F8s..), - // then the 8n FMAs. - steps := func(n int) { - for s := 0; s < n; s++ { - b.Raw("VLE32V (X12), V%d", 16+2*s).Raw("ADD $64, X12, X12") - } - for s := 0; s < n; s++ { - for r := 0; r < 8; r++ { - b.Raw("MOVF %d(X11), F%d", 32*s+4*r, 8*s+r) - } - } - for s := 0; s < n; s++ { - for r := 0; r < 8; r++ { - b.Raw("VFMACCVF V%d, F%d, V%d", 16+2*s, 8*s+r, 2*r) - } - } - b.Raw("ADD $%d, X11, X11", 32*n) - } - if unroll > 1 { - b.Raw("ANDI $%d, X10, X5", unroll-1).Raw("SUB X5, X10, X10").Raw("BEQZ X5, main") - b.Raw("tail:") - steps(1) - b.Raw("ADD $-1, X5, X5").Raw("BNEZ X5, tail") - b.Raw("main:").Raw("BEQZ X10, epi") - } - b.Raw("loop:") - steps(unroll) - b.Raw("ADD $%d, X10, X10", -unroll).Raw("BNEZ X10, loop") - b.Raw("epi:").Raw("SLLI $2, X14, X14") // row stride in bytes - for r := 0; r < 8; r++ { - b.Raw("VLE32V (X13), V28"). - Raw("VFADDVV V%d, V28, V28", 2*r). // C row += acc_r - Raw("VSE32V V28, (X13)") - if r < 7 { - b.Raw("ADD X14, X13, X13") - } - } - return b.Ret().Func() -} - -// gemmVLENBKernel builds gemmVLENB() int, the vector register length in bytes: -// VSETVLI with AVL = VLMAX (rs1 = x0) at E8, LMUL=1 returns it in vl. -func gemmVLENBKernel() *emit.Function { - sig := riscv64.Layout(nil, nil, []string{"ret"}, []riscv64.Type{riscv64.Int64}) - b := riscv64.NewFunc("gemmVLENB", sig, 0) - b.Raw("VSETVLI X0, E8, M1, TA, MA, X5").StoreRet("X5", "ret") - return b.Ret().Func() -} - // blocks8Kernel builds blocks8RVV(r, a *float32, blocks int): for each of the // blocks of eight float32 at a, in order, r[j] += a[8i+j] (NumPy's // pairwise-sum block). VSETIVLI $8 with E32 and M2 gives vl = 8 on every diff --git a/internal/kernels/asmgen/s390x/gen.go b/internal/kernels/asmgen/s390x/gen.go index 2e4ede2..28cb80c 100644 --- a/internal/kernels/asmgen/s390x/gen.go +++ b/internal/kernels/asmgen/s390x/gen.go @@ -36,8 +36,6 @@ func main() { } { f.Add(bin32Kernel(k.name[:3]+"32VXE", k.name[:3], k.sca+"S")) } - f.Add(gemmKernel()) - f.Add(gemm32Kernel()) f.Add(extremumKernel("maxVXE", "VFMAXDB", "WFMAXDB")) f.Add(extremumKernel("minVXE", "VFMINDB", "WFMINDB")) f.Add(blocks8Kernel()) @@ -162,86 +160,6 @@ func bin32Kernel(name, op, sca string) *emit.Function { return b.Func() } -// gemmKernel builds gemmMicro4x8VX(kc int, pa, pb, c *float64, ldc int) for -// kc >= 1: C[r][0:8] += sum_p pa[p*4+r] * pb[p*8:p*8+8] for r < 4, the rows of C -// ldc float64s apart. Sixteen two-lane accumulators V0..V15 (row r in -// V4r..V4r+3); each step loads the B row in one VLM, replicates each A value -// across a register with VLREPG, and fuses it in with VFMADB. -func gemmKernel() *emit.Function { - sig := s390x.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, - []s390x.Type{s390x.Int64, s390x.Ptr, s390x.Ptr, s390x.Ptr, s390x.Int64}, nil, nil) - b := s390x.NewFunc("gemmMicro4x8VX", sig, 0) - b.LoadArg("kc", "R5").LoadArg("pa", "R2").LoadArg("pb", "R3"). - LoadArg("c", "R1").LoadArg("ldc", "R4") - for r := 0; r < 16; r++ { - b.Raw("VZERO V%d", r) - } - b.Raw("loop:").Raw("VLM (R3), V16, V19") // the B row - for r := 0; r < 4; r++ { - b.Raw("VLREPG %d(R2), V%d", 8*r, 20+r) // a_r in both lanes - } - for r := 0; r < 4; r++ { - for j := 0; j < 4; j++ { - b.Raw("VFMADB V%d, V%d, V%d, V%d", 20+r, 16+j, 4*r+j, 4*r+j) // acc += a_r * b - } - } - b.Raw("ADD $32, R2").Raw("ADD $64, R3").Raw("SUB $1, R5"). - Raw("CMP R5, $0").Raw("BNE loop"). - Raw("SLD $3, R4") // row stride in bytes - for r := 0; r < 4; r++ { - b.Raw("VLM (R1), V24, V27") - for j := 0; j < 4; j++ { - b.Raw("VFADB V%d, V%d, V%d", 4*r+j, 24+j, 24+j) // C row += acc_r - } - b.Raw("VSTM V24, V27, (R1)") - if r < 3 { - b.Raw("ADD R4, R1") - } - } - return b.Ret().Func() -} - -// gemm32Kernel builds gemmMicro4x16VXE(kc int, pa, pb, c *float32, ldc int) -// for kc >= 1: C[r][0:16] += sum_p pa[p*4+r] * pb[p*16:p*16+16] for r < 4, the -// rows of C ldc float32s apart. gemmKernel at twice the lanes: sixteen -// four-lane accumulators V0..V15 (row r in V4r..V4r+3), the B row in one VLM, -// each A value replicated across a register with VLREPF, and VFMASB, the -// vector-enhancements facility 1 (z14) fused multiply-add, which go-asmgen -// encodes because Go's assembler has no short-format vector float op. -func gemm32Kernel() *emit.Function { - sig := s390x.Layout([]string{"kc", "pa", "pb", "c", "ldc"}, - []s390x.Type{s390x.Int64, s390x.Ptr, s390x.Ptr, s390x.Ptr, s390x.Int64}, nil, nil) - b := s390x.NewFunc("gemmMicro4x16VXE", sig, 0) - b.LoadArg("kc", "R5").LoadArg("pa", "R2").LoadArg("pb", "R3"). - LoadArg("c", "R1").LoadArg("ldc", "R4") - for r := 0; r < 16; r++ { - b.Raw("VZERO V%d", r) - } - b.Raw("loop:").Raw("VLM (R3), V16, V19") // the B row: 16 floats - for r := 0; r < 4; r++ { - b.Raw("VLREPF %d(R2), V%d", 4*r, 20+r) // a_r in all four lanes - } - for r := 0; r < 4; r++ { - for j := 0; j < 4; j++ { - b.VFMASB(4*r+j, 20+r, 16+j, 4*r+j) // acc += a_r * b - } - } - b.Raw("ADD $16, R2").Raw("ADD $64, R3").Raw("SUB $1, R5"). - Raw("CMP R5, $0").Raw("BNE loop"). - Raw("SLD $2, R4") // row stride in bytes - for r := 0; r < 4; r++ { - b.Raw("VLM (R1), V24, V27") - for j := 0; j < 4; j++ { - b.VFASB(24+j, 24+j, 4*r+j) // C row += acc_r - } - b.Raw("VSTM V24, V27, (R1)") - if r < 3 { - b.Raw("ADD R4, R1") - } - } - return b.Ret().Func() -} - // extremumKernel builds NAME(a *float64, n int) float64, the maximum (or // minimum) of a, for n >= 8. VFMAXDB/VFMINDB and their scalar WF forms come // with the vector-enhancements facility 1 (z14), so the Go side gates these diff --git a/internal/kernels/barrier_test.go b/internal/kernels/barrier_test.go index 6378cbd..974a026 100644 --- a/internal/kernels/barrier_test.go +++ b/internal/kernels/barrier_test.go @@ -22,20 +22,3 @@ func TestSpinUntil(t *testing.T) { spinIters = saved } } - -// TestGemmParallelYielding runs the parallel GEMM with every wait yielding at -// once, as on an oversubscribed machine, and with more workers than CPUs. -func TestGemmParallelYielding(t *testing.T) { - saved := spinIters - spinIters = 0 - defer func() { spinIters = saved }() - withMaxProcs(8, func() { - withThresholds(1<<14, 1, func() { - for _, s := range []struct{ m, k, n int }{{50, 70, 90}, {13, 300, 7}, {97, 31, 600}} { - a, b := intMat(s.m, s.k, 3), intMat(s.k, s.n, 4) - assertGemm(t, s.m, s.k, s.n, a, b) - withBlocks(MR, 8, 2*NR, func() { assertGemm(t, s.m, s.k, s.n, a, b) }) - } - }) - }) -} diff --git a/internal/kernels/dot_amd64.go b/internal/kernels/dot_amd64.go new file mode 100644 index 0000000..b2ed5d1 --- /dev/null +++ b/internal/kernels/dot_amd64.go @@ -0,0 +1,21 @@ +package kernels + +// useFMA selects the FMA dot kernel. It is a var so tests can force the pure-Go +// path on hardware that has FMA (never the reverse: that would fault). +var useFMA = hasFMA() + +// dotSIMD is the inner product of two equal-length slices: the AVX2/FMA +// kernel when the CPU has FMA, else dotRange. +func dotSIMD(a, b []float64) float64 { + if !useFMA || len(a) == 0 { + return dotRange(a, b) + } + _ = b[len(a)-1] // the kernel reads len(a) of b + return dotFMA(&a[0], &b[0], len(a)) +} + +//go:noescape +func dotFMA(a, b *float64, n int) float64 + +// hasFMA reports CPU and OS support for the FMA kernel (go-asmgen FeatureProbe). +func hasFMA() bool diff --git a/internal/kernels/dot_arm64.go b/internal/kernels/dot_arm64.go new file mode 100644 index 0000000..4fa5d52 --- /dev/null +++ b/internal/kernels/dot_arm64.go @@ -0,0 +1,13 @@ +package kernels + +// dotSIMD is the inner product of two equal-length slices (NEON kernel). +func dotSIMD(a, b []float64) float64 { + if len(a) == 0 { + return 0 + } + _ = b[len(a)-1] // the kernel reads len(a) of b + return dotNEON(&a[0], &b[0], len(a)) +} + +//go:noescape +func dotNEON(a, b *float64, n int) float64 diff --git a/internal/kernels/dot_loong64.go b/internal/kernels/dot_loong64.go new file mode 100644 index 0000000..037bc56 --- /dev/null +++ b/internal/kernels/dot_loong64.go @@ -0,0 +1,13 @@ +package kernels + +// dotSIMD is the inner product (LASX kernel, four fused accumulators). +func dotSIMD(a, b []float64) float64 { + if !useLASX || len(a) == 0 { + return dotRange(a, b) + } + _ = b[len(a)-1] // the kernel reads len(a) of b + return dotLASX(&a[0], &b[0], len(a)) +} + +//go:noescape +func dotLASX(a, b *float64, n int) float64 diff --git a/internal/kernels/dot_ppc64le.go b/internal/kernels/dot_ppc64le.go new file mode 100644 index 0000000..3dcd9d8 --- /dev/null +++ b/internal/kernels/dot_ppc64le.go @@ -0,0 +1,14 @@ +package kernels + +// dotSIMD is the inner product of two equal-length slices (VSX kernel, four +// fused multiply-add accumulators). +func dotSIMD(a, b []float64) float64 { + if len(a) == 0 { + return 0 + } + _ = b[len(a)-1] // the kernel reads len(a) of b + return dotVSX(&a[0], &b[0], len(a)) +} + +//go:noescape +func dotVSX(a, b *float64, n int) float64 diff --git a/internal/kernels/fence_amd64_test.go b/internal/kernels/fence_amd64_test.go deleted file mode 100644 index 7b32f75..0000000 --- a/internal/kernels/fence_amd64_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package kernels - -// forEachFallback runs f again with the SSE2 kernels the FMA ones replace. -func forEachFallback(f func()) { - saved := useFMA - useFMA = false - defer func() { useFMA = saved }() - f() -} diff --git a/internal/kernels/fence_fallback_test.go b/internal/kernels/fence_fallback_test.go deleted file mode 100644 index 387527c..0000000 --- a/internal/kernels/fence_fallback_test.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build arm64 || ppc64le - -package kernels - -// forEachFallback has nothing to do: these kernels have no CPU-dependent -// alternative. -func forEachFallback(func()) {} diff --git a/internal/kernels/fence_loong64_test.go b/internal/kernels/fence_loong64_test.go deleted file mode 100644 index e2f8dcb..0000000 --- a/internal/kernels/fence_loong64_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package kernels - -// forEachFallback runs f again with the scalar kernel used without LASX. -func forEachFallback(f func()) { - saved := useLASX - useLASX = false - defer func() { useLASX = saved }() - f() -} diff --git a/internal/kernels/fence_riscv64_test.go b/internal/kernels/fence_riscv64_test.go deleted file mode 100644 index b12a954..0000000 --- a/internal/kernels/fence_riscv64_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package kernels - -// forEachFallback runs f again with the scalar kernels used without V. -func forEachFallback(f func()) { - saved := useRVV - useRVV = false - defer func() { useRVV = saved }() - f() -} diff --git a/internal/kernels/fence_s390x_test.go b/internal/kernels/fence_s390x_test.go deleted file mode 100644 index 5d30fac..0000000 --- a/internal/kernels/fence_s390x_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package kernels - -// forEachFallback runs f again with the scalar kernels used without the vector facility. -func forEachFallback(f func()) { - saved, savedE := useVX, useVXE - useVX, useVXE = false, false - defer func() { useVX, useVXE = saved, savedE }() - f() -} diff --git a/internal/kernels/fence_test.go b/internal/kernels/fence_test.go index 3b243ad..c75c460 100644 --- a/internal/kernels/fence_test.go +++ b/internal/kernels/fence_test.go @@ -116,76 +116,3 @@ func TestKernelsStayInsideTheirSlices(t *testing.T) { } } } - -func TestGemmMicroStaysInsideItsOperands(t *testing.T) { - run := func() { - for kc := 1; kc <= 9; kc++ { - for _, ldc := range []int{NR, NR + 3, 64} { - for _, pa := range fences(t, kc*MR) { - for _, pb := range fences(t, kc*NR) { - for _, dst := range fences(t, (MR-1)*ldc+NR) { - gemmMicro(kc, pa, pb, dst, ldc) - } - } - } - } - } - } - run() - forEachFallback(run) - if gemm64.microUA == nil { - return - } - for kc := 1; kc <= 9; kc++ { - for _, ldc := range []int{NR, NR + 3} { - for _, ldb := range []int{NR, NR + 5} { // a packed panel, a row-major B in place - b := fences(t, (kc-1)*ldb+NR)[1] - for mr := 1; mr <= MR; mr++ { - for _, lda := range []int{kc, kc + 5} { - for _, a := range fences(t, (mr-1)*lda+kc) { - for _, dst := range fences(t, (mr-1)*ldc+NR) { - gemm64.microUA(kc, a, lda, b, ldb, dst, ldc, mr) - } - } - } - } - } - } - } -} - -// TestGemmMicro32StaysInsideItsOperands fences the float32 GEMM kernel's -// panels and C tile, where there is one. -func TestGemmMicro32StaysInsideItsOperands(t *testing.T) { - g := gemm32 - if g == nil { - t.Skip("no float32 GEMM kernel here") - } - for kc := 1; kc <= 9; kc++ { - // a row-major B read in place by microUA, rows g.nr+5 apart, ending - // flush against the guard - bs := fences32(t, (kc-1)*(g.nr+5)+g.nr)[1] - for _, ldc := range []int{g.nr, g.nr + 3, 64} { - for _, pa := range fences32(t, kc*g.mr) { - for _, pb := range fences32(t, kc*g.nr) { - for _, dst := range fences32(t, (g.mr-1)*ldc+g.nr) { - g.micro(kc, pa, pb, dst, ldc) - } - if g.microUA == nil { - continue - } - for mr := 1; mr <= g.mr; mr++ { - for _, lda := range []int{kc, kc + 5} { - for _, a := range fences32(t, (mr-1)*lda+kc) { - for _, dst := range fences32(t, (mr-1)*ldc+g.nr) { - g.microUA(kc, a, lda, pb, g.nr, dst, ldc, mr) - g.microUA(kc, a, lda, bs, g.nr+5, dst, ldc, mr) - } - } - } - } - } - } - } - } -} diff --git a/internal/kernels/gemm.go b/internal/kernels/gemm.go deleted file mode 100644 index 1358f0d..0000000 --- a/internal/kernels/gemm.go +++ /dev/null @@ -1,707 +0,0 @@ -package kernels - -import ( - "runtime" - "sync" - "sync/atomic" -) - -// Packed, cache-blocked GEMM (the OpenBLAS/BLIS structure). -// -// The earlier register-blocked kernel read A and B straight from the source -// matrices, so on power-of-two row strides its four destination rows collided in -// the L1 cache and it lost to the autovectorized ikj kernel for large n. The fix -// is the technique every tuned BLAS uses: *packing*. A is copied into MR-tall row -// panels and B into NR-wide column panels in contiguous, unit-stride scratch, so -// the micro-kernel always streams conflict-free memory regardless of the source -// matrices' strides. A cache-blocking loop nest (NC over columns, KC over the -// contraction, MC over rows) keeps the packed B panel resident in L2/L3 and each -// packed A panel in L1, and a register-blocked SIMD-FMA micro-kernel -// (gemmMicro, arch-specific: NEON 4x8 on arm64, AVX2/FMA 6x8 or SSE2 on amd64, -// scalar 4x4 elsewhere) does the inner work. Parallelised across row bands. -// -// The driver is generic over the element type: float64 everywhere (gemm64), and -// float32 where a float32 micro-kernel exists (gemm32: NEON 8x8 on arm64, -// AVX2/FMA 6x16 on amd64), which accumulates in float32 as a BLAS sgemm does. -// Each element type brings its own tile and its own pack-buffer pools. -// -// The arithmetic is the ikj accumulation dst[i][j] += a[i][p]*b[p][j]: within -// one KC block the products are summed in p order, and each block's sum is then -// added into dst. For k <= KC that is the scalar oracle's order exactly; above -// it the block partials regroup the sum, a valid reordering that can move the -// last place (as every blocked BLAS does). Kernels that fuse the multiply-add -// (arm64, amd64 with FMA) round once per step where an unfused one rounds -// twice. On data whose products and sums are exact (the integer-valued tests) -// every path equals the oracle bit for bit. - -// Block sizes for the cache-blocking loop nest. They are vars so tests can pin -// them (forcing the multi-block paths on small inputs) and so they can be tuned -// per machine. Defaults are measured on the arm64 bench VM (Apple silicon): -// - KC*NC packed-B panel (256*512*8 = 1 MiB) stays L2-resident, -// - MC*KC packed-A panel (256*256*8 = 512 KiB) feeds the micro-kernel, -// - they divide evenly into MR/NR tiles at the edges via zero padding. -var ( - blockMC = 256 - blockKC = 256 - blockNC = 512 -) - -// bandTilesMR is the height, in MR-row micro-tiles, of one dynamic work band in -// MatMulP (so a band is bandTilesMR*MR rows). It is sized so the m rows dice into -// several bands per core: enough oversubscription for the atomic work-counter to -// balance the M-series P/E cores, while large enough that the per-band A-pack and -// goroutine-loop overhead stays amortized. A var so it can be tuned/pinned. -var bandTilesMR = 8 - -// gemmFloat is the element types the packed GEMM multiplies. -type gemmFloat interface{ float32 | float64 } - -// MatOf is a read-only strided view of a matrix: element (i, j) is -// Data[Off + i*RS + j*CS]. Packing reads the operands through it, so a -// transposed, sliced or offset view is multiplied in place — the packing copy -// the GEMM makes anyway does the gather — instead of being materialised into -// a contiguous copy first (what BLAS's trans flags buy). Strides may be -// negative or zero; every (i, j) in range must index Data. -type MatOf[T gemmFloat] struct { - Data []T - Off, RS, CS int -} - -// Mat and Mat32 are the float64 and float32 operand views. -type ( - Mat = MatOf[float64] - Mat32 = MatOf[float32] -) - -// rowMajor views a contiguous row-major matrix with ld elements per row. -func rowMajor[T gemmFloat](data []T, ld int) MatOf[T] { - return MatOf[T]{Data: data, RS: ld, CS: 1} -} - -// gemmKernel is one element type's micro-kernel, the mr x nr tile it computes, -// and the pools of pack buffers sized for that tile. micro adds the mr x nr tile -// sum_p pa[p*mr..]*pb[p*nr..] into dst, whose rows are ldc apart; see gemmMicro. -// -// microUA, where there is one, is the same tile with A read in place: rows -// lda > 0 elements apart from a[0], k steps along each row, for the top mr -// (1..mr) rows only, against B rows ldb elements apart (nr for a packed panel). -// gemmBand then never packs a row-major A, and the serial path reads the full -// panels of a row-major B in place too (see packGemmRows). -type gemmKernel[T gemmFloat] struct { - mr, nr int - micro func(kc int, pa, pb, dst []T, ldc int) - microUA func(kc int, a []T, lda int, b []T, ldb int, dst []T, ldc, mr int) - paPool, pbPool sync.Pool -} - -// unpackA lets the GEMM read a row-major A in place where the kernel can, and -// unpackB a row-major B on the serial path. They are vars so tests can drive -// the packed paths on the same inputs. -var unpackA, unpackB = true, true - -// gemm64 is the float64 GEMM, on the per-architecture gemmMicro. -var gemm64 = &gemmKernel[float64]{mr: MR, nr: NR, micro: gemmMicro, microUA: gemmMicroUA} - -// packBuf is a pair of pack buffers: pa for the MC*KC A panel, pb for the KC*NC -// B panel. The two come from separate pools, because the parallel GEMM needs -// one pa per worker but only two pb in all: drawing a pair per worker held -// 16 MiB of unused B buffers at 16 workers, which the garbage collector empties -// out of the pool and the next call zeroes again. -type packBuf[T gemmFloat] struct { - pa []T - pb []T -} - -// roundUp returns x rounded up to the next multiple of m (m >= 1). -func roundUp(x, m int) int { return (x + m - 1) / m * m } - -// paCap / pbCap are the pack-buffer capacities. The A buffer holds -// ceil(MC/MR)*MR-tall panels (each MC row block is tiled into MR-tall panels and -// the last, partial panel is zero-padded to a full MR), so its row extent must be -// rounded UP to a multiple of MR — otherwise the bottom edge panel of an MC block -// whose height is not a multiple of MR would write past the buffer. Likewise the -// B buffer rounds the column extent up to a multiple of NR. (With the old MR=4 -// and MC=256 the rounding was a no-op since 4 | 256; MR=6 makes it load-bearing.) -// -// The A buffer also carries, past the panels, the mr x nr scratch tile gemmEdge -// runs the kernel into. The tile cannot live on gemmEdge's stack: the kernel is -// called through a func value, so escape analysis would move it to the heap at -// every edge tile. -func (g *gemmKernel[T]) paCap() int { return roundUp(blockMC, g.mr)*blockKC + g.mr*g.nr } -func (g *gemmKernel[T]) pbCap() int { return blockKC * roundUp(blockNC, g.nr) } - -// getPA and getPB return a pooled pack buffer sized for the current block -// parameters, allocating one when the pool is empty or its buffer is too small -// (a test may raise blockMC/KC/NC). putPA and putPB return it. -func (g *gemmKernel[T]) getPA() []T { return getScratch[T](&g.paPool, g.paCap()) } -func (g *gemmKernel[T]) getPB() []T { return getScratch[T](&g.pbPool, g.pbCap()) } -func (g *gemmKernel[T]) putPA(b []T) { g.paPool.Put(&b) } -func (g *gemmKernel[T]) putPB(b []T) { g.pbPool.Put(&b) } - -func getScratch[T gemmFloat](p *sync.Pool, n int) []T { - if b, ok := p.Get().(*[]T); ok && cap(*b) >= n { - return (*b)[:n] - } - return make([]T, n) -} - -// getPackBuf returns a pooled pair, for the single-worker paths. -func (g *gemmKernel[T]) getPackBuf() *packBuf[T] { - return &packBuf[T]{pa: g.getPA(), pb: g.getPB()} -} - -func (g *gemmKernel[T]) putPackBuf(b *packBuf[T]) { - g.putPA(b.pa) - g.putPB(b.pb) -} - -// packGemmRows computes the row band [r0,r1) of dst = a(m x k) * b(k x n) with -// the packed, cache-blocked algorithm, using buf for scratch. dst is the FULL -// (m x n) destination (the band writes only rows [r0,r1)); a and b are the full -// operands. dst must be pre-zeroed by the caller. The band must be MR-aligned at -// r0 (the parallel splitter guarantees this) so packed A panels never straddle a -// band boundary; r1 may be m. This is the single-worker entry: it packs its own -// B panel. The multi-worker path (gemmBand) shares one packed B across workers. -// -// When both operands are row-major and the kernel reads A in place, B's full -// NR-wide panels are read in place as well (BLIS's sup kernels pack neither -// operand) and only the partial last panel is packed, for its zero padding. -// This is the small-product path (below GemmThreshold), where packing B was a -// measurable share of the time. -func (g *gemmKernel[T]) packGemmRows(r0, r1 int, dst []T, a, b MatOf[T], k, n int, buf *packBuf[T]) { - pa, pb := buf.pa, buf.pb - var bIn *MatOf[T] - if g.microUA != nil && unpackA && unpackB && a.CS == 1 && a.RS > 0 && b.CS == 1 && b.RS > 0 { - bIn = &b - } - for jc := 0; jc < n; jc += blockNC { // L3 column block of B - nc := min(blockNC, n-jc) - for pc := 0; pc < k; pc += blockKC { // L2 contraction block - kc := min(blockKC, k-pc) - if bIn == nil { - g.packB(b, pc, kc, jc, nc, pb) // pack B(kc x nc) -> NR-wide panels - } else if full := nc - nc%g.nr; full < nc { - g.packB(b, pc, kc, jc+full, nc-full, pb[full*kc:]) // the partial last panel only - } - g.gemmBand(r0, r1, jc, nc, pc, kc, dst, a, n, pa, pb, bIn) - } - } -} - -// gemmBand runs the macro-kernel for the row band [r0,r1) against an -// already-packed B panel pb (the kc x nc block at source columns [jc,jc+nc), -// source contraction rows [pc,pc+kc)). It packs each MC-row block of A into pa -// (the worker's private buffer) and streams the register-blocked micro-kernel -// over the MR x NR tiles, accumulating into the band's rows of dst. Splitting B -// packing (done once by the caller) from this per-band A packing + macro-kernel -// is what lets several workers share one packed B: the redundant full-B repack -// each worker used to do — 1 MiB copied GOMAXPROCS times per kc block — was the -// dominant non-scaling cost of the old row-split parallelism. -// -// bIn, when not nil, is a row-major B whose full panels the A-in-place kernel -// reads directly; pb then holds only the partial last panel (packGemmRows). -func (g *gemmKernel[T]) gemmBand(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pa, pb []T, bIn *MatOf[T]) { - mr0, nr0 := g.mr, g.nr - tile := pa[len(pa)-mr0*nr0:] // see paCap - if g.microUA != nil && unpackA && a.CS == 1 && a.RS > 0 { - g.gemmBandUA(r0, r1, jc, nc, pc, kc, dst, a, n, pb, tile, bIn) - return - } - for ic := r0; ic < r1; ic += blockMC { // L1 row block of A - mc := min(blockMC, r1-ic) - g.packA(a, ic, mc, pc, kc, pa) // pack A(mc x kc) -> MR-tall panels - for jr := 0; jr < nc; jr += nr0 { - nr := min(nr0, nc-jr) - for ir := 0; ir < mc; ir += mr0 { - mr := min(mr0, mc-ir) - c0 := (ic+ir)*n + jc + jr - if mr == mr0 && nr == nr0 { - g.micro(kc, pa[ir*kc:], pb[jr*kc:], dst[c0:], n) - } else { - g.gemmEdge(kc, pa[ir*kc:], pb[jr*kc:], dst, c0, n, mr, nr, tile) - } - } - } - } -} - -// gemmBandUA is gemmBand for a row-major A read in place by microUA: no A -// panel is packed. Each tile's A is the mr rows starting at row ic+ir, column -// pc, a.RS apart. A tile narrower than NR (the last B panel, zero-padded) runs -// into the scratch tile and only its nr columns are added into dst; a tile -// shorter than MR uses the kernel for its own height, so no row past A's last -// is read. With bIn, a full panel's B is read in place, bIn.RS apart. -func (g *gemmKernel[T]) gemmBandUA(r0, r1, jc, nc, pc, kc int, dst []T, a MatOf[T], n int, pb, tile []T, bIn *MatOf[T]) { - mr0, nr0 := g.mr, g.nr - for jr := 0; jr < nc; jr += nr0 { - nr := min(nr0, nc-jr) - bp, ldb := pb[jr*kc:], nr0 - if bIn != nil && nr == nr0 { - bp, ldb = bIn.Data[bIn.Off+pc*bIn.RS+jc+jr:], bIn.RS - } - for i := r0; i < r1; i += mr0 { - mr := min(mr0, r1-i) - ar := a.Data[a.Off+i*a.RS+pc:] - c0 := i*n + jc + jr - if nr == nr0 { - g.microUA(kc, ar, a.RS, bp, ldb, dst[c0:], n, mr) - continue - } - clear(tile) - g.microUA(kc, ar, a.RS, bp, ldb, tile, nr0, mr) - for r := 0; r < mr; r++ { - row := dst[c0+r*n : c0+r*n+nr] - for c := range row { - row[c] += tile[r*nr0+c] - } - } - } - } -} - -// packB copies B[pc:pc+kc][jc:jc+nc] into NR-wide column panels: for each panel -// jr the layout is pb[jr*kc + p*NR + c] = B(pc+p, jc+jr+c), with the c >= nr -// columns of an edge panel zero-filled so the micro-kernel reads a full NR. -// A unit column stride copies row runs; any other (a transposed B) walks each -// column down its own unit-or-not row stride, writing the panel column-wise. -func (g *gemmKernel[T]) packB(b MatOf[T], pc, kc, jc, nc int, pb []T) { - NR := g.nr - for jr := 0; jr < nc; jr += NR { - nr := min(NR, nc-jr) - dst := pb[jr*kc:] - if b.CS == 1 { - if nr == NR && packRowsWhole(dst, b.Data, b.Off+pc*b.RS+jc+jr, b.RS, kc, NR) { - continue - } - // Element loops, not copy and clear: on runs this short (NR - // elements) each of those is a runtime call. - for p := 0; p < kc; p++ { - srcRow := b.Data[b.Off+(pc+p)*b.RS+jc+jr:] - d := dst[p*NR:] - for c := 0; c < nr; c++ { - d[c] = srcRow[c] - } - for c := nr; c < NR; c++ { - d[c] = 0 - } - } - continue - } - for c := 0; c < nr; c++ { - src := b.Off + pc*b.RS + (jc+jr+c)*b.CS - for p := 0; p < kc; p++ { - dst[p*NR+c] = b.Data[src+p*b.RS] - } - } - for p := 0; p < kc; p++ { - d := dst[p*NR:] - for c := nr; c < NR; c++ { - d[c] = 0 - } - } - } -} - -// packRowsWhole copies kc runs of nr elements, the p-th starting at -// src[s+p*rs], to dst[p*nr:], each as one fixed-size array assignment, which -// the compiler lowers to a few vector moves instead of a loop or a runtime -// call. It reports false, copying nothing, for a width it has no array type -// for; the caller then copies element by element. -func packRowsWhole[T gemmFloat](dst, src []T, s, rs, kc, nr int) bool { - switch nr { - case 8: - for p := 0; p < kc; p++ { - *(*[8]T)(dst[p*8:]) = *(*[8]T)(src[s+p*rs:]) - } - case 16: - for p := 0; p < kc; p++ { - *(*[16]T)(dst[p*16:]) = *(*[16]T)(src[s+p*rs:]) - } - default: - return false - } - return true -} - -// packRows8 packs a full 8-row panel of a row-major A: dst[p*8+r] = a[s+r*rs+p] -// for p < kc, r < 8. It reads the eight rows as eight sequential streams and -// writes the panel in order, eight elements per step, where the row-by-row -// scatter walks the whole panel eight times with a stride of eight elements -// (4 ns per element on an in-order core, the cost of 64² float32 packing). -func packRows8[T gemmFloat](dst, a []T, s, rs, kc int) { - r0 := a[s : s+kc] - r1 := a[s+rs : s+rs+kc] - r2 := a[s+2*rs : s+2*rs+kc] - r3 := a[s+3*rs : s+3*rs+kc] - r4 := a[s+4*rs : s+4*rs+kc] - r5 := a[s+5*rs : s+5*rs+kc] - r6 := a[s+6*rs : s+6*rs+kc] - r7 := a[s+7*rs : s+7*rs+kc] - dst = dst[:kc*8] - for p := range r0 { - d := (*[8]T)(dst[p*8:]) - d[0], d[1], d[2], d[3] = r0[p], r1[p], r2[p], r3[p] - d[4], d[5], d[6], d[7] = r4[p], r5[p], r6[p], r7[p] - } -} - -// packA copies A[ic:ic+mc][pc:pc+kc] into MR-tall row panels: for each panel ir -// the layout is pa[ir*kc + p*MR + r] = A(ic+ir+r, pc+p), with the r >= mr rows -// of an edge panel zero-filled so the micro-kernel reads a full MR. -func (g *gemmKernel[T]) packA(a MatOf[T], ic, mc, pc, kc int, pa []T) { - MR := g.mr - for ir := 0; ir < mc; ir += MR { - mr := min(MR, mc-ir) - dst := pa[ir*kc:] - src := a.Off + (ic+ir)*a.RS + pc*a.CS - if a.CS == 1 && MR == 8 && mr == 8 { - packRows8(dst, a.Data, src, a.RS, kc) - continue - } - if a.CS == 1 { - // Row-major A: read each row as one contiguous run and scatter it - // into its lane of the panel. The column-wise walk below spends - // its loop and bounds checks on every MR elements; at 64² it - // took longer than the micro-kernel. - for r := 0; r < mr; r++ { - row := a.Data[src+r*a.RS : src+r*a.RS+kc] - d := dst[r : r+(kc-1)*MR+1] - for p, v := range row { - d[p*MR] = v - } - } - for r := mr; r < MR; r++ { - for p := 0; p < kc; p++ { - dst[p*MR+r] = 0 - } - } - continue - } - for p := 0; p < kc; p++ { - d := dst[p*MR:] - at := src + p*a.CS - for r := 0; r < mr; r++ { - d[r] = a.Data[at+r*a.RS] - } - for r := mr; r < MR; r++ { - d[r] = 0 - } - } - } -} - -// gemmEdge computes a partial (mr < MR or nr < NR) tile with the micro-kernel -// itself, the way BLIS does: the packed panels are already zero-padded to a full -// MR x NR, so the kernel runs into a zeroed MR x NR scratch tile and only the -// valid mr x nr corner is added into dst. Every element of C is therefore summed -// by the same kernel, edge or not. The scalar loop this replaces ran ~20x slower -// per element, and on amd64 (MR=6) every 256-row MC block ends in a 4-row edge -// panel: it measured 40% of a 512^2 product there. -func (g *gemmKernel[T]) gemmEdge(kc int, pa, pb, dst []T, c0, ldc, mr, nr int, tile []T) { - NR := g.nr - clear(tile) - g.micro(kc, pa, pb, tile, NR) - for r := 0; r < mr; r++ { - row := dst[c0+r*ldc : c0+r*ldc+nr] - for c := range row { - row[c] += tile[r*NR+c] - } - } -} - -// GemmThreshold is the minimum number of result elements (m*n) at which the -// packed GEMM fans out across goroutines. Small products run single-threaded -// (one worker, one pooled buffer) so the goroutine launch never dominates. -// -// 6000 is the measured crossover on the Apple-silicon bench host: below it the -// goroutine launch + join costs more than the parallel speedup buys, above it -// the fan-out wins. Concretely, 64x64 (4096 results) is ~14us serial vs ~17us -// parallel (serial wins) while 80x80 (6400 results) is ~26us serial vs ~22us -// parallel and 96x96 (9216) is ~46us serial vs ~34us parallel (parallel wins by -// 15-30%). The old 1<<14 (16384) floor left the whole 80x80..120x120 band — the -// shapes that benefit most from the cores — stuck on the slow serial path; it -// was sized for MatVecP (memory-bound, see matVecThreshold), not for the -// compute-bound GEMM. 32x32 (1024) and 64x64 (4096) stay serial, where the -// register-blocked single-core kernel is fastest. -var GemmThreshold = 6000 - -// matVecThreshold is the m*k element count above which MatVecP fans the row-dots -// across cores. Mat-vec is a single bandwidth-bound streaming pass over A (one -// multiply-add per element, no reuse), so its parallel crossover is far higher -// than the compute-bound GEMM's: forking it on a few-thousand-element A is a net -// loss (100x100 mat-vec measures ~2.5us serial vs ~5.8us parallel). It is kept -// at the original 1<<14 floor — lowering GemmThreshold for the GEMM must not -// drag mat-vec onto the parallel path in the band where it regresses. It is a -// var so tests can pin it. -var matVecThreshold = 1 << 14 - -// MatMulP computes dst = a(m x k) * b(k x n) with the packed, cache-blocked GEMM, -// fanning the m output rows across cores above GemmThreshold. dst must be zeroed -// by the caller. -// -// Parallelism is the BLIS "loop-3" (ic, row-block) split with DYNAMIC, work- -// stealing scheduling: for every (jc, pc) cache block the B panel is packed ONCE -// into a shared buffer, then the m rows are diced into MR-aligned bands of MC -// rows and the workers pull bands off a shared atomic counter until exhausted, -// each packing only its own A panel and running the macro-kernel (gemmBand) over -// the band against the single shared packed B. Every band writes a disjoint, -// MR-aligned region of dst (b and the shared pb are read-only during the band), -// so the result is identical to the serial computation. -// -// Two properties together gave the large-N scaling: -// - Packing B once (not once per worker) removes the O(P) redundant 1-MiB -// repack the old row-split paid per kc block. -// - Dynamic banding handles the M-series big.LITTLE asymmetry. A static -// one-band-per-core split makes every fast P-core wait on the slow E-cores -// at the join (measured: 16 cores was SLOWER than 10 on 1024², the 4 E-cores -// straggling). With many small bands the P-cores simply grab more of them, so -// adding the E-cores helps instead of hurting. -func MatMulP(dst, a, b []float64, m, k, n int) { - MatMulStridedP(dst, rowMajor(a, k), rowMajor(b, n), m, k, n) -} - -// MatMulStridedP is MatMulP over strided operand views (see Mat): dst (m x n, -// contiguous, zeroed by the caller) = a (m x k) * b (k x n). The views are -// only read, through the packing, so a transposed or sliced operand costs no -// copy beyond the one the packed GEMM makes anyway. -func MatMulStridedP(dst []float64, a, b Mat, m, k, n int) { - gemm64.matMul(dst, a, b, m, k, n) -} - -// HasGemm32 reports whether this architecture and CPU have a float32 GEMM -// micro-kernel, which MatMul32StridedP requires. Without one, the caller -// multiplies in float64. -func HasGemm32() bool { return gemm32 != nil } - -// MatMul32StridedP is MatMulStridedP in float32, accumulating in float32 as a -// BLAS sgemm does. It requires HasGemm32. -func MatMul32StridedP(dst []float32, a, b Mat32, m, k, n int) { - gemm32.matMul(dst, a, b, m, k, n) -} - -func (g *gemmKernel[T]) matMul(dst []T, a, b MatOf[T], m, k, n int) { - if m*n < GemmThreshold { - buf := g.getPackBuf() - g.packGemmRows(0, m, dst, a, b, k, n, buf) - g.putPackBuf(buf) - return - } - MR, NR := g.mr, g.nr - w := runtime.GOMAXPROCS(0) - if w > m { - w = m - } - if w == 1 { - // One worker: the serial path, without the block list, the progress - // counters and the worker closure, which cost four allocations per call - // and linalg's blocked factorizations make one call per panel (#104). - buf := g.getPackBuf() - g.packGemmRows(0, m, dst, a, b, k, n, buf) - g.putPackBuf(buf) - return - } - // Dice the m rows into MR-aligned bands a few rows of micro-tiles tall, so - // there are several bands per worker. That oversubscription is what lets the - // dynamic counter balance the M-series big.LITTLE cores: fast P-cores grab - // more bands while the slow E-cores grab fewer, instead of every P-core - // waiting on an E-core at a static join. bandRows is bandTilesMR micro-rows - // (a multiple of MR so no packed A panel straddles a band boundary) but at - // least one, and clamped so there are at least ~4 bands per worker when the - // row count allows. Bands stay <= MC so each A panel still fits L1. - bandRows := bandTilesMR * MR - if want := roundUp(max(1, (m+(4*w)-1)/(4*w)), MR); want < bandRows { - bandRows = want // small m: shrink bands so every worker still gets one - } - if bandRows > blockMC { - bandRows = blockMC - } - nBands := (m + bandRows - 1) / bandRows - if w > nBands { - w = nBands // never spawn more goroutines than there is work for - } - - // The workers are started once per call and stay for every (jc, pc) block, - // instead of being forked and joined per block. A fork/join wakes the - // parked worker threads one after another; measured on a 16-vCPU Zen 3 - // guest it costs 26 µs at 2 workers, 165 µs at 8 and 330 µs at 16, against - // ~0.8 ms of work per block of a 1024² product, and there were two per - // block (pack B, then the bands): ~40% of the wall time. - // - // The workers synchronise on work done, never on workers arrived: a block's - // bands start once all of its B panels are packed and all bands of the - // previous block are done. A worker still waking up holds nobody back, so - // the wake-up chain stays off the critical path; a barrier that waited for - // every worker put it right back on it for small products (96³ on 8 POWER9 - // cores: 0.57x). Claimed work never waits, so the scheme cannot deadlock. - // - // pb is double-buffered, so block i can be packed while block i-1 is being - // multiplied. Tiles of C are still accumulated in pc order, since block i's - // bands wait for block i-1's. - pbs := [2][]T{g.getPB(), g.getPB()} - defer g.putPB(pbs[0]) - defer g.putPB(pbs[1]) - - type block struct{ jc, nc, pc, kc, chunks int } - const chunk = 4 // B panels packed per claim - var blocks []block - for jc := 0; jc < n; jc += blockNC { // L3 column block of B - nc := min(blockNC, n-jc) - panels := (nc + NR - 1) / NR - for pc := 0; pc < k; pc += blockKC { // L2 contraction block - blocks = append(blocks, block{jc, nc, pc, min(blockKC, k-pc), (panels + chunk - 1) / chunk}) - } - } - type progress struct { - packNext, packDone, bandNext, bandDone atomic.Int64 - _ [32]byte // keep blocks' counters off one cache line - } - prog := make([]progress, len(blocks)) - bandsDone := func(i int) bool { return i < 0 || prog[i].bandDone.Load() == int64(nBands) } - - work := func() { - pa := g.getPA() - defer g.putPA(pa) - for i, bl := range blocks { - // pbs[i%2] is free: its last reader was block i-2, and to get here - // this worker passed block i-1's wait, which requires every band of - // block i-2 done. - pr, pb := &prog[i], pbs[i%2] - for { - c := int(pr.packNext.Add(1)) - 1 - if c >= bl.chunks { - break - } - j0, j1 := c*chunk*NR, min((c+1)*chunk*NR, bl.nc) - g.packB(b, bl.pc, bl.kc, bl.jc+j0, j1-j0, pb[j0*bl.kc:]) - pr.packDone.Add(1) - } - spinUntil(func() bool { return pr.packDone.Load() == int64(bl.chunks) && bandsDone(i-1) }) - for { - bi := int(pr.bandNext.Add(1)) - 1 - if bi >= nBands { - break - } - r0 := bi * bandRows - g.gemmBand(r0, min(r0+bandRows, m), bl.jc, bl.nc, bl.pc, bl.kc, dst, a, n, pa, pb, nil) - pr.bandDone.Add(1) - } - } - } - var wg sync.WaitGroup - for range w - 1 { - wg.Add(1) - go func() { defer wg.Done(); work() }() - } - work() // the caller is worker 0 - wg.Wait() -} - -// MatMul computes dst = a(m x k) * b(k x n) serially with the packed GEMM (the -// single-worker path of MatMulP). dst must be zeroed by the caller. It is kept as -// the named serial entry point used by callers and tests that want the exact -// non-parallel computation. -func MatMul(dst, a, b []float64, m, k, n int) { - buf := gemm64.getPackBuf() - gemm64.packGemmRows(0, m, dst, rowMajor(a, k), rowMajor(b, n), k, n, buf) - gemm64.putPackBuf(buf) -} - -// dotRange returns sum(a[i]*b[i]) over equal-length slices, using four -// independent accumulators, so four multiply-add chains are in flight instead of -// one (gc does not vectorise this loop; the gain is latency hiding). The -// four-way grouping is a fixed -// reassociation of the sum; MatVecP/Dot1DP document the ULP trade-off where it is -// observable. It is the contiguous building block for both mat·vec and the 1-D -// dot — neither needs to go through the packing GEMM. -func dotRange(a, b []float64) float64 { - var s0, s1, s2, s3 float64 - n := len(a) - i := 0 - for ; i+4 <= n; i += 4 { - s0 += a[i] * b[i] - s1 += a[i+1] * b[i+1] - s2 += a[i+2] * b[i+2] - s3 += a[i+3] * b[i+3] - } - for ; i < n; i++ { - s0 += a[i] * b[i] - } - return (s0 + s1) + (s2 + s3) -} - -// MatVecP computes dst = a(m x k) * v(k,) as m independent contiguous row-dots, -// fanned across cores above matVecThreshold. Each output dst[i] is the dot of -// row i of a (the contiguous slice a[i*k:(i+1)*k]) with v, so the access pattern -// is pure unit-stride streaming — far cheaper than routing a single-column GEMM -// through the packer. dst need not be pre-zeroed (each entry is written, not -// accumulated). Rows are disjoint, so the parallel result equals the serial one. -func MatVecP(dst, a, v []float64, m, k int) { MatVecStridedP(dst, a, k, v, m, k) } - -// MatVecStridedP is MatVecP on rows lda elements apart: row i of a is -// a[i*lda : i*lda+k], so a sub-block of a larger row-major matrix is -// multiplied in place. -func MatVecStridedP(dst, a []float64, lda int, v []float64, m, k int) { - body := func(lo, hi int) { - for i := lo; i < hi; i++ { - dst[i] = dotSIMD(a[i*lda:i*lda+k], v) - } - } - if m*k < matVecThreshold { - body(0, m) - return - } - w := runtime.GOMAXPROCS(0) - if w > m { - w = m - } - parallelFor(m, w, body) -} - -// VecMatP computes dst = v(k,) * a(k x n) = column sums weighted by v, i.e. -// dst[j] = sum_p v[p]*a[p*n+j]. It accumulates row by row so the inner loop over -// j streams a contiguous a row and the contiguous dst, which is the cache- -// friendly traversal (the transpose-free analogue of mat·vec for the 1-D·2-D -// case). dst must be zeroed by the caller. -func VecMatP(dst, v, a []float64, k, n int) { - for p := 0; p < k; p++ { - vp := v[p] - row := a[p*n : p*n+n] - for j := 0; j < n; j++ { - dst[j] += vp * row[j] - } - } -} - -// Dot1DP returns the inner product of two equal-length contiguous vectors, -// fanned across cores above ParThreshold: each chunk is dot-producted with the -// SIMD kernel and the partials are summed. The chunked four-way -// grouping reassociates the sum (a few ULP, like SumP / numpy pairwise); callers -// needing the exact left-to-right value use the scalar Dot1D. -func Dot1DP(a, b []float64) float64 { - n := len(a) - if n < ParThreshold { - return dotSIMD(a, b) - } - w := numWorkers(n) - parts := min(4*w, max(w, n/reduceGrain)) // see mapReduceP - partials := make([]float64, parts) - chunk := (n + parts - 1) / parts - parallelFor(parts, w, func(lo, hi int) { - for idx := lo; idx < hi; idx++ { - s := idx * chunk - if s >= n { - // chunk rounding can leave trailing workers past the end (e.g. - // many workers over a short, threshold-forced input); their - // partial is just the additive identity. - partials[idx] = 0 - continue - } - e := s + chunk - if e > n { - e = n - } - partials[idx] = dotSIMD(a[s:e], b[s:e]) - } - }) - return Sum(partials) -} diff --git a/internal/kernels/gemm32_amd64.go b/internal/kernels/gemm32_amd64.go deleted file mode 100644 index cadad56..0000000 --- a/internal/kernels/gemm32_amd64.go +++ /dev/null @@ -1,56 +0,0 @@ -package kernels - -// gemm32 is the float32 GEMM on the AVX2/FMA 6x16 kernel (gemmMicro6x16FMA in -// sum_amd64.s, generated by go-asmgen; see asmgen/amd64/gen.go). Without FMA -// there is none, and float32 products are computed in float64. -var gemm32 = newGemm32(useFMA) - -func newGemm32(fma bool) *gemmKernel[float32] { - if !fma { - return nil - } - return &gemmKernel[float32]{mr: 6, nr: 16, micro: gemmMicro32, microUA: gemmMicroUA32} -} - -// gemmUA32 holds the unpacked-A kernels by tile height. -var gemmUA32 = [...]func(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int){ - 1: gemmUA1x16FMA, 2: gemmUA2x16FMA, 3: gemmUA3x16FMA, - 4: gemmUA4x16FMA, 5: gemmUA5x16FMA, 6: gemmUA6x16FMA, -} - -// gemmMicroUA32 adds the mr x 16 tile sum_p a[r*lda+p]*b[p*ldb..] into dst, -// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0) and B -// rows ldb > 0 apart (16 for a packed panel). -func gemmMicroUA32(kc int, a []float32, lda int, b []float32, ldb int, dst []float32, ldc, mr int) { - _, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+15], dst[(mr-1)*ldc+15] // the kernel's whole reach - gemmUA32[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc) -} - -// gemmMicro32 adds the 6 x 16 tile sum_p pa[p*6..]*pb[p*16..] into the C block -// at dst[0:], whose rows are ldc apart. pa, pb hold at least kc*6 and kc*16 -// packed (zero-padded) floats; dst has at least 5*ldc + 16 elements. -func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { - _, _, _ = pa[kc*6-1], pb[kc*16-1], dst[5*ldc+15] // the kernel's whole reach - gemmMicro6x16FMA(kc, &pa[0], &pb[0], &dst[0], ldc) -} - -//go:noescape -func gemmMicro6x16FMA(kc int, pa, pb, c *float32, ldc int) - -//go:noescape -func gemmUA1x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA2x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA3x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA4x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA5x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA6x16FMA(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) diff --git a/internal/kernels/gemm32_arm64.go b/internal/kernels/gemm32_arm64.go deleted file mode 100644 index 155ff42..0000000 --- a/internal/kernels/gemm32_arm64.go +++ /dev/null @@ -1,54 +0,0 @@ -package kernels - -// gemm32 is the float32 GEMM on the NEON 8x8 kernel (gemmMicro8x8S in -// sum_arm64.s, generated by go-asmgen; see asmgen/arm64/gen.go). -var gemm32 = &gemmKernel[float32]{mr: 8, nr: 8, micro: gemmMicro32, microUA: gemmMicroUA32} - -// gemmUA32 holds the unpacked-A kernels by tile height. -var gemmUA32 = [...]func(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int){ - 1: gemmUA1x8S, 2: gemmUA2x8S, 3: gemmUA3x8S, 4: gemmUA4x8S, - 5: gemmUA5x8S, 6: gemmUA6x8S, 7: gemmUA7x8S, 8: gemmUA8x8S, -} - -// gemmMicroUA32 adds the mr x 8 tile sum_p a[r*lda+p]*b[p*ldb..] into dst, -// whose rows are ldc apart, reading A in place (mr in 1..8, lda > 0) and B -// rows ldb > 0 apart (8 for a packed panel). -func gemmMicroUA32(kc int, a []float32, lda int, b []float32, ldb int, dst []float32, ldc, mr int) { - _, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+7], dst[(mr-1)*ldc+7] // the kernel's whole reach - gemmUA32[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc) -} - -//go:noescape -func gemmUA1x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA2x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA3x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA4x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA5x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA6x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA7x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -//go:noescape -func gemmUA8x8S(kc int, a *float32, lda int, pb *float32, ldb int, c *float32, ldc int) - -// gemmMicro32 adds the 8 x 8 tile sum_p pa[p*8..]*pb[p*8..] into the C block at -// dst[0:], whose rows are ldc apart. pa, pb hold at least kc*8 packed -// (zero-padded) floats; dst has at least 7*ldc + 8 elements. -func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { - _, _, _ = pa[kc*8-1], pb[kc*8-1], dst[7*ldc+7] // the kernel's whole reach - gemmMicro8x8S(kc, &pa[0], &pb[0], &dst[0], ldc) -} - -//go:noescape -func gemmMicro8x8S(kc int, pa, pb, c *float32, ldc int) diff --git a/internal/kernels/gemm32_other.go b/internal/kernels/gemm32_other.go deleted file mode 100644 index 5e46b0d..0000000 --- a/internal/kernels/gemm32_other.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !amd64 && !arm64 && !ppc64le && !s390x && !riscv64 - -package kernels - -// gemm32 is nil here: no float32 micro-kernel yet, so HasGemm32 is false and -// float32 products are computed in float64. -var gemm32 *gemmKernel[float32] diff --git a/internal/kernels/gemm32_parts_riscv64_test.go b/internal/kernels/gemm32_parts_riscv64_test.go deleted file mode 100644 index a9a17f8..0000000 --- a/internal/kernels/gemm32_parts_riscv64_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package kernels - -import ( - "fmt" - "testing" -) - -// BenchmarkGemm32Parts times the pieces of a float32 n x n product on the -// current kernel: the whole MatMul32StridedP, packing A, packing B, and the -// micro-kernel tiles alone (packed panels, no packing). -func BenchmarkGemm32Parts(b *testing.B) { - if gemm32 == nil { - b.Skip("no float32 GEMM kernel here") - } - g := gemm32 - for _, n := range []int{64, 256} { - a := rowMajor(intMat32(n, n, 1), n) - bm := rowMajor(intMat32(n, n, 2), n) - dst := make([]float32, n*n) - kc := min(n, blockKC) - pa := make([]float32, g.paCap()) - pb := make([]float32, g.pbCap()) - b.Run(fmt.Sprintf("full/n=%d", n), func(b *testing.B) { - for i := 0; i < b.N; i++ { - clear(dst) - MatMul32StridedP(dst, a, bm, n, n, n) - } - }) - var sink []float32 - b.Run(fmt.Sprintf("alloc/n=%d", n), func(b *testing.B) { - for i := 0; i < b.N; i++ { - sink = make([]float32, n*n) - } - }) - _ = sink - b.Run(fmt.Sprintf("clear/n=%d", n), func(b *testing.B) { - for i := 0; i < b.N; i++ { - clear(dst) - } - }) - b.Run(fmt.Sprintf("packA/n=%d", n), func(b *testing.B) { - for i := 0; i < b.N; i++ { - g.packA(a, 0, min(n, blockMC), 0, kc, pa) - } - }) - b.Run(fmt.Sprintf("packB/n=%d", n), func(b *testing.B) { - for i := 0; i < b.N; i++ { - g.packB(bm, 0, kc, 0, min(n, blockNC), pb) - } - }) - b.Run(fmt.Sprintf("micro/n=%d", n), func(b *testing.B) { - mc, nc := min(n, blockMC), min(n, blockNC) - for i := 0; i < b.N; i++ { - for jr := 0; jr+g.nr <= nc; jr += g.nr { - for ir := 0; ir+g.mr <= mc; ir += g.mr { - g.micro(kc, pa[ir*kc:], pb[jr*kc:], dst[ir*n+jr:], n) - } - } - } - }) - } -} diff --git a/internal/kernels/gemm32_ppc64le.go b/internal/kernels/gemm32_ppc64le.go deleted file mode 100644 index e8d5e9e..0000000 --- a/internal/kernels/gemm32_ppc64le.go +++ /dev/null @@ -1,16 +0,0 @@ -package kernels - -// gemm32 is the float32 GEMM on the VSX 8x16 kernel (gemmMicro8x16VSX in -// sum_ppc64le.s, generated by go-asmgen; see asmgen/ppc64le/gen.go). -var gemm32 = &gemmKernel[float32]{mr: 8, nr: 16, micro: gemmMicro32} - -// gemmMicro32 adds the 8 x 16 tile sum_p pa[p*8..]*pb[p*16..] into the C block -// at dst[0:], whose rows are ldc apart. pa, pb hold at least kc*8 and kc*16 -// packed (zero-padded) floats; dst has at least 7*ldc + 16 elements. -func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { - _, _, _ = pa[kc*8-1], pb[kc*16-1], dst[7*ldc+15] // the kernel's whole reach - gemmMicro8x16VSX(kc, &pa[0], &pb[0], &dst[0], ldc) -} - -//go:noescape -func gemmMicro8x16VSX(kc int, pa, pb, c *float32, ldc int) diff --git a/internal/kernels/gemm32_riscv64.go b/internal/kernels/gemm32_riscv64.go deleted file mode 100644 index 8ef69c3..0000000 --- a/internal/kernels/gemm32_riscv64.go +++ /dev/null @@ -1,55 +0,0 @@ -package kernels - -// gemm32 is the float32 GEMM on an RVV kernel (gemmMicro8x16RVV or -// gemmMicro6x16RVV in sum_riscv64.s, generated by go-asmgen; see -// asmgen/riscv64/gen.go). Without the V extension there is none, and float32 -// products are computed in float64. -var gemm32 = newGemm32(useRVV) - -// gemm32WideVLEN is the vector register length in bytes from which the 8x16 -// kernel runs: with E32 and LMUL=2 it needs vl = 16, which a group of two -// registers holds only when VLEN >= 256. A var so tests can pin the narrow -// kernel on a wide core. -var gemm32WideVLEN = 32 - -// vlenb is the vector register length in bytes (0 without the V extension). -var vlenb = func() int { - if !useRVV { - return 0 - } - return gemmVLENB() -}() - -func newGemm32(rvv bool) *gemmKernel[float32] { - if !rvv { - return nil - } - if vlenb >= gemm32WideVLEN { - return &gemmKernel[float32]{mr: 8, nr: 16, micro: gemmMicro32x8} - } - return &gemmKernel[float32]{mr: 6, nr: 16, micro: gemmMicro32} -} - -// gemmMicro32 adds the 6 x 16 tile sum_p pa[p*6..]*pb[p*16..] into the C block -// at dst[0:], whose rows are ldc apart. pa, pb hold at least kc*6 and kc*16 -// packed (zero-padded) floats; dst has at least 5*ldc + 16 elements. -func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { - _, _, _ = pa[kc*6-1], pb[kc*16-1], dst[5*ldc+15] // the kernel's whole reach - gemmMicro6x16RVV(kc, &pa[0], &pb[0], &dst[0], ldc) -} - -// gemmMicro32x8 is gemmMicro32 for the 8 x 16 tile: pa holds at least kc*8 -// floats, dst at least 7*ldc + 16 elements. -func gemmMicro32x8(kc int, pa, pb, dst []float32, ldc int) { - _, _, _ = pa[kc*8-1], pb[kc*16-1], dst[7*ldc+15] // the kernel's whole reach - gemmMicro8x16RVV(kc, &pa[0], &pb[0], &dst[0], ldc) -} - -//go:noescape -func gemmMicro6x16RVV(kc int, pa, pb, c *float32, ldc int) - -//go:noescape -func gemmMicro8x16RVV(kc int, pa, pb, c *float32, ldc int) - -// gemmVLENB returns the vector register length in bytes. It requires V. -func gemmVLENB() int diff --git a/internal/kernels/gemm32_riscv64_test.go b/internal/kernels/gemm32_riscv64_test.go deleted file mode 100644 index 1b2a48b..0000000 --- a/internal/kernels/gemm32_riscv64_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package kernels - -import "testing" - -// TestGemm32NeedsRVV: without the V extension there is no float32 kernel, and -// HasGemm32 sends float32 products to the float64 GEMM. With it, the tile is -// 8x16 on a core whose vector registers hold sixteen floats in two (VLEN >= 256) -// and 6x16 on a narrower one. -func TestGemm32NeedsRVV(t *testing.T) { - if newGemm32(false) != nil { - t.Fatal("a float32 GEMM without V") - } - want := 6 - if vlenb >= gemm32WideVLEN { - want = 8 - } - t.Logf("vlenb = %d", vlenb) - if g := newGemm32(true); g == nil || g.mr != want || g.nr != 16 { - t.Fatalf("newGemm32(true) = %+v, want mr %d", g, want) - } -} - -// TestGemm32NarrowKernel runs the 6x16 kernel, the one a VLEN-128 core uses, on -// whatever core this is, through the float32 GEMM tests. -func TestGemm32NarrowKernel(t *testing.T) { - if !useRVV { - t.Skip("no V extension") - } - saved := gemm32 - defer func() { gemm32 = saved }() - gemm32 = &gemmKernel[float32]{mr: 6, nr: 16, micro: gemmMicro32} - TestGemm32Correctness(t) - TestGemm32Accuracy(t) - TestGemm32Specials(t) -} - -// TestGemm32WideKernel runs the 8x16 kernel when the core can: VLEN >= 256. -func TestGemm32WideKernel(t *testing.T) { - if vlenb < gemm32WideVLEN { - t.Skip("VLEN < 256") - } - saved := gemm32 - defer func() { gemm32 = saved }() - gemm32 = &gemmKernel[float32]{mr: 8, nr: 16, micro: gemmMicro32x8} - TestGemm32Correctness(t) - TestGemm32Accuracy(t) - TestGemm32Specials(t) -} diff --git a/internal/kernels/gemm32_s390x.go b/internal/kernels/gemm32_s390x.go deleted file mode 100644 index 9677a5f..0000000 --- a/internal/kernels/gemm32_s390x.go +++ /dev/null @@ -1,25 +0,0 @@ -package kernels - -// gemm32 is the float32 GEMM on the 4x16 vector kernel (gemmMicro4x16VXE in -// sum_s390x.s, generated by go-asmgen; see asmgen/s390x/gen.go). Its VFMASB is -// the vector-enhancements facility 1 (z14): without it there is none, and -// float32 products are computed in float64. -var gemm32 = newGemm32(useVXE) - -func newGemm32(vxe bool) *gemmKernel[float32] { - if !vxe { - return nil - } - return &gemmKernel[float32]{mr: 4, nr: 16, micro: gemmMicro32} -} - -// gemmMicro32 adds the 4 x 16 tile sum_p pa[p*4..]*pb[p*16..] into the C block -// at dst[0:], whose rows are ldc apart. pa, pb hold at least kc*4 and kc*16 -// packed (zero-padded) floats; dst has at least 3*ldc + 16 elements. -func gemmMicro32(kc int, pa, pb, dst []float32, ldc int) { - _, _, _ = pa[kc*4-1], pb[kc*16-1], dst[3*ldc+15] // the kernel's whole reach - gemmMicro4x16VXE(kc, &pa[0], &pb[0], &dst[0], ldc) -} - -//go:noescape -func gemmMicro4x16VXE(kc int, pa, pb, c *float32, ldc int) diff --git a/internal/kernels/gemm32_test.go b/internal/kernels/gemm32_test.go deleted file mode 100644 index 48e1f56..0000000 --- a/internal/kernels/gemm32_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package kernels - -import ( - "math" - "math/rand" - "testing" -) - -// gemmOracle32 is gemmOracle in float32 arithmetic: the ikj order, each product -// and sum rounded to float32. -func gemmOracle32(dst []float32, a, b MatOf[float32], m, k, n int) { - for i := 0; i < m; i++ { - for p := 0; p < k; p++ { - av := a.Data[a.Off+i*a.RS+p*a.CS] - for j := 0; j < n; j++ { - dst[i*n+j] += av * b.Data[b.Off+p*b.RS+j*b.CS] - } - } - } -} - -func intMat32(m, k int, seed int64) []float32 { - r := rand.New(rand.NewSource(seed)) - a := make([]float32, m*k) - for i := range a { - a[i] = float32(r.Intn(17) - 8) - } - return a -} - -// assertGemm32 checks the float32 GEMM, serial and with the parallel band split -// forced, against the oracle bit for bit on integer-valued operands, whose -// products and sums are exact in float32 at these sizes. -func assertGemm32(t *testing.T, m, k, n int, a, b Mat32) { - t.Helper() - want := make([]float32, m*n) - gemmOracle32(want, a, b, m, k, n) - savedA, savedB := unpackA, unpackB - defer func() { unpackA, unpackB = savedA, savedB }() - for _, p := range [][2]bool{{true, true}, {true, false}, {false, false}} { - unpackA, unpackB = p[0], p[1] - for _, gemm := range []int{1 << 30, 1} { - withThresholds(1<<20, gemm, func() { - got := make([]float32, m*n) - MatMul32StridedP(got, a, b, m, k, n) - for i := range want { - if got[i] != want[i] { - t.Fatalf("m=%d k=%d n=%d gemm=%d unpackA,B=%v [%d]: %v != %v", - m, k, n, gemm, p, i, got[i], want[i]) - } - } - }) - } - } -} - -// TestGemm32Correctness is TestGemmCorrectness for float32: every shape and -// block configuration that drives the edge tiles and the loop nest, on -// row-major operands and on a transposed, offset B view. -func TestGemm32Correctness(t *testing.T) { - g := gemm32 - if !HasGemm32() { - t.Skip("no float32 GEMM kernel here") - } - MR, NR := g.mr, g.nr - shapes := []struct{ m, k, n int }{ - {1, 1, 1}, {2, 3, 4}, {6, 4, 16}, {8, 8, 8}, {5, 7, 9}, {7, 7, 17}, - {9, 5, 11}, {16, 16, 16}, {17, 13, 19}, {3, 10, 1}, {1, 10, 3}, - {33, 33, 33}, {32, 64, 48}, - } - for _, s := range shapes { - a := rowMajor(intMat32(s.m, s.k, int64(s.m*100+s.k)), s.k) - b := rowMajor(intMat32(s.k, s.n, int64(s.n*7+s.k)), s.n) - // bt holds B transposed behind a 3-element offset; the view reads it - // back as B, through the packB column walk (CS != 1). - bt := Mat32{Data: make([]float32, 3+s.k*s.n), Off: 3, RS: 1, CS: s.k} - for p := 0; p < s.k; p++ { - for j := 0; j < s.n; j++ { - bt.Data[3+j*s.k+p] = b.Data[p*s.n+j] - } - } - // as holds A behind a 2-element offset, rows k+3 apart: the - // unpacked-A kernels read it through lda != k. - as := Mat32{Data: make([]float32, 2+s.m*(s.k+3)), Off: 2, RS: s.k + 3, CS: 1} - for i := 0; i < s.m; i++ { - copy(as.Data[2+i*(s.k+3):], a.Data[i*s.k:(i+1)*s.k]) - } - assertGemm32(t, s.m, s.k, s.n, as, b) - for _, bv := range []Mat32{b, bt} { - assertGemm32(t, s.m, s.k, s.n, a, bv) - withBlocks(MR, 2, NR, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) - withBlocks(2*MR, 3, 2*NR, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) - withBlocks(MR+1, 2, NR+1, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) - withBlocks(3*MR-1, 5, 3*NR-1, func() { assertGemm32(t, s.m, s.k, s.n, a, bv) }) - } - } - // Default blocks with MR not dividing blockMC, across the band split. - m, k, n := 300, 200, 260 - assertGemm32(t, m, k, n, rowMajor(intMat32(m, k, 1), k), rowMajor(intMat32(k, n, 2), n)) -} - -// TestGemm32Accuracy bounds the float32 GEMM on random data by the classical -// error bound of a float32 dot product, k*eps32*sum|a||b|, against the product -// computed in float64. -func TestGemm32Accuracy(t *testing.T) { - if gemm32 == nil { - t.Skip("no float32 GEMM kernel here") - } - r := rand.New(rand.NewSource(5)) - m, k, n := 70, 300, 50 - a, b := make([]float32, m*k), make([]float32, k*n) - for i := range a { - a[i] = float32(r.NormFloat64()) - } - for i := range b { - b[i] = float32(r.NormFloat64()) - } - got := make([]float32, m*n) - MatMul32StridedP(got, rowMajor(a, k), rowMajor(b, n), m, k, n) - for i := 0; i < m; i++ { - for j := 0; j < n; j++ { - var exact, abs float64 - for p := 0; p < k; p++ { - x := float64(a[i*k+p]) * float64(b[p*n+j]) - exact += x - abs += math.Abs(x) - } - if d := math.Abs(float64(got[i*n+j]) - exact); d > float64(k)*0x1p-24*abs { - t.Fatalf("[%d,%d]: %v vs %v (off %g, bound %g)", i, j, got[i*n+j], exact, d, float64(k)*0x1p-24*abs) - } - } - } -} - -// TestGemm32Specials: NaN and Inf in an operand reach exactly the outputs -// whose dot products they enter. -func TestGemm32Specials(t *testing.T) { - if gemm32 == nil { - t.Skip("no float32 GEMM kernel here") - } - m, k, n := 13, 9, 21 - for _, special := range []float32{float32(math.NaN()), float32(math.Inf(1))} { - a, b := intMat32(m, k, 3), intMat32(k, n, 4) - for i := range b { - if b[i] == 0 { - b[i] = 1 // keep Inf*0 out: the row of a meets every b entry - } - } - a[4*k+2] = special // row 4 - got := make([]float32, m*n) - MatMul32StridedP(got, rowMajor(a, k), rowMajor(b, n), m, k, n) - for i := 0; i < m; i++ { - for j := 0; j < n; j++ { - v := float64(got[i*n+j]) - bad := math.IsNaN(v) || math.IsInf(v, 0) - if bad != (i == 4) { - t.Fatalf("%v at row 4: [%d,%d] = %v", special, i, j, v) - } - } - } - } -} diff --git a/internal/kernels/gemm_amd64.go b/internal/kernels/gemm_amd64.go deleted file mode 100644 index 0315ca9..0000000 --- a/internal/kernels/gemm_amd64.go +++ /dev/null @@ -1,107 +0,0 @@ -package kernels - -// amd64 GEMM micro-kernel: a 6x8 register-blocked tile (gemm in sum_amd64.s, -// generated by go-asmgen), the BLIS Haswell dgemm shape. Two kernels share the -// one 6x8 packing: -// -// - gemmMicro6x8FMA, AVX2-era FMA3: 12 YMM accumulators, VBROADCASTSD of each -// A value and VFMADD231PD against the two-YMM B row. Taken when hasFMA -// (CPUID FMA, AVX and OSXSAVE, and XCR0 saving YMM state) says so — every -// Intel and AMD part since 2013. -// - gemmMicro6x4SSE2, the GOAMD64=v1 baseline, called twice per tile (columns -// 0-3, then 4-7): explicit MULPD+ADDPD, since SSE2 has no FMA. -// -// The FMA kernel rounds each a*b+c once, as tuned BLAS and arm64's kernel do; the -// SSE2 one rounds twice, as the scalar oracle does at GOAMD64=v1. On data whose -// products and partial sums are exact (the integer-valued tests) the two agree -// bit-for-bit with the oracle; elsewhere they can differ in the last place. - -// MR, NR are the micro-kernel's register-block dimensions: it computes an -// MR-row x NR-col tile of C per call. The packing in gemm.go matches. -const ( - MR = 6 - NR = 8 -) - -// useFMA selects the FMA kernels (GEMM, dot). It is a var so tests can force -// the SSE2 / pure-Go paths on hardware that has FMA (never the reverse: that -// would fault). -var useFMA = hasFMA() - -// gemmMicro adds the MR x NR tile sum_p pa[p*MR..]*pb[p*NR..] into the C block at -// dst[0:], whose rows are ldc apart. pa, pb hold at least kc*MR and kc*NR packed -// (zero-padded) doubles; dst has at least (MR-1)*ldc + NR elements. -func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { - _, _, _ = pa[kc*MR-1], pb[kc*NR-1], dst[(MR-1)*ldc+NR-1] // the kernel's whole reach - if useFMA { - gemmMicro6x8FMA(kc, &pa[0], &pb[0], &dst[0], ldc) - return - } - gemmMicro6x4SSE2(kc, &pa[0], &pb[0], &dst[0], ldc) - gemmMicro6x4SSE2(kc, &pa[0], &pb[4], &dst[4], ldc) -} - -// gemmMicroUA is the unpacked-A float64 kernel (see gemmKernel.microUA), on -// FMA CPUs only; without FMA, A is packed for the SSE2 kernel. -var gemmMicroUA = microUAIf(useFMA) - -func microUAIf(fma bool) func(kc int, a []float64, lda int, b []float64, ldb int, dst []float64, ldc, mr int) { - if !fma { - return nil - } - return gemmMicroUA64 -} - -// gemmUA64 holds the unpacked-A kernels by tile height. -var gemmUA64 = [...]func(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int){ - 1: gemmUA1x8FMA, 2: gemmUA2x8FMA, 3: gemmUA3x8FMA, - 4: gemmUA4x8FMA, 5: gemmUA5x8FMA, 6: gemmUA6x8FMA, -} - -// gemmMicroUA64 adds the mr x 8 tile sum_p a[r*lda+p]*b[p*ldb..] into dst, -// whose rows are ldc apart, reading A in place (mr in 1..6, lda > 0) and B -// rows ldb > 0 apart (NR for a packed panel). -func gemmMicroUA64(kc int, a []float64, lda int, b []float64, ldb int, dst []float64, ldc, mr int) { - _, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+NR-1], dst[(mr-1)*ldc+NR-1] // the kernel's whole reach - gemmUA64[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc) -} - -//go:noescape -func gemmUA1x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmUA2x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmUA3x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmUA4x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmUA5x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmUA6x8FMA(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmMicro6x8FMA(kc int, pa, pb, c *float64, ldc int) - -//go:noescape -func gemmMicro6x4SSE2(kc int, pa, pb, c *float64, ldc int) - -// dotSIMD is the inner product of two equal-length slices: the AVX2/FMA -// kernel when the CPU has FMA, else dotRange. -func dotSIMD(a, b []float64) float64 { - if !useFMA || len(a) == 0 { - return dotRange(a, b) - } - _ = b[len(a)-1] // the kernel reads len(a) of b - return dotFMA(&a[0], &b[0], len(a)) -} - -//go:noescape -func dotFMA(a, b *float64, n int) float64 - -// hasFMA reports CPU and OS support for the FMA kernel (go-asmgen FeatureProbe). -func hasFMA() bool diff --git a/internal/kernels/gemm_amd64_test.go b/internal/kernels/gemm_amd64_test.go deleted file mode 100644 index 7ada228..0000000 --- a/internal/kernels/gemm_amd64_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package kernels - -import "testing" - -// TestGemmSSE2Fallback runs the GEMM suite on the baseline SSE2 kernel whatever -// the host supports, so a CPU with FMA still exercises the path a pre-2013 CPU -// (or a hypervisor that masks FMA) takes. The FMA path is the default and is -// exercised by the rest of the suite wherever hasFMA is true. -func TestGemmSSE2Fallback(t *testing.T) { - saved, savedUA := useFMA, unpackA - useFMA, unpackA = false, false // the SSE2 kernel reads packed A only - defer func() { useFMA, unpackA = saved, savedUA }() - if microUAIf(false) != nil || microUAIf(true) == nil { - t.Fatal("microUAIf: an unpacked-A kernel without FMA, or none with it") - } - t.Run("correctness", TestGemmCorrectness) - t.Run("bands", TestGemmBandSizing) - t.Run("float", TestGemmFloatTolerance) - t.Run("dot", TestDotSIMD) -} - -// TestGemm32NeedsFMA: without FMA there is no float32 kernel, and HasGemm32 -// sends float32 products to the float64 GEMM. -func TestGemm32NeedsFMA(t *testing.T) { - if newGemm32(false) != nil { - t.Fatal("a float32 GEMM without FMA") - } - if g := newGemm32(true); g == nil || g.mr != 6 || g.nr != 16 { - t.Fatalf("newGemm32(true) = %+v", g) - } -} diff --git a/internal/kernels/gemm_arm64.go b/internal/kernels/gemm_arm64.go deleted file mode 100644 index ea9f71d..0000000 --- a/internal/kernels/gemm_arm64.go +++ /dev/null @@ -1,71 +0,0 @@ -package kernels - -// arm64 GEMM micro-kernel: a NEON 4x8 register-blocked tile (gemm in -// sum_arm64.s, generated by go-asmgen). See asmgen/arm64/gen.go for the register -// map. It uses the WORD-encoded by-element double FMLA (FMLA Vd.2D, Vn.2D, -// Vm.D[i] — the one instruction Go's arm64 assembler cannot name), so each A -// scalar folds straight from a loaded vector lane, eliminating the MOVD-to-GP + -// VDUP round-trip the prior 4x8 kernel paid per A value every k-step. The 4-row x -// 8-col tile (16 D2 accumulators) fits in one pass with B in 4 vectors and the A -// column in 2, and 4|N, 8|N aligns with the power-of-two sizes so there is no -// scalar edge. MR x NR = 4 x 8 is the tile the packing produces. - -// MR, NR are the micro-kernel's register-block dimensions: it computes an -// MR-row x NR-col tile of C per call. The packing in gemm.go produces MR-tall A -// panels and NR-wide B panels to match. -const ( - MR = 4 - NR = 8 -) - -// gemmMicro adds the MR x NR tile sum_p pa[p*MR..]*pb[p*NR..] into the C block at -// dst[0:], whose rows are ldc apart, using the NEON kernel. pa, pb hold at least -// kc*MR and kc*NR packed (zero-padded) doubles; dst has at least (MR-1)*ldc + NR -// elements. Bit-identical to the scalar ikj accumulation (see gemmEdge). -func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { - _, _, _ = pa[kc*MR-1], pb[kc*NR-1], dst[(MR-1)*ldc+NR-1] // the kernel's whole reach - gemmMicro4x8(kc, &pa[0], &pb[0], &dst[0], ldc) -} - -// dotSIMD is the inner product of two equal-length slices (NEON kernel). -func dotSIMD(a, b []float64) float64 { - if len(a) == 0 { - return 0 - } - _ = b[len(a)-1] // the kernel reads len(a) of b - return dotNEON(&a[0], &b[0], len(a)) -} - -//go:noescape -func dotNEON(a, b *float64, n int) float64 - -//go:noescape -func gemmMicro4x8(kc int, pa, pb, c *float64, ldc int) - -// gemmMicroUA is the unpacked-A float64 kernel (see gemmKernel.microUA). -var gemmMicroUA = gemmMicroUA64 - -// gemmUA64 holds the unpacked-A kernels by tile height. -var gemmUA64 = [...]func(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int){ - 1: gemmUA1x8D, 2: gemmUA2x8D, 3: gemmUA3x8D, 4: gemmUA4x8D, -} - -// gemmMicroUA64 adds the mr x 8 tile sum_p a[r*lda+p]*b[p*ldb..] into dst, -// whose rows are ldc apart, reading A in place (mr in 1..4, lda > 0) and B -// rows ldb > 0 apart (NR for a packed panel). -func gemmMicroUA64(kc int, a []float64, lda int, b []float64, ldb int, dst []float64, ldc, mr int) { - _, _, _ = a[(mr-1)*lda+kc-1], b[(kc-1)*ldb+NR-1], dst[(mr-1)*ldc+NR-1] // the kernel's whole reach - gemmUA64[mr](kc, &a[0], lda, &b[0], ldb, &dst[0], ldc) -} - -//go:noescape -func gemmUA1x8D(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmUA2x8D(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmUA3x8D(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) - -//go:noescape -func gemmUA4x8D(kc int, a *float64, lda int, pb *float64, ldb int, c *float64, ldc int) diff --git a/internal/kernels/gemm_bench_test.go b/internal/kernels/gemm_bench_test.go deleted file mode 100644 index fcf2904..0000000 --- a/internal/kernels/gemm_bench_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package kernels - -import ( - "strconv" - "testing" -) - -// BenchmarkGemmMicro times the register-blocked micro-kernel alone on -// L1-resident packed panels (kc=256): its GFLOP/s is the per-core ceiling the -// blocked driver can approach, separate from packing and scheduling. -func BenchmarkGemmMicro(b *testing.B) { - const kc = 256 - pa := make([]float64, kc*MR) - pb := make([]float64, kc*NR) - for i := range pa { - pa[i] = float64(i%7) * 0.5 - } - for i := range pb { - pb[i] = float64(i%5) * 0.25 - } - c := make([]float64, MR*NR) - b.SetBytes(0) - b.ResetTimer() - for i := 0; i < b.N; i++ { - gemmMicro(kc, pa, pb, c, NR) - } - b.ReportMetric(float64(2*MR*NR*kc)*float64(b.N)/b.Elapsed().Seconds()/1e9, "GFLOP/s") -} - -// BenchmarkMatMulSerial times the whole serial GEMM (packing + blocking + -// micro-kernel, one core) at a few sizes. -func BenchmarkMatMulSerial(b *testing.B) { - for _, n := range []int{32, 64, 128, 256, 512} { - a, bb := intMat(n, n, 1), intMat(n, n, 2) - dst := make([]float64, n*n) - b.Run(strconv.Itoa(n), func(b *testing.B) { - for i := 0; i < b.N; i++ { - clear(dst) - MatMul(dst, a, bb, n, n, n) - } - b.ReportMetric(float64(2*n*n*n)*float64(b.N)/b.Elapsed().Seconds()/1e9, "GFLOP/s") - }) - } -} diff --git a/internal/kernels/gemm_generic.go b/internal/kernels/gemm_generic.go deleted file mode 100644 index 98f8629..0000000 --- a/internal/kernels/gemm_generic.go +++ /dev/null @@ -1,75 +0,0 @@ -//go:build !arm64 && !amd64 && !ppc64le && !loong64 && !riscv64 && !s390x - -package kernels - -// Generic GEMM micro-kernel for the targets without a vector-double SIMD -// micro-kernel (s390x and the 32-bit ones): a portable scalar 4x4 register -// tile over the packed panels. There is no hand-vectorized tile here yet (Go's -// assembler has vector-double arithmetic for s390x, see docs/perf.md) — but the kernel still gets the two structural wins of the packed -// GEMM: the panels are contiguous unit-stride (so the loads are conflict-free, -// the L1-collision fix), and the cache-blocking loop nest plus the multicore -// fan-out (MatMulP) keep it well ahead of an unblocked scalar loop and beat -// single-threaded numpy on large products. - -// MR, NR are the register-block dimensions; 4x4 keeps 16 + a few scalars live in -// the GP/FP register file on these targets without spilling. -const ( - MR = 4 - NR = 4 -) - -// gemmMicro adds the MR x NR tile sum_p pa[p*MR+r]*pb[p*NR+c] into the C block at -// dst[c0 + r*ldc + c], reading the packed (zero-padded) panels. The accumulation -// is the same ikj order as gemmEdge and the scalar oracle, so it is bit-identical -// across all arches. The 16 tile accumulators are explicit locals so the compiler -// keeps them in registers across the kc loop. -func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { - var ( - c00, c01, c02, c03 float64 - c10, c11, c12, c13 float64 - c20, c21, c22, c23 float64 - c30, c31, c32, c33 float64 - ) - for p := 0; p < kc; p++ { - a := pa[p*MR : p*MR+4 : p*MR+4] - b := pb[p*NR : p*NR+4 : p*NR+4] - a0, a1, a2, a3 := a[0], a[1], a[2], a[3] - b0, b1, b2, b3 := b[0], b[1], b[2], b[3] - c00 += a0 * b0 - c01 += a0 * b1 - c02 += a0 * b2 - c03 += a0 * b3 - c10 += a1 * b0 - c11 += a1 * b1 - c12 += a1 * b2 - c13 += a1 * b3 - c20 += a2 * b0 - c21 += a2 * b1 - c22 += a2 * b2 - c23 += a2 * b3 - c30 += a3 * b0 - c31 += a3 * b1 - c32 += a3 * b2 - c33 += a3 * b3 - } - r0 := dst[0:4:4] - r0[0] += c00 - r0[1] += c01 - r0[2] += c02 - r0[3] += c03 - r1 := dst[ldc : ldc+4 : ldc+4] - r1[0] += c10 - r1[1] += c11 - r1[2] += c12 - r1[3] += c13 - r2 := dst[2*ldc : 2*ldc+4 : 2*ldc+4] - r2[0] += c20 - r2[1] += c21 - r2[2] += c22 - r2[3] += c23 - r3 := dst[3*ldc : 3*ldc+4 : 3*ldc+4] - r3[0] += c30 - r3[1] += c31 - r3[2] += c32 - r3[3] += c33 -} diff --git a/internal/kernels/gemm_loong64.go b/internal/kernels/gemm_loong64.go deleted file mode 100644 index 076e7e1..0000000 --- a/internal/kernels/gemm_loong64.go +++ /dev/null @@ -1,59 +0,0 @@ -package kernels - -// loong64 GEMM micro-kernel: an 8x8 LASX tile (gemmMicro8x8LASX in -// sum_loong64.s, generated by go-asmgen): 16 four-lane accumulators, the B row -// in two vectors, each A value broadcast by xvldrepl.d (XVMOVQ .V4) and a -// fused xvfmadd.d. -// Without LASX, gemmMicroGo computes the same tile in scalar code. - -// MR, NR are the micro-kernel's register-block dimensions. -const ( - MR = 8 - NR = 8 -) - -func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { - if useLASX { - _, _, _ = pa[kc*MR-1], pb[kc*NR-1], dst[(MR-1)*ldc+NR-1] // the kernel's whole reach - gemmMicro8x8LASX(kc, &pa[0], &pb[0], &dst[0], ldc) - return - } - gemmMicroGo(kc, pa, pb, dst, ldc) -} - -// gemmMicroGo is the scalar MR x NR tile: dst[r*ldc+c] += sum_p -// pa[p*MR+r]*pb[p*NR+c], in p order per element. -func gemmMicroGo(kc int, pa, pb, dst []float64, ldc int) { - var acc [MR * NR]float64 - for p := 0; p < kc; p++ { - a := pa[p*MR : p*MR+MR : p*MR+MR] - b := pb[p*NR : p*NR+NR : p*NR+NR] - for r, av := range a { - row := acc[r*NR : r*NR+NR : r*NR+NR] - for c, bv := range b { - row[c] += av * bv - } - } - } - for r := 0; r < MR; r++ { - d := dst[r*ldc : r*ldc+NR : r*ldc+NR] - for c := range d { - d[c] += acc[r*NR+c] - } - } -} - -// dotSIMD is the inner product (LASX kernel, four fused accumulators). -func dotSIMD(a, b []float64) float64 { - if !useLASX || len(a) == 0 { - return dotRange(a, b) - } - _ = b[len(a)-1] // the kernel reads len(a) of b - return dotLASX(&a[0], &b[0], len(a)) -} - -//go:noescape -func gemmMicro8x8LASX(kc int, pa, pb, c *float64, ldc int) - -//go:noescape -func dotLASX(a, b *float64, n int) float64 diff --git a/internal/kernels/gemm_loong64_test.go b/internal/kernels/gemm_loong64_test.go deleted file mode 100644 index 38623c7..0000000 --- a/internal/kernels/gemm_loong64_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package kernels - -import "testing" - -// TestScalarFallbackLoong64 runs the GEMM, dot and elementwise suites with -// LASX forced off, so the path a CPU without LASX takes is exercised on one -// that has it. -func TestScalarFallbackLoong64(t *testing.T) { - saved := useLASX - useLASX = false - defer func() { useLASX = saved }() - t.Run("gemm", TestGemmCorrectness) - t.Run("dot", TestDotSIMD) - t.Run("bin", TestBinSIMD) -} - -func TestLASXDetected(t *testing.T) { - t.Logf("useLASX = %v (AT_HWCAP)", useLASX) -} diff --git a/internal/kernels/gemm_oneworker_test.go b/internal/kernels/gemm_oneworker_test.go deleted file mode 100644 index cb94786..0000000 --- a/internal/kernels/gemm_oneworker_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package kernels - -import ( - "math/rand/v2" - "runtime" - "testing" -) - -// TestGemmOneWorkerSameBits pins the one-worker path of matMul (#104): at -// GOMAXPROCS=1 a product above GemmThreshold takes packGemmRows instead of the -// banded parallel path, and on non-integer data the two must agree bit for -// bit (same tiles, same pc order), in float64 and float32, including m = 1, -// which takes the same branch at any GOMAXPROCS. -func TestGemmOneWorkerSameBits(t *testing.T) { - r := rand.New(rand.NewPCG(11, 12)) - for _, s := range [][3]int{{97, 131, 83}, {1, 300, 7000}, {256, 64, 200}} { - m, k, n := s[0], s[1], s[2] - if m*n < GemmThreshold { - t.Fatalf("%v is below GemmThreshold %d: the case would not reach the branch", s, GemmThreshold) - } - a, b := make([]float64, m*k), make([]float64, k*n) - for i := range a { - a[i] = r.NormFloat64() - } - for i := range b { - b[i] = r.NormFloat64() - } - run := func(procs int) []float64 { - defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(procs)) - dst := make([]float64, m*n) - MatMulStridedP(dst, rowMajor(a, k), rowMajor(b, n), m, k, n) - return dst - } - one, many := run(1), run(4) - for i := range one { - if one[i] != many[i] { - t.Fatalf("float64 %v: element %d is %v on one worker, %v on four", s, i, one[i], many[i]) - } - } - if !HasGemm32() { - continue - } - a32, b32 := make([]float32, m*k), make([]float32, k*n) - for i := range a32 { - a32[i] = float32(a[i]) - } - for i := range b32 { - b32[i] = float32(b[i]) - } - run32 := func(procs int) []float32 { - defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(procs)) - dst := make([]float32, m*n) - MatMul32StridedP(dst, MatOf[float32]{Data: a32, RS: k, CS: 1}, MatOf[float32]{Data: b32, RS: n, CS: 1}, m, k, n) - return dst - } - one32, many32 := run32(1), run32(4) - for i := range one32 { - if one32[i] != many32[i] { - t.Fatalf("float32 %v: element %d is %v on one worker, %v on four", s, i, one32[i], many32[i]) - } - } - } -} diff --git a/internal/kernels/gemm_ppc64le.go b/internal/kernels/gemm_ppc64le.go deleted file mode 100644 index 7f14907..0000000 --- a/internal/kernels/gemm_ppc64le.go +++ /dev/null @@ -1,37 +0,0 @@ -package kernels - -// ppc64le GEMM micro-kernel: an 8x8 VSX tile (gemmMicro8x8VSX in -// sum_ppc64le.s, generated by go-asmgen): 32 two-lane accumulators, the B row -// loaded with LXVD2X, each A value splatted with LXVDSX, and a fused -// XVMADDADP per (row, column pair). Fused like the arm64 and amd64 FMA -// kernels: on integer-valued data it equals the scalar ikj oracle exactly. - -// MR, NR are the micro-kernel's register-block dimensions: it computes an -// MR-row x NR-col tile of C per call. The packing in gemm.go matches. -const ( - MR = 8 - NR = 8 -) - -// gemmMicro adds the MR x NR tile sum_p pa[p*MR..]*pb[p*NR..] into the C block -// at dst[0:], whose rows are ldc apart. -func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { - _, _, _ = pa[kc*MR-1], pb[kc*NR-1], dst[(MR-1)*ldc+NR-1] // the kernel's whole reach - gemmMicro8x8VSX(kc, &pa[0], &pb[0], &dst[0], ldc) -} - -// dotSIMD is the inner product of two equal-length slices (VSX kernel, four -// fused multiply-add accumulators). -func dotSIMD(a, b []float64) float64 { - if len(a) == 0 { - return 0 - } - _ = b[len(a)-1] // the kernel reads len(a) of b - return dotVSX(&a[0], &b[0], len(a)) -} - -//go:noescape -func gemmMicro8x8VSX(kc int, pa, pb, c *float64, ldc int) - -//go:noescape -func dotVSX(a, b *float64, n int) float64 diff --git a/internal/kernels/gemm_riscv64.go b/internal/kernels/gemm_riscv64.go deleted file mode 100644 index a83dcbf..0000000 --- a/internal/kernels/gemm_riscv64.go +++ /dev/null @@ -1,47 +0,0 @@ -package kernels - -// riscv64 GEMM micro-kernel: a 6x8 RVV tile (gemmMicro6x8RVV in -// sum_riscv64.s, generated by go-asmgen): six accumulator groups of eight -// float64 (LMUL=4, eight lanes on any VLEN >= 128), the B row loaded once per -// step and each A value fused in with vfmacc.vf. Without V, gemmMicroGo -// computes the same tile in scalar code. - -// MR, NR are the micro-kernel's register-block dimensions. -const ( - MR = 6 - NR = 8 -) - -func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { - if useRVV { - _, _, _ = pa[kc*MR-1], pb[kc*NR-1], dst[(MR-1)*ldc+NR-1] // the kernel's whole reach - gemmMicro6x8RVV(kc, &pa[0], &pb[0], &dst[0], ldc) - return - } - gemmMicroGo(kc, pa, pb, dst, ldc) -} - -// gemmMicroGo is the scalar MR x NR tile: dst[r*ldc+c] += sum_p -// pa[p*MR+r]*pb[p*NR+c], in p order per element. -func gemmMicroGo(kc int, pa, pb, dst []float64, ldc int) { - var acc [MR * NR]float64 - for p := 0; p < kc; p++ { - a := pa[p*MR : p*MR+MR : p*MR+MR] - b := pb[p*NR : p*NR+NR : p*NR+NR] - for r, av := range a { - row := acc[r*NR : r*NR+NR : r*NR+NR] - for c, bv := range b { - row[c] += av * bv - } - } - } - for r := 0; r < MR; r++ { - d := dst[r*ldc : r*ldc+NR : r*ldc+NR] - for c := range d { - d[c] += acc[r*NR+c] - } - } -} - -//go:noescape -func gemmMicro6x8RVV(kc int, pa, pb, c *float64, ldc int) diff --git a/internal/kernels/gemm_riscv64_test.go b/internal/kernels/gemm_riscv64_test.go deleted file mode 100644 index 85b0a82..0000000 --- a/internal/kernels/gemm_riscv64_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package kernels - -import "testing" - -// TestRVVDetected logs whether this machine runs the RVV kernels, and runs the -// SIMD-vs-scalar checks again with them forced off. -func TestRVVDetected(t *testing.T) { - t.Logf("useRVV = %v (AT_HWCAP)", useRVV) - saved := useRVV - useRVV = false - defer func() { useRVV = saved }() - t.Run("scalar", func(t *testing.T) { - TestBinSIMD(t) - TestSumSIMD(t) - TestGemmParallelYielding(t) // gemmMicroGo against the oracle - }) -} diff --git a/internal/kernels/gemm_s390x.go b/internal/kernels/gemm_s390x.go deleted file mode 100644 index 0a261b1..0000000 --- a/internal/kernels/gemm_s390x.go +++ /dev/null @@ -1,47 +0,0 @@ -package kernels - -// s390x GEMM micro-kernel: a 4x8 vector-facility tile (gemmMicro4x8VX in -// sum_s390x.s, generated by go-asmgen): sixteen two-lane accumulators, the B -// row loaded once per step with VLM, each A value replicated with VLREPG and -// fused in with VFMADB. Without the facility, gemmMicroGo computes the same -// tile in scalar code. - -// MR, NR are the micro-kernel's register-block dimensions. -const ( - MR = 4 - NR = 8 -) - -func gemmMicro(kc int, pa, pb, dst []float64, ldc int) { - if useVX { - _, _, _ = pa[kc*MR-1], pb[kc*NR-1], dst[(MR-1)*ldc+NR-1] // the kernel's whole reach - gemmMicro4x8VX(kc, &pa[0], &pb[0], &dst[0], ldc) - return - } - gemmMicroGo(kc, pa, pb, dst, ldc) -} - -// gemmMicroGo is the scalar MR x NR tile: dst[r*ldc+c] += sum_p -// pa[p*MR+r]*pb[p*NR+c], in p order per element. -func gemmMicroGo(kc int, pa, pb, dst []float64, ldc int) { - var acc [MR * NR]float64 - for p := 0; p < kc; p++ { - a := pa[p*MR : p*MR+MR : p*MR+MR] - b := pb[p*NR : p*NR+NR : p*NR+NR] - for r, av := range a { - row := acc[r*NR : r*NR+NR : r*NR+NR] - for c, bv := range b { - row[c] += av * bv - } - } - } - for r := 0; r < MR; r++ { - d := dst[r*ldc : r*ldc+NR : r*ldc+NR] - for c := range d { - d[c] += acc[r*NR+c] - } - } -} - -//go:noescape -func gemmMicro4x8VX(kc int, pa, pb, c *float64, ldc int) diff --git a/internal/kernels/gemm_test.go b/internal/kernels/gemm_test.go deleted file mode 100644 index d8ec9fe..0000000 --- a/internal/kernels/gemm_test.go +++ /dev/null @@ -1,437 +0,0 @@ -package kernels - -import ( - "math" - "math/rand" - "testing" -) - -// gemmOracle is the plain triple-loop reference GEMM in ikj order, the exact -// summation order the packed kernel reproduces. dst is pre-zeroed. -func gemmOracle(dst, a, b []float64, m, k, n int) { - for i := 0; i < m; i++ { - for p := 0; p < k; p++ { - av := a[i*k+p] - for j := 0; j < n; j++ { - dst[i*n+j] += av * b[p*n+j] - } - } - } -} - -// withBlocks runs f with the GEMM block sizes forced to (mc,kc,nc) and restores -// them after, so a test can force the multi-block loop nest on small inputs. -func withBlocks(mc, kc, nc int, f func()) { - omc, okc, onc := blockMC, blockKC, blockNC - blockMC, blockKC, blockNC = mc, kc, nc - defer func() { blockMC, blockKC, blockNC = omc, okc, onc }() - f() -} - -// withBandTiles runs f with the MatMulP dynamic-band height (in MR-tiles) forced -// to bt and restores it after, so a test can drive the band-sizing clamps. -func withBandTiles(bt int, f func()) { - o := bandTilesMR - bandTilesMR = bt - defer func() { bandTilesMR = o }() - f() -} - -// assertGemm checks both MatMul (serial) and MatMulP (parallel forced on) equal -// the oracle bit-for-bit for the given shape, using the integer-valued operands -// that make the ikj order reproduce the oracle exactly. -func assertGemm(t *testing.T, m, k, n int, a, b []float64) { - t.Helper() - savedA, savedB := unpackA, unpackB - defer func() { unpackA, unpackB = savedA, savedB }() - for _, p := range [][2]bool{{savedA, savedB}, {savedA, false}, {false, false}} { - unpackA, unpackB = p[0], p[1] - assertGemmPath(t, m, k, n, a, b) - } - // A in place behind an offset, rows k+3 apart (lda != k). - as := Mat{Data: make([]float64, 2+m*(k+3)), Off: 2, RS: k + 3, CS: 1} - for i := 0; i < m; i++ { - copy(as.Data[2+i*(k+3):], a[i*k:(i+1)*k]) - } - want := make([]float64, m*n) - gemmOracle(want, a, b, m, k, n) - got := make([]float64, m*n) - MatMulStridedP(got, as, rowMajor(b, n), m, k, n) - for i := range want { - if got[i] != want[i] { - t.Fatalf("strided A m=%d k=%d n=%d [%d]: %v != %v", m, k, n, i, got[i], want[i]) - } - } -} - -func assertGemmPath(t *testing.T, m, k, n int, a, b []float64) { - t.Helper() - want := make([]float64, m*n) - gemmOracle(want, a, b, m, k, n) - - gotS := make([]float64, m*n) - MatMul(gotS, a, b, m, k, n) - for i := range want { - if gotS[i] != want[i] { - t.Fatalf("MatMul m=%d k=%d n=%d [%d]: %v != %v", m, k, n, i, gotS[i], want[i]) - } - } - withThresholds(1<<20, 1, func() { // force the parallel band split - gotP := make([]float64, m*n) - MatMulP(gotP, a, b, m, k, n) - for i := range want { - if gotP[i] != want[i] { - t.Fatalf("MatMulP m=%d k=%d n=%d [%d]: %v != %v", m, k, n, i, gotP[i], want[i]) - } - } - }) -} - -// intMat returns an m*k matrix of small integer-valued doubles (exactly -// representable, and their sums of products are exact within float64 for these -// sizes) so the packed kernel must match the oracle bit-for-bit. -func intMat(m, k int, seed int64) []float64 { - r := rand.New(rand.NewSource(seed)) - a := make([]float64, m*k) - for i := range a { - a[i] = float64(r.Intn(17) - 8) - } - return a -} - -// TestGemmCorrectness validates the packed GEMM against the ikj oracle across a -// wide range of shapes: tiny, MR/NR-aligned and ragged (edge tiles on both the -// row and column borders), tall, wide, and sizes that cross several cache -// blocks once the block params are pinned small. -func TestGemmCorrectness(t *testing.T) { - shapes := []struct{ m, k, n int }{ - {1, 1, 1}, {2, 3, 4}, {4, 4, 4}, {4, 4, 8}, {4, 1, 8}, - {5, 7, 9}, {7, 7, 7}, {8, 8, 8}, {9, 5, 11}, {16, 16, 16}, - {17, 13, 19}, {3, 10, 1}, {1, 10, 3}, {33, 33, 33}, {32, 64, 48}, - } - for _, s := range shapes { - a, b := intMat(s.m, s.k, int64(s.m*100+s.k)), intMat(s.k, s.n, int64(s.n*7+s.k)) - // Default blocks. - assertGemm(t, s.m, s.k, s.n, a, b) - // Tiny blocks force the NC/KC/MC loop nest and many edge tiles. - withBlocks(MR, 2, NR, func() { assertGemm(t, s.m, s.k, s.n, a, b) }) - // Asymmetric small blocks. - withBlocks(2*MR, 3, 2*NR, func() { assertGemm(t, s.m, s.k, s.n, a, b) }) - // Block dims that are NOT multiples of MR/NR: the bottom A panel and the - // right B panel of each block are then partial and must be zero-padded to - // a full MR/NR tile in scratch sized via roundUp — this is the regression - // guard for the buffer-overflow bug a non-MR-aligned blockMC exposed. - withBlocks(MR+1, 2, NR+1, func() { assertGemm(t, s.m, s.k, s.n, a, b) }) - withBlocks(3*MR-1, 5, 3*NR-1, func() { assertGemm(t, s.m, s.k, s.n, a, b) }) - } - // A size large enough that the DEFAULT (256/256/512) blocks apply with the - // real MR=6 not dividing blockMC=256 — the production configuration that - // panicked before the roundUp fix. Crosses the parallel band split too. - withThresholds(1<<14, 1, func() { - m, k, n := 300, 200, 260 - a, b := intMat(m, k, 1), intMat(k, n, 2) - assertGemm(t, m, k, n, a, b) - }) -} - -// TestGemmBandSizing drives the two band-sizing clamps in MatMulP's parallel -// path against the oracle: (a) the small-m shrink (band wider than m/(4w) rounds -// down so every worker still gets a band), and (b) the band > blockMC cap (a -// band taller than the MC row block is clamped to MC so each A panel still fits). -// Both must leave the result bit-identical to the serial GEMM. -func TestGemmBandSizing(t *testing.T) { - // (a) Small-m shrink: tiny row count, default band height would exceed - // m/(4w); the want blockMC cap: pin blockMC below the band height and feed enough - // rows that the m/(4w) shrink does NOT pre-empt the cap (want >= bandRows), - // so the bandRows>blockMC clamp is the one that fires. - withMaxProcs(2, func() { - withBandTiles(2, func() { // bandRows = 2*MR - withBlocks(MR, 3, 2*NR, func() { // blockMC = MR < bandRows - withThresholds(1<<14, 1, func() { - m, k, n := 40*MR, 5, 2*NR - a, b := intMat(m, k, 23), intMat(k, n, 24) - assertGemm(t, m, k, n, a, b) - }) - }) - }) - }) -} - -// TestGemmSharedPackB runs MatMulP's parallel path on small products, where -// the workers share the packing of B in chunks of NR-wide panels: the chunks — -// including a ragged last panel and more workers than panels — must reassemble -// the exact packed block the serial packB writes, in both pb buffers. -func TestGemmSharedPackB(t *testing.T) { - withMaxProcs(4, func() { - withThresholds(1<<14, 1, func() { // force MatMulP's parallel path - for _, s := range []struct{ m, k, n int }{ - {13, 9, 3 * NR}, // whole panels - {20, 17, 5*NR + 3}, // ragged last panel - {7, 5, NR - 1}, // one partial panel, more workers than panels - {33, 40, 11*NR + 1}, - } { - a, b := intMat(s.m, s.k, int64(s.n)), intMat(s.k, s.n, int64(s.m)) - assertGemm(t, s.m, s.k, s.n, a, b) - withBlocks(MR, 3, 2*NR, func() { assertGemm(t, s.m, s.k, s.n, a, b) }) - } - }) - }) -} - -// TestGemmFloatTolerance validates the packed GEMM against the oracle for -// genuinely floating-point (non-integer) operands. The packed kernel uses the -// identical ikj summation order, so it is in fact bit-identical here too; we -// assert a tight tolerance to document that no reordering occurs. -func TestGemmFloatTolerance(t *testing.T) { - r := rand.New(rand.NewSource(99)) - m, k, n := 40, 50, 30 - a := make([]float64, m*k) - b := make([]float64, k*n) - for i := range a { - a[i] = r.NormFloat64() - } - for i := range b { - b[i] = r.NormFloat64() - } - want := make([]float64, m*n) - gemmOracle(want, a, b, m, k, n) - got := make([]float64, m*n) - MatMul(got, a, b, m, k, n) - for i := range want { - if math.Abs(got[i]-want[i]) > 1e-9*(math.Abs(want[i])+1) { - t.Fatalf("[%d]: %v vs %v", i, got[i], want[i]) - } - } -} - -// TestPackRowsWhole: the whole-run copy moves exactly kc runs of 8 or 16 -// strided elements, and declines any other width without writing. -func TestPackRowsWhole(t *testing.T) { - src := randVec(200, 3) - for _, nr := range []int{8, 16} { - const s, rs, kc = 5, 21, 7 - dst := make([]float64, kc*nr+1) - if !packRowsWhole(dst, src, s, rs, kc, nr) { - t.Fatalf("nr=%d declined", nr) - } - for p := 0; p < kc; p++ { - for c := 0; c < nr; c++ { - if dst[p*nr+c] != src[s+p*rs+c] { - t.Fatalf("nr=%d [%d][%d] = %v, want %v", nr, p, c, dst[p*nr+c], src[s+p*rs+c]) - } - } - } - if dst[kc*nr] != 0 { - t.Fatalf("nr=%d wrote past its %d runs", nr, kc) - } - } - dst := make([]float64, 40) - if packRowsWhole(dst, src, 0, 10, 4, 5) || dst[0] != 0 { - t.Fatal("nr=5 copied") - } -} - -// TestGemmBufferRealloc forces block sizes larger than the pooled buffers were -// first sized for, exercising the reallocation branch in getPackBuf, then a -// normal-sized call to confirm the pool still serves valid scratch. -func TestGemmBufferRealloc(t *testing.T) { - // Prime the pool at the default size. - MatMul(make([]float64, 16), intMat(4, 4, 1), intMat(4, 4, 2), 4, 4, 4) - // Now demand much larger blocks: getPackBuf must grow both buffers. - withBlocks(64, 64, 256, func() { - a, b := intMat(40, 40, 5), intMat(40, 40, 6) - assertGemm(t, 40, 40, 40, a, b) - }) - // Back to default blocks; pool buffers are oversized but still valid. - a, b := intMat(8, 8, 7), intMat(8, 8, 8) - assertGemm(t, 8, 8, 8, a, b) -} - -// TestGemmThresholdPaths exercises the serial (below GemmThreshold) and parallel -// (above) branches of MatMulP explicitly, plus the m==0 and w>m clamps. -func TestGemmThresholdPaths(t *testing.T) { - a, b := intMat(7, 4, 9), intMat(4, 9, 10) - want := make([]float64, 7*9) - gemmOracle(want, a, b, 7, 4, 9) - check := func(gemm int) { - withThresholds(1<<14, gemm, func() { - got := make([]float64, 7*9) - MatMulP(got, a, b, 7, 4, 9) - for i := range want { - if got[i] != want[i] { - t.Fatalf("gemm=%d [%d]: %v != %v", gemm, i, got[i], want[i]) - } - } - }) - } - check(1 << 20) // serial path - check(1) // parallel path (also w>m for the small row count) - - // m==0 is a no-op and must not panic. - withThresholds(1<<14, 1, func() { - MatMulP(nil, nil, b, 0, 4, 9) - }) -} - -// TestDot1D checks the exact left-to-right scalar dot oracle (the reference the -// unrolled/parallel variants document their ULP trade-off against). -func TestDot1D(t *testing.T) { - for _, n := range []int{0, 1, 4, 9, 100} { - a, b := randVec(n, int64(7*n+1)), randVec(n, int64(7*n+2)) - var want float64 - for i := range a { - want += a[i] * b[i] - } - if got := Dot1D(a, b); got != want { - t.Fatalf("Dot1D n=%d: %v != %v", n, got, want) - } - } -} - -// TestDotRange checks the unrolled contiguous dot against a plain reference for -// lengths that exercise the 4-wide body and every remainder (0..3 trailing). -func TestDotRange(t *testing.T) { - for _, n := range []int{0, 1, 2, 3, 4, 5, 7, 8, 13, 1000} { - a, b := randVec(n, int64(n+1)), randVec(n, int64(n+2)) - var want float64 - for i := range a { - want += a[i] * b[i] - } - got := dotRange(a, b) - if math.Abs(got-want) > 1e-9*(1+math.Abs(want)) { - t.Fatalf("dotRange n=%d: %v != %v", n, got, want) - } - } -} - -// TestMatVecP checks mat·vec over both the serial (below GemmThreshold) and -// parallel (above, plus the w>m clamp) paths against a row-dot reference. -func TestMatVecP(t *testing.T) { - m, k := 7, 13 - a, v := randVec(m*k, 3), randVec(k, 4) - want := make([]float64, m) - for i := 0; i < m; i++ { - for p := 0; p < k; p++ { - want[i] += a[i*k+p] * v[p] - } - } - check := func(gemm int) { - withThresholds(1<<14, gemm, func() { - got := make([]float64, m) - MatVecP(got, a, v, m, k) - for i := range want { - if math.Abs(got[i]-want[i]) > 1e-9*(1+math.Abs(want[i])) { - t.Fatalf("MatVecP gemm=%d [%d]: %v != %v", gemm, i, got[i], want[i]) - } - } - }) - } - check(1 << 20) // serial - // Pin GOMAXPROCS above m so the worker>m clamp fires on any-core CI. - withMaxProcs(128, func() { check(1) }) // parallel + w>m clamp -} - -// TestMatVecStridedP checks mat·vec on a sub-block of a wider matrix: only -// the first k of every lda elements are read. -func TestMatVecStridedP(t *testing.T) { - m, k, lda := 5, 6, 11 - a, v := randVec(m*lda, 5), randVec(k, 6) - got := make([]float64, m) - MatVecStridedP(got, a, lda, v, m, k) - for i := 0; i < m; i++ { - want := 0.0 - for p := 0; p < k; p++ { - want += a[i*lda+p] * v[p] - } - if math.Abs(got[i]-want) > 1e-9*(1+math.Abs(want)) { - t.Fatalf("MatVecStridedP [%d]: %v != %v", i, got[i], want) - } - } -} - -// TestVecMatP checks vec·mat (1-D · 2-D) against a column-accumulation reference. -func TestVecMatP(t *testing.T) { - k, n := 11, 9 - v, a := randVec(k, 5), randVec(k*n, 6) - want := make([]float64, n) - for p := 0; p < k; p++ { - for j := 0; j < n; j++ { - want[j] += v[p] * a[p*n+j] - } - } - got := make([]float64, n) - VecMatP(got, v, a, k, n) - for j := range want { - if math.Abs(got[j]-want[j]) > 1e-9*(1+math.Abs(want[j])) { - t.Fatalf("VecMatP [%d]: %v != %v", j, got[j], want[j]) - } - } -} - -// TestDot1DP checks the parallel 1-D dot below and above ParThreshold against a -// straight reference sum. -func TestDot1DP(t *testing.T) { - for _, n := range []int{1, 1000, 100000} { - a, b := randVec(n, int64(n)), randVec(n, int64(2*n)) - var want float64 - for i := range a { - want += a[i] * b[i] - } - for _, par := range []int{1 << 20, 4} { // serial, then parallel - withMaxProcs(128, func() { - withThresholds(par, 1<<14, func() { - got := Dot1DP(a, b) - if math.Abs(got-want) > 1e-6*(1+math.Abs(want)) { - t.Fatalf("Dot1DP n=%d par=%d: %v != %v", n, par, got, want) - } - }) - }) - } - } -} - -// TestDotSIMD holds the SIMD inner product (AVX2/FMA, NEON, or dotRange on the -// other targets) to a sequential oracle: exactly on integer data for every -// length 0..70 (main loop, folds and tail), within the regrouping bound -// n*eps*sum|a_i*b_i| on random data, and NaN/Inf propagation. -func TestDotSIMD(t *testing.T) { - naive := func(a, b []float64) (s, abs float64) { - for i := range a { - s += a[i] * b[i] - abs += math.Abs(a[i] * b[i]) - } - return s, abs - } - for n := 0; n <= 70; n++ { - a, b := intMat(1, n, int64(n)), intMat(1, n, int64(n+100)) - if want, _ := naive(a, b); dotSIMD(a, b) != want { - t.Fatalf("n=%d: %v != %v", n, dotSIMD(a, b), want) - } - } - for _, n := range []int{17, 1000, 4099} { - a, b := randVec(n, int64(n)), randVec(n, int64(2*n)) - want, abs := naive(a, b) - if got := dotSIMD(a, b); math.Abs(got-want) > float64(n)*0x1p-52*abs { - t.Errorf("n=%d: %v vs %v", n, got, want) - } - } - for _, special := range []float64{math.NaN(), math.Inf(1)} { - for _, at := range []int{0, 20, 37} { - a := intMat(1, 38, 5) - a[at] = special - b := intMat(1, 38, 6) - b[at] = 1 - got := dotSIMD(a, b) - if math.IsNaN(special) != math.IsNaN(got) || !math.IsNaN(special) && !math.IsInf(got, 1) { - t.Errorf("%v at %d: dot = %v", special, at, got) - } - } - } -} diff --git a/internal/kernels/gemmua_other.go b/internal/kernels/gemmua_other.go deleted file mode 100644 index 5b53a81..0000000 --- a/internal/kernels/gemmua_other.go +++ /dev/null @@ -1,6 +0,0 @@ -//go:build !amd64 && !arm64 - -package kernels - -// gemmMicroUA is nil here: no unpacked-A float64 kernel, so A is packed. -var gemmMicroUA func(kc int, a []float64, lda int, b []float64, ldb int, dst []float64, ldc, mr int) diff --git a/internal/kernels/generate.go b/internal/kernels/generate.go index 4104d8c..1044b2e 100644 --- a/internal/kernels/generate.go +++ b/internal/kernels/generate.go @@ -2,9 +2,10 @@ package kernels // The amd64 and arm64 assembly of this package is generated by go-asmgen. The // float64 sum, sqrt, max/min and elementwise kernels, and the exp/log kernels, -// live in go-simd/floats since K1 of the go-scikit performance campaign; what is -// generated here is GEMM, the float64 dot, and the float32 elementwise, sqrt -// and pairwise-block kernels. The float32 extrema (Max32/Min32) moved to +// live in go-simd/floats since K1 of the go-scikit performance campaign, and the +// GEMM micro-kernels in github.com/go-blas/blas (which generates them); what is +// generated here is the float64 dot, and the float32 elementwise, sqrt and +// pairwise-block kernels. The float32 extrema (Max32/Min32) moved to // go-simd/floats as Float32Max/Float32Min (go-ndarray/ndarray#98). // //go:generate sh -c "cd asmgen/amd64 && GOFLAGS=-mod=mod go run gen.go && mv sum_amd64.s ../../sum_amd64.s" diff --git a/internal/kernels/guard_test.go b/internal/kernels/guard_test.go index e16b8b1..31dd010 100644 --- a/internal/kernels/guard_test.go +++ b/internal/kernels/guard_test.go @@ -9,7 +9,6 @@ import "testing" // past a slice inside the kernel. func TestWrappersRefuseShortOperands(t *testing.T) { long, short := make([]float64, 8), make([]float64, 7) - ldc := NR for name, f := range map[string]func(){ "sqrt src": func() { sqrtSIMD(long, short) }, "add a": func() { addBin(long, short, long) }, @@ -17,9 +16,6 @@ func TestWrappersRefuseShortOperands(t *testing.T) { "mul a": func() { mulBin(long, short, long) }, "div b": func() { divBin(long, long, short) }, "dot b": func() { dotSIMD(long, short) }, - "gemm pa": func() { gemmMicro(1, make([]float64, MR-1), make([]float64, NR), make([]float64, (MR-1)*ldc+NR), ldc) }, - "gemm pb": func() { gemmMicro(1, make([]float64, MR), make([]float64, NR-1), make([]float64, (MR-1)*ldc+NR), ldc) }, - "gemm dst": func() { gemmMicro(1, make([]float64, MR), make([]float64, NR), make([]float64, (MR-1)*ldc+NR-1), ldc) }, } { func() { defer func() { diff --git a/internal/kernels/matvec.go b/internal/kernels/matvec.go new file mode 100644 index 0000000..e0fa737 --- /dev/null +++ b/internal/kernels/matvec.go @@ -0,0 +1,114 @@ +package kernels + +import "runtime" + +// Mat-vec and dot kernels. The packed GEMM that used to share a file with them +// lives in github.com/go-blas/blas; these need no packing. + +// matVecThreshold is the m*k element count above which MatVecP fans the row-dots +// across cores. Mat-vec is a single bandwidth-bound streaming pass over A (one +// multiply-add per element, no reuse), so its parallel crossover is far higher +// than the compute-bound GEMM's: forking it on a few-thousand-element A is a net +// loss (100x100 mat-vec measures ~2.5us serial vs ~5.8us parallel). It is a var +// so tests can pin it. +var matVecThreshold = 1 << 14 + +// dotRange returns sum(a[i]*b[i]) over equal-length slices, using four +// independent accumulators, so four multiply-add chains are in flight instead of +// one (gc does not vectorise this loop; the gain is latency hiding). The +// four-way grouping is a fixed +// reassociation of the sum; MatVecP/Dot1DP document the ULP trade-off where it is +// observable. It is the contiguous building block for both mat·vec and the 1-D +// dot — neither needs to go through the packing GEMM. +func dotRange(a, b []float64) float64 { + var s0, s1, s2, s3 float64 + n := len(a) + i := 0 + for ; i+4 <= n; i += 4 { + s0 += a[i] * b[i] + s1 += a[i+1] * b[i+1] + s2 += a[i+2] * b[i+2] + s3 += a[i+3] * b[i+3] + } + for ; i < n; i++ { + s0 += a[i] * b[i] + } + return (s0 + s1) + (s2 + s3) +} + +// MatVecP computes dst = a(m x k) * v(k,) as m independent contiguous row-dots, +// fanned across cores above matVecThreshold. Each output dst[i] is the dot of +// row i of a (the contiguous slice a[i*k:(i+1)*k]) with v, so the access pattern +// is pure unit-stride streaming — far cheaper than routing a single-column GEMM +// through the packer. dst need not be pre-zeroed (each entry is written, not +// accumulated). Rows are disjoint, so the parallel result equals the serial one. +func MatVecP(dst, a, v []float64, m, k int) { MatVecStridedP(dst, a, k, v, m, k) } + +// MatVecStridedP is MatVecP on rows lda elements apart: row i of a is +// a[i*lda : i*lda+k], so a sub-block of a larger row-major matrix is +// multiplied in place. +func MatVecStridedP(dst, a []float64, lda int, v []float64, m, k int) { + body := func(lo, hi int) { + for i := lo; i < hi; i++ { + dst[i] = dotSIMD(a[i*lda:i*lda+k], v) + } + } + if m*k < matVecThreshold { + body(0, m) + return + } + w := runtime.GOMAXPROCS(0) + if w > m { + w = m + } + parallelFor(m, w, body) +} + +// VecMatP computes dst = v(k,) * a(k x n) = column sums weighted by v, i.e. +// dst[j] = sum_p v[p]*a[p*n+j]. It accumulates row by row so the inner loop over +// j streams a contiguous a row and the contiguous dst, which is the cache- +// friendly traversal (the transpose-free analogue of mat·vec for the 1-D·2-D +// case). dst must be zeroed by the caller. +func VecMatP(dst, v, a []float64, k, n int) { + for p := 0; p < k; p++ { + vp := v[p] + row := a[p*n : p*n+n] + for j := 0; j < n; j++ { + dst[j] += vp * row[j] + } + } +} + +// Dot1DP returns the inner product of two equal-length contiguous vectors, +// fanned across cores above ParThreshold: each chunk is dot-producted with the +// SIMD kernel and the partials are summed. The chunked four-way +// grouping reassociates the sum (a few ULP, like SumP / numpy pairwise); callers +// needing the exact left-to-right value use the scalar Dot1D. +func Dot1DP(a, b []float64) float64 { + n := len(a) + if n < ParThreshold { + return dotSIMD(a, b) + } + w := numWorkers(n) + parts := min(4*w, max(w, n/reduceGrain)) // see mapReduceP + partials := make([]float64, parts) + chunk := (n + parts - 1) / parts + parallelFor(parts, w, func(lo, hi int) { + for idx := lo; idx < hi; idx++ { + s := idx * chunk + if s >= n { + // chunk rounding can leave trailing workers past the end (e.g. + // many workers over a short, threshold-forced input); their + // partial is just the additive identity. + partials[idx] = 0 + continue + } + e := s + chunk + if e > n { + e = n + } + partials[idx] = dotSIMD(a[s:e], b[s:e]) + } + }) + return Sum(partials) +} diff --git a/internal/kernels/matvec_test.go b/internal/kernels/matvec_test.go new file mode 100644 index 0000000..b7bdf7d --- /dev/null +++ b/internal/kernels/matvec_test.go @@ -0,0 +1,175 @@ +package kernels + +import ( + "math" + "math/rand" + "testing" +) + +// intMat returns an m*k matrix of small integer-valued doubles (exactly +// representable, and their sums of products are exact within float64 for these +// sizes) so a kernel must match its oracle bit-for-bit. +func intMat(m, k int, seed int64) []float64 { + r := rand.New(rand.NewSource(seed)) + a := make([]float64, m*k) + for i := range a { + a[i] = float64(r.Intn(17) - 8) + } + return a +} + +// TestDot1D checks the exact left-to-right scalar dot oracle (the reference the +// unrolled/parallel variants document their ULP trade-off against). +func TestDot1D(t *testing.T) { + for _, n := range []int{0, 1, 4, 9, 100} { + a, b := randVec(n, int64(7*n+1)), randVec(n, int64(7*n+2)) + var want float64 + for i := range a { + want += a[i] * b[i] + } + if got := Dot1D(a, b); got != want { + t.Fatalf("Dot1D n=%d: %v != %v", n, got, want) + } + } +} + +// TestDotRange checks the unrolled contiguous dot against a plain reference for +// lengths that exercise the 4-wide body and every remainder (0..3 trailing). +func TestDotRange(t *testing.T) { + for _, n := range []int{0, 1, 2, 3, 4, 5, 7, 8, 13, 1000} { + a, b := randVec(n, int64(n+1)), randVec(n, int64(n+2)) + var want float64 + for i := range a { + want += a[i] * b[i] + } + got := dotRange(a, b) + if math.Abs(got-want) > 1e-9*(1+math.Abs(want)) { + t.Fatalf("dotRange n=%d: %v != %v", n, got, want) + } + } +} + +// TestMatVecP checks mat·vec over both the serial (below matVecThreshold) and +// parallel (above, plus the w>m clamp) paths against a row-dot reference. +func TestMatVecP(t *testing.T) { + m, k := 7, 13 + a, v := randVec(m*k, 3), randVec(k, 4) + want := make([]float64, m) + for i := 0; i < m; i++ { + for p := 0; p < k; p++ { + want[i] += a[i*k+p] * v[p] + } + } + check := func(gemm int) { + withThresholds(1<<14, gemm, func() { + got := make([]float64, m) + MatVecP(got, a, v, m, k) + for i := range want { + if math.Abs(got[i]-want[i]) > 1e-9*(1+math.Abs(want[i])) { + t.Fatalf("MatVecP gemm=%d [%d]: %v != %v", gemm, i, got[i], want[i]) + } + } + }) + } + check(1 << 20) // serial + // Pin GOMAXPROCS above m so the worker>m clamp fires on any-core CI. + withMaxProcs(128, func() { check(1) }) // parallel + w>m clamp +} + +// TestMatVecStridedP checks mat·vec on a sub-block of a wider matrix: only +// the first k of every lda elements are read. +func TestMatVecStridedP(t *testing.T) { + m, k, lda := 5, 6, 11 + a, v := randVec(m*lda, 5), randVec(k, 6) + got := make([]float64, m) + MatVecStridedP(got, a, lda, v, m, k) + for i := 0; i < m; i++ { + want := 0.0 + for p := 0; p < k; p++ { + want += a[i*lda+p] * v[p] + } + if math.Abs(got[i]-want) > 1e-9*(1+math.Abs(want)) { + t.Fatalf("MatVecStridedP [%d]: %v != %v", i, got[i], want) + } + } +} + +// TestVecMatP checks vec·mat (1-D · 2-D) against a column-accumulation reference. +func TestVecMatP(t *testing.T) { + k, n := 11, 9 + v, a := randVec(k, 5), randVec(k*n, 6) + want := make([]float64, n) + for p := 0; p < k; p++ { + for j := 0; j < n; j++ { + want[j] += v[p] * a[p*n+j] + } + } + got := make([]float64, n) + VecMatP(got, v, a, k, n) + for j := range want { + if math.Abs(got[j]-want[j]) > 1e-9*(1+math.Abs(want[j])) { + t.Fatalf("VecMatP [%d]: %v != %v", j, got[j], want[j]) + } + } +} + +// TestDot1DP checks the parallel 1-D dot below and above ParThreshold against a +// straight reference sum. +func TestDot1DP(t *testing.T) { + for _, n := range []int{1, 1000, 100000} { + a, b := randVec(n, int64(n)), randVec(n, int64(2*n)) + var want float64 + for i := range a { + want += a[i] * b[i] + } + for _, par := range []int{1 << 20, 4} { // serial, then parallel + withMaxProcs(128, func() { + withThresholds(par, 1<<14, func() { + got := Dot1DP(a, b) + if math.Abs(got-want) > 1e-6*(1+math.Abs(want)) { + t.Fatalf("Dot1DP n=%d par=%d: %v != %v", n, par, got, want) + } + }) + }) + } + } +} + +// TestDotSIMD holds the SIMD inner product (AVX2/FMA, NEON, or dotRange on the +// other targets) to a sequential oracle: exactly on integer data for every +// length 0..70 (main loop, folds and tail), within the regrouping bound +// n*eps*sum|a_i*b_i| on random data, and NaN/Inf propagation. +func TestDotSIMD(t *testing.T) { + naive := func(a, b []float64) (s, abs float64) { + for i := range a { + s += a[i] * b[i] + abs += math.Abs(a[i] * b[i]) + } + return s, abs + } + for n := 0; n <= 70; n++ { + a, b := intMat(1, n, int64(n)), intMat(1, n, int64(n+100)) + if want, _ := naive(a, b); dotSIMD(a, b) != want { + t.Fatalf("n=%d: %v != %v", n, dotSIMD(a, b), want) + } + } + for _, n := range []int{17, 1000, 4099} { + a, b := randVec(n, int64(n)), randVec(n, int64(2*n)) + want, abs := naive(a, b) + if got := dotSIMD(a, b); math.Abs(got-want) > float64(n)*0x1p-52*abs { + t.Errorf("n=%d: %v vs %v", n, got, want) + } + } + for _, special := range []float64{math.NaN(), math.Inf(1)} { + for _, at := range []int{0, 20, 37} { + a := intMat(1, 38, 5) + a[at] = special + b := intMat(1, 38, 6) + b[at] = 1 + got := dotSIMD(a, b) + if math.IsNaN(special) != math.IsNaN(got) || !math.IsNaN(special) && !math.IsInf(got, 1) { + t.Errorf("%v at %d: dot = %v", special, at, got) + } + } + } +} diff --git a/internal/kernels/packrows8_test.go b/internal/kernels/packrows8_test.go deleted file mode 100644 index ce8f853..0000000 --- a/internal/kernels/packrows8_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package kernels - -import "testing" - -// TestPackRows8 checks the 8-row fast path of packA against the layout it -// documents, pa[ir*kc + p*8 + r] = A(ic+ir+r, pc+p), for row strides larger -// than the packed width, an offset, and an edge panel (mr < 8) that takes the -// general path, in both element types. -func TestPackRows8(t *testing.T) { - testPackRows8[float32](t) - testPackRows8[float64](t) -} - -func testPackRows8[T gemmFloat](t *testing.T) { - g := &gemmKernel[T]{mr: 8, nr: 8} - const m, k, rs, off = 21, 13, 17, 3 - data := make([]T, off+m*rs) - for i := range data { - data[i] = T(i + 1) - } - a := MatOf[T]{Data: data, Off: off, RS: rs, CS: 1} - for _, c := range []struct{ ic, mc, pc, kc int }{ - {0, 16, 0, k}, {0, 21, 2, 9}, {5, 16, 1, 1}, {8, 13, 0, 12}, - } { - pa := make([]T, roundUp(c.mc, 8)*c.kc) - for i := range pa { - pa[i] = -1 // a sentinel no packed value takes, so a missing zero-fill fails - } - g.packA(a, c.ic, c.mc, c.pc, c.kc, pa) - for ir := 0; ir < roundUp(c.mc, 8); ir += 8 { - for p := 0; p < c.kc; p++ { - for r := 0; r < 8; r++ { - var want T - if ir+r < c.mc { - want = a.Data[off+(c.ic+ir+r)*rs+c.pc+p] - } - if got := pa[ir*c.kc+p*8+r]; got != want { - t.Fatalf("%T %+v: pa[ir=%d p=%d r=%d] = %v, want %v", want, c, ir, p, r, got, want) - } - } - } - } - } -} diff --git a/internal/kernels/parallel.go b/internal/kernels/parallel.go index dba06c5..07973b1 100644 --- a/internal/kernels/parallel.go +++ b/internal/kernels/parallel.go @@ -430,6 +430,8 @@ func RunAxisP(k axisKernel, dst, src []float64, outer, axisLen, inner int) { } } +func roundUp(x, m int) int { return (x + m - 1) / m * m } + // reduceGrain is the fewest elements a reduction partial covers (64 KiB). const reduceGrain = 8192 diff --git a/internal/kernels/parallel_test.go b/internal/kernels/parallel_test.go index 589d65b..0334fb9 100644 --- a/internal/kernels/parallel_test.go +++ b/internal/kernels/parallel_test.go @@ -10,14 +10,13 @@ import ( // withThresholds runs f with the parallel thresholds forced and restores them // after, so a test can exercise both the serial (below-threshold) and the // parallel (above-threshold) branches deterministically. The gemm argument pins -// BOTH compute thresholds — GemmThreshold (MatMulP) and matVecThreshold -// (MatVecP) — to the same value, so a single low/high gemm flips every -// GEMM-family kernel onto the serial or parallel path; ParThreshold drives the -// elementwise/reduction kernels. +// matVecThreshold (MatVecP), so a single low/high value flips the mat-vec +// kernel onto the serial or parallel path; ParThreshold drives the +// elementwise/reduction kernels. (The GEMM has its own threshold, in go-blas.) func withThresholds(par, gemm int, f func()) { - op, og, omv := ParThreshold, GemmThreshold, matVecThreshold - ParThreshold, GemmThreshold, matVecThreshold = par, gemm, gemm - defer func() { ParThreshold, GemmThreshold, matVecThreshold = op, og, omv }() + op, omv := ParThreshold, matVecThreshold + ParThreshold, matVecThreshold = par, gemm + defer func() { ParThreshold, matVecThreshold = op, omv }() f() } @@ -164,31 +163,6 @@ func TestRunAxisP(t *testing.T) { } } -// TestParallelMatMul checks MatMulP equals the serial MatMul for non-square -// shapes, both below and above the (forced-low) GEMM threshold. -func TestParallelMatMul(t *testing.T) { - // The {2,..} case with GOMAXPROCS>2 and the parallel threshold forced to 1 - // exercises the w>m clamp (more cores than output rows). {0,..} hits the - // m==0 path; the rest cover assorted non-square bands. - cases := []struct{ m, k, n int }{{0, 3, 4}, {1, 1, 1}, {2, 6, 8}, {3, 5, 2}, {7, 4, 9}, {64, 48, 40}} - for _, c := range cases { - a, b := randVec(c.m*c.k, 10), randVec(c.k*c.n, 11) - for _, gemm := range []int{1 << 20, 1} { // serial, then parallel - withThresholds(1<<14, gemm, func() { - gotP := make([]float64, c.m*c.n) - gotS := make([]float64, c.m*c.n) - MatMulP(gotP, a, b, c.m, c.k, c.n) - MatMul(gotS, a, b, c.m, c.k, c.n) - for i := range gotP { - if gotP[i] != gotS[i] { - t.Fatalf("MatMulP %v gemm=%d [%d]: %v != %v", c, gemm, i, gotP[i], gotS[i]) - } - } - }) - } - } -} - // TestSqrtP checks the parallel sqrt wrapper is bit-identical to a scalar // math.Sqrt loop on both the serial (below-threshold) and parallel branches. func TestSqrtP(t *testing.T) { diff --git a/internal/kernels/stress_test.go b/internal/kernels/stress_test.go index 1b04ae7..ebff36c 100644 --- a/internal/kernels/stress_test.go +++ b/internal/kernels/stress_test.go @@ -22,9 +22,9 @@ func TestParallelStress(t *testing.T) { if testing.Short() { dur = 200 * time.Millisecond } - op, og, omv := ParThreshold, GemmThreshold, matVecThreshold - ParThreshold, GemmThreshold, matVecThreshold = 64, 64, 64 // parallel from tiny sizes - defer func() { ParThreshold, GemmThreshold, matVecThreshold = op, og, omv }() + op, omv := ParThreshold, matVecThreshold + ParThreshold, matVecThreshold = 64, 64 // parallel from tiny sizes + defer func() { ParThreshold, matVecThreshold = op, omv }() defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(0)) stop := make(chan struct{}) diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index 2df07cd..8773818 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -4,1136 +4,6 @@ #include "textflag.h" -TEXT ·gemmMicro6x8FMA(SB), NOSPLIT, $0-40 - MOVQ kc+0(FP), CX - MOVQ pa+8(FP), SI - MOVQ pb+16(FP), DI - MOVQ c+24(FP), DX - MOVQ ldc+32(FP), R8 - SHLQ $3, R8 - MOVQ DX, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - VXORPD Y0, Y0, Y0 - VXORPD Y1, Y1, Y1 - VXORPD Y2, Y2, Y2 - VXORPD Y3, Y3, Y3 - VXORPD Y4, Y4, Y4 - VXORPD Y5, Y5, Y5 - VXORPD Y6, Y6, Y6 - VXORPD Y7, Y7, Y7 - VXORPD Y8, Y8, Y8 - VXORPD Y9, Y9, Y9 - VXORPD Y10, Y10, Y10 - VXORPD Y11, Y11, Y11 - MOVQ CX, R10 - SHRQ $1, R10 - JZ ftail - fkloop: - PREFETCHT0 384(SI) - VMOVUPD 0(DI), Y12 - VMOVUPD 32(DI), Y13 - VBROADCASTSD 0(SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - VBROADCASTSD 8(SI), Y14 - VFMADD231PD Y12, Y14, Y2 - VFMADD231PD Y13, Y14, Y3 - VBROADCASTSD 16(SI), Y14 - VFMADD231PD Y12, Y14, Y4 - VFMADD231PD Y13, Y14, Y5 - VBROADCASTSD 24(SI), Y14 - VFMADD231PD Y12, Y14, Y6 - VFMADD231PD Y13, Y14, Y7 - VBROADCASTSD 32(SI), Y14 - VFMADD231PD Y12, Y14, Y8 - VFMADD231PD Y13, Y14, Y9 - VBROADCASTSD 40(SI), Y14 - VFMADD231PD Y12, Y14, Y10 - VFMADD231PD Y13, Y14, Y11 - VMOVUPD 64(DI), Y12 - VMOVUPD 96(DI), Y13 - VBROADCASTSD 48(SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - VBROADCASTSD 56(SI), Y14 - VFMADD231PD Y12, Y14, Y2 - VFMADD231PD Y13, Y14, Y3 - VBROADCASTSD 64(SI), Y14 - VFMADD231PD Y12, Y14, Y4 - VFMADD231PD Y13, Y14, Y5 - VBROADCASTSD 72(SI), Y14 - VFMADD231PD Y12, Y14, Y6 - VFMADD231PD Y13, Y14, Y7 - VBROADCASTSD 80(SI), Y14 - VFMADD231PD Y12, Y14, Y8 - VFMADD231PD Y13, Y14, Y9 - VBROADCASTSD 88(SI), Y14 - VFMADD231PD Y12, Y14, Y10 - VFMADD231PD Y13, Y14, Y11 - ADDQ $96, SI - ADDQ $128, DI - DECQ R10 - JNZ fkloop - ftail: - TESTQ $1, CX - JZ fstore - VMOVUPD 0(DI), Y12 - VMOVUPD 32(DI), Y13 - VBROADCASTSD 0(SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - VBROADCASTSD 8(SI), Y14 - VFMADD231PD Y12, Y14, Y2 - VFMADD231PD Y13, Y14, Y3 - VBROADCASTSD 16(SI), Y14 - VFMADD231PD Y12, Y14, Y4 - VFMADD231PD Y13, Y14, Y5 - VBROADCASTSD 24(SI), Y14 - VFMADD231PD Y12, Y14, Y6 - VFMADD231PD Y13, Y14, Y7 - VBROADCASTSD 32(SI), Y14 - VFMADD231PD Y12, Y14, Y8 - VFMADD231PD Y13, Y14, Y9 - VBROADCASTSD 40(SI), Y14 - VFMADD231PD Y12, Y14, Y10 - VFMADD231PD Y13, Y14, Y11 - fstore: - VADDPD (DX), Y0, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y1, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R8, DX - VADDPD (DX), Y2, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y3, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R8, DX - VADDPD (DX), Y4, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y5, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R8, DX - VADDPD (DX), Y6, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y7, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R8, DX - VADDPD (DX), Y8, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y9, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R8, DX - VADDPD (DX), Y10, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y11, Y15 - VMOVUPD Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmMicro6x16FMA(SB), NOSPLIT, $0-40 - MOVQ kc+0(FP), CX - MOVQ pa+8(FP), SI - MOVQ pb+16(FP), DI - MOVQ c+24(FP), DX - MOVQ ldc+32(FP), R8 - SHLQ $2, R8 - MOVQ DX, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - ADDQ R8, R9 - PREFETCHT0 (R9) - PREFETCHT0 63(R9) - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - VXORPS Y4, Y4, Y4 - VXORPS Y5, Y5, Y5 - VXORPS Y6, Y6, Y6 - VXORPS Y7, Y7, Y7 - VXORPS Y8, Y8, Y8 - VXORPS Y9, Y9, Y9 - VXORPS Y10, Y10, Y10 - VXORPS Y11, Y11, Y11 - MOVQ CX, R10 - SHRQ $1, R10 - JZ stail - skloop: - PREFETCHT0 384(SI) - VMOVUPS 0(DI), Y12 - VMOVUPS 32(DI), Y13 - VBROADCASTSS 0(SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - VBROADCASTSS 4(SI), Y14 - VFMADD231PS Y12, Y14, Y2 - VFMADD231PS Y13, Y14, Y3 - VBROADCASTSS 8(SI), Y14 - VFMADD231PS Y12, Y14, Y4 - VFMADD231PS Y13, Y14, Y5 - VBROADCASTSS 12(SI), Y14 - VFMADD231PS Y12, Y14, Y6 - VFMADD231PS Y13, Y14, Y7 - VBROADCASTSS 16(SI), Y14 - VFMADD231PS Y12, Y14, Y8 - VFMADD231PS Y13, Y14, Y9 - VBROADCASTSS 20(SI), Y14 - VFMADD231PS Y12, Y14, Y10 - VFMADD231PS Y13, Y14, Y11 - VMOVUPS 64(DI), Y12 - VMOVUPS 96(DI), Y13 - VBROADCASTSS 24(SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - VBROADCASTSS 28(SI), Y14 - VFMADD231PS Y12, Y14, Y2 - VFMADD231PS Y13, Y14, Y3 - VBROADCASTSS 32(SI), Y14 - VFMADD231PS Y12, Y14, Y4 - VFMADD231PS Y13, Y14, Y5 - VBROADCASTSS 36(SI), Y14 - VFMADD231PS Y12, Y14, Y6 - VFMADD231PS Y13, Y14, Y7 - VBROADCASTSS 40(SI), Y14 - VFMADD231PS Y12, Y14, Y8 - VFMADD231PS Y13, Y14, Y9 - VBROADCASTSS 44(SI), Y14 - VFMADD231PS Y12, Y14, Y10 - VFMADD231PS Y13, Y14, Y11 - ADDQ $48, SI - ADDQ $128, DI - DECQ R10 - JNZ skloop - stail: - TESTQ $1, CX - JZ sstore - VMOVUPS 0(DI), Y12 - VMOVUPS 32(DI), Y13 - VBROADCASTSS 0(SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - VBROADCASTSS 4(SI), Y14 - VFMADD231PS Y12, Y14, Y2 - VFMADD231PS Y13, Y14, Y3 - VBROADCASTSS 8(SI), Y14 - VFMADD231PS Y12, Y14, Y4 - VFMADD231PS Y13, Y14, Y5 - VBROADCASTSS 12(SI), Y14 - VFMADD231PS Y12, Y14, Y6 - VFMADD231PS Y13, Y14, Y7 - VBROADCASTSS 16(SI), Y14 - VFMADD231PS Y12, Y14, Y8 - VFMADD231PS Y13, Y14, Y9 - VBROADCASTSS 20(SI), Y14 - VFMADD231PS Y12, Y14, Y10 - VFMADD231PS Y13, Y14, Y11 - sstore: - VADDPS (DX), Y0, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y1, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R8, DX - VADDPS (DX), Y2, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y3, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R8, DX - VADDPS (DX), Y4, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y5, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R8, DX - VADDPS (DX), Y6, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y7, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R8, DX - VADDPS (DX), Y8, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y9, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R8, DX - VADDPS (DX), Y10, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y11, Y15 - VMOVUPS Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA1x16FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $2, R8 - SHLQ $2, R13 - SHLQ $2, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPS (DI), Y12 - VMOVUPS 32(DI), Y13 - VBROADCASTSS (SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - ADDQ $4, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPS (DX), Y0, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y1, Y15 - VMOVUPS Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA1x8FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $3, R8 - SHLQ $3, R13 - SHLQ $3, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPD Y0, Y0, Y0 - VXORPD Y1, Y1, Y1 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPD (DI), Y12 - VMOVUPD 32(DI), Y13 - VBROADCASTSD (SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - ADDQ $8, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPD (DX), Y0, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y1, Y15 - VMOVUPD Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA2x16FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $2, R8 - SHLQ $2, R13 - SHLQ $2, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPS (DI), Y12 - VMOVUPS 32(DI), Y13 - VBROADCASTSS (SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - VBROADCASTSS (SI)(R8*1), Y14 - VFMADD231PS Y12, Y14, Y2 - VFMADD231PS Y13, Y14, Y3 - ADDQ $4, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPS (DX), Y0, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y1, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y2, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y3, Y15 - VMOVUPS Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA2x8FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $3, R8 - SHLQ $3, R13 - SHLQ $3, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPD Y0, Y0, Y0 - VXORPD Y1, Y1, Y1 - VXORPD Y2, Y2, Y2 - VXORPD Y3, Y3, Y3 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPD (DI), Y12 - VMOVUPD 32(DI), Y13 - VBROADCASTSD (SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - VBROADCASTSD (SI)(R8*1), Y14 - VFMADD231PD Y12, Y14, Y2 - VFMADD231PD Y13, Y14, Y3 - ADDQ $8, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPD (DX), Y0, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y1, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y2, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y3, Y15 - VMOVUPD Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA3x16FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $2, R8 - SHLQ $2, R13 - SHLQ $2, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - VXORPS Y4, Y4, Y4 - VXORPS Y5, Y5, Y5 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPS (DI), Y12 - VMOVUPS 32(DI), Y13 - VBROADCASTSS (SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - VBROADCASTSS (SI)(R8*1), Y14 - VFMADD231PS Y12, Y14, Y2 - VFMADD231PS Y13, Y14, Y3 - VBROADCASTSS (SI)(R8*2), Y14 - VFMADD231PS Y12, Y14, Y4 - VFMADD231PS Y13, Y14, Y5 - ADDQ $4, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPS (DX), Y0, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y1, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y2, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y3, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y4, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y5, Y15 - VMOVUPS Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA3x8FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $3, R8 - SHLQ $3, R13 - SHLQ $3, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPD Y0, Y0, Y0 - VXORPD Y1, Y1, Y1 - VXORPD Y2, Y2, Y2 - VXORPD Y3, Y3, Y3 - VXORPD Y4, Y4, Y4 - VXORPD Y5, Y5, Y5 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPD (DI), Y12 - VMOVUPD 32(DI), Y13 - VBROADCASTSD (SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - VBROADCASTSD (SI)(R8*1), Y14 - VFMADD231PD Y12, Y14, Y2 - VFMADD231PD Y13, Y14, Y3 - VBROADCASTSD (SI)(R8*2), Y14 - VFMADD231PD Y12, Y14, Y4 - VFMADD231PD Y13, Y14, Y5 - ADDQ $8, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPD (DX), Y0, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y1, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y2, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y3, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y4, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y5, Y15 - VMOVUPD Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA4x16FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $2, R8 - SHLQ $2, R13 - SHLQ $2, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - VXORPS Y4, Y4, Y4 - VXORPS Y5, Y5, Y5 - VXORPS Y6, Y6, Y6 - VXORPS Y7, Y7, Y7 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPS (DI), Y12 - VMOVUPS 32(DI), Y13 - VBROADCASTSS (SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - VBROADCASTSS (SI)(R8*1), Y14 - VFMADD231PS Y12, Y14, Y2 - VFMADD231PS Y13, Y14, Y3 - VBROADCASTSS (SI)(R8*2), Y14 - VFMADD231PS Y12, Y14, Y4 - VFMADD231PS Y13, Y14, Y5 - VBROADCASTSS (SI)(R11*1), Y14 - VFMADD231PS Y12, Y14, Y6 - VFMADD231PS Y13, Y14, Y7 - ADDQ $4, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPS (DX), Y0, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y1, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y2, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y3, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y4, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y5, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y6, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y7, Y15 - VMOVUPS Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA4x8FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $3, R8 - SHLQ $3, R13 - SHLQ $3, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPD Y0, Y0, Y0 - VXORPD Y1, Y1, Y1 - VXORPD Y2, Y2, Y2 - VXORPD Y3, Y3, Y3 - VXORPD Y4, Y4, Y4 - VXORPD Y5, Y5, Y5 - VXORPD Y6, Y6, Y6 - VXORPD Y7, Y7, Y7 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPD (DI), Y12 - VMOVUPD 32(DI), Y13 - VBROADCASTSD (SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - VBROADCASTSD (SI)(R8*1), Y14 - VFMADD231PD Y12, Y14, Y2 - VFMADD231PD Y13, Y14, Y3 - VBROADCASTSD (SI)(R8*2), Y14 - VFMADD231PD Y12, Y14, Y4 - VFMADD231PD Y13, Y14, Y5 - VBROADCASTSD (SI)(R11*1), Y14 - VFMADD231PD Y12, Y14, Y6 - VFMADD231PD Y13, Y14, Y7 - ADDQ $8, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPD (DX), Y0, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y1, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y2, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y3, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y4, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y5, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y6, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y7, Y15 - VMOVUPD Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA5x16FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $2, R8 - SHLQ $2, R13 - SHLQ $2, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - VXORPS Y4, Y4, Y4 - VXORPS Y5, Y5, Y5 - VXORPS Y6, Y6, Y6 - VXORPS Y7, Y7, Y7 - VXORPS Y8, Y8, Y8 - VXORPS Y9, Y9, Y9 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPS (DI), Y12 - VMOVUPS 32(DI), Y13 - VBROADCASTSS (SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - VBROADCASTSS (SI)(R8*1), Y14 - VFMADD231PS Y12, Y14, Y2 - VFMADD231PS Y13, Y14, Y3 - VBROADCASTSS (SI)(R8*2), Y14 - VFMADD231PS Y12, Y14, Y4 - VFMADD231PS Y13, Y14, Y5 - VBROADCASTSS (SI)(R11*1), Y14 - VFMADD231PS Y12, Y14, Y6 - VFMADD231PS Y13, Y14, Y7 - VBROADCASTSS (SI)(R8*4), Y14 - VFMADD231PS Y12, Y14, Y8 - VFMADD231PS Y13, Y14, Y9 - ADDQ $4, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPS (DX), Y0, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y1, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y2, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y3, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y4, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y5, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y6, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y7, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y8, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y9, Y15 - VMOVUPS Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA5x8FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $3, R8 - SHLQ $3, R13 - SHLQ $3, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPD Y0, Y0, Y0 - VXORPD Y1, Y1, Y1 - VXORPD Y2, Y2, Y2 - VXORPD Y3, Y3, Y3 - VXORPD Y4, Y4, Y4 - VXORPD Y5, Y5, Y5 - VXORPD Y6, Y6, Y6 - VXORPD Y7, Y7, Y7 - VXORPD Y8, Y8, Y8 - VXORPD Y9, Y9, Y9 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPD (DI), Y12 - VMOVUPD 32(DI), Y13 - VBROADCASTSD (SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - VBROADCASTSD (SI)(R8*1), Y14 - VFMADD231PD Y12, Y14, Y2 - VFMADD231PD Y13, Y14, Y3 - VBROADCASTSD (SI)(R8*2), Y14 - VFMADD231PD Y12, Y14, Y4 - VFMADD231PD Y13, Y14, Y5 - VBROADCASTSD (SI)(R11*1), Y14 - VFMADD231PD Y12, Y14, Y6 - VFMADD231PD Y13, Y14, Y7 - VBROADCASTSD (SI)(R8*4), Y14 - VFMADD231PD Y12, Y14, Y8 - VFMADD231PD Y13, Y14, Y9 - ADDQ $8, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPD (DX), Y0, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y1, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y2, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y3, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y4, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y5, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y6, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y7, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y8, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y9, Y15 - VMOVUPD Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA6x16FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $2, R8 - SHLQ $2, R13 - SHLQ $2, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - VXORPS Y4, Y4, Y4 - VXORPS Y5, Y5, Y5 - VXORPS Y6, Y6, Y6 - VXORPS Y7, Y7, Y7 - VXORPS Y8, Y8, Y8 - VXORPS Y9, Y9, Y9 - VXORPS Y10, Y10, Y10 - VXORPS Y11, Y11, Y11 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPS (DI), Y12 - VMOVUPS 32(DI), Y13 - VBROADCASTSS (SI), Y14 - VFMADD231PS Y12, Y14, Y0 - VFMADD231PS Y13, Y14, Y1 - VBROADCASTSS (SI)(R8*1), Y14 - VFMADD231PS Y12, Y14, Y2 - VFMADD231PS Y13, Y14, Y3 - VBROADCASTSS (SI)(R8*2), Y14 - VFMADD231PS Y12, Y14, Y4 - VFMADD231PS Y13, Y14, Y5 - VBROADCASTSS (SI)(R11*1), Y14 - VFMADD231PS Y12, Y14, Y6 - VFMADD231PS Y13, Y14, Y7 - VBROADCASTSS (SI)(R8*4), Y14 - VFMADD231PS Y12, Y14, Y8 - VFMADD231PS Y13, Y14, Y9 - VBROADCASTSS (SI)(R12*1), Y14 - VFMADD231PS Y12, Y14, Y10 - VFMADD231PS Y13, Y14, Y11 - ADDQ $4, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPS (DX), Y0, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y1, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y2, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y3, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y4, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y5, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y6, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y7, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y8, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y9, Y15 - VMOVUPS Y15, 32(DX) - ADDQ R9, DX - VADDPS (DX), Y10, Y15 - VMOVUPS Y15, (DX) - VADDPS 32(DX), Y11, Y15 - VMOVUPS Y15, 32(DX) - VZEROUPPER - RET - -TEXT ·gemmUA6x8FMA(SB), NOSPLIT, $0-56 - MOVQ kc+0(FP), CX - MOVQ a+8(FP), SI - MOVQ lda+16(FP), R8 - MOVQ pb+24(FP), DI - MOVQ ldb+32(FP), R13 - MOVQ c+40(FP), DX - MOVQ ldc+48(FP), R9 - SHLQ $3, R8 - SHLQ $3, R13 - SHLQ $3, R9 - LEAQ (R8)(R8*2), R11 - LEAQ (R8)(R8*4), R12 - MOVQ DX, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - ADDQ R9, R10 - PREFETCHT0 (R10) - PREFETCHT0 63(R10) - VXORPD Y0, Y0, Y0 - VXORPD Y1, Y1, Y1 - VXORPD Y2, Y2, Y2 - VXORPD Y3, Y3, Y3 - VXORPD Y4, Y4, Y4 - VXORPD Y5, Y5, Y5 - VXORPD Y6, Y6, Y6 - VXORPD Y7, Y7, Y7 - VXORPD Y8, Y8, Y8 - VXORPD Y9, Y9, Y9 - VXORPD Y10, Y10, Y10 - VXORPD Y11, Y11, Y11 - TESTQ CX, CX - JZ uastore - uakloop: - VMOVUPD (DI), Y12 - VMOVUPD 32(DI), Y13 - VBROADCASTSD (SI), Y14 - VFMADD231PD Y12, Y14, Y0 - VFMADD231PD Y13, Y14, Y1 - VBROADCASTSD (SI)(R8*1), Y14 - VFMADD231PD Y12, Y14, Y2 - VFMADD231PD Y13, Y14, Y3 - VBROADCASTSD (SI)(R8*2), Y14 - VFMADD231PD Y12, Y14, Y4 - VFMADD231PD Y13, Y14, Y5 - VBROADCASTSD (SI)(R11*1), Y14 - VFMADD231PD Y12, Y14, Y6 - VFMADD231PD Y13, Y14, Y7 - VBROADCASTSD (SI)(R8*4), Y14 - VFMADD231PD Y12, Y14, Y8 - VFMADD231PD Y13, Y14, Y9 - VBROADCASTSD (SI)(R12*1), Y14 - VFMADD231PD Y12, Y14, Y10 - VFMADD231PD Y13, Y14, Y11 - ADDQ $8, SI - ADDQ R13, DI - DECQ CX - JNZ uakloop - uastore: - VADDPD (DX), Y0, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y1, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y2, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y3, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y4, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y5, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y6, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y7, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y8, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y9, Y15 - VMOVUPD Y15, 32(DX) - ADDQ R9, DX - VADDPD (DX), Y10, Y15 - VMOVUPD Y15, (DX) - VADDPD 32(DX), Y11, Y15 - VMOVUPD Y15, 32(DX) - VZEROUPPER - RET - TEXT ·dotFMA(SB), NOSPLIT, $0-32 MOVQ a+0(FP), SI MOVQ b+8(FP), DI @@ -1179,114 +49,6 @@ TEXT ·dotFMA(SB), NOSPLIT, $0-32 MOVSD X0, ret+24(FP) RET -TEXT ·gemmMicro6x4SSE2(SB), NOSPLIT, $0-40 - MOVQ kc+0(FP), CX - MOVQ pa+8(FP), SI - MOVQ pb+16(FP), DI - MOVQ c+24(FP), DX - MOVQ ldc+32(FP), R8 - SHLQ $3, R8 - XORPS X0, X0 - XORPS X1, X1 - XORPS X2, X2 - XORPS X3, X3 - XORPS X4, X4 - XORPS X5, X5 - XORPS X6, X6 - XORPS X7, X7 - XORPS X8, X8 - XORPS X9, X9 - XORPS X10, X10 - XORPS X11, X11 - TESTQ CX, CX - JZ hstore - hkloop: - MOVUPD (DI), X12 - MOVUPD 16(DI), X13 - MOVDDUP 0(SI), X14 - MOVAPS X12, X15 - MULPD X14, X15 - ADDPD X15, X0 - MULPD X13, X14 - ADDPD X14, X1 - MOVDDUP 8(SI), X14 - MOVAPS X12, X15 - MULPD X14, X15 - ADDPD X15, X2 - MULPD X13, X14 - ADDPD X14, X3 - MOVDDUP 16(SI), X14 - MOVAPS X12, X15 - MULPD X14, X15 - ADDPD X15, X4 - MULPD X13, X14 - ADDPD X14, X5 - MOVDDUP 24(SI), X14 - MOVAPS X12, X15 - MULPD X14, X15 - ADDPD X15, X6 - MULPD X13, X14 - ADDPD X14, X7 - MOVDDUP 32(SI), X14 - MOVAPS X12, X15 - MULPD X14, X15 - ADDPD X15, X8 - MULPD X13, X14 - ADDPD X14, X9 - MOVDDUP 40(SI), X14 - MOVAPS X12, X15 - MULPD X14, X15 - ADDPD X15, X10 - MULPD X13, X14 - ADDPD X14, X11 - ADDQ $48, SI - ADDQ $64, DI - DECQ CX - JNZ hkloop - hstore: - MOVUPD (DX), X14 - ADDPD X0, X14 - MOVUPD X14, (DX) - MOVUPD 16(DX), X15 - ADDPD X1, X15 - MOVUPD X15, 16(DX) - ADDQ R8, DX - MOVUPD (DX), X14 - ADDPD X2, X14 - MOVUPD X14, (DX) - MOVUPD 16(DX), X15 - ADDPD X3, X15 - MOVUPD X15, 16(DX) - ADDQ R8, DX - MOVUPD (DX), X14 - ADDPD X4, X14 - MOVUPD X14, (DX) - MOVUPD 16(DX), X15 - ADDPD X5, X15 - MOVUPD X15, 16(DX) - ADDQ R8, DX - MOVUPD (DX), X14 - ADDPD X6, X14 - MOVUPD X14, (DX) - MOVUPD 16(DX), X15 - ADDPD X7, X15 - MOVUPD X15, 16(DX) - ADDQ R8, DX - MOVUPD (DX), X14 - ADDPD X8, X14 - MOVUPD X14, (DX) - MOVUPD 16(DX), X15 - ADDPD X9, X15 - MOVUPD X15, 16(DX) - ADDQ R8, DX - MOVUPD (DX), X14 - ADDPD X10, X14 - MOVUPD X14, (DX) - MOVUPD 16(DX), X15 - ADDPD X11, X15 - MOVUPD X15, 16(DX) - RET - TEXT ·hasFMA(SB), NOSPLIT, $0-1 MOVB $0, ret+0(FP) MOVL $1, AX diff --git a/internal/kernels/sum_arm64.s b/internal/kernels/sum_arm64.s index c9deb8f..9e0c469 100644 --- a/internal/kernels/sum_arm64.s +++ b/internal/kernels/sum_arm64.s @@ -196,1494 +196,6 @@ TEXT ·sqrt32NEON(SB), NOSPLIT, $0-24 done: RET -TEXT ·gemmMicro4x8(SB), NOSPLIT, $0-40 - MOVD kc+0(FP), R0 - MOVD pa+8(FP), R1 - MOVD pb+16(FP), R2 - MOVD c+24(FP), R3 - MOVD ldc+32(FP), R4 - LSL $3, R4, R4 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - VEOR V8.B16, V8.B16, V8.B16 - VEOR V9.B16, V9.B16, V9.B16 - VEOR V10.B16, V10.B16, V10.B16 - VEOR V11.B16, V11.B16, V11.B16 - VEOR V12.B16, V12.B16, V12.B16 - VEOR V13.B16, V13.B16, V13.B16 - VEOR V14.B16, V14.B16, V14.B16 - VEOR V15.B16, V15.B16, V15.B16 - CBZ R0, gstore - gkloop: - VLD1.P 64(R2), [V16.D2, V17.D2, V18.D2, V19.D2] - VLD1.P 32(R1), [V20.D2, V21.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - WORD $0x4FD41A04 // FMLA V4.2D, V16.2D, V20.D[1] - WORD $0x4FD41A25 // FMLA V5.2D, V17.2D, V20.D[1] - WORD $0x4FD41A46 // FMLA V6.2D, V18.2D, V20.D[1] - WORD $0x4FD41A67 // FMLA V7.2D, V19.2D, V20.D[1] - WORD $0x4FD51208 // FMLA V8.2D, V16.2D, V21.D[0] - WORD $0x4FD51229 // FMLA V9.2D, V17.2D, V21.D[0] - WORD $0x4FD5124A // FMLA V10.2D, V18.2D, V21.D[0] - WORD $0x4FD5126B // FMLA V11.2D, V19.2D, V21.D[0] - WORD $0x4FD51A0C // FMLA V12.2D, V16.2D, V21.D[1] - WORD $0x4FD51A2D // FMLA V13.2D, V17.2D, V21.D[1] - WORD $0x4FD51A4E // FMLA V14.2D, V18.2D, V21.D[1] - WORD $0x4FD51A6F // FMLA V15.2D, V19.2D, V21.D[1] - SUB $1, R0 - CBNZ R0, gkloop - gstore: - MOVD $0x3FF0000000000000, R6 - VDUP R6, V31.D2 - MOVD R3, R5 - VLD1 (R5), [V16.D2, V17.D2, V18.D2, V19.D2] - VFMLA V0.D2, V31.D2, V16.D2 - VFMLA V1.D2, V31.D2, V17.D2 - VFMLA V2.D2, V31.D2, V18.D2 - VFMLA V3.D2, V31.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5) - ADD R4, R3, R3 - MOVD R3, R5 - VLD1 (R5), [V16.D2, V17.D2, V18.D2, V19.D2] - VFMLA V4.D2, V31.D2, V16.D2 - VFMLA V5.D2, V31.D2, V17.D2 - VFMLA V6.D2, V31.D2, V18.D2 - VFMLA V7.D2, V31.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5) - ADD R4, R3, R3 - MOVD R3, R5 - VLD1 (R5), [V16.D2, V17.D2, V18.D2, V19.D2] - VFMLA V8.D2, V31.D2, V16.D2 - VFMLA V9.D2, V31.D2, V17.D2 - VFMLA V10.D2, V31.D2, V18.D2 - VFMLA V11.D2, V31.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5) - ADD R4, R3, R3 - MOVD R3, R5 - VLD1 (R5), [V16.D2, V17.D2, V18.D2, V19.D2] - VFMLA V12.D2, V31.D2, V16.D2 - VFMLA V13.D2, V31.D2, V17.D2 - VFMLA V14.D2, V31.D2, V18.D2 - VFMLA V15.D2, V31.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R5) - RET - -TEXT ·gemmMicro8x8S(SB), NOSPLIT, $0-40 - MOVD kc+0(FP), R0 - MOVD pa+8(FP), R1 - MOVD pb+16(FP), R2 - MOVD c+24(FP), R3 - MOVD ldc+32(FP), R4 - LSL $2, R4, R4 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - VEOR V8.B16, V8.B16, V8.B16 - VEOR V9.B16, V9.B16, V9.B16 - VEOR V10.B16, V10.B16, V10.B16 - VEOR V11.B16, V11.B16, V11.B16 - VEOR V12.B16, V12.B16, V12.B16 - VEOR V13.B16, V13.B16, V13.B16 - VEOR V14.B16, V14.B16, V14.B16 - VEOR V15.B16, V15.B16, V15.B16 - CBZ R0, sstore - skloop: - VLD1.P 32(R2), [V16.S4, V17.S4] - VLD1.P 32(R1), [V18.S4, V19.S4] - WORD $0x4F921200 // FMLA V0.4S, V16.4S, V18.S[0] - WORD $0x4F921221 // FMLA V1.4S, V17.4S, V18.S[0] - WORD $0x4FB21202 // FMLA V2.4S, V16.4S, V18.S[1] - WORD $0x4FB21223 // FMLA V3.4S, V17.4S, V18.S[1] - WORD $0x4F921A04 // FMLA V4.4S, V16.4S, V18.S[2] - WORD $0x4F921A25 // FMLA V5.4S, V17.4S, V18.S[2] - WORD $0x4FB21A06 // FMLA V6.4S, V16.4S, V18.S[3] - WORD $0x4FB21A27 // FMLA V7.4S, V17.4S, V18.S[3] - WORD $0x4F931208 // FMLA V8.4S, V16.4S, V19.S[0] - WORD $0x4F931229 // FMLA V9.4S, V17.4S, V19.S[0] - WORD $0x4FB3120A // FMLA V10.4S, V16.4S, V19.S[1] - WORD $0x4FB3122B // FMLA V11.4S, V17.4S, V19.S[1] - WORD $0x4F931A0C // FMLA V12.4S, V16.4S, V19.S[2] - WORD $0x4F931A2D // FMLA V13.4S, V17.4S, V19.S[2] - WORD $0x4FB31A0E // FMLA V14.4S, V16.4S, V19.S[3] - WORD $0x4FB31A2F // FMLA V15.4S, V17.4S, V19.S[3] - SUB $1, R0 - CBNZ R0, skloop - sstore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V2.S4, V16.S4, V16.S4 - VFADD V3.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V4.S4, V16.S4, V16.S4 - VFADD V5.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V6.S4, V16.S4, V16.S4 - VFADD V7.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V8.S4, V16.S4, V16.S4 - VFADD V9.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V10.S4, V16.S4, V16.S4 - VFADD V11.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V12.S4, V16.S4, V16.S4 - VFADD V13.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V14.S4, V16.S4, V16.S4 - VFADD V15.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - -TEXT ·gemmUA1x8S(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $2, R2, R2 - LSL $2, R16, R16 - LSL $2, R4, R4 - MOVD R1, R5 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - LSR $2, R0, R14 - AND $3, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.S4] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] - WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] - WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] - WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVS.P 4(R5), F20 - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - -TEXT ·gemmUA1x8D(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $3, R2, R2 - LSL $3, R16, R16 - LSL $3, R4, R4 - MOVD R1, R5 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - LSR $1, R0, R14 - AND $1, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.D2] - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] - WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] - WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] - WORD $0x4FD41A63 // FMLA V3.2D, V19.2D, V20.D[1] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVD.P 8(R5), F20 - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V0.D2, V16.D2, V16.D2 - VFADD V1.D2, V17.D2, V17.D2 - VFADD V2.D2, V18.D2, V18.D2 - VFADD V3.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - RET - -TEXT ·gemmUA2x8S(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $2, R2, R2 - LSL $2, R16, R16 - LSL $2, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - LSR $2, R0, R14 - AND $3, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.S4] - VLD1.P 16(R6), [V21.S4] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] - WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] - WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] - WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] - WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] - WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] - WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] - WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVS.P 4(R5), F20 - FMOVS.P 4(R6), F21 - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V2.S4, V16.S4, V16.S4 - VFADD V3.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - -TEXT ·gemmUA2x8D(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $3, R2, R2 - LSL $3, R16, R16 - LSL $3, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - LSR $1, R0, R14 - AND $1, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.D2] - VLD1.P 16(R6), [V21.D2] - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] - WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] - WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] - WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] - WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] - WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] - WORD $0x4FD41A63 // FMLA V3.2D, V19.2D, V20.D[1] - WORD $0x4FD51A04 // FMLA V4.2D, V16.2D, V21.D[1] - WORD $0x4FD51A25 // FMLA V5.2D, V17.2D, V21.D[1] - WORD $0x4FD51A46 // FMLA V6.2D, V18.2D, V21.D[1] - WORD $0x4FD51A67 // FMLA V7.2D, V19.2D, V21.D[1] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVD.P 8(R5), F20 - FMOVD.P 8(R6), F21 - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] - WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] - WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] - WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V0.D2, V16.D2, V16.D2 - VFADD V1.D2, V17.D2, V17.D2 - VFADD V2.D2, V18.D2, V18.D2 - VFADD V3.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V4.D2, V16.D2, V16.D2 - VFADD V5.D2, V17.D2, V17.D2 - VFADD V6.D2, V18.D2, V18.D2 - VFADD V7.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - RET - -TEXT ·gemmUA3x8S(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $2, R2, R2 - LSL $2, R16, R16 - LSL $2, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - ADD R2, R6, R7 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - LSR $2, R0, R14 - AND $3, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.S4] - VLD1.P 16(R6), [V21.S4] - VLD1.P 16(R7), [V22.S4] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] - WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] - WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] - WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] - WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] - WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] - WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] - WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] - WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] - WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] - WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] - WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] - WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] - WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVS.P 4(R5), F20 - FMOVS.P 4(R6), F21 - FMOVS.P 4(R7), F22 - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V2.S4, V16.S4, V16.S4 - VFADD V3.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V4.S4, V16.S4, V16.S4 - VFADD V5.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - -TEXT ·gemmUA3x8D(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $3, R2, R2 - LSL $3, R16, R16 - LSL $3, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - ADD R2, R6, R7 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - VEOR V8.B16, V8.B16, V8.B16 - VEOR V9.B16, V9.B16, V9.B16 - VEOR V10.B16, V10.B16, V10.B16 - VEOR V11.B16, V11.B16, V11.B16 - LSR $1, R0, R14 - AND $1, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.D2] - VLD1.P 16(R6), [V21.D2] - VLD1.P 16(R7), [V22.D2] - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] - WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] - WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] - WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] - WORD $0x4FD61208 // FMLA V8.2D, V16.2D, V22.D[0] - WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] - WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] - WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] - WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] - WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] - WORD $0x4FD41A63 // FMLA V3.2D, V19.2D, V20.D[1] - WORD $0x4FD51A04 // FMLA V4.2D, V16.2D, V21.D[1] - WORD $0x4FD51A25 // FMLA V5.2D, V17.2D, V21.D[1] - WORD $0x4FD51A46 // FMLA V6.2D, V18.2D, V21.D[1] - WORD $0x4FD51A67 // FMLA V7.2D, V19.2D, V21.D[1] - WORD $0x4FD61A08 // FMLA V8.2D, V16.2D, V22.D[1] - WORD $0x4FD61A29 // FMLA V9.2D, V17.2D, V22.D[1] - WORD $0x4FD61A4A // FMLA V10.2D, V18.2D, V22.D[1] - WORD $0x4FD61A6B // FMLA V11.2D, V19.2D, V22.D[1] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVD.P 8(R5), F20 - FMOVD.P 8(R6), F21 - FMOVD.P 8(R7), F22 - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] - WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] - WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] - WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] - WORD $0x4FD61208 // FMLA V8.2D, V16.2D, V22.D[0] - WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] - WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] - WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V0.D2, V16.D2, V16.D2 - VFADD V1.D2, V17.D2, V17.D2 - VFADD V2.D2, V18.D2, V18.D2 - VFADD V3.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V4.D2, V16.D2, V16.D2 - VFADD V5.D2, V17.D2, V17.D2 - VFADD V6.D2, V18.D2, V18.D2 - VFADD V7.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V8.D2, V16.D2, V16.D2 - VFADD V9.D2, V17.D2, V17.D2 - VFADD V10.D2, V18.D2, V18.D2 - VFADD V11.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - RET - -TEXT ·gemmUA4x8S(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $2, R2, R2 - LSL $2, R16, R16 - LSL $2, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - ADD R2, R6, R7 - ADD R2, R7, R8 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - LSR $2, R0, R14 - AND $3, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.S4] - VLD1.P 16(R6), [V21.S4] - VLD1.P 16(R7), [V22.S4] - VLD1.P 16(R8), [V23.S4] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] - WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] - WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] - WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] - WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] - WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] - WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] - WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] - WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] - WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] - WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] - WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] - WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] - WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] - WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] - WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] - WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] - WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] - WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] - WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVS.P 4(R5), F20 - FMOVS.P 4(R6), F21 - FMOVS.P 4(R7), F22 - FMOVS.P 4(R8), F23 - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V2.S4, V16.S4, V16.S4 - VFADD V3.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V4.S4, V16.S4, V16.S4 - VFADD V5.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V6.S4, V16.S4, V16.S4 - VFADD V7.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - -TEXT ·gemmUA4x8D(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $3, R2, R2 - LSL $3, R16, R16 - LSL $3, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - ADD R2, R6, R7 - ADD R2, R7, R8 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - VEOR V8.B16, V8.B16, V8.B16 - VEOR V9.B16, V9.B16, V9.B16 - VEOR V10.B16, V10.B16, V10.B16 - VEOR V11.B16, V11.B16, V11.B16 - VEOR V12.B16, V12.B16, V12.B16 - VEOR V13.B16, V13.B16, V13.B16 - VEOR V14.B16, V14.B16, V14.B16 - VEOR V15.B16, V15.B16, V15.B16 - LSR $1, R0, R14 - AND $1, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.D2] - VLD1.P 16(R6), [V21.D2] - VLD1.P 16(R7), [V22.D2] - VLD1.P 16(R8), [V23.D2] - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] - WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] - WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] - WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] - WORD $0x4FD61208 // FMLA V8.2D, V16.2D, V22.D[0] - WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] - WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] - WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] - WORD $0x4FD7120C // FMLA V12.2D, V16.2D, V23.D[0] - WORD $0x4FD7122D // FMLA V13.2D, V17.2D, V23.D[0] - WORD $0x4FD7124E // FMLA V14.2D, V18.2D, V23.D[0] - WORD $0x4FD7126F // FMLA V15.2D, V19.2D, V23.D[0] - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41A00 // FMLA V0.2D, V16.2D, V20.D[1] - WORD $0x4FD41A21 // FMLA V1.2D, V17.2D, V20.D[1] - WORD $0x4FD41A42 // FMLA V2.2D, V18.2D, V20.D[1] - WORD $0x4FD41A63 // FMLA V3.2D, V19.2D, V20.D[1] - WORD $0x4FD51A04 // FMLA V4.2D, V16.2D, V21.D[1] - WORD $0x4FD51A25 // FMLA V5.2D, V17.2D, V21.D[1] - WORD $0x4FD51A46 // FMLA V6.2D, V18.2D, V21.D[1] - WORD $0x4FD51A67 // FMLA V7.2D, V19.2D, V21.D[1] - WORD $0x4FD61A08 // FMLA V8.2D, V16.2D, V22.D[1] - WORD $0x4FD61A29 // FMLA V9.2D, V17.2D, V22.D[1] - WORD $0x4FD61A4A // FMLA V10.2D, V18.2D, V22.D[1] - WORD $0x4FD61A6B // FMLA V11.2D, V19.2D, V22.D[1] - WORD $0x4FD71A0C // FMLA V12.2D, V16.2D, V23.D[1] - WORD $0x4FD71A2D // FMLA V13.2D, V17.2D, V23.D[1] - WORD $0x4FD71A4E // FMLA V14.2D, V18.2D, V23.D[1] - WORD $0x4FD71A6F // FMLA V15.2D, V19.2D, V23.D[1] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVD.P 8(R5), F20 - FMOVD.P 8(R6), F21 - FMOVD.P 8(R7), F22 - FMOVD.P 8(R8), F23 - VLD1.P (R13)(R16), [V16.D2, V17.D2, V18.D2, V19.D2] - WORD $0x4FD41200 // FMLA V0.2D, V16.2D, V20.D[0] - WORD $0x4FD41221 // FMLA V1.2D, V17.2D, V20.D[0] - WORD $0x4FD41242 // FMLA V2.2D, V18.2D, V20.D[0] - WORD $0x4FD41263 // FMLA V3.2D, V19.2D, V20.D[0] - WORD $0x4FD51204 // FMLA V4.2D, V16.2D, V21.D[0] - WORD $0x4FD51225 // FMLA V5.2D, V17.2D, V21.D[0] - WORD $0x4FD51246 // FMLA V6.2D, V18.2D, V21.D[0] - WORD $0x4FD51267 // FMLA V7.2D, V19.2D, V21.D[0] - WORD $0x4FD61208 // FMLA V8.2D, V16.2D, V22.D[0] - WORD $0x4FD61229 // FMLA V9.2D, V17.2D, V22.D[0] - WORD $0x4FD6124A // FMLA V10.2D, V18.2D, V22.D[0] - WORD $0x4FD6126B // FMLA V11.2D, V19.2D, V22.D[0] - WORD $0x4FD7120C // FMLA V12.2D, V16.2D, V23.D[0] - WORD $0x4FD7122D // FMLA V13.2D, V17.2D, V23.D[0] - WORD $0x4FD7124E // FMLA V14.2D, V18.2D, V23.D[0] - WORD $0x4FD7126F // FMLA V15.2D, V19.2D, V23.D[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V0.D2, V16.D2, V16.D2 - VFADD V1.D2, V17.D2, V17.D2 - VFADD V2.D2, V18.D2, V18.D2 - VFADD V3.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V4.D2, V16.D2, V16.D2 - VFADD V5.D2, V17.D2, V17.D2 - VFADD V6.D2, V18.D2, V18.D2 - VFADD V7.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V8.D2, V16.D2, V16.D2 - VFADD V9.D2, V17.D2, V17.D2 - VFADD V10.D2, V18.D2, V18.D2 - VFADD V11.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.D2, V17.D2, V18.D2, V19.D2] - VFADD V12.D2, V16.D2, V16.D2 - VFADD V13.D2, V17.D2, V17.D2 - VFADD V14.D2, V18.D2, V18.D2 - VFADD V15.D2, V19.D2, V19.D2 - VST1 [V16.D2, V17.D2, V18.D2, V19.D2], (R3) - RET - -TEXT ·gemmUA5x8S(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $2, R2, R2 - LSL $2, R16, R16 - LSL $2, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - ADD R2, R6, R7 - ADD R2, R7, R8 - ADD R2, R8, R9 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - VEOR V8.B16, V8.B16, V8.B16 - VEOR V9.B16, V9.B16, V9.B16 - LSR $2, R0, R14 - AND $3, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.S4] - VLD1.P 16(R6), [V21.S4] - VLD1.P 16(R7), [V22.S4] - VLD1.P 16(R8), [V23.S4] - VLD1.P 16(R9), [V24.S4] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] - WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] - WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] - WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] - WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] - WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] - WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] - WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] - WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] - WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] - WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] - WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] - WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] - WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] - WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] - WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] - WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] - WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] - WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] - WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] - WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] - WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] - WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] - WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] - WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] - WORD $0x4FB81A08 // FMLA V8.4S, V16.4S, V24.S[3] - WORD $0x4FB81A29 // FMLA V9.4S, V17.4S, V24.S[3] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVS.P 4(R5), F20 - FMOVS.P 4(R6), F21 - FMOVS.P 4(R7), F22 - FMOVS.P 4(R8), F23 - FMOVS.P 4(R9), F24 - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] - WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V2.S4, V16.S4, V16.S4 - VFADD V3.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V4.S4, V16.S4, V16.S4 - VFADD V5.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V6.S4, V16.S4, V16.S4 - VFADD V7.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V8.S4, V16.S4, V16.S4 - VFADD V9.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - -TEXT ·gemmUA6x8S(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $2, R2, R2 - LSL $2, R16, R16 - LSL $2, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - ADD R2, R6, R7 - ADD R2, R7, R8 - ADD R2, R8, R9 - ADD R2, R9, R10 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - VEOR V8.B16, V8.B16, V8.B16 - VEOR V9.B16, V9.B16, V9.B16 - VEOR V10.B16, V10.B16, V10.B16 - VEOR V11.B16, V11.B16, V11.B16 - LSR $2, R0, R14 - AND $3, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.S4] - VLD1.P 16(R6), [V21.S4] - VLD1.P 16(R7), [V22.S4] - VLD1.P 16(R8), [V23.S4] - VLD1.P 16(R9), [V24.S4] - VLD1.P 16(R10), [V25.S4] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] - WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] - WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] - WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] - WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] - WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] - WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] - WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] - WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] - WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] - WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] - WORD $0x4FB9120A // FMLA V10.4S, V16.4S, V25.S[1] - WORD $0x4FB9122B // FMLA V11.4S, V17.4S, V25.S[1] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] - WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] - WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] - WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] - WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] - WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] - WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] - WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] - WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] - WORD $0x4F991A0A // FMLA V10.4S, V16.4S, V25.S[2] - WORD $0x4F991A2B // FMLA V11.4S, V17.4S, V25.S[2] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] - WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] - WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] - WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] - WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] - WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] - WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] - WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] - WORD $0x4FB81A08 // FMLA V8.4S, V16.4S, V24.S[3] - WORD $0x4FB81A29 // FMLA V9.4S, V17.4S, V24.S[3] - WORD $0x4FB91A0A // FMLA V10.4S, V16.4S, V25.S[3] - WORD $0x4FB91A2B // FMLA V11.4S, V17.4S, V25.S[3] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVS.P 4(R5), F20 - FMOVS.P 4(R6), F21 - FMOVS.P 4(R7), F22 - FMOVS.P 4(R8), F23 - FMOVS.P 4(R9), F24 - FMOVS.P 4(R10), F25 - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] - WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] - WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V2.S4, V16.S4, V16.S4 - VFADD V3.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V4.S4, V16.S4, V16.S4 - VFADD V5.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V6.S4, V16.S4, V16.S4 - VFADD V7.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V8.S4, V16.S4, V16.S4 - VFADD V9.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V10.S4, V16.S4, V16.S4 - VFADD V11.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - -TEXT ·gemmUA7x8S(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $2, R2, R2 - LSL $2, R16, R16 - LSL $2, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - ADD R2, R6, R7 - ADD R2, R7, R8 - ADD R2, R8, R9 - ADD R2, R9, R10 - ADD R2, R10, R11 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - VEOR V8.B16, V8.B16, V8.B16 - VEOR V9.B16, V9.B16, V9.B16 - VEOR V10.B16, V10.B16, V10.B16 - VEOR V11.B16, V11.B16, V11.B16 - VEOR V12.B16, V12.B16, V12.B16 - VEOR V13.B16, V13.B16, V13.B16 - LSR $2, R0, R14 - AND $3, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.S4] - VLD1.P 16(R6), [V21.S4] - VLD1.P 16(R7), [V22.S4] - VLD1.P 16(R8), [V23.S4] - VLD1.P 16(R9), [V24.S4] - VLD1.P 16(R10), [V25.S4] - VLD1.P 16(R11), [V26.S4] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] - WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] - WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] - WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] - WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] - WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] - WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] - WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] - WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] - WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] - WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] - WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] - WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] - WORD $0x4FB9120A // FMLA V10.4S, V16.4S, V25.S[1] - WORD $0x4FB9122B // FMLA V11.4S, V17.4S, V25.S[1] - WORD $0x4FBA120C // FMLA V12.4S, V16.4S, V26.S[1] - WORD $0x4FBA122D // FMLA V13.4S, V17.4S, V26.S[1] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] - WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] - WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] - WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] - WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] - WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] - WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] - WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] - WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] - WORD $0x4F991A0A // FMLA V10.4S, V16.4S, V25.S[2] - WORD $0x4F991A2B // FMLA V11.4S, V17.4S, V25.S[2] - WORD $0x4F9A1A0C // FMLA V12.4S, V16.4S, V26.S[2] - WORD $0x4F9A1A2D // FMLA V13.4S, V17.4S, V26.S[2] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] - WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] - WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] - WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] - WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] - WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] - WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] - WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] - WORD $0x4FB81A08 // FMLA V8.4S, V16.4S, V24.S[3] - WORD $0x4FB81A29 // FMLA V9.4S, V17.4S, V24.S[3] - WORD $0x4FB91A0A // FMLA V10.4S, V16.4S, V25.S[3] - WORD $0x4FB91A2B // FMLA V11.4S, V17.4S, V25.S[3] - WORD $0x4FBA1A0C // FMLA V12.4S, V16.4S, V26.S[3] - WORD $0x4FBA1A2D // FMLA V13.4S, V17.4S, V26.S[3] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVS.P 4(R5), F20 - FMOVS.P 4(R6), F21 - FMOVS.P 4(R7), F22 - FMOVS.P 4(R8), F23 - FMOVS.P 4(R9), F24 - FMOVS.P 4(R10), F25 - FMOVS.P 4(R11), F26 - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] - WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] - WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] - WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] - WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V2.S4, V16.S4, V16.S4 - VFADD V3.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V4.S4, V16.S4, V16.S4 - VFADD V5.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V6.S4, V16.S4, V16.S4 - VFADD V7.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V8.S4, V16.S4, V16.S4 - VFADD V9.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V10.S4, V16.S4, V16.S4 - VFADD V11.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V12.S4, V16.S4, V16.S4 - VFADD V13.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - -TEXT ·gemmUA8x8S(SB), NOSPLIT, $0-56 - MOVD kc+0(FP), R0 - MOVD a+8(FP), R1 - MOVD lda+16(FP), R2 - MOVD pb+24(FP), R13 - MOVD ldb+32(FP), R16 - MOVD c+40(FP), R3 - MOVD ldc+48(FP), R4 - LSL $2, R2, R2 - LSL $2, R16, R16 - LSL $2, R4, R4 - MOVD R1, R5 - ADD R2, R5, R6 - ADD R2, R6, R7 - ADD R2, R7, R8 - ADD R2, R8, R9 - ADD R2, R9, R10 - ADD R2, R10, R11 - ADD R2, R11, R12 - VEOR V0.B16, V0.B16, V0.B16 - VEOR V1.B16, V1.B16, V1.B16 - VEOR V2.B16, V2.B16, V2.B16 - VEOR V3.B16, V3.B16, V3.B16 - VEOR V4.B16, V4.B16, V4.B16 - VEOR V5.B16, V5.B16, V5.B16 - VEOR V6.B16, V6.B16, V6.B16 - VEOR V7.B16, V7.B16, V7.B16 - VEOR V8.B16, V8.B16, V8.B16 - VEOR V9.B16, V9.B16, V9.B16 - VEOR V10.B16, V10.B16, V10.B16 - VEOR V11.B16, V11.B16, V11.B16 - VEOR V12.B16, V12.B16, V12.B16 - VEOR V13.B16, V13.B16, V13.B16 - VEOR V14.B16, V14.B16, V14.B16 - VEOR V15.B16, V15.B16, V15.B16 - LSR $2, R0, R14 - AND $3, R0, R15 - CBZ R14, uatail - uakloop: - VLD1.P 16(R5), [V20.S4] - VLD1.P 16(R6), [V21.S4] - VLD1.P 16(R7), [V22.S4] - VLD1.P 16(R8), [V23.S4] - VLD1.P 16(R9), [V24.S4] - VLD1.P 16(R10), [V25.S4] - VLD1.P 16(R11), [V26.S4] - VLD1.P 16(R12), [V27.S4] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] - WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] - WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] - WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] - WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] - WORD $0x4F9B120E // FMLA V14.4S, V16.4S, V27.S[0] - WORD $0x4F9B122F // FMLA V15.4S, V17.4S, V27.S[0] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41200 // FMLA V0.4S, V16.4S, V20.S[1] - WORD $0x4FB41221 // FMLA V1.4S, V17.4S, V20.S[1] - WORD $0x4FB51202 // FMLA V2.4S, V16.4S, V21.S[1] - WORD $0x4FB51223 // FMLA V3.4S, V17.4S, V21.S[1] - WORD $0x4FB61204 // FMLA V4.4S, V16.4S, V22.S[1] - WORD $0x4FB61225 // FMLA V5.4S, V17.4S, V22.S[1] - WORD $0x4FB71206 // FMLA V6.4S, V16.4S, V23.S[1] - WORD $0x4FB71227 // FMLA V7.4S, V17.4S, V23.S[1] - WORD $0x4FB81208 // FMLA V8.4S, V16.4S, V24.S[1] - WORD $0x4FB81229 // FMLA V9.4S, V17.4S, V24.S[1] - WORD $0x4FB9120A // FMLA V10.4S, V16.4S, V25.S[1] - WORD $0x4FB9122B // FMLA V11.4S, V17.4S, V25.S[1] - WORD $0x4FBA120C // FMLA V12.4S, V16.4S, V26.S[1] - WORD $0x4FBA122D // FMLA V13.4S, V17.4S, V26.S[1] - WORD $0x4FBB120E // FMLA V14.4S, V16.4S, V27.S[1] - WORD $0x4FBB122F // FMLA V15.4S, V17.4S, V27.S[1] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941A00 // FMLA V0.4S, V16.4S, V20.S[2] - WORD $0x4F941A21 // FMLA V1.4S, V17.4S, V20.S[2] - WORD $0x4F951A02 // FMLA V2.4S, V16.4S, V21.S[2] - WORD $0x4F951A23 // FMLA V3.4S, V17.4S, V21.S[2] - WORD $0x4F961A04 // FMLA V4.4S, V16.4S, V22.S[2] - WORD $0x4F961A25 // FMLA V5.4S, V17.4S, V22.S[2] - WORD $0x4F971A06 // FMLA V6.4S, V16.4S, V23.S[2] - WORD $0x4F971A27 // FMLA V7.4S, V17.4S, V23.S[2] - WORD $0x4F981A08 // FMLA V8.4S, V16.4S, V24.S[2] - WORD $0x4F981A29 // FMLA V9.4S, V17.4S, V24.S[2] - WORD $0x4F991A0A // FMLA V10.4S, V16.4S, V25.S[2] - WORD $0x4F991A2B // FMLA V11.4S, V17.4S, V25.S[2] - WORD $0x4F9A1A0C // FMLA V12.4S, V16.4S, V26.S[2] - WORD $0x4F9A1A2D // FMLA V13.4S, V17.4S, V26.S[2] - WORD $0x4F9B1A0E // FMLA V14.4S, V16.4S, V27.S[2] - WORD $0x4F9B1A2F // FMLA V15.4S, V17.4S, V27.S[2] - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4FB41A00 // FMLA V0.4S, V16.4S, V20.S[3] - WORD $0x4FB41A21 // FMLA V1.4S, V17.4S, V20.S[3] - WORD $0x4FB51A02 // FMLA V2.4S, V16.4S, V21.S[3] - WORD $0x4FB51A23 // FMLA V3.4S, V17.4S, V21.S[3] - WORD $0x4FB61A04 // FMLA V4.4S, V16.4S, V22.S[3] - WORD $0x4FB61A25 // FMLA V5.4S, V17.4S, V22.S[3] - WORD $0x4FB71A06 // FMLA V6.4S, V16.4S, V23.S[3] - WORD $0x4FB71A27 // FMLA V7.4S, V17.4S, V23.S[3] - WORD $0x4FB81A08 // FMLA V8.4S, V16.4S, V24.S[3] - WORD $0x4FB81A29 // FMLA V9.4S, V17.4S, V24.S[3] - WORD $0x4FB91A0A // FMLA V10.4S, V16.4S, V25.S[3] - WORD $0x4FB91A2B // FMLA V11.4S, V17.4S, V25.S[3] - WORD $0x4FBA1A0C // FMLA V12.4S, V16.4S, V26.S[3] - WORD $0x4FBA1A2D // FMLA V13.4S, V17.4S, V26.S[3] - WORD $0x4FBB1A0E // FMLA V14.4S, V16.4S, V27.S[3] - WORD $0x4FBB1A2F // FMLA V15.4S, V17.4S, V27.S[3] - SUB $1, R14 - CBNZ R14, uakloop - uatail: - CBZ R15, uastore - uatloop: - FMOVS.P 4(R5), F20 - FMOVS.P 4(R6), F21 - FMOVS.P 4(R7), F22 - FMOVS.P 4(R8), F23 - FMOVS.P 4(R9), F24 - FMOVS.P 4(R10), F25 - FMOVS.P 4(R11), F26 - FMOVS.P 4(R12), F27 - VLD1.P (R13)(R16), [V16.S4, V17.S4] - WORD $0x4F941200 // FMLA V0.4S, V16.4S, V20.S[0] - WORD $0x4F941221 // FMLA V1.4S, V17.4S, V20.S[0] - WORD $0x4F951202 // FMLA V2.4S, V16.4S, V21.S[0] - WORD $0x4F951223 // FMLA V3.4S, V17.4S, V21.S[0] - WORD $0x4F961204 // FMLA V4.4S, V16.4S, V22.S[0] - WORD $0x4F961225 // FMLA V5.4S, V17.4S, V22.S[0] - WORD $0x4F971206 // FMLA V6.4S, V16.4S, V23.S[0] - WORD $0x4F971227 // FMLA V7.4S, V17.4S, V23.S[0] - WORD $0x4F981208 // FMLA V8.4S, V16.4S, V24.S[0] - WORD $0x4F981229 // FMLA V9.4S, V17.4S, V24.S[0] - WORD $0x4F99120A // FMLA V10.4S, V16.4S, V25.S[0] - WORD $0x4F99122B // FMLA V11.4S, V17.4S, V25.S[0] - WORD $0x4F9A120C // FMLA V12.4S, V16.4S, V26.S[0] - WORD $0x4F9A122D // FMLA V13.4S, V17.4S, V26.S[0] - WORD $0x4F9B120E // FMLA V14.4S, V16.4S, V27.S[0] - WORD $0x4F9B122F // FMLA V15.4S, V17.4S, V27.S[0] - SUB $1, R15 - CBNZ R15, uatloop - uastore: - VLD1 (R3), [V16.S4, V17.S4] - VFADD V0.S4, V16.S4, V16.S4 - VFADD V1.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V2.S4, V16.S4, V16.S4 - VFADD V3.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V4.S4, V16.S4, V16.S4 - VFADD V5.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V6.S4, V16.S4, V16.S4 - VFADD V7.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V8.S4, V16.S4, V16.S4 - VFADD V9.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V10.S4, V16.S4, V16.S4 - VFADD V11.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V12.S4, V16.S4, V16.S4 - VFADD V13.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - ADD R4, R3, R3 - VLD1 (R3), [V16.S4, V17.S4] - VFADD V14.S4, V16.S4, V16.S4 - VFADD V15.S4, V17.S4, V17.S4 - VST1 [V16.S4, V17.S4], (R3) - RET - TEXT ·blocks8NEON(SB), NOSPLIT, $0-24 MOVD r+0(FP), R0 MOVD a+8(FP), R1 diff --git a/internal/kernels/sum_loong64.s b/internal/kernels/sum_loong64.s index 8701d53..b50c362 100644 --- a/internal/kernels/sum_loong64.s +++ b/internal/kernels/sum_loong64.s @@ -231,119 +231,6 @@ ftail: fdone: RET -TEXT ·gemmMicro8x8LASX(SB), NOSPLIT, $0-40 - MOVV kc+0(FP), R5 - MOVV pa+8(FP), R6 - MOVV pb+16(FP), R7 - MOVV c+24(FP), R8 - MOVV ldc+32(FP), R9 - SLLV $3, R9 - XVXORV X0, X0, X0 - XVXORV X1, X1, X1 - XVXORV X2, X2, X2 - XVXORV X3, X3, X3 - XVXORV X4, X4, X4 - XVXORV X5, X5, X5 - XVXORV X6, X6, X6 - XVXORV X7, X7, X7 - XVXORV X8, X8, X8 - XVXORV X9, X9, X9 - XVXORV X10, X10, X10 - XVXORV X11, X11, X11 - XVXORV X12, X12, X12 - XVXORV X13, X13, X13 - XVXORV X14, X14, X14 - XVXORV X15, X15, X15 - BEQ R5, gstore -gloop: - XVMOVQ 0(R7), X16 - XVMOVQ 32(R7), X17 - XVMOVQ 0(R6), X18.V4 - XVMOVQ 8(R6), X19.V4 - XVMOVQ 16(R6), X20.V4 - XVMOVQ 24(R6), X21.V4 - XVMOVQ 32(R6), X22.V4 - XVMOVQ 40(R6), X23.V4 - XVMOVQ 48(R6), X24.V4 - XVMOVQ 56(R6), X25.V4 - WORD $0x0a204240 // XVFMADDD X0, X16, X18, X0 - WORD $0x0a20c641 // XVFMADDD X1, X17, X18, X1 - WORD $0x0a214262 // XVFMADDD X2, X16, X19, X2 - WORD $0x0a21c663 // XVFMADDD X3, X17, X19, X3 - WORD $0x0a224284 // XVFMADDD X4, X16, X20, X4 - WORD $0x0a22c685 // XVFMADDD X5, X17, X20, X5 - WORD $0x0a2342a6 // XVFMADDD X6, X16, X21, X6 - WORD $0x0a23c6a7 // XVFMADDD X7, X17, X21, X7 - WORD $0x0a2442c8 // XVFMADDD X8, X16, X22, X8 - WORD $0x0a24c6c9 // XVFMADDD X9, X17, X22, X9 - WORD $0x0a2542ea // XVFMADDD X10, X16, X23, X10 - WORD $0x0a25c6eb // XVFMADDD X11, X17, X23, X11 - WORD $0x0a26430c // XVFMADDD X12, X16, X24, X12 - WORD $0x0a26c70d // XVFMADDD X13, X17, X24, X13 - WORD $0x0a27432e // XVFMADDD X14, X16, X25, X14 - WORD $0x0a27c72f // XVFMADDD X15, X17, X25, X15 - ADDV $64, R6 - ADDV $64, R7 - ADDV $-1, R5 - BNE R5, gloop -gstore: - XVMOVQ 0(R8), X16 - XVMOVQ 32(R8), X17 - XVADDD X0, X16, X16 - XVADDD X1, X17, X17 - XVMOVQ X16, 0(R8) - XVMOVQ X17, 32(R8) - ADDV R9, R8 - XVMOVQ 0(R8), X16 - XVMOVQ 32(R8), X17 - XVADDD X2, X16, X16 - XVADDD X3, X17, X17 - XVMOVQ X16, 0(R8) - XVMOVQ X17, 32(R8) - ADDV R9, R8 - XVMOVQ 0(R8), X16 - XVMOVQ 32(R8), X17 - XVADDD X4, X16, X16 - XVADDD X5, X17, X17 - XVMOVQ X16, 0(R8) - XVMOVQ X17, 32(R8) - ADDV R9, R8 - XVMOVQ 0(R8), X16 - XVMOVQ 32(R8), X17 - XVADDD X6, X16, X16 - XVADDD X7, X17, X17 - XVMOVQ X16, 0(R8) - XVMOVQ X17, 32(R8) - ADDV R9, R8 - XVMOVQ 0(R8), X16 - XVMOVQ 32(R8), X17 - XVADDD X8, X16, X16 - XVADDD X9, X17, X17 - XVMOVQ X16, 0(R8) - XVMOVQ X17, 32(R8) - ADDV R9, R8 - XVMOVQ 0(R8), X16 - XVMOVQ 32(R8), X17 - XVADDD X10, X16, X16 - XVADDD X11, X17, X17 - XVMOVQ X16, 0(R8) - XVMOVQ X17, 32(R8) - ADDV R9, R8 - XVMOVQ 0(R8), X16 - XVMOVQ 32(R8), X17 - XVADDD X12, X16, X16 - XVADDD X13, X17, X17 - XVMOVQ X16, 0(R8) - XVMOVQ X17, 32(R8) - ADDV R9, R8 - XVMOVQ 0(R8), X16 - XVMOVQ 32(R8), X17 - XVADDD X14, X16, X16 - XVADDD X15, X17, X17 - XVMOVQ X16, 0(R8) - XVMOVQ X17, 32(R8) - RET - TEXT ·blocks8LSX(SB), NOSPLIT, $0-24 MOVV r+0(FP), R4 MOVV a+8(FP), R5 diff --git a/internal/kernels/sum_ppc64le.s b/internal/kernels/sum_ppc64le.s index 4389c01..b04a8b9 100644 --- a/internal/kernels/sum_ppc64le.s +++ b/internal/kernels/sum_ppc64le.s @@ -243,332 +243,6 @@ ftail: fdone: RET -TEXT ·gemmMicro8x8VSX(SB), NOSPLIT, $0-40 - MOVD kc+0(FP), R3 - MOVD pa+8(FP), R4 - MOVD pb+16(FP), R5 - MOVD c+24(FP), R6 - MOVD ldc+32(FP), R7 - SLD $3, R7 - MOVD $16, R14 - MOVD $32, R15 - MOVD $48, R16 - XXLXOR VS0, VS0, VS0 - XXLXOR VS1, VS1, VS1 - XXLXOR VS2, VS2, VS2 - XXLXOR VS3, VS3, VS3 - XXLXOR VS4, VS4, VS4 - XXLXOR VS5, VS5, VS5 - XXLXOR VS6, VS6, VS6 - XXLXOR VS7, VS7, VS7 - XXLXOR VS8, VS8, VS8 - XXLXOR VS9, VS9, VS9 - XXLXOR VS10, VS10, VS10 - XXLXOR VS11, VS11, VS11 - XXLXOR VS12, VS12, VS12 - XXLXOR VS13, VS13, VS13 - XXLXOR VS14, VS14, VS14 - XXLXOR VS15, VS15, VS15 - XXLXOR VS16, VS16, VS16 - XXLXOR VS17, VS17, VS17 - XXLXOR VS18, VS18, VS18 - XXLXOR VS19, VS19, VS19 - XXLXOR VS20, VS20, VS20 - XXLXOR VS21, VS21, VS21 - XXLXOR VS22, VS22, VS22 - XXLXOR VS23, VS23, VS23 - XXLXOR VS24, VS24, VS24 - XXLXOR VS25, VS25, VS25 - XXLXOR VS26, VS26, VS26 - XXLXOR VS27, VS27, VS27 - XXLXOR VS28, VS28, VS28 - XXLXOR VS29, VS29, VS29 - XXLXOR VS30, VS30, VS30 - XXLXOR VS31, VS31, VS31 - CMP R3, $0 - BEQ gemmMicro8x8VSXstore -gemmMicro8x8VSXloop2: - CMP R3, $2 - BLT gemmMicro8x8VSXone - LXVD2X (R5)(R0), VS32 - LXVD2X (R4)(R0), VS36 - LXVD2X (R5)(R14), VS33 - LXVD2X (R4)(R14), VS37 - LXVD2X (R5)(R15), VS34 - LXVD2X (R4)(R15), VS38 - LXVD2X (R5)(R16), VS35 - LXVD2X (R4)(R16), VS39 - ADD $64, R4 - ADD $64, R5 - XXPERMDI VS36, VS36, $2, VS40 - XXPERMDI VS37, VS37, $2, VS41 - XXPERMDI VS38, VS38, $2, VS42 - XXPERMDI VS39, VS39, $2, VS43 - WORD $0xf004030e // XVMADDADP VS36, VS32, VS0 - WORD $0xf028030e // XVMADDADP VS40, VS32, VS1 - WORD $0xf0440b0e // XVMADDADP VS36, VS33, VS2 - WORD $0xf0680b0e // XVMADDADP VS40, VS33, VS3 - LXVD2X (R5)(R0), VS44 - WORD $0xf084130e // XVMADDADP VS36, VS34, VS4 - WORD $0xf0a8130e // XVMADDADP VS40, VS34, VS5 - WORD $0xf0c41b0e // XVMADDADP VS36, VS35, VS6 - WORD $0xf0e81b0e // XVMADDADP VS40, VS35, VS7 - LXVD2X (R4)(R0), VS48 - WORD $0xf105030e // XVMADDADP VS37, VS32, VS8 - WORD $0xf129030e // XVMADDADP VS41, VS32, VS9 - WORD $0xf1450b0e // XVMADDADP VS37, VS33, VS10 - WORD $0xf1690b0e // XVMADDADP VS41, VS33, VS11 - LXVD2X (R5)(R14), VS45 - WORD $0xf185130e // XVMADDADP VS37, VS34, VS12 - WORD $0xf1a9130e // XVMADDADP VS41, VS34, VS13 - WORD $0xf1c51b0e // XVMADDADP VS37, VS35, VS14 - WORD $0xf1e91b0e // XVMADDADP VS41, VS35, VS15 - LXVD2X (R4)(R14), VS49 - WORD $0xf206030e // XVMADDADP VS38, VS32, VS16 - WORD $0xf22a030e // XVMADDADP VS42, VS32, VS17 - WORD $0xf2460b0e // XVMADDADP VS38, VS33, VS18 - WORD $0xf26a0b0e // XVMADDADP VS42, VS33, VS19 - LXVD2X (R5)(R15), VS46 - WORD $0xf286130e // XVMADDADP VS38, VS34, VS20 - WORD $0xf2aa130e // XVMADDADP VS42, VS34, VS21 - WORD $0xf2c61b0e // XVMADDADP VS38, VS35, VS22 - WORD $0xf2ea1b0e // XVMADDADP VS42, VS35, VS23 - LXVD2X (R4)(R15), VS50 - WORD $0xf307030e // XVMADDADP VS39, VS32, VS24 - WORD $0xf32b030e // XVMADDADP VS43, VS32, VS25 - WORD $0xf3470b0e // XVMADDADP VS39, VS33, VS26 - WORD $0xf36b0b0e // XVMADDADP VS43, VS33, VS27 - LXVD2X (R5)(R16), VS47 - WORD $0xf387130e // XVMADDADP VS39, VS34, VS28 - WORD $0xf3ab130e // XVMADDADP VS43, VS34, VS29 - WORD $0xf3c71b0e // XVMADDADP VS39, VS35, VS30 - WORD $0xf3eb1b0e // XVMADDADP VS43, VS35, VS31 - LXVD2X (R4)(R16), VS51 - ADD $64, R4 - ADD $64, R5 - XXPERMDI VS48, VS48, $2, VS52 - XXPERMDI VS49, VS49, $2, VS53 - XXPERMDI VS50, VS50, $2, VS54 - XXPERMDI VS51, VS51, $2, VS55 - WORD $0xf010630e // XVMADDADP VS48, VS44, VS0 - WORD $0xf034630e // XVMADDADP VS52, VS44, VS1 - WORD $0xf0506b0e // XVMADDADP VS48, VS45, VS2 - WORD $0xf0746b0e // XVMADDADP VS52, VS45, VS3 - WORD $0xf090730e // XVMADDADP VS48, VS46, VS4 - WORD $0xf0b4730e // XVMADDADP VS52, VS46, VS5 - WORD $0xf0d07b0e // XVMADDADP VS48, VS47, VS6 - WORD $0xf0f47b0e // XVMADDADP VS52, VS47, VS7 - WORD $0xf111630e // XVMADDADP VS49, VS44, VS8 - WORD $0xf135630e // XVMADDADP VS53, VS44, VS9 - WORD $0xf1516b0e // XVMADDADP VS49, VS45, VS10 - WORD $0xf1756b0e // XVMADDADP VS53, VS45, VS11 - WORD $0xf191730e // XVMADDADP VS49, VS46, VS12 - WORD $0xf1b5730e // XVMADDADP VS53, VS46, VS13 - WORD $0xf1d17b0e // XVMADDADP VS49, VS47, VS14 - WORD $0xf1f57b0e // XVMADDADP VS53, VS47, VS15 - WORD $0xf212630e // XVMADDADP VS50, VS44, VS16 - WORD $0xf236630e // XVMADDADP VS54, VS44, VS17 - WORD $0xf2526b0e // XVMADDADP VS50, VS45, VS18 - WORD $0xf2766b0e // XVMADDADP VS54, VS45, VS19 - WORD $0xf292730e // XVMADDADP VS50, VS46, VS20 - WORD $0xf2b6730e // XVMADDADP VS54, VS46, VS21 - WORD $0xf2d27b0e // XVMADDADP VS50, VS47, VS22 - WORD $0xf2f67b0e // XVMADDADP VS54, VS47, VS23 - WORD $0xf313630e // XVMADDADP VS51, VS44, VS24 - WORD $0xf337630e // XVMADDADP VS55, VS44, VS25 - WORD $0xf3536b0e // XVMADDADP VS51, VS45, VS26 - WORD $0xf3776b0e // XVMADDADP VS55, VS45, VS27 - WORD $0xf393730e // XVMADDADP VS51, VS46, VS28 - WORD $0xf3b7730e // XVMADDADP VS55, VS46, VS29 - WORD $0xf3d37b0e // XVMADDADP VS51, VS47, VS30 - WORD $0xf3f77b0e // XVMADDADP VS55, VS47, VS31 - ADD $-2, R3 - BR gemmMicro8x8VSXloop2 -gemmMicro8x8VSXone: - CMP R3, $0 - BEQ gemmMicro8x8VSXstore - LXVD2X (R5)(R0), VS32 - LXVD2X (R4)(R0), VS36 - LXVD2X (R5)(R14), VS33 - LXVD2X (R4)(R14), VS37 - LXVD2X (R5)(R15), VS34 - LXVD2X (R4)(R15), VS38 - LXVD2X (R5)(R16), VS35 - LXVD2X (R4)(R16), VS39 - XXPERMDI VS36, VS36, $2, VS40 - XXPERMDI VS37, VS37, $2, VS41 - XXPERMDI VS38, VS38, $2, VS42 - XXPERMDI VS39, VS39, $2, VS43 - WORD $0xf004030e // XVMADDADP VS36, VS32, VS0 - WORD $0xf028030e // XVMADDADP VS40, VS32, VS1 - WORD $0xf0440b0e // XVMADDADP VS36, VS33, VS2 - WORD $0xf0680b0e // XVMADDADP VS40, VS33, VS3 - WORD $0xf084130e // XVMADDADP VS36, VS34, VS4 - WORD $0xf0a8130e // XVMADDADP VS40, VS34, VS5 - WORD $0xf0c41b0e // XVMADDADP VS36, VS35, VS6 - WORD $0xf0e81b0e // XVMADDADP VS40, VS35, VS7 - WORD $0xf105030e // XVMADDADP VS37, VS32, VS8 - WORD $0xf129030e // XVMADDADP VS41, VS32, VS9 - WORD $0xf1450b0e // XVMADDADP VS37, VS33, VS10 - WORD $0xf1690b0e // XVMADDADP VS41, VS33, VS11 - WORD $0xf185130e // XVMADDADP VS37, VS34, VS12 - WORD $0xf1a9130e // XVMADDADP VS41, VS34, VS13 - WORD $0xf1c51b0e // XVMADDADP VS37, VS35, VS14 - WORD $0xf1e91b0e // XVMADDADP VS41, VS35, VS15 - WORD $0xf206030e // XVMADDADP VS38, VS32, VS16 - WORD $0xf22a030e // XVMADDADP VS42, VS32, VS17 - WORD $0xf2460b0e // XVMADDADP VS38, VS33, VS18 - WORD $0xf26a0b0e // XVMADDADP VS42, VS33, VS19 - WORD $0xf286130e // XVMADDADP VS38, VS34, VS20 - WORD $0xf2aa130e // XVMADDADP VS42, VS34, VS21 - WORD $0xf2c61b0e // XVMADDADP VS38, VS35, VS22 - WORD $0xf2ea1b0e // XVMADDADP VS42, VS35, VS23 - WORD $0xf307030e // XVMADDADP VS39, VS32, VS24 - WORD $0xf32b030e // XVMADDADP VS43, VS32, VS25 - WORD $0xf3470b0e // XVMADDADP VS39, VS33, VS26 - WORD $0xf36b0b0e // XVMADDADP VS43, VS33, VS27 - WORD $0xf387130e // XVMADDADP VS39, VS34, VS28 - WORD $0xf3ab130e // XVMADDADP VS43, VS34, VS29 - WORD $0xf3c71b0e // XVMADDADP VS39, VS35, VS30 - WORD $0xf3eb1b0e // XVMADDADP VS43, VS35, VS31 -gemmMicro8x8VSXstore: - XXPERMDI VS0, VS1, $1, VS56 - LXVD2X (R6)(R0), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R0) - XXPERMDI VS2, VS3, $1, VS56 - LXVD2X (R6)(R14), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R14) - XXPERMDI VS4, VS5, $1, VS56 - LXVD2X (R6)(R15), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R15) - XXPERMDI VS6, VS7, $1, VS56 - LXVD2X (R6)(R16), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R16) - ADD R7, R6 - XXPERMDI VS1, VS0, $1, VS56 - LXVD2X (R6)(R0), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R0) - XXPERMDI VS3, VS2, $1, VS56 - LXVD2X (R6)(R14), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R14) - XXPERMDI VS5, VS4, $1, VS56 - LXVD2X (R6)(R15), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R15) - XXPERMDI VS7, VS6, $1, VS56 - LXVD2X (R6)(R16), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R16) - ADD R7, R6 - XXPERMDI VS8, VS9, $1, VS56 - LXVD2X (R6)(R0), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R0) - XXPERMDI VS10, VS11, $1, VS56 - LXVD2X (R6)(R14), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R14) - XXPERMDI VS12, VS13, $1, VS56 - LXVD2X (R6)(R15), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R15) - XXPERMDI VS14, VS15, $1, VS56 - LXVD2X (R6)(R16), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R16) - ADD R7, R6 - XXPERMDI VS9, VS8, $1, VS56 - LXVD2X (R6)(R0), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R0) - XXPERMDI VS11, VS10, $1, VS56 - LXVD2X (R6)(R14), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R14) - XXPERMDI VS13, VS12, $1, VS56 - LXVD2X (R6)(R15), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R15) - XXPERMDI VS15, VS14, $1, VS56 - LXVD2X (R6)(R16), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R16) - ADD R7, R6 - XXPERMDI VS16, VS17, $1, VS56 - LXVD2X (R6)(R0), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R0) - XXPERMDI VS18, VS19, $1, VS56 - LXVD2X (R6)(R14), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R14) - XXPERMDI VS20, VS21, $1, VS56 - LXVD2X (R6)(R15), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R15) - XXPERMDI VS22, VS23, $1, VS56 - LXVD2X (R6)(R16), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R16) - ADD R7, R6 - XXPERMDI VS17, VS16, $1, VS56 - LXVD2X (R6)(R0), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R0) - XXPERMDI VS19, VS18, $1, VS56 - LXVD2X (R6)(R14), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R14) - XXPERMDI VS21, VS20, $1, VS56 - LXVD2X (R6)(R15), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R15) - XXPERMDI VS23, VS22, $1, VS56 - LXVD2X (R6)(R16), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R16) - ADD R7, R6 - XXPERMDI VS24, VS25, $1, VS56 - LXVD2X (R6)(R0), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R0) - XXPERMDI VS26, VS27, $1, VS56 - LXVD2X (R6)(R14), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R14) - XXPERMDI VS28, VS29, $1, VS56 - LXVD2X (R6)(R15), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R15) - XXPERMDI VS30, VS31, $1, VS56 - LXVD2X (R6)(R16), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R16) - ADD R7, R6 - XXPERMDI VS25, VS24, $1, VS56 - LXVD2X (R6)(R0), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R0) - XXPERMDI VS27, VS26, $1, VS56 - LXVD2X (R6)(R14), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R14) - XXPERMDI VS29, VS28, $1, VS56 - LXVD2X (R6)(R15), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R15) - XXPERMDI VS31, VS30, $1, VS56 - LXVD2X (R6)(R16), VS57 - WORD $0xf339c307 // XVADDDP VS57, VS56, VS57 - STXVD2X VS57, (R6)(R16) - RET - TEXT ·blocks8VSX(SB), NOSPLIT, $0-24 MOVD r+0(FP), R3 MOVD a+8(FP), R4 @@ -772,306 +446,6 @@ minVSXfold: MOVD R6, ret+24(FP) RET -TEXT ·gemmMicro8x16VSX(SB), NOSPLIT, $0-40 - MOVD kc+0(FP), R3 - MOVD pa+8(FP), R4 - MOVD pb+16(FP), R5 - MOVD c+24(FP), R6 - MOVD ldc+32(FP), R7 - SLD $2, R7 - MOVD $16, R14 - MOVD $32, R15 - MOVD $48, R16 - MOVD $oddw<>+0(SB), R8 - LXVW4X (R8)(R0), VS48 - MOVD $highw<>+0(SB), R8 - LXVW4X (R8)(R0), VS49 - XXLXOR VS0, VS0, VS0 - XXLXOR VS1, VS1, VS1 - XXLXOR VS2, VS2, VS2 - XXLXOR VS3, VS3, VS3 - XXLXOR VS4, VS4, VS4 - XXLXOR VS5, VS5, VS5 - XXLXOR VS6, VS6, VS6 - XXLXOR VS7, VS7, VS7 - XXLXOR VS8, VS8, VS8 - XXLXOR VS9, VS9, VS9 - XXLXOR VS10, VS10, VS10 - XXLXOR VS11, VS11, VS11 - XXLXOR VS12, VS12, VS12 - XXLXOR VS13, VS13, VS13 - XXLXOR VS14, VS14, VS14 - XXLXOR VS15, VS15, VS15 - XXLXOR VS16, VS16, VS16 - XXLXOR VS17, VS17, VS17 - XXLXOR VS18, VS18, VS18 - XXLXOR VS19, VS19, VS19 - XXLXOR VS20, VS20, VS20 - XXLXOR VS21, VS21, VS21 - XXLXOR VS22, VS22, VS22 - XXLXOR VS23, VS23, VS23 - XXLXOR VS24, VS24, VS24 - XXLXOR VS25, VS25, VS25 - XXLXOR VS26, VS26, VS26 - XXLXOR VS27, VS27, VS27 - XXLXOR VS28, VS28, VS28 - XXLXOR VS29, VS29, VS29 - XXLXOR VS30, VS30, VS30 - XXLXOR VS31, VS31, VS31 - CMP R3, $0 - BEQ gemmMicro8x16VSXstore -gemmMicro8x16VSXloop: - LXVW4X (R4)(R0), VS32 - LXVW4X (R4)(R14), VS36 - LXVW4X (R5)(R0), VS40 - LXVW4X (R5)(R14), VS41 - LXVW4X (R5)(R15), VS42 - LXVW4X (R5)(R16), VS43 - XXSLDWI VS32, VS32, $1, VS33 - XXSLDWI VS32, VS32, $2, VS34 - XXSLDWI VS32, VS32, $3, VS35 - XXSLDWI VS36, VS36, $1, VS37 - XXSLDWI VS36, VS36, $2, VS38 - XXSLDWI VS36, VS36, $3, VS39 - WORD $0xf000420e // XVMADDASP VS32, VS40, VS0 - WORD $0xf021420e // XVMADDASP VS33, VS40, VS1 - WORD $0xf042420e // XVMADDASP VS34, VS40, VS2 - WORD $0xf063420e // XVMADDASP VS35, VS40, VS3 - WORD $0xf0804a0e // XVMADDASP VS32, VS41, VS4 - WORD $0xf0a14a0e // XVMADDASP VS33, VS41, VS5 - WORD $0xf0c24a0e // XVMADDASP VS34, VS41, VS6 - WORD $0xf0e34a0e // XVMADDASP VS35, VS41, VS7 - WORD $0xf100520e // XVMADDASP VS32, VS42, VS8 - WORD $0xf121520e // XVMADDASP VS33, VS42, VS9 - WORD $0xf142520e // XVMADDASP VS34, VS42, VS10 - WORD $0xf163520e // XVMADDASP VS35, VS42, VS11 - WORD $0xf1805a0e // XVMADDASP VS32, VS43, VS12 - WORD $0xf1a15a0e // XVMADDASP VS33, VS43, VS13 - WORD $0xf1c25a0e // XVMADDASP VS34, VS43, VS14 - WORD $0xf1e35a0e // XVMADDASP VS35, VS43, VS15 - WORD $0xf204420e // XVMADDASP VS36, VS40, VS16 - WORD $0xf225420e // XVMADDASP VS37, VS40, VS17 - WORD $0xf246420e // XVMADDASP VS38, VS40, VS18 - WORD $0xf267420e // XVMADDASP VS39, VS40, VS19 - WORD $0xf2844a0e // XVMADDASP VS36, VS41, VS20 - WORD $0xf2a54a0e // XVMADDASP VS37, VS41, VS21 - WORD $0xf2c64a0e // XVMADDASP VS38, VS41, VS22 - WORD $0xf2e74a0e // XVMADDASP VS39, VS41, VS23 - WORD $0xf304520e // XVMADDASP VS36, VS42, VS24 - WORD $0xf325520e // XVMADDASP VS37, VS42, VS25 - WORD $0xf346520e // XVMADDASP VS38, VS42, VS26 - WORD $0xf367520e // XVMADDASP VS39, VS42, VS27 - WORD $0xf3845a0e // XVMADDASP VS36, VS43, VS28 - WORD $0xf3a55a0e // XVMADDASP VS37, VS43, VS29 - WORD $0xf3c65a0e // XVMADDASP VS38, VS43, VS30 - WORD $0xf3e75a0e // XVMADDASP VS39, VS43, VS31 - ADD $32, R4 - ADD $64, R5 - ADD $-1, R3 - CMP R3, $0 - BNE gemmMicro8x16VSXloop -gemmMicro8x16VSXstore: - XXSEL VS0, VS3, VS48, VS44 - XXSEL VS2, VS1, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R0), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R0) - XXSEL VS4, VS7, VS48, VS44 - XXSEL VS6, VS5, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R14), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R14) - XXSEL VS8, VS11, VS48, VS44 - XXSEL VS10, VS9, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R15), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R15) - XXSEL VS12, VS15, VS48, VS44 - XXSEL VS14, VS13, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R16), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R16) - ADD R7, R6 - XXSEL VS1, VS0, VS48, VS44 - XXSEL VS3, VS2, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R0), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R0) - XXSEL VS5, VS4, VS48, VS44 - XXSEL VS7, VS6, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R14), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R14) - XXSEL VS9, VS8, VS48, VS44 - XXSEL VS11, VS10, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R15), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R15) - XXSEL VS13, VS12, VS48, VS44 - XXSEL VS15, VS14, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R16), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R16) - ADD R7, R6 - XXSEL VS2, VS1, VS48, VS44 - XXSEL VS0, VS3, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R0), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R0) - XXSEL VS6, VS5, VS48, VS44 - XXSEL VS4, VS7, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R14), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R14) - XXSEL VS10, VS9, VS48, VS44 - XXSEL VS8, VS11, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R15), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R15) - XXSEL VS14, VS13, VS48, VS44 - XXSEL VS12, VS15, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R16), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R16) - ADD R7, R6 - XXSEL VS3, VS2, VS48, VS44 - XXSEL VS1, VS0, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R0), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R0) - XXSEL VS7, VS6, VS48, VS44 - XXSEL VS5, VS4, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R14), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R14) - XXSEL VS11, VS10, VS48, VS44 - XXSEL VS9, VS8, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R15), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R15) - XXSEL VS15, VS14, VS48, VS44 - XXSEL VS13, VS12, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R16), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R16) - ADD R7, R6 - XXSEL VS16, VS19, VS48, VS44 - XXSEL VS18, VS17, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R0), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R0) - XXSEL VS20, VS23, VS48, VS44 - XXSEL VS22, VS21, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R14), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R14) - XXSEL VS24, VS27, VS48, VS44 - XXSEL VS26, VS25, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R15), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R15) - XXSEL VS28, VS31, VS48, VS44 - XXSEL VS30, VS29, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R16), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R16) - ADD R7, R6 - XXSEL VS17, VS16, VS48, VS44 - XXSEL VS19, VS18, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R0), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R0) - XXSEL VS21, VS20, VS48, VS44 - XXSEL VS23, VS22, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R14), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R14) - XXSEL VS25, VS24, VS48, VS44 - XXSEL VS27, VS26, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R15), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R15) - XXSEL VS29, VS28, VS48, VS44 - XXSEL VS31, VS30, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R16), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R16) - ADD R7, R6 - XXSEL VS18, VS17, VS48, VS44 - XXSEL VS16, VS19, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R0), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R0) - XXSEL VS22, VS21, VS48, VS44 - XXSEL VS20, VS23, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R14), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R14) - XXSEL VS26, VS25, VS48, VS44 - XXSEL VS24, VS27, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R15), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R15) - XXSEL VS30, VS29, VS48, VS44 - XXSEL VS28, VS31, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R16), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R16) - ADD R7, R6 - XXSEL VS19, VS18, VS48, VS44 - XXSEL VS17, VS16, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R0), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R0) - XXSEL VS23, VS22, VS48, VS44 - XXSEL VS21, VS20, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R14), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R14) - XXSEL VS27, VS26, VS48, VS44 - XXSEL VS25, VS24, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R15), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R15) - XXSEL VS31, VS30, VS48, VS44 - XXSEL VS29, VS28, VS48, VS45 - XXSEL VS44, VS45, VS49, VS46 - LXVW4X (R6)(R16), VS47 - WORD $0xf1ef7207 // XVADDSP VS47, VS46, VS47 - STXVW4X VS47, (R6)(R16) - RET - DATA abs64<>+0(SB)/1, $0xff DATA abs64<>+1(SB)/1, $0xff DATA abs64<>+2(SB)/1, $0xff @@ -1108,39 +482,3 @@ DATA inf64<>+14(SB)/1, $0xf0 DATA inf64<>+15(SB)/1, $0x7f GLOBL inf64<>(SB), RODATA|NOPTR, $16 -DATA oddw<>+0(SB)/1, $0x00 -DATA oddw<>+1(SB)/1, $0x00 -DATA oddw<>+2(SB)/1, $0x00 -DATA oddw<>+3(SB)/1, $0x00 -DATA oddw<>+4(SB)/1, $0xff -DATA oddw<>+5(SB)/1, $0xff -DATA oddw<>+6(SB)/1, $0xff -DATA oddw<>+7(SB)/1, $0xff -DATA oddw<>+8(SB)/1, $0x00 -DATA oddw<>+9(SB)/1, $0x00 -DATA oddw<>+10(SB)/1, $0x00 -DATA oddw<>+11(SB)/1, $0x00 -DATA oddw<>+12(SB)/1, $0xff -DATA oddw<>+13(SB)/1, $0xff -DATA oddw<>+14(SB)/1, $0xff -DATA oddw<>+15(SB)/1, $0xff -GLOBL oddw<>(SB), RODATA|NOPTR, $16 - -DATA highw<>+0(SB)/1, $0x00 -DATA highw<>+1(SB)/1, $0x00 -DATA highw<>+2(SB)/1, $0x00 -DATA highw<>+3(SB)/1, $0x00 -DATA highw<>+4(SB)/1, $0x00 -DATA highw<>+5(SB)/1, $0x00 -DATA highw<>+6(SB)/1, $0x00 -DATA highw<>+7(SB)/1, $0x00 -DATA highw<>+8(SB)/1, $0xff -DATA highw<>+9(SB)/1, $0xff -DATA highw<>+10(SB)/1, $0xff -DATA highw<>+11(SB)/1, $0xff -DATA highw<>+12(SB)/1, $0xff -DATA highw<>+13(SB)/1, $0xff -DATA highw<>+14(SB)/1, $0xff -DATA highw<>+15(SB)/1, $0xff -GLOBL highw<>(SB), RODATA|NOPTR, $16 - diff --git a/internal/kernels/sum_riscv64.s b/internal/kernels/sum_riscv64.s index 4c51c30..c0d384d 100644 --- a/internal/kernels/sum_riscv64.s +++ b/internal/kernels/sum_riscv64.s @@ -104,242 +104,6 @@ TEXT ·div32RVV(SB), NOSPLIT, $0-32 BNEZ X13, loop RET -TEXT ·gemmMicro6x8RVV(SB), NOSPLIT, $0-40 - MOV kc+0(FP), X10 - MOV pa+8(FP), X11 - MOV pb+16(FP), X12 - MOV c+24(FP), X13 - MOV ldc+32(FP), X14 - VSETIVLI $8, E64, M4, TA, MA, X15 - VMVVI $0, V0 - VMVVI $0, V4 - VMVVI $0, V8 - VMVVI $0, V12 - VMVVI $0, V16 - VMVVI $0, V20 - loop: - VLE64V (X12), V24 - MOVD 0(X11), F0 - MOVD 8(X11), F1 - MOVD 16(X11), F2 - MOVD 24(X11), F3 - MOVD 32(X11), F4 - MOVD 40(X11), F5 - VFMACCVF V24, F0, V0 - VFMACCVF V24, F1, V4 - VFMACCVF V24, F2, V8 - VFMACCVF V24, F3, V12 - VFMACCVF V24, F4, V16 - VFMACCVF V24, F5, V20 - ADD $48, X11, X11 - ADD $64, X12, X12 - ADD $-1, X10, X10 - BNEZ X10, loop - SLLI $3, X14, X14 - VLE64V (X13), V28 - VFADDVV V0, V28, V28 - VSE64V V28, (X13) - ADD X14, X13, X13 - VLE64V (X13), V28 - VFADDVV V4, V28, V28 - VSE64V V28, (X13) - ADD X14, X13, X13 - VLE64V (X13), V28 - VFADDVV V8, V28, V28 - VSE64V V28, (X13) - ADD X14, X13, X13 - VLE64V (X13), V28 - VFADDVV V12, V28, V28 - VSE64V V28, (X13) - ADD X14, X13, X13 - VLE64V (X13), V28 - VFADDVV V16, V28, V28 - VSE64V V28, (X13) - ADD X14, X13, X13 - VLE64V (X13), V28 - VFADDVV V20, V28, V28 - VSE64V V28, (X13) - RET - -TEXT ·gemmMicro6x16RVV(SB), NOSPLIT, $0-40 - MOV kc+0(FP), X10 - MOV pa+8(FP), X11 - MOV pb+16(FP), X12 - MOV c+24(FP), X13 - MOV ldc+32(FP), X14 - VSETIVLI $16, E32, M4, TA, MA, X15 - VMVVI $0, V0 - VMVVI $0, V4 - VMVVI $0, V8 - VMVVI $0, V12 - VMVVI $0, V16 - VMVVI $0, V20 - loop: - VLE32V (X12), V24 - MOVF 0(X11), F0 - MOVF 4(X11), F1 - MOVF 8(X11), F2 - MOVF 12(X11), F3 - MOVF 16(X11), F4 - MOVF 20(X11), F5 - VFMACCVF V24, F0, V0 - VFMACCVF V24, F1, V4 - VFMACCVF V24, F2, V8 - VFMACCVF V24, F3, V12 - VFMACCVF V24, F4, V16 - VFMACCVF V24, F5, V20 - ADD $24, X11, X11 - ADD $64, X12, X12 - ADD $-1, X10, X10 - BNEZ X10, loop - SLLI $2, X14, X14 - VLE32V (X13), V28 - VFADDVV V0, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V4, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V8, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V12, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V16, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V20, V28, V28 - VSE32V V28, (X13) - RET - -TEXT ·gemmMicro8x16RVV(SB), NOSPLIT, $0-40 - MOV kc+0(FP), X10 - MOV pa+8(FP), X11 - MOV pb+16(FP), X12 - MOV c+24(FP), X13 - MOV ldc+32(FP), X14 - VSETIVLI $16, E32, M2, TA, MA, X15 - VMVVI $0, V0 - VMVVI $0, V2 - VMVVI $0, V4 - VMVVI $0, V6 - VMVVI $0, V8 - VMVVI $0, V10 - VMVVI $0, V12 - VMVVI $0, V14 - ANDI $1, X10, X5 - SUB X5, X10, X10 - BEQZ X5, main - tail: - VLE32V (X12), V16 - ADD $64, X12, X12 - MOVF 0(X11), F0 - MOVF 4(X11), F1 - MOVF 8(X11), F2 - MOVF 12(X11), F3 - MOVF 16(X11), F4 - MOVF 20(X11), F5 - MOVF 24(X11), F6 - MOVF 28(X11), F7 - VFMACCVF V16, F0, V0 - VFMACCVF V16, F1, V2 - VFMACCVF V16, F2, V4 - VFMACCVF V16, F3, V6 - VFMACCVF V16, F4, V8 - VFMACCVF V16, F5, V10 - VFMACCVF V16, F6, V12 - VFMACCVF V16, F7, V14 - ADD $32, X11, X11 - ADD $-1, X5, X5 - BNEZ X5, tail - main: - BEQZ X10, epi - loop: - VLE32V (X12), V16 - ADD $64, X12, X12 - VLE32V (X12), V18 - ADD $64, X12, X12 - MOVF 0(X11), F0 - MOVF 4(X11), F1 - MOVF 8(X11), F2 - MOVF 12(X11), F3 - MOVF 16(X11), F4 - MOVF 20(X11), F5 - MOVF 24(X11), F6 - MOVF 28(X11), F7 - MOVF 32(X11), F8 - MOVF 36(X11), F9 - MOVF 40(X11), F10 - MOVF 44(X11), F11 - MOVF 48(X11), F12 - MOVF 52(X11), F13 - MOVF 56(X11), F14 - MOVF 60(X11), F15 - VFMACCVF V16, F0, V0 - VFMACCVF V16, F1, V2 - VFMACCVF V16, F2, V4 - VFMACCVF V16, F3, V6 - VFMACCVF V16, F4, V8 - VFMACCVF V16, F5, V10 - VFMACCVF V16, F6, V12 - VFMACCVF V16, F7, V14 - VFMACCVF V18, F8, V0 - VFMACCVF V18, F9, V2 - VFMACCVF V18, F10, V4 - VFMACCVF V18, F11, V6 - VFMACCVF V18, F12, V8 - VFMACCVF V18, F13, V10 - VFMACCVF V18, F14, V12 - VFMACCVF V18, F15, V14 - ADD $64, X11, X11 - ADD $-2, X10, X10 - BNEZ X10, loop - epi: - SLLI $2, X14, X14 - VLE32V (X13), V28 - VFADDVV V0, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V2, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V4, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V6, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V8, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V10, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V12, V28, V28 - VSE32V V28, (X13) - ADD X14, X13, X13 - VLE32V (X13), V28 - VFADDVV V14, V28, V28 - VSE32V V28, (X13) - RET - -TEXT ·gemmVLENB(SB), NOSPLIT, $0-8 - VSETVLI X0, E8, M1, TA, MA, X5 - MOV X5, ret+0(FP) - RET - TEXT ·blocks8RVV(SB), NOSPLIT, $0-24 MOV r+0(FP), X10 MOV a+8(FP), X11 diff --git a/internal/kernels/sum_s390x.s b/internal/kernels/sum_s390x.s index 4cda6ce..d599028 100644 --- a/internal/kernels/sum_s390x.s +++ b/internal/kernels/sum_s390x.s @@ -186,164 +186,6 @@ TEXT ·div32VXE(SB), NOSPLIT, $0-32 done: RET -TEXT ·gemmMicro4x8VX(SB), NOSPLIT, $0-40 - MOVD kc+0(FP), R5 - MOVD pa+8(FP), R2 - MOVD pb+16(FP), R3 - MOVD c+24(FP), R1 - MOVD ldc+32(FP), R4 - VZERO V0 - VZERO V1 - VZERO V2 - VZERO V3 - VZERO V4 - VZERO V5 - VZERO V6 - VZERO V7 - VZERO V8 - VZERO V9 - VZERO V10 - VZERO V11 - VZERO V12 - VZERO V13 - VZERO V14 - VZERO V15 - loop: - VLM (R3), V16, V19 - VLREPG 0(R2), V20 - VLREPG 8(R2), V21 - VLREPG 16(R2), V22 - VLREPG 24(R2), V23 - VFMADB V20, V16, V0, V0 - VFMADB V20, V17, V1, V1 - VFMADB V20, V18, V2, V2 - VFMADB V20, V19, V3, V3 - VFMADB V21, V16, V4, V4 - VFMADB V21, V17, V5, V5 - VFMADB V21, V18, V6, V6 - VFMADB V21, V19, V7, V7 - VFMADB V22, V16, V8, V8 - VFMADB V22, V17, V9, V9 - VFMADB V22, V18, V10, V10 - VFMADB V22, V19, V11, V11 - VFMADB V23, V16, V12, V12 - VFMADB V23, V17, V13, V13 - VFMADB V23, V18, V14, V14 - VFMADB V23, V19, V15, V15 - ADD $32, R2 - ADD $64, R3 - SUB $1, R5 - CMP R5, $0 - BNE loop - SLD $3, R4 - VLM (R1), V24, V27 - VFADB V0, V24, V24 - VFADB V1, V25, V25 - VFADB V2, V26, V26 - VFADB V3, V27, V27 - VSTM V24, V27, (R1) - ADD R4, R1 - VLM (R1), V24, V27 - VFADB V4, V24, V24 - VFADB V5, V25, V25 - VFADB V6, V26, V26 - VFADB V7, V27, V27 - VSTM V24, V27, (R1) - ADD R4, R1 - VLM (R1), V24, V27 - VFADB V8, V24, V24 - VFADB V9, V25, V25 - VFADB V10, V26, V26 - VFADB V11, V27, V27 - VSTM V24, V27, (R1) - ADD R4, R1 - VLM (R1), V24, V27 - VFADB V12, V24, V24 - VFADB V13, V25, V25 - VFADB V14, V26, V26 - VFADB V15, V27, V27 - VSTM V24, V27, (R1) - RET - -TEXT ·gemmMicro4x16VXE(SB), NOSPLIT, $0-40 - MOVD kc+0(FP), R5 - MOVD pa+8(FP), R2 - MOVD pb+16(FP), R3 - MOVD c+24(FP), R1 - MOVD ldc+32(FP), R4 - VZERO V0 - VZERO V1 - VZERO V2 - VZERO V3 - VZERO V4 - VZERO V5 - VZERO V6 - VZERO V7 - VZERO V8 - VZERO V9 - VZERO V10 - VZERO V11 - VZERO V12 - VZERO V13 - VZERO V14 - VZERO V15 - loop: - VLM (R3), V16, V19 - VLREPF 0(R2), V20 - VLREPF 4(R2), V21 - VLREPF 8(R2), V22 - VLREPF 12(R2), V23 - WORD $0xe7040200; BYTE $0x06; BYTE $0x8f // VFMASB V20, V16, V0, V0 - WORD $0xe7141200; BYTE $0x16; BYTE $0x8f // VFMASB V20, V17, V1, V1 - WORD $0xe7242200; BYTE $0x26; BYTE $0x8f // VFMASB V20, V18, V2, V2 - WORD $0xe7343200; BYTE $0x36; BYTE $0x8f // VFMASB V20, V19, V3, V3 - WORD $0xe7450200; BYTE $0x46; BYTE $0x8f // VFMASB V21, V16, V4, V4 - WORD $0xe7551200; BYTE $0x56; BYTE $0x8f // VFMASB V21, V17, V5, V5 - WORD $0xe7652200; BYTE $0x66; BYTE $0x8f // VFMASB V21, V18, V6, V6 - WORD $0xe7753200; BYTE $0x76; BYTE $0x8f // VFMASB V21, V19, V7, V7 - WORD $0xe7860200; BYTE $0x86; BYTE $0x8f // VFMASB V22, V16, V8, V8 - WORD $0xe7961200; BYTE $0x96; BYTE $0x8f // VFMASB V22, V17, V9, V9 - WORD $0xe7a62200; BYTE $0xa6; BYTE $0x8f // VFMASB V22, V18, V10, V10 - WORD $0xe7b63200; BYTE $0xb6; BYTE $0x8f // VFMASB V22, V19, V11, V11 - WORD $0xe7c70200; BYTE $0xc6; BYTE $0x8f // VFMASB V23, V16, V12, V12 - WORD $0xe7d71200; BYTE $0xd6; BYTE $0x8f // VFMASB V23, V17, V13, V13 - WORD $0xe7e72200; BYTE $0xe6; BYTE $0x8f // VFMASB V23, V18, V14, V14 - WORD $0xe7f73200; BYTE $0xf6; BYTE $0x8f // VFMASB V23, V19, V15, V15 - ADD $16, R2 - ADD $64, R3 - SUB $1, R5 - CMP R5, $0 - BNE loop - SLD $2, R4 - VLM (R1), V24, V27 - WORD $0xe7880000; BYTE $0x2c; BYTE $0xe3 // VFASB V24, V0, V24 - WORD $0xe7991000; BYTE $0x2c; BYTE $0xe3 // VFASB V25, V1, V25 - WORD $0xe7aa2000; BYTE $0x2c; BYTE $0xe3 // VFASB V26, V2, V26 - WORD $0xe7bb3000; BYTE $0x2c; BYTE $0xe3 // VFASB V27, V3, V27 - VSTM V24, V27, (R1) - ADD R4, R1 - VLM (R1), V24, V27 - WORD $0xe7884000; BYTE $0x2c; BYTE $0xe3 // VFASB V24, V4, V24 - WORD $0xe7995000; BYTE $0x2c; BYTE $0xe3 // VFASB V25, V5, V25 - WORD $0xe7aa6000; BYTE $0x2c; BYTE $0xe3 // VFASB V26, V6, V26 - WORD $0xe7bb7000; BYTE $0x2c; BYTE $0xe3 // VFASB V27, V7, V27 - VSTM V24, V27, (R1) - ADD R4, R1 - VLM (R1), V24, V27 - WORD $0xe7888000; BYTE $0x2c; BYTE $0xe3 // VFASB V24, V8, V24 - WORD $0xe7999000; BYTE $0x2c; BYTE $0xe3 // VFASB V25, V9, V25 - WORD $0xe7aaa000; BYTE $0x2c; BYTE $0xe3 // VFASB V26, V10, V26 - WORD $0xe7bbb000; BYTE $0x2c; BYTE $0xe3 // VFASB V27, V11, V27 - VSTM V24, V27, (R1) - ADD R4, R1 - VLM (R1), V24, V27 - WORD $0xe788c000; BYTE $0x2c; BYTE $0xe3 // VFASB V24, V12, V24 - WORD $0xe799d000; BYTE $0x2c; BYTE $0xe3 // VFASB V25, V13, V25 - WORD $0xe7aae000; BYTE $0x2c; BYTE $0xe3 // VFASB V26, V14, V26 - WORD $0xe7bbf000; BYTE $0x2c; BYTE $0xe3 // VFASB V27, V15, V27 - VSTM V24, V27, (R1) - RET - TEXT ·maxVXE(SB), NOSPLIT, $0-24 MOVD a+0(FP), R1 MOVD n+8(FP), R2 diff --git a/internal/kernels/vx_s390x_test.go b/internal/kernels/vx_s390x_test.go index dc5482f..4c4977f 100644 --- a/internal/kernels/vx_s390x_test.go +++ b/internal/kernels/vx_s390x_test.go @@ -12,22 +12,9 @@ func TestVXDetected(t *testing.T) { t.Run("scalar", func(t *testing.T) { TestBinSIMD(t) TestSumSIMD(t) - TestGemmParallelYielding(t) // gemmMicroGo against the oracle - }) } -// TestGemm32NeedsVXE: without the vector-enhancements facility 1 there is no -// float32 kernel, and HasGemm32 sends float32 products to the float64 GEMM. -func TestGemm32NeedsVXE(t *testing.T) { - if newGemm32(false) != nil { - t.Fatal("a float32 GEMM without VXE") - } - if g := newGemm32(true); g == nil || g.mr != 4 || g.nr != 16 { - t.Fatalf("newGemm32(true) = %+v", g) - } -} - // TestVXEDetected runs the max/min checks with the vector kernel forced off // too, and logs which path this machine takes. func TestVXEDetected(t *testing.T) { diff --git a/linalg.go b/linalg.go index bc432e9..272df82 100644 --- a/linalg.go +++ b/linalg.go @@ -3,6 +3,7 @@ package ndarray import ( "fmt" + "github.com/go-ndarray/ndarray/internal/blasmm" "github.com/go-ndarray/ndarray/internal/kernels" ) @@ -15,8 +16,8 @@ var ErrLinalg = fmt.Errorf("ndarray: incompatible shapes for linear algebra") // array. Either operand may be any 2-D view: the GEMM packs it through its // strides (matView), so nothing is materialised first. func matmul2D(a, b *Array, m, k, n int) *Array { - dst := a.ws.alloc(m*n, true) // MatMulStridedP accumulates into dst - kernels.MatMulStridedP(dst, a.matView(), b.matView(), m, k, n) + dst := a.ws.alloc(m*n, false) // blasmm.MatMul overwrites dst + blasmm.MatMul(dst, a.matView(), b.matView(), m, k, n) shape := []int{m, n} return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape), ws: a.ws} } @@ -24,8 +25,8 @@ func matmul2D(a, b *Array, m, k, n int) *Array { // matView is a 2-D array as the GEMM's strided operand view. The packing // reads it in place, so a transposed or sliced operand (Inner's bᵀ, a[:, ::2]) // is never materialised into a contiguous copy first. -func (a *Array) matView() kernels.Mat { - return kernels.Mat{Data: a.data, Off: a.offset, RS: a.strides[0], CS: a.strides[1]} +func (a *Array) matView() blasmm.Mat[float64] { + return blasmm.Mat[float64]{Data: a.data, Off: a.offset, RS: a.strides[0], CS: a.strides[1]} } // MatMul returns the matrix product of two 2-D arrays, matching numpy.matmul @@ -78,24 +79,24 @@ func matmulTyped(a, b *Array) *Array { // useGemm32 selects the float32 GEMM. It is a var so tests can force the // widening path on hardware that has the kernel (never the reverse). -var useGemm32 = kernels.HasGemm32() +var useGemm32 = blasmm.HasGemm32() // matmulFloat32 multiplies two validated 2-D operands whose common dtype is // Float32 on the float32 GEMM. A Float32 operand is read in place through its // strides, like matView; another dtype is converted first. func matmulFloat32(a, b *Array) *Array { m, k, n := a.shape[0], a.shape[1], b.shape[1] - dst := a.ws.alloc32(m*n, true) // MatMul32StridedP accumulates into dst - kernels.MatMul32StridedP(dst, a.matView32(), b.matView32(), m, k, n) + dst := a.ws.alloc32(m*n, false) // blasmm.MatMul32 overwrites dst + blasmm.MatMul32(dst, a.matView32(), b.matView32(), m, k, n) return a.result(dst, []int{m, n}) } // matView32 is matView for an operand of the float32 GEMM. -func (a *Array) matView32() kernels.Mat32 { +func (a *Array) matView32() blasmm.Mat[float32] { if a.dtype != Float32 { - return kernels.Mat32{Data: a.operandStore(Float32, a.shape).([]float32), RS: a.shape[1], CS: 1} + return blasmm.Mat[float32]{Data: a.operandStore(Float32, a.shape).([]float32), RS: a.shape[1], CS: 1} } - return kernels.Mat32{Data: typed[float32](a), Off: a.offset, RS: a.strides[0], CS: a.strides[1]} + return blasmm.Mat[float32]{Data: typed[float32](a), Off: a.offset, RS: a.strides[0], CS: a.strides[1]} } // matmulFloat64 is matmul2D on two validated 2-D Float64 arrays. diff --git a/linalg/blas.go b/linalg/blas.go index d35fcd6..c1a857b 100644 --- a/linalg/blas.go +++ b/linalg/blas.go @@ -4,7 +4,7 @@ import ( "math" "math/cmplx" - "github.com/go-ndarray/ndarray/internal/kernels" + "github.com/go-ndarray/ndarray/internal/blasmm" ) // The algorithms are written once, generic over the two computation types. @@ -209,7 +209,7 @@ func fastGemm[T scalar](m, n, k int, alpha T, a, b mview[T], c []T, ldc int) { tmp := *tp if cf, ok := any(c).([]float64); ok { av, bv := any(a).(mview[float64]), any(b).(mview[float64]) - kernels.MatMulStridedP(tmp, kmat(av), kmat(bv), m, k, n) + blasmm.MatMul(tmp, kmat(av), kmat(bv), m, k, n) al := any(alpha).(float64) for i := 0; i < m; i++ { row := cf[i*ldc : i*ldc+n] @@ -223,17 +223,16 @@ func fastGemm[T scalar](m, n, k int, alpha T, a, b mview[T], c []T, ldc int) { ar, ai := split(m, k, a) br, bi := split(k, n, b) ti := make([]float64, m*n) - ra, rb := kernels.Mat{Data: ar, RS: k, CS: 1}, kernels.Mat{Data: br, RS: n, CS: 1} - ia, ib := kernels.Mat{Data: ai, RS: k, CS: 1}, kernels.Mat{Data: bi, RS: n, CS: 1} - kernels.MatMulStridedP(tmp, ra, rb, m, k, n) // Re·Re - kernels.MatMulStridedP(ti, ia, ib, m, k, n) // Im·Im + ra, rb := blasmm.Mat[float64]{Data: ar, RS: k, CS: 1}, blasmm.Mat[float64]{Data: br, RS: n, CS: 1} + ia, ib := blasmm.Mat[float64]{Data: ai, RS: k, CS: 1}, blasmm.Mat[float64]{Data: bi, RS: n, CS: 1} + blasmm.MatMul(tmp, ra, rb, m, k, n) // Re·Re + blasmm.MatMul(ti, ia, ib, m, k, n) // Im·Im for i := range tmp { tmp[i] -= ti[i] } t3 := make([]float64, m*n) - clear(ti) - kernels.MatMulStridedP(ti, ra, ib, m, k, n) // Re·Im - kernels.MatMulStridedP(t3, ia, rb, m, k, n) // Im·Re + blasmm.MatMul(ti, ra, ib, m, k, n) // Re·Im + blasmm.MatMul(t3, ia, rb, m, k, n) // Im·Re cz := any(c).([]complex128) al := any(alpha).(complex128) for i := 0; i < m; i++ { @@ -244,8 +243,8 @@ func fastGemm[T scalar](m, n, k int, alpha T, a, b mview[T], c []T, ldc int) { } } -func kmat(v mview[float64]) kernels.Mat { - return kernels.Mat{Data: v.d, Off: v.off, RS: v.rs, CS: v.cs} +func kmat(v mview[float64]) blasmm.Mat[float64] { + return blasmm.Mat[float64]{Data: v.d, Off: v.off, RS: v.rs, CS: v.cs} } // split copies an r×c complex operand into contiguous real and imaginary diff --git a/matmul_workers_test.go b/matmul_workers_test.go new file mode 100644 index 0000000..d4af9ac --- /dev/null +++ b/matmul_workers_test.go @@ -0,0 +1,65 @@ +package ndarray + +import ( + "math/rand/v2" + "runtime" + "testing" +) + +// TestMatMulOneWorkerSameBits is TestGemmOneWorkerSameBits of the GEMM, which +// moved to go-blas/blas with it, asked of MatMul: on non-integer data a product +// above the parallel threshold at GOMAXPROCS=1 (one worker, the single-pass +// path) must equal, bit for bit, the product on four workers, for float64 and, +// where there is a float32 kernel, for float32; with a transposed and a +// decimated operand, which go-blas reads through their strides. +func TestMatMulOneWorkerSameBits(t *testing.T) { + r := rand.New(rand.NewPCG(11, 12)) + random := func(rows, cols int) *Array { + d := make([]float64, rows*cols) + for i := range d { + d[i] = r.NormFloat64() + } + a, err := FromData(d, rows, cols) + if err != nil { + t.Fatal(err) + } + return a + } + for _, s := range [][3]int{{97, 131, 83}, {1, 300, 7000}, {256, 64, 200}} { + m, k, n := s[0], s[1], s[2] + wide := random(m, 2*k) + decimated, err := wide.Slice(All(), Step(2)) + if err != nil { + t.Fatal(err) + } + for name, ab := range map[string][2]*Array{ + "contiguous": {random(m, k), random(k, n)}, + "transposed": {random(k, m).Transpose(), random(n, k).Transpose()}, + "decimated": {decimated, random(k, n)}, + } { + dts := []DType{Float64} + if useGemm32 { + dts = append(dts, Float32) + } + for _, dt := range dts { + a, b := ab[0].AsType(dt), ab[1].AsType(dt) + run := func(procs int) *Array { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(procs)) + c, err := a.MatMul(b) + if err != nil { + t.Fatal(err) + } + return c + } + one, many := run(1), run(4) + for i := 0; i < m; i++ { + for j := 0; j < n; j++ { + if one.At(i, j) != many.At(i, j) { + t.Fatalf("%v %s %v: element (%d,%d) is %v on one worker, %v on four", s, name, dt, i, j, one.At(i, j), many.At(i, j)) + } + } + } + } + } + } +} From 660334ed7222c720ff8b58bd3921778db223e49d Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 15 Nov 2026 16:36:57 +0100 Subject: [PATCH 2/4] build: go-blas/blas v0.2.1 v0.2.1 scales C a contiguous run at a time in DgemmStrided/SgemmStrided (go-blas/blas#3): MatMul through v0.2.0 was 1.15 to 1.45x slower than the GEMM it replaces at n = 64..256 because of that loop. Co-Authored-By: Claude Sonnet 5.5 --- bench/go.mod | 2 +- bench/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bench/go.mod b/bench/go.mod index 6b02d4b..b222337 100644 --- a/bench/go.mod +++ b/bench/go.mod @@ -8,7 +8,7 @@ require ( ) require ( - github.com/go-blas/blas v0.2.0 // indirect + github.com/go-blas/blas v0.2.1 // indirect github.com/go-fft/fft v0.29.0 // indirect github.com/go-fleettools/fleettools v0.81.0 // indirect github.com/go-simd/floats v0.17.0 // indirect diff --git a/bench/go.sum b/bench/go.sum index 0d66bd2..43fe0a2 100644 --- a/bench/go.sum +++ b/bench/go.sum @@ -1,5 +1,5 @@ -github.com/go-blas/blas v0.2.0 h1:kLWNy4ZKhUEHz8vXWIC7n6CP0ytNuEWTYBuASdF+Sxo= -github.com/go-blas/blas v0.2.0/go.mod h1:WjhlqzQ+RA8tS473f18QtcJl5Q0GonPs3RJIxT5G7Bk= +github.com/go-blas/blas v0.2.1 h1:5uZfmvXVZn9CjE+I3DalALqWHdKz+rbBs8VffH3dAZo= +github.com/go-blas/blas v0.2.1/go.mod h1:WjhlqzQ+RA8tS473f18QtcJl5Q0GonPs3RJIxT5G7Bk= github.com/go-fft/fft v0.29.0 h1:WtJb5K3dmbLWREvvZ50od0fxtjIBiXrg7mVybz/nPoA= github.com/go-fft/fft v0.29.0/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s= github.com/go-fleettools/fleettools v0.81.0 h1:UnMflmNw5b82Z+7Ums7UVaW5RrGuACCVmSwytwT1hxw= diff --git a/go.mod b/go.mod index 5028998..03f28b9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/go-ndarray/ndarray go 1.27.2 require ( - github.com/go-blas/blas v0.2.0 + github.com/go-blas/blas v0.2.1 github.com/go-fft/fft v0.29.0 github.com/go-simd/floats v0.17.0 ) diff --git a/go.sum b/go.sum index a2c19a6..c9567c6 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/go-blas/blas v0.2.0 h1:kLWNy4ZKhUEHz8vXWIC7n6CP0ytNuEWTYBuASdF+Sxo= -github.com/go-blas/blas v0.2.0/go.mod h1:WjhlqzQ+RA8tS473f18QtcJl5Q0GonPs3RJIxT5G7Bk= +github.com/go-blas/blas v0.2.1 h1:5uZfmvXVZn9CjE+I3DalALqWHdKz+rbBs8VffH3dAZo= +github.com/go-blas/blas v0.2.1/go.mod h1:WjhlqzQ+RA8tS473f18QtcJl5Q0GonPs3RJIxT5G7Bk= github.com/go-fft/fft v0.29.0 h1:WtJb5K3dmbLWREvvZ50od0fxtjIBiXrg7mVybz/nPoA= github.com/go-fft/fft v0.29.0/go.mod h1:GN5V4TWCjV05ECqH1ONfjFM6Z1m2shqCdssxKGqQ41s= github.com/go-simd/floats v0.17.0 h1:Dv0ZmZ3k0IavvHwQgstpUz+TeIf/gGJ9HEGQ/UART8Q= From cd4d69c2f8769d2a179e6337db6d777789cef801 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 15 Nov 2026 16:59:41 +0100 Subject: [PATCH 3/4] test: MatMul over every pair of eleven operand layouts; SECURITY names v0.2.1 TestMatMulLayouts multiplies 16 shapes x 11 x 11 layouts (contiguous, transposed, reversed, broadcast, decimated with an offset, extent-1 axes with garbage strides) in float32 and float64 against a naive loop within a bound, and requires the same bits from the heap and from a NaN-filled Workspace and at GOMAXPROCS 1 and N. TestMatMulConcurrentViews runs 16 goroutines on the same views. -short drops the three largest shapes (0.6 s against 12 s). SECURITY.md's 2026-11-15 entry said go-blas/blas v0.2.0; go.mod pins v0.2.1. Co-Authored-By: Claude Sonnet 5.5 --- SECURITY.md | 2 +- matmul_layouts_test.go | 226 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 matmul_layouts_test.go diff --git a/SECURITY.md b/SECURITY.md index 0c37e14..5399d2f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -71,7 +71,7 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log **2026-11-15 (unreleased).** The GEMM moved to -[go-blas/blas](https://github.com/go-blas/blas) v0.2.0 (the micro-kernels, their +[go-blas/blas](https://github.com/go-blas/blas) v0.2.1 (the micro-kernels, their wrappers, the packing and their fence tests, byte for byte; the audit of that code is that repository's). What ndarray adds is `internal/blasmm`, which hands an operand's row and column strides to `DgemmStrided`/`SgemmStrided`. go-blas diff --git a/matmul_layouts_test.go b/matmul_layouts_test.go new file mode 100644 index 0000000..18340c4 --- /dev/null +++ b/matmul_layouts_test.go @@ -0,0 +1,226 @@ +package ndarray + +import ( + "fmt" + "hash/fnv" + "math" + "runtime" + "testing" +) + +// MatMul reads each operand through its strides (go-blas/blas), whatever the +// view: this test multiplies every pair of eleven layouts (contiguous, +// transposed, reversed on either or both axes, broadcast on either or both, +// decimated with an offset, extent-1 axes with garbage strides), for float32 +// and float64 and for shapes with empty, unit and ragged extents, and checks +// +// - every element against a naive triple loop, within a bound; +// - that the result's bits are the same from the heap and from a Workspace +// pre-filled with NaN (nothing reads the destination), and at GOMAXPROCS 1 +// and N (the result does not depend on the worker count); +// - that sixteen goroutines multiplying the same views at once agree. +// +// Under -short (the emulated CI lanes) the two largest shapes are left out. + +type matLayout struct { + name string + mk func(rows, cols int) (n, rs, cs, off int) +} + +var rvLayouts = []matLayout{ + {"contig", func(r, c int) (int, int, int, int) { return r * c, c, 1, 0 }}, + {"trans", func(r, c int) (int, int, int, int) { return r * c, 1, r, 0 }}, + {"negrow", func(r, c int) (int, int, int, int) { return r * c, -c, 1, (r - 1) * c }}, + {"negcol", func(r, c int) (int, int, int, int) { return r * c, c, -1, c - 1 }}, + {"negboth", func(r, c int) (int, int, int, int) { return r * c, -c, -1, r*c - 1 }}, + {"transneg", func(r, c int) (int, int, int, int) { return r * c, -1, r, r - 1 }}, + {"bcastrow", func(r, c int) (int, int, int, int) { return c, 0, 1, 0 }}, + {"bcastcol", func(r, c int) (int, int, int, int) { return r, 1, 0, 0 }}, + {"bcastall", func(r, c int) (int, int, int, int) { return 1, 0, 0, 0 }}, + {"step2off", func(r, c int) (int, int, int, int) { + rs := 2*c + 3 + return 5 + (r-1)*rs + (c-1)*2 + 1, rs, 2, 5 + }}, + {"odd1", func(r, c int) (int, int, int, int) { return r * c, c, 1, 0 }}, +} + +func rvMake(l matLayout, rows, cols int, f32 bool, seed int) *Array { + n, rs, cs, off := l.mk(rows, cols) + if rows == 0 || cols == 0 { + n, rs, cs, off = 0, max(cols, 1), 1, 0 + } + if l.name == "odd1" { + // extent-1 axes carry garbage strides + if rows == 1 { + rs = -7 + } + if cols == 1 { + cs = 0 + } + } + a := &Array{shape: []int{rows, cols}, strides: []int{rs, cs}, offset: off} + vals := make([]float64, n) + for i := range vals { + vals[i] = float64((i*37+seed*11)%101-50) / 7 + } + if f32 { + d := make([]float32, n) + for i := range d { + d[i] = float32(vals[i]) + } + a.ext, a.dtype = d, Float32 + } else { + a.data, a.dtype = vals, Float64 + } + return a +} + +func rvAt(a *Array, i, j int) float64 { + p := a.offset + i*a.strides[0] + j*a.strides[1] + if a.dtype == Float32 { + return float64(a.ext.([]float32)[p]) + } + return a.data[p] +} + +func rvHash(r *Array) uint64 { + h := fnv.New64a() + m, n := r.shape[0], r.shape[1] + for i := 0; i < m; i++ { + for j := 0; j < n; j++ { + var b [8]byte + u := math.Float64bits(rvAt(r, i, j)) + for q := 0; q < 8; q++ { + b[q] = byte(u >> (8 * q)) + } + h.Write(b[:]) + } + } + return h.Sum64() +} + +func rvRun(t *testing.T, ws *Workspace) []string { + shapes := [][3]int{{0, 3, 4}, {3, 0, 4}, {3, 4, 0}, {1, 1, 1}, {1, 5, 1}, {5, 1, 7}, {1, 9, 6}, {6, 9, 1}, + {3, 4, 5}, {17, 13, 11}, {33, 31, 29}, {64, 64, 64}, {70, 1, 90}} + if !testing.Short() { + shapes = append(shapes, [3]int{1, 300, 200}, [3]int{130, 67, 150}, [3]int{257, 129, 65}) + } + var out []string + for _, f32 := range []bool{false, true} { + for _, s := range shapes { + m, k, n := s[0], s[1], s[2] + for _, la := range rvLayouts { + for _, lb := range rvLayouts { + a := rvMake(la, m, k, f32, 1) + b := rvMake(lb, k, n, f32, 2) + if ws != nil { + a = ws.Use(a) + } + r, err := a.MatMul(b) + if err != nil { + t.Fatalf("%v %s %s: %v", s, la.name, lb.name, err) + } + // reference + var worst float64 + for i := 0; i < m; i++ { + for j := 0; j < n; j++ { + var ref, mag float64 + for p := 0; p < k; p++ { + ref += rvAt(a, i, p) * rvAt(b, p, j) + mag += math.Abs(rvAt(a, i, p) * rvAt(b, p, j)) + } + got := rvAt(r, i, j) + tol := 1e-12 * (mag + 1) + if f32 { + tol = 1e-5 * (mag + 1) + } + if d := math.Abs(got - ref); !(d <= tol) { + worst = math.Max(worst, d) + if math.IsNaN(d) { + worst = math.Inf(1) + } + } + } + } + key := fmt.Sprintf("f32=%v %dx%dx%d %s*%s", f32, m, k, n, la.name, lb.name) + if worst != 0 { + t.Errorf("%s: off by %g", key, worst) + } + out = append(out, fmt.Sprintf("%s %016x", key, rvHash(r))) + if ws != nil { + ws.Reset() + } + } + } + } + } + return out +} + +func rvCompare(t *testing.T, label string, x, y []string) { + if len(x) != len(y) { + t.Fatalf("%s: %d vs %d lines", label, len(x), len(y)) + } + bad := 0 + for i := range x { + if x[i] != y[i] { + bad++ + if bad <= 10 { + t.Errorf("%s: %q vs %q", label, x[i], y[i]) + } + } + } + t.Logf("%s: %d lines, %d differ", label, len(x), bad) +} + +func TestMatMulConcurrentViews(t *testing.T) { + a := rvMake(rvLayouts[1], 130, 67, false, 1) + b := rvMake(rvLayouts[3], 67, 150, false, 2) + a32 := rvMake(rvLayouts[9], 130, 67, true, 1) + b32 := rvMake(rvLayouts[0], 67, 150, true, 2) + r0, _ := a.MatMul(b) + s0, _ := a32.MatMul(b32) + want, want32 := rvHash(r0), rvHash(s0) + ws := NewWorkspace() + done := make(chan string, 16) + for g := 0; g < 16; g++ { + go func(g int) { + msg := "" + for it := 0; it < 20; it++ { + x := a + if g%2 == 0 { + x = ws.Use(a) + } + r, _ := x.MatMul(b) + s, _ := a32.MatMul(b32) + if rvHash(r) != want || rvHash(s) != want32 { + msg = fmt.Sprintf("goroutine %d iter %d differs", g, it) + } + } + done <- msg + }(g) + } + for g := 0; g < 16; g++ { + if m := <-done; m != "" { + t.Error(m) + } + } +} + +func TestMatMulLayouts(t *testing.T) { + heap := rvRun(t, nil) + // A workspace whose memory holds NaN: a product that read its destination + // would show it. + ws := NewWorkspace() + junk := ws.alloc(1<<20, false) + for i := range junk { + junk[i] = math.NaN() + } + ws.Reset() + wsr := rvRun(t, ws) + rvCompare(t, "heap vs dirty workspace", heap, wsr) + old := runtime.GOMAXPROCS(1) + one := rvRun(t, nil) + runtime.GOMAXPROCS(old) + rvCompare(t, "GOMAXPROCS N vs 1", heap, one) +} From ce110fed7ba41606dc12c78c79688dab88c2eae7 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 15 Nov 2026 17:17:50 +0100 Subject: [PATCH 4/4] test: shorten TestMatMulLayouts so the -race coverage job stays under its timeout The two largest shapes (1.3M and 2.1M multiplications per layout pair) took the test job past 10 minutes under -race. They are replaced by {100, 40, 70}, which is past the parallel threshold (m*n >= 6000), and the naive reference runs only in the heap run, the workspace and one-worker runs being compared with it by hash: 25 s under -race on a loaded machine, 0.6 s with -short. Co-Authored-By: Claude Sonnet 5.5 --- matmul_layouts_test.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/matmul_layouts_test.go b/matmul_layouts_test.go index 18340c4..2388ee4 100644 --- a/matmul_layouts_test.go +++ b/matmul_layouts_test.go @@ -20,7 +20,8 @@ import ( // and N (the result does not depend on the worker count); // - that sixteen goroutines multiplying the same views at once agree. // -// Under -short (the emulated CI lanes) the two largest shapes are left out. +// Under -short (the emulated CI lanes) the two shapes past the parallel +// threshold are left out. type matLayout struct { name string @@ -99,11 +100,16 @@ func rvHash(r *Array) uint64 { return h.Sum64() } -func rvRun(t *testing.T, ws *Workspace) []string { +// rvRun multiplies every case, from the workspace ws if not nil, and returns a +// hash of each result; with check it also compares each result with the naive +// loop (the other runs are compared with that one by their hashes). +func rvRun(t *testing.T, ws *Workspace, check bool) []string { shapes := [][3]int{{0, 3, 4}, {3, 0, 4}, {3, 4, 0}, {1, 1, 1}, {1, 5, 1}, {5, 1, 7}, {1, 9, 6}, {6, 9, 1}, {3, 4, 5}, {17, 13, 11}, {33, 31, 29}, {64, 64, 64}, {70, 1, 90}} if !testing.Short() { - shapes = append(shapes, [3]int{1, 300, 200}, [3]int{130, 67, 150}, [3]int{257, 129, 65}) + // Past the GEMM's parallel threshold (m*n >= 6000) and past its KC block + // (k > 256). Larger shapes cost minutes under -race and coverage. + shapes = append(shapes, [3]int{1, 300, 200}, [3]int{100, 40, 70}) } var out []string for _, f32 := range []bool{false, true} { @@ -120,9 +126,8 @@ func rvRun(t *testing.T, ws *Workspace) []string { if err != nil { t.Fatalf("%v %s %s: %v", s, la.name, lb.name, err) } - // reference var worst float64 - for i := 0; i < m; i++ { + for i := 0; check && i < m; i++ { for j := 0; j < n; j++ { var ref, mag float64 for p := 0; p < k; p++ { @@ -208,7 +213,7 @@ func TestMatMulConcurrentViews(t *testing.T) { } func TestMatMulLayouts(t *testing.T) { - heap := rvRun(t, nil) + heap := rvRun(t, nil, true) // A workspace whose memory holds NaN: a product that read its destination // would show it. ws := NewWorkspace() @@ -217,10 +222,10 @@ func TestMatMulLayouts(t *testing.T) { junk[i] = math.NaN() } ws.Reset() - wsr := rvRun(t, ws) + wsr := rvRun(t, ws, false) rvCompare(t, "heap vs dirty workspace", heap, wsr) old := runtime.GOMAXPROCS(1) - one := rvRun(t, nil) + one := rvRun(t, nil, false) runtime.GOMAXPROCS(old) rvCompare(t, "GOMAXPROCS N vs 1", heap, one) }