diff --git a/README.md b/README.md index d606f15..d5540ca 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ `ZerosOf`, `FromSlice`, typed access with `Data[T]`/`Item[T]`, weak Go scalars with `Scalar`). Float64 is the default and the most SIMD-accelerated one. Float32 has SIMD `Sum`, `Add`/`Sub`/`Mul`/`Div` and `Sqrt` on every - 64-bit target, and `Max`/`Min` on amd64, arm64 and riscv64, bit-identical to its + 64-bit target, and `Max`/`Min` on amd64, arm64, riscv64 and s390x, bit-identical to its scalar loops; its `MatMul` runs on the float64 GEMM (accumulated in float64, rounded once) ([docs/design-dtypes.md](docs/design-dtypes.md)). diff --git a/SECURITY.md b/SECURITY.md index 89e8059..2e0e318 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,12 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-10-26 (v0.23.0).** The s390x float32 extremum kernels (`max32VXE`, +`min32VXE`) read `len(a)` elements of a slice of at least 16, behind the +`useVXE` gate. On a real IBM Z the PROT_NONE fence and the sequential-oracle +test pass, and a sabotaged kernel fails the latter. The CI s390x lane runs +VX and VXE (`NDARRAY_REQUIRE_VXE`) since #68. + **2026-10-26 (v0.22.0).** - **Correctness defect.** On riscv64 with V, `MeanAxis(1)` of the int64 row diff --git a/docs/perf.md b/docs/perf.md index 636387f..20283dd 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1047,7 +1047,34 @@ Still to do: - ppc64le and loong64 need registry entries in go-asmgen first. Go's assemblers have no VSX float compare (for the NaN mask), and on loong64 no vector float max, min or compare at all. -- s390x (`VFMAXSB` mode 1) waits for a z machine to test on. +- s390x: done in v0.23.0, below. + +### s390x float32 `Max`/`Min`, and the z14 float32 kernels measured (2026-10-26) + +The s390x kernels use `VFMAXSB`/`VFMINSB` in mode 1 (Java's Math.max/min): +a NaN operand gives NaN, and −0 orders below +0. Four four-lane accumulators +are folded by an 8-byte `VSLDB` rotation, then lane 1 onto lane 0, behind the +`useVXE` gate (z14). + +The IBM Z (LinuxONE) had looked unreachable since 2026-10-25, but the access +had not expired. After the macOS upgrade the SSH agent held no key, and the +`id_ed25519` the server accepts needs a passphrase. With the key loaded +again, everything was measured on the real machine (`abbench`, 6 rounds, +alternated, pinned core): + +| float32, IBM Z | 2¹⁴ | 2²⁰ | base | +|---|---:|---:|---| +| `Max` | **−98.3%** | **−97.2%** | v0.22.0 (Go loop) | +| `Add` | −80.2% | −85.3% | v0.17.0 | +| `AddInto` | −91.9% | −89.1% | v0.17.0 | +| `Sqrt` | −81.7% | −82.7% | v0.17.0 | + +The float64 controls are flat in both runs. + +On the machine, a sabotaged kernel (one `VFMINSB` lane in `max32VXE`) +fails the sequential-oracle test. The kernel suite passes with +`NDARRAY_REQUIRE_VXE=1`, and so does the root suite. Since #68 the s390x qemu +lane in CI also runs VX and VXE, on qemu's `max` CPU. - riscv64: done in v0.22.0, below. ### riscv64 `Max`/`Min` and a sum defect (2026-10-26) diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index 8224633..bdd4f34 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -157,9 +157,9 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). `fft` bridge shipped in v0.10.0 (package `fft`, over go-fft). float32 has SIMD kernels on all six 64-bit targets for `Sum` (v0.14.0–v0.17.0) and for `Add`/`Sub`/`Mul`/`Div` (v0.18.0) and `Sqrt` (v0.19.0). Still pure Go - for float32: `Max`/`Min` on ppc64le, loong64 and s390x (v0.20.0 vectorised - them on amd64 and arm64 and stopped converting to float64 everywhere, - v0.22.0 on riscv64 for float32 and float64), and the transcendental ufuncs. `MatMul` runs on the float64 + for float32: `Max`/`Min` on ppc64le and loong64 (v0.20.0 vectorised them + on amd64 and arm64 and stopped converting to float64 everywhere, v0.22.0 + on riscv64 for float32 and float64, v0.23.0 on s390x), and the transcendental ufuncs. `MatMul` runs on the float64 GEMM since v0.21.0, 77–90% faster than the triple loop; a native float32 GEMM is still to write. - **Phase 3 — broadcasting ufuncs — MOSTLY DONE.** Broadcasting arithmetic, diff --git a/internal/kernels/asmgen/s390x/gen.go b/internal/kernels/asmgen/s390x/gen.go index 75e55ef..fc55c1a 100644 --- a/internal/kernels/asmgen/s390x/gen.go +++ b/internal/kernels/asmgen/s390x/gen.go @@ -42,6 +42,8 @@ func main() { f.Add(extremumKernel("minVXE", "VFMINDB", "WFMINDB")) f.Add(blocks8Kernel()) f.Add(sqrt32Kernel()) + f.Add(extremum32Kernel("max32VXE", "VFMAXSB", "WFMAXSB")) + f.Add(extremum32Kernel("min32VXE", "VFMINSB", "WFMINSB")) if err := os.WriteFile("sum_s390x.s", []byte(f.String()), 0o644); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -145,6 +147,45 @@ func binKernel(name, vec, sca string) *emit.Function { return b.Func() } +// extremum32Kernel builds NAME(a *float32, n int) float32, the maximum (or +// minimum) of a, for n >= 16. VFMAXSB/VFMINSB and their scalar W forms come +// with the vector-enhancements facility 1 (z14), so the Go side gates them on +// useVXE. In mode 1 (Java's Math.max/min) a NaN operand gives NaN and -0 +// orders below +0, Go's rule. The operation is associative under it, so four +// four-lane accumulators seeded with the first 16 elements, folded at the end +// (lanes 2-3 onto 0-1 by an 8-byte VSLDB rotation, then lane 1 onto 0), give +// the sequential result. +func extremum32Kernel(name, vec, sca string) *emit.Function { + sig := s390x.Layout([]string{"a", "n"}, []s390x.Type{s390x.Ptr, s390x.Int64}, + []string{"ret"}, []s390x.Type{s390x.Float32}) + b := s390x.NewFunc(name, sig, 0) + b.LoadArg("a", "R1").LoadArg("n", "R2"). + Raw("VLM (R1), V16, V19"). + Raw("ADD $64, R1").Raw("SUB $16, R2"). + Raw("loop:").Raw("CMP R2, $16").Raw("BLT fold"). + Raw("VLM (R1), V0, V3") + for r := 0; r < 4; r++ { + b.Raw("%s $1, V%d, V%d, V%d", vec, r, 16+r, 16+r) + } + b.Raw("ADD $64, R1").Raw("SUB $16, R2").Raw("BR loop"). + Raw("fold:"). + Raw("%s $1, V17, V16, V16", vec). + Raw("%s $1, V19, V18, V18", vec). + Raw("%s $1, V18, V16, V0", vec). + Raw("VSLDB $8, V0, V0, V1"). // V1 = e2 e3 e0 e1: lanes 0,1 meet 2,3 + Raw("%s $1, V1, V0, V0", vec). + Raw("VREPF $1, V0, V1"). + Raw("%s $1, F1, F0, F0", sca). // F0 = extremum of every lane + Raw("tail:").Raw("CMP R2, $0").Raw("BEQ done"). + Raw("FMOVS (R1), F1"). + Raw("%s $1, F1, F0, F0", sca). + Raw("ADD $4, R1").Raw("SUB $1, R2").Raw("BR tail"). + Raw("done:"). + StoreRet("F0", "ret"). + Ret() + return b.Func() +} + // sqrt32Kernel builds sqrt32VXE(dst, src *float32, n int): vfsqsb (z14, // WORD+BYTE+BYTE from go-asmgen) over 16 floats per iteration, FSQRTS for the // tail; the correctly rounded float32 square root, bit-identical to diff --git a/internal/kernels/extrema32_generic.go b/internal/kernels/extrema32_generic.go index f7baef8..7a7e558 100644 --- a/internal/kernels/extrema32_generic.go +++ b/internal/kernels/extrema32_generic.go @@ -1,4 +1,4 @@ -//go:build !amd64 && !arm64 && !riscv64 +//go:build !amd64 && !arm64 && !riscv64 && !s390x package kernels diff --git a/internal/kernels/extrema32_s390x.go b/internal/kernels/extrema32_s390x.go new file mode 100644 index 0000000..a7bddaa --- /dev/null +++ b/internal/kernels/extrema32_s390x.go @@ -0,0 +1,25 @@ +package kernels + +// max32 runs the VXE kernel (sum_s390x.s: VFMAXSB in Java mode, NaN-propagating +// with -0 below +0) from 16 elements up on a z14 or later, the four-chain Go +// loop otherwise. +func max32(a []float32) float32 { + if !useVXE || len(a) < 16 { + return max32Go(a) + } + return max32VXE(&a[0], len(a)) +} + +// min32 is the dual of max32, with VFMINSB. +func min32(a []float32) float32 { + if !useVXE || len(a) < 16 { + return min32Go(a) + } + return min32VXE(&a[0], len(a)) +} + +//go:noescape +func max32VXE(a *float32, n int) float32 + +//go:noescape +func min32VXE(a *float32, n int) float32 diff --git a/internal/kernels/sum_s390x.s b/internal/kernels/sum_s390x.s index 1e1a84b..c991894 100644 --- a/internal/kernels/sum_s390x.s +++ b/internal/kernels/sum_s390x.s @@ -589,3 +589,77 @@ TEXT ·sqrt32VXE(SB), NOSPLIT, $0-24 done: RET +TEXT ·max32VXE(SB), NOSPLIT, $0-20 + MOVD a+0(FP), R1 + MOVD n+8(FP), R2 + VLM (R1), V16, V19 + ADD $64, R1 + SUB $16, R2 + loop: + CMP R2, $16 + BLT fold + VLM (R1), V0, V3 + VFMAXSB $1, V0, V16, V16 + VFMAXSB $1, V1, V17, V17 + VFMAXSB $1, V2, V18, V18 + VFMAXSB $1, V3, V19, V19 + ADD $64, R1 + SUB $16, R2 + BR loop + fold: + VFMAXSB $1, V17, V16, V16 + VFMAXSB $1, V19, V18, V18 + VFMAXSB $1, V18, V16, V0 + VSLDB $8, V0, V0, V1 + VFMAXSB $1, V1, V0, V0 + VREPF $1, V0, V1 + WFMAXSB $1, F1, F0, F0 + tail: + CMP R2, $0 + BEQ done + FMOVS (R1), F1 + WFMAXSB $1, F1, F0, F0 + ADD $4, R1 + SUB $1, R2 + BR tail + done: + FMOVS F0, ret+16(FP) + RET + +TEXT ·min32VXE(SB), NOSPLIT, $0-20 + MOVD a+0(FP), R1 + MOVD n+8(FP), R2 + VLM (R1), V16, V19 + ADD $64, R1 + SUB $16, R2 + loop: + CMP R2, $16 + BLT fold + VLM (R1), V0, V3 + VFMINSB $1, V0, V16, V16 + VFMINSB $1, V1, V17, V17 + VFMINSB $1, V2, V18, V18 + VFMINSB $1, V3, V19, V19 + ADD $64, R1 + SUB $16, R2 + BR loop + fold: + VFMINSB $1, V17, V16, V16 + VFMINSB $1, V19, V18, V18 + VFMINSB $1, V18, V16, V0 + VSLDB $8, V0, V0, V1 + VFMINSB $1, V1, V0, V0 + VREPF $1, V0, V1 + WFMINSB $1, F1, F0, F0 + tail: + CMP R2, $0 + BEQ done + FMOVS (R1), F1 + WFMINSB $1, F1, F0, F0 + ADD $4, R1 + SUB $1, R2 + BR tail + done: + FMOVS F0, ret+16(FP) + RET +