From 11f8d7ef3603bdfb55cb53ab68705b204a1b8629 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 1 Nov 2026 11:02:35 +0100 Subject: [PATCH 1/2] perf: float32 Exp and Log on the float64 kernels (2-3.5x faster) float32 Exp and Log called math.Exp/math.Log per element, 6-14x slower than NumPy on one Zen 3 core. They now run this package's float64 kernels (the Arm optimized-routines ports) 256 elements at a time through a stack buffer, rounding each result once: within 0.5 + 2^-29 float32 ULP, no allocation. abbench, 8 rounds, one Zen 3 core: Exp 297 -> 84 us and Log 198 -> 99 us at 16 Ki; Exp 18.8 -> 5.2 ms and Log 12.4 -> 6.1 ms at 1 Mi. Still about 3x slower than NumPy's float32 SIMD, recorded as an open defect in docs/perf.md. Co-Authored-By: Claude Opus 5.5 --- SECURITY.md | 6 ++ benchmarks/dtype_bench_test.go | 26 ++++++ docs/perf.md | 29 +++++++ docs/plan-ndarray.md | 2 +- internal/kernels/via64.go | 70 ++++++++++++++++ internal/kernels/via64_test.go | 146 +++++++++++++++++++++++++++++++++ ufunc.go | 16 ++++ 7 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 internal/kernels/via64.go create mode 100644 internal/kernels/via64_test.go diff --git a/SECURITY.md b/SECURITY.md index 6d7c7b3..2ce3685 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,12 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-11-01 (v0.30.0).** float32 `Exp` and `Log` widen their input 256 +elements at a time into a fixed stack buffer and call the float64 kernels in +place. There is no allocation proportional to the input beyond the result, +and no index arithmetic beyond slices bounded by `len(dst)`. The source is +resliced to `len(dst)`, so a shorter source panics instead of being read past. + **2026-10-31 (v0.29.1).** The A-in-place GEMM kernels now take B's row stride (`ldb`) and may read a row-major B operand in place: `kc` rows `ldb` elements apart, `NR` elements each. The wrappers bounds-check diff --git a/benchmarks/dtype_bench_test.go b/benchmarks/dtype_bench_test.go index 49d4de2..f2c9747 100644 --- a/benchmarks/dtype_bench_test.go +++ b/benchmarks/dtype_bench_test.go @@ -56,6 +56,32 @@ func BenchmarkDTypeSqrt(b *testing.B) { } } +// BenchmarkDTypeExpLog times Exp and Log on 0.25..24.25 (positive, so Log +// takes its common path; exp stays finite in float32). +func BenchmarkDTypeExpLog(b *testing.B) { + for _, dt := range []nd.DType{nd.Float64, nd.Float32} { + for _, n := range []int{1 << 14, 1 << 20} { + d := make([]float64, n) + for i := range d { + d[i] = 0.25 + float64(i%97)*0.25 + } + x0, _ := nd.FromData(d, n) + x := x0.AsType(dt) + for _, f := range []struct { + name string + op func(*nd.Array) *nd.Array + }{{"Exp", (*nd.Array).Exp}, {"Log", (*nd.Array).Log}} { + b.Run(fmt.Sprintf("%s/%v/n=%d", f.name, dt, n), func(b *testing.B) { + b.SetBytes(int64(n * dt.Size())) + for i := 0; i < b.N; i++ { + _ = f.op(x) + } + }) + } + } + } +} + func BenchmarkDTypeMax(b *testing.B) { for _, dt := range []nd.DType{nd.Float64, nd.Float32} { for _, n := range []int{1 << 14, 1 << 20} { diff --git a/docs/perf.md b/docs/perf.md index aa68100..0cb7088 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1418,6 +1418,35 @@ What does remove it: - **Tune the GC:** `GOGC` or `GOMEMLIMIT` for a program dominated by short-lived results. +### float32 `Exp` and `Log` on the float64 kernels (2026-11-01) + +float32 `Exp` and `Log` called `math.Exp` and `math.Log` per element: 6–14× +slower than NumPy's float32 SIMD on one Zen 3 core. They now run this +package's float64 kernels (the ports of Arm's optimized-routines, 0.504 ULP +worst case in float64). Elements go 256 at a time through a stack buffer: +widened (exact), computed in place, rounded once. Nothing is allocated. The +result is within 0.5 + 2⁻²⁹ float32 ULP, so correctly rounded except +next to a rounding midpoint. Overflow and underflow come from the rounding: +exp(89) is finite in float64 and rounds to +Inf. + +| float32, one Zen 3 core (`abbench`, 8 rounds) | before | after | NumPy 2.5.4 | after / NumPy | +|---|---:|---:|---:|---:| +| `Exp`, 16 Ki | 297 µs | 84 µs | 26 µs | 3.2 | +| `Log`, 16 Ki | 198 µs | 99 µs | 32 µs | 3.1 | +| `Exp`, 1 Mi | 18.8 ms | 5.2 ms | 1.73 ms | 3.0 | +| `Log`, 1 Mi | 12.4 ms | 6.1 ms | 2.02 ms | 3.0 | + +The tests hold the results to `float32(math.Exp/Log(x))`, within one ULP and +equal on at least 99.9% of 100 000 inputs. They also check the edges +(±0, ±Inf, NaN, overflow, the subnormals, log of a negative), every length +across the chunk boundaries, in-place use and the parallel split. + +**Open defect: still 3× slower than NumPy.** NumPy computes float32 exp and +log in float32 SIMD (`simd_exp_f32`/`simd_log_f32` in +`loops_exponent_log.dispatch.c.src`). Porting those kernels per architecture +is the next step. float64 `Log` is also 1.23× slower than NumPy at 16 Ki on +Zen 3 (85 µs against 69 µs), while float64 `Exp` is 16% faster. + ## SIMD coverage - **amd64 (SSE2)** ships hand-vectorized `sum` (4-accumulator `ADDPD`), `sqrt` diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index c409991..76202db 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -158,7 +158,7 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). SIMD kernels on all six 64-bit targets for `Sum` (v0.14.0–v0.17.0) and for `Add`/`Sub`/`Mul`/`Div` (v0.18.0) and `Sqrt` (v0.19.0). Still pure Go for float32: nothing in `Max`/`Min` any more (v0.20.0 to v0.25.0 - vectorised them on all six 64-bit targets, for float32 and float64), and the transcendental ufuncs. `MatMul` runs on the float64 + vectorised them on all six 64-bit targets, for float32 and float64), and the transcendental ufuncs (float32 `Exp`/`Log` run the float64 kernels since v0.30.0, 2–3.5× faster but still 3× behind NumPy's float32 SIMD). `MatMul` runs on the float64 GEMM since v0.21.0, 77–90% faster than the triple loop, and since v0.26.0 on a native float32 GEMM on arm64 (NEON 8×8) and amd64 with FMA (6×16), a further 53–61% faster, and since v0.28.0 on ppc64le (VSX 8×16, 37–59% diff --git a/internal/kernels/via64.go b/internal/kernels/via64.go new file mode 100644 index 0000000..3dcf985 --- /dev/null +++ b/internal/kernels/via64.go @@ -0,0 +1,70 @@ +package kernels + +// float32 transcendental functions on the float64 kernels. +// +// float32 Exp and Log went through math.Exp and math.Log per element, 6-14x +// slower than NumPy's float32 SIMD on one Zen 3 core. The float64 kernels of +// this package (ports of Arm's optimized-routines, 0.504 ULP worst case in +// float64) are 3-4x faster than math's. A float32 element is widened (exact), +// computed in float64 and rounded once. The result is the correctly rounded +// float32 unless the exact value lies within the float64 kernel's error +// (about half a float64 ULP, 2^-29 of a float32 ULP) of a float32 rounding +// midpoint; it is within 0.5 + 2^-29 float32 ULP everywhere. Overflow and +// underflow follow from the rounding: exp(89) is finite in float64 and +// rounds to +Inf. +// +// The elements go through a 256-element stack buffer, widened, computed in +// place and rounded back, so nothing is allocated. Exp and Log are called +// directly, not through a func value, which would make the buffer escape. + +const via64Chunk = 256 + +type via64Op uint8 + +const ( + via64Exp via64Op = iota + via64Log +) + +func via64(op via64Op, dst, src []float32) { + var buf [via64Chunk]float64 + src = src[:len(dst)] + for i := 0; i < len(dst); i += via64Chunk { + b := buf[:min(via64Chunk, len(dst)-i)] + s := src[i : i+len(b)] + for j := range b { + b[j] = float64(s[j]) + } + switch op { + case via64Exp: + Exp(b, b) + case via64Log: + Log(b, b) + } + d := dst[i : i+len(b)] + for j, v := range b { + d[j] = float32(v) + } + } +} + +// Exp32 writes exp(src[i]) into dst[i] for float32 elements. +func Exp32(dst, src []float32) { via64(via64Exp, dst, src) } + +// Log32 writes log(src[i]) into dst[i] for float32 elements. +func Log32(dst, src []float32) { via64(via64Log, dst, src) } + +// Exp32P is Exp32, split across cores above ParThreshold. +func Exp32P(dst, src []float32) { via64P(via64Exp, dst, src) } + +// Log32P is Log32, split across cores above ParThreshold. +func Log32P(dst, src []float32) { via64P(via64Log, dst, src) } + +func via64P(op via64Op, dst, src []float32) { + n := len(dst) + if n < ParThreshold { + via64(op, dst, src) + return + } + parallelFor(n, numWorkers(n), func(lo, hi int) { via64(op, dst[lo:hi], src[lo:hi]) }) +} diff --git a/internal/kernels/via64_test.go b/internal/kernels/via64_test.go new file mode 100644 index 0000000..da20764 --- /dev/null +++ b/internal/kernels/via64_test.go @@ -0,0 +1,146 @@ +package kernels + +import ( + "math" + "math/rand" + "testing" +) + +// ulps32 is the distance between two float32s in units in the last place +// (0 for equal values, including two NaNs). +func ulps32(a, b float32) int64 { + if a == b || (a != a && b != b) { + return 0 + } + ia, ib := int64(math.Float32bits(a)), int64(math.Float32bits(b)) + if ia < 0 { + ia = math.MinInt32 - ia + } + if ib < 0 { + ib = math.MinInt32 - ib + } + if d := ia - ib; d < 0 { + return -d + } else { + return d + } +} + +// TestVia64MatchesMath holds Exp32 and Log32 to float32(math.Exp/Log(x)): +// both round a float64 result once, so they may differ by one float32 ULP +// where math's float64 result (0.88 ULP) and ours (0.504) fall on opposite +// sides of a rounding midpoint, and must agree on at least 99.9% of inputs. +// Every length 0..600 crosses the 256-element chunk boundaries. +func TestVia64MatchesMath(t *testing.T) { + r := rand.New(rand.NewSource(7)) + for _, f := range []struct { + name string + k func(dst, src []float32) + ref func(float64) float64 + gen func() float32 + }{ + {"Exp", Exp32, math.Exp, func() float32 { return float32(r.Float64()*200 - 110) }}, + {"Log", Log32, math.Log, func() float32 { + return float32(math.Exp(r.Float64()*170 - 85)) + }}, + } { + const n = 100000 + src, dst := make([]float32, n), make([]float32, n) + for i := range src { + src[i] = f.gen() + } + f.k(dst, src) + var off int + for i, x := range src { + want := float32(f.ref(float64(x))) + switch d := ulps32(dst[i], want); { + case d > 1: + t.Fatalf("%s(%v) = %v, math gives %v (%d ULP)", f.name, x, dst[i], want, d) + case d == 1: + off++ + } + } + if off > n/1000 { + t.Errorf("%s: %d of %d differ from math by one ULP", f.name, off, n) + } + for m := 0; m <= 600; m++ { + d := make([]float32, m) + f.k(d, src[:m]) + for i := range d { + if d[i] != dst[i] { + t.Fatalf("%s length %d [%d]: %v != %v", f.name, m, i, d[i], dst[i]) + } + } + } + } +} + +// TestVia64Specials checks the edges that come from rounding the float64 +// result: overflow to +Inf, underflow through the subnormals to 0, and the +// IEEE special values. +func TestVia64Specials(t *testing.T) { + inf, nan := float32(math.Inf(1)), float32(math.NaN()) + negZero := float32(math.Copysign(0, -1)) + for _, c := range []struct { + name string + k func(dst, src []float32) + in, out float32 + }{ + {"Exp", Exp32, 0, 1}, + {"Exp", Exp32, negZero, 1}, + {"Exp", Exp32, 89, inf}, // finite in float64, overflows float32 + {"Exp", Exp32, -inf, 0}, + {"Exp", Exp32, inf, inf}, + {"Exp", Exp32, -104, 0}, // below the smallest float32 subnormal + {"Exp", Exp32, -100, float32(math.Exp(-100))}, + {"Log", Log32, 1, 0}, + {"Log", Log32, 0, -inf}, + {"Log", Log32, negZero, -inf}, + {"Log", Log32, inf, inf}, + {"Log", Log32, math.SmallestNonzeroFloat32, float32(math.Log(math.SmallestNonzeroFloat32))}, + } { + d := []float32{0} + c.k(d, []float32{c.in}) + if d[0] != c.out || math.Signbit(float64(d[0])) != math.Signbit(float64(c.out)) { + t.Errorf("%s(%v) = %v, want %v", c.name, c.in, d[0], c.out) + } + } + for _, in := range []float32{nan, -1, -inf} { + d := []float32{0} + Log32(d, []float32{in}) + if d[0] == d[0] { + t.Errorf("Log(%v) = %v, want NaN", in, d[0]) + } + } + d := []float32{0} + Exp32(d, []float32{nan}) + if d[0] == d[0] { + t.Errorf("Exp(NaN) = %v", d[0]) + } +} + +// TestVia64InPlaceAndParallel: dst may be src, and the parallel entries give +// the serial bits. +func TestVia64InPlaceAndParallel(t *testing.T) { + src := make([]float32, 3*ParThreshold+17) + for i := range src { + src[i] = float32(i%97) * 0.25 + } + want := make([]float32, len(src)) + Exp32(want, src) + got := append([]float32(nil), src...) + Exp32(got, got) + par := make([]float32, len(src)) + withMaxProcs(4, func() { Exp32P(par, src) }) + for i := range want { + if got[i] != want[i] || par[i] != want[i] { + t.Fatalf("[%d]: in place %v, parallel %v, serial %v", i, got[i], par[i], want[i]) + } + } + Log32P(par, src) // the serial path below ParThreshold is covered by the tests above + for i, x := range src { + if par[i] != float32(math.Log(float64(x))) && ulps32(par[i], float32(math.Log(float64(x)))) > 1 { + t.Fatalf("Log32P [%d]: %v", i, par[i]) + } + } +} diff --git a/ufunc.go b/ufunc.go index c17d1fc..a8c4d83 100644 --- a/ufunc.go +++ b/ufunc.go @@ -59,6 +59,9 @@ func (a *Array) SqrtInto(out *Array) error { // element, and finite up to ln(MaxFloat64) on amd64, where math.Exp returns // +Inf from x ~ 709.436 (golang/go#81995; Go 1.26.8 and 1.27.1). func (a *Array) Exp() *Array { + if a.dtype == Float32 { + return a.unary32(kernels.Exp32P) + } if a.dtype != Float64 { return a.mathUnary(uExp, (*Array).Exp) } @@ -76,12 +79,25 @@ func (a *Array) Exp() *Array { // correct for subnormal inputs on amd64, where math.Log is not (log(5e-324) // gives -709.09 instead of -744.44; golang/go#56600). func (a *Array) Log() *Array { + if a.dtype == Float32 { + return a.unary32(kernels.Log32P) + } if a.dtype != Float64 { return a.mathUnary(uLog, (*Array).Log) } return a.unary(kernels.LogP) } +// unary32 applies a float32 kernel to a Float32 array: the float64 kernels of +// Exp and Log, widened and rounded once (kernels.Exp32P, Log32P), at the +// float64 kernels' speed instead of math.Exp's and math.Log's. +func (a *Array) unary32(f func(dst, src []float32)) *Array { + src := a.contiguousStore().([]float32) + dst := make([]float32, len(src)) + f(dst, src) + return a.result(dst, append([]int(nil), a.shape...)) +} + // Log2 returns the elementwise base-2 logarithm, within 2 ULP of numpy's. func (a *Array) Log2() *Array { if a.dtype != Float64 { From 7d65e22a2a44b3f8f99bfdf0e4fcfa9ae40f9a0b Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 1 Nov 2026 11:14:55 +0100 Subject: [PATCH 2/2] docs: the float32 Exp/Log release is v0.31.0 (v0.30.0 was taken by #84) Co-Authored-By: Claude Opus 5.5 --- SECURITY.md | 2 +- docs/plan-ndarray.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 2ce3685..eeb477a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,7 +66,7 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log -**2026-11-01 (v0.30.0).** float32 `Exp` and `Log` widen their input 256 +**2026-11-01 (v0.31.0).** float32 `Exp` and `Log` widen their input 256 elements at a time into a fixed stack buffer and call the float64 kernels in place. There is no allocation proportional to the input beyond the result, and no index arithmetic beyond slices bounded by `len(dst)`. The source is diff --git a/docs/plan-ndarray.md b/docs/plan-ndarray.md index 76202db..50c68f8 100644 --- a/docs/plan-ndarray.md +++ b/docs/plan-ndarray.md @@ -158,7 +158,7 @@ suite (plus the per-arch CI jobs already wired in `.github/workflows/ci.yml`). SIMD kernels on all six 64-bit targets for `Sum` (v0.14.0–v0.17.0) and for `Add`/`Sub`/`Mul`/`Div` (v0.18.0) and `Sqrt` (v0.19.0). Still pure Go for float32: nothing in `Max`/`Min` any more (v0.20.0 to v0.25.0 - vectorised them on all six 64-bit targets, for float32 and float64), and the transcendental ufuncs (float32 `Exp`/`Log` run the float64 kernels since v0.30.0, 2–3.5× faster but still 3× behind NumPy's float32 SIMD). `MatMul` runs on the float64 + vectorised them on all six 64-bit targets, for float32 and float64), and the transcendental ufuncs (float32 `Exp`/`Log` run the float64 kernels since v0.31.0, 2–3.5× faster but still 3× behind NumPy's float32 SIMD). `MatMul` runs on the float64 GEMM since v0.21.0, 77–90% faster than the triple loop, and since v0.26.0 on a native float32 GEMM on arm64 (NEON 8×8) and amd64 with FMA (6×16), a further 53–61% faster, and since v0.28.0 on ppc64le (VSX 8×16, 37–59%