From 008774d842e88dc6fd4b31e465c4f0fcee78f29e Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 2 Nov 2026 11:44:52 +0100 Subject: [PATCH 1/4] kernels: expCR, Arm's correctly rounded expf algorithm, proven over every float32 expCR transcribes the double-precision exp of Arm optimized-routines' cr_expf (math/aarch64/advsimd/cr_expf.c, MIT) one operation per line, every vfmaq as math.FMA, so a SIMD kernel can repeat it bit for bit: x/ln2 rounded to 1/64 by a shift, a 64-entry table of 2^(i/64) (re-derived at 300 bits by TestExpCRTable), and a degree-5 polynomial. Its float64 error is up to 1.006 ULP, so the exhaustive certifier takes a per-function margin (4 ULP here; 2 for the 0.504-ULP kernels). Over all 2^32 float32 inputs: 7 decided by the 300-bit reference, 0 incorrectly rounded. No exception table needed. This is the target for the SIMD float32 exp. Co-Authored-By: Claude Opus 5.5 --- internal/kernels/expcr.go | 94 +++++++++++++++++++++++ internal/kernels/expcr_test.go | 53 +++++++++++++ internal/kernels/via64_exhaustive_test.go | 21 +++-- 3 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 internal/kernels/expcr.go create mode 100644 internal/kernels/expcr_test.go diff --git a/internal/kernels/expcr.go b/internal/kernels/expcr.go new file mode 100644 index 0000000..dee70cf --- /dev/null +++ b/internal/kernels/expcr.go @@ -0,0 +1,94 @@ +package kernels + +import "math" + +// expCR is the double-precision exp of Arm's correctly rounded vector expf +// (optimized-routines, math/aarch64/advsimd/cr_expf.c and cr_expf_utils.h, +// MIT), transcribed one operation per line so that a SIMD kernel can repeat +// it bit for bit: every vfmaq/vfmsq is math.FMA. +// +// x/ln2 is rounded to a multiple of 1/64 by adding a shift whose ULP is 2^-6; +// the low 6 bits of the sum index a table of 2^(i/64), and the bits above +// (with the bias 0x3ff folded into the shift) are the exponent of the scale. +// exp(r) - 1 on |r| <= ln2/128 is a degree-5 polynomial. Valid for +// |x| <= 512, which covers every float32 whose exp is neither 0 nor +Inf. +func expCR(x float64) float64 { + z := math.FMA(x, crInvLn2, crShift) + n := z - crShift + r := math.FMA(-n, crLn2Hi, x) + r = math.FMA(-n, crLn2Lo, r) + r2 := r * r + p01 := math.FMA(r, crC1, crC0) + p23 := math.FMA(r, crC3, crC2) + p04 := math.FMA(r2, p23, p01) + y := math.FMA(r2, p04, r) + u := math.Float64bits(z) + s := math.Float64frombits(u<<46&0xfff0000000000000 | crMant[u&63]) + return math.FMA(s, y, s) +} + +const ( + crShift = 0x1.800000000ffc0p+46 + crInvLn2 = 0x1.71547652b82fep+0 + crLn2Hi = 0x1.62e42fefa39efp-1 + crLn2Lo = 0x1.abc9e3b39803fp-56 + crC0 = 0x1.fffffffffdbcep-2 + crC1 = 0x1.55555555543c2p-3 + crC2 = 0x1.555573c64f2e3p-5 + crC3 = 0x1.111126b4eff73p-7 +) + +// crMant are the mantissas of 2^(i/64), i = 0..63 (cr_expf_utils.h). +var crMant = [64]uint64{ + 0x0000000000000, 0x02c9a3e778061, 0x059b0d3158574, 0x0874518759bc8, + 0x0b5586cf9890f, 0x0e3ec32d3d1a2, 0x11301d0125b51, 0x1429aaea92de0, + 0x172b83c7d517b, 0x1a35beb6fcb75, 0x1d4873168b9aa, 0x2063b88628cd6, + 0x2387a6e756238, 0x26b4565e27cdd, 0x29e9df51fdee1, 0x2d285a6e4030b, + 0x306fe0a31b715, 0x33c08b26416ff, 0x371a7373aa9cb, 0x3a7db34e59ff7, + 0x3dea64c123422, 0x4160a21f72e2a, 0x44e086061892d, 0x486a2b5c13cd0, + 0x4bfdad5362a27, 0x4f9b2769d2ca7, 0x5342b569d4f82, 0x56f4736b527da, + 0x5ab07dd485429, 0x5e76f15ad2148, 0x6247eb03a5585, 0x6623882552225, + 0x6a09e667f3bcd, 0x6dfb23c651a2f, 0x71f75e8ec5f74, 0x75feb564267c9, + 0x7a11473eb0187, 0x7e2f336cf4e62, 0x82589994cce13, 0x868d99b4492ed, + 0x8ace5422aa0db, 0x8f1ae99157736, 0x93737b0cdc5e5, 0x97d829fde4e50, + 0x9c49182a3f090, 0xa0c667b5de565, 0xa5503b23e255d, 0xa9e6b5579fdbf, + 0xae89f995ad3ad, 0xb33a2b84f15fb, 0xb7f76f2fb5e47, 0xbcc1e904bc1d2, + 0xc199bdd85529c, 0xc67f12e57d14b, 0xcb720dcef9069, 0xd072d4a07897c, + 0xd5818dcfba487, 0xda9e603db3285, 0xdfc97337b9b5f, 0xe502ee78b3ff6, + 0xea4afa2a490da, 0xefa1bee615a27, 0xf50765b6e4540, 0xfa7c1819e90d8, +} + +// expCRSlice writes expCR(src[i]) into dst[i] for |src[i]| <= 512, and the +// limits outside: +Inf above, 0 below, NaN for NaN (the float32 results +// there are the same). +func expCRSlice(dst, src []float64) { + dst = dst[:len(src)] + for i, x := range src { + switch { + case math.Abs(x) <= 512: + dst[i] = expCR(x) + case x > 0: + dst[i] = math.Inf(1) + case x < 0: + dst[i] = 0 + default: + dst[i] = x // NaN + } + } +} + +// exp32CR is float32 exp on expCR, rounded once. +func exp32CR(dst, src []float32) { + src = src[:len(dst)] + var buf [via64Chunk]float64 + for i := 0; i < len(dst); i += via64Chunk { + b := buf[:min(via64Chunk, len(dst)-i)] + for j := range b { + b[j] = float64(src[i+j]) + } + expCRSlice(b, b) + for j, v := range b { + dst[i+j] = float32(v) + } + } +} diff --git a/internal/kernels/expcr_test.go b/internal/kernels/expcr_test.go new file mode 100644 index 0000000..dea44ae --- /dev/null +++ b/internal/kernels/expcr_test.go @@ -0,0 +1,53 @@ +package kernels + +import ( + "math" + "math/big" + "math/rand" + "testing" +) + +// TestExpCRULP bounds expCR's float64 error against the 300-bit reference +// over the inputs float32 exp needs (|x| <= 104, the float32 overflow and +// underflow edges included). Measured worst: 1.006 ULP. The exhaustive +// float32 test certifies an expCR result only when it is more than 4 float64 +// ULP from a rounding midpoint, so this bound must stay below 2 (half the +// margin). +func TestExpCRULP(t *testing.T) { + r := rand.New(rand.NewSource(11)) + var worst float64 + var at float64 + for i := 0; i < 200000; i++ { + x := float64(float32(r.Float64()*193 - 104)) // a float32 input, widened + if e := ulpErr(expCR(x), refExp(x)); e > worst { + worst, at = e, x + } + } + t.Logf("expCR: worst %.3f float64 ULP at x = %v", worst, at) + if worst >= 2 { + t.Errorf("expCR error %.3f ULP at %v: the certifier's 4-ULP margin needs it below 2", worst, at) + } +} + +// TestExp32CRCorrectlyRounded: the prototype, through the same exhaustive +// certifier as Exp32 (NDARRAY_EXHAUSTIVE=1 for all 2^32 inputs). +func TestExp32CRCorrectlyRounded(t *testing.T) { + runExhaustive(t, exhaustiveCase{"expCR", exp32CR, expCRSlice, refExp, exp32Special, 4}, exhaustiveStride()) +} + +// TestExpCRTable re-derives the 64 mantissas of 2^(i/64) from the 300-bit +// reference: the table was copied, not trusted. +func TestExpCRTable(t *testing.T) { + root := new(big.Float).SetPrec(refPrec).SetInt64(2) + for range 6 { // 2^(1/64) + root.Sqrt(root) + } + e := new(big.Float).SetPrec(refPrec).SetInt64(1) + for i, m := range crMant { + want, _ := e.Float64() + if got := math.Float64frombits(0x3ff0000000000000 | m); got != want { + t.Errorf("crMant[%d] gives %v, the nearest double to 2^(%d/64) is %v", i, got, i, want) + } + e.Mul(e, root) + } +} diff --git a/internal/kernels/via64_exhaustive_test.go b/internal/kernels/via64_exhaustive_test.go index feda285..8ba90f3 100644 --- a/internal/kernels/via64_exhaustive_test.go +++ b/internal/kernels/via64_exhaustive_test.go @@ -29,13 +29,17 @@ import ( // certified32 rounds the float64 intermediate y to float32 and reports // whether that rounding is provably the correct rounding of the exact value. -func certified32(y float64) (float32, bool) { +func certified32(y float64) (float32, bool) { return certified32Margin(y, 2) } + +// certified32Margin is certified32 for an intermediate whose error may reach +// margin/2 float64 ULP (the margin is kept at twice the measured worst). +func certified32Margin(y float64, margin float64) (float32, bool) { f := float32(y) ay := math.Abs(y) if math.IsNaN(y) || math.IsInf(y, 0) || ay > 0x1p129 || ay < 0x1p-152 { return f, true } - eps := 2 * (math.Nextafter(ay, math.Inf(1)) - ay) + eps := margin * (math.Nextafter(ay, math.Inf(1)) - ay) if math.IsInf(float64(f), 0) { // y rounded to ±Inf although |y| <= 2^129: the only boundary is // half an ULP past MaxFloat32, where ties-to-even goes to Inf. @@ -66,6 +70,9 @@ type exhaustiveCase struct { // special returns the expected result for inputs the reference does not // take (NaN, ±Inf, and for log 0 and negatives), and ok=false otherwise. special func(x float32) (want float32, isNaN, ok bool) + // margin is the certifier's distance to a midpoint, in float64 ULP + // (0 means 2, for the 0.504-ULP kernels). + margin float64 } func exp32Special(x float32) (float32, bool, bool) { @@ -140,7 +147,11 @@ func runExhaustive(t *testing.T, c exhaustiveCase, stride uint64) { } want = w } else { - f, sure := certified32(y64[i]) + margin := c.margin + if margin == 0 { + margin = 2 + } + f, sure := certified32Margin(y64[i], margin) if sure { want = f } else { @@ -195,11 +206,11 @@ func exhaustiveStride() uint64 { } func TestExp32CorrectlyRounded(t *testing.T) { - runExhaustive(t, exhaustiveCase{"exp", Exp32, Exp, refExp, exp32Special}, exhaustiveStride()) + runExhaustive(t, exhaustiveCase{"exp", Exp32, Exp, refExp, exp32Special, 2}, exhaustiveStride()) } func TestLog32CorrectlyRounded(t *testing.T) { - runExhaustive(t, exhaustiveCase{"log", Log32, Log, refLog, log32Special}, exhaustiveStride()) + runExhaustive(t, exhaustiveCase{"log", Log32, Log, refLog, log32Special, 2}, exhaustiveStride()) } // TestLog32Exceptions runs Log32 on every input of the exception table (the From cb5f3cbeb2625b2251c302378e4de3bf4c1be281 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 2 Nov 2026 11:52:42 +0100 Subject: [PATCH 2/4] perf: float32 Exp in AVX2, correctly rounded and as fast as NumPy Every architecture's float32 Exp now runs expCR, the transcription of Arm's correctly rounded expf (proven over all 2^32 inputs in the previous commit). On amd64 with AVX2+FMA, exp32CRAVX2 (go-asmgen) repeats its operations eight floats at a time: VCVTPS2PD, the FMA sequence, VPGATHERQQ for the 2^(i/64) table, the exponent from z<<46, then one VCVTPD2PS. Inputs are clamped to +-512 (same float32 result) and NaN lanes restored. The kernel is bit-identical to expCR (100 000 inputs, specials, lengths 0-70) and fenced. On Zen 3 the exhaustive test, which runs it, finds 0 incorrectly rounded of 2^32. Zen 3, one core, abbench 8 rounds: 16 Ki 90 -> 26.9 us (NumPy 27.1), 1 Mi 5.8 -> 1.48 ms (NumPy 1.66). Parity, then 11% faster than NumPy, which is at 2.52 ULP; we are correctly rounded. Log moves to its own log32 (unchanged behaviour); the old via64 exp path is gone. Co-Authored-By: Claude Opus 5.5 --- SECURITY.md | 7 ++ docs/perf.md | 44 +++++++++-- internal/kernels/asmgen/amd64/gen.go | 78 ++++++++++++++++++++ internal/kernels/expcr_amd64.go | 50 +++++++++++++ internal/kernels/expcr_amd64_test.go | 59 +++++++++++++++ internal/kernels/expcr_other.go | 7 ++ internal/kernels/sum_amd64.s | 89 +++++++++++++++++++++++ internal/kernels/via64.go | 82 ++++++++------------- internal/kernels/via64_exhaustive_test.go | 3 +- 9 files changed, 363 insertions(+), 56 deletions(-) create mode 100644 internal/kernels/expcr_amd64.go create mode 100644 internal/kernels/expcr_amd64_test.go create mode 100644 internal/kernels/expcr_other.go diff --git a/SECURITY.md b/SECURITY.md index 4b49f23..020f9d4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -66,6 +66,13 @@ standard library's known vulnerabilities; `govulncheck ./...` reports none. ## Audit log +**2026-11-02 (v0.33.0).** The AVX2 float32 exp kernel (`exp32CRAVX2`) +reads and writes `n` floats, a positive multiple of 8 chosen by the Go +wrapper. It gathers table entries at `z & 63`, so the index cannot leave the +64-entry table, whatever the input. It reads a constant block the wrapper +passes. It is gated on AVX2 and FMA (CPU and OS) and fenced with PROT_NONE +on both sides of each operand. + **2026-11-01 (v0.32.0).** float32 `Log` checks each result for a float32 rounding midpoint with one integer comparison. On a hit it looks the input up by binary search in a fixed 24-entry table, compiled in and read only. diff --git a/docs/perf.md b/docs/perf.md index 2ecd0a9..3e25f1c 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1481,11 +1481,45 @@ So float32 `Exp` and `Log` are **correctly rounded**: every result is the float32 nearest the exact value. NumPy's float32 SIMD kernels are documented at 2.52 and 3.83 ULP in their sources. -**Open defect: still 3× slower than NumPy.** NumPy computes float32 exp and -log in float32 SIMD (`simd_exp_f32`/`simd_log_f32` in -`loops_exponent_log.dispatch.c.src`). Porting those kernels per architecture -is the next step. float64 `Log` is also 1.23× slower than NumPy at 16 Ki on -Zen 3 (85 µs against 69 µs), while float64 `Exp` is 16% faster. +#### float32 `Exp` in AVX2, correctly rounded and as fast as NumPy (2026-11-02) + +NumPy's float32 exp gets its speed in float32 SIMD and gives up accuracy to +do it: 2.52 ULP in its source. We wanted both. Arm's optimized-routines has a +vector `expf` that is correctly rounded (`math/aarch64/advsimd/cr_expf.c`, +MIT). It widens to float64 and uses a 64-entry table of 2^(i/64) plus a +degree-5 polynomial, cheap enough for SIMD. + +- **`expCR`** transcribes it one operation per line, every fused + multiply-add as `math.FMA`, so a kernel can repeat it bit for bit. Its + table is re-derived at 300 bits (`TestExpCRTable`). Its float64 error is + 1.006 ULP, so the exhaustive certifier uses a 4-ULP margin for it. + Over all 2³² float32 inputs: 7 decided by the 300-bit reference, + **0 incorrectly rounded**, with no exception table needed. +- **`exp32CRAVX2`** (go-asmgen, amd64 with AVX2 and FMA) runs the same + operations eight floats at a time. It widens with `VCVTPS2PD`, gathers the + table with `VPGATHERQQ`, assembles the exponent from `z << 46`, and rounds + once with `VCVTPD2PS`. Inputs are clamped to [−512, 512], which keeps the + float32 result: +Inf above, 0 below. NaN lanes are restored from the input. + It is bit-identical to `expCR` on 100 000 inputs, every special value and + every length 0–70, it is fenced, and on a Zen 3 the exhaustive test, which + runs it, finds **0 incorrectly rounded** of 4 294 967 296. +- Every architecture now runs `expCR`: the AVX2 kernel where there is one, + the Go transcription elsewhere. So the bits are the same everywhere. + +| float32 `Exp`, one Zen 3 core | v0.32.0 | AVX2 | NumPy 2.5.4 | AVX2 / NumPy | +|---|---:|---:|---:|---:| +| 16 Ki | 90 µs | **26.9 µs** | 27.1 µs | **0.99** | +| 1 Mi | 5.8 ms | **1.48 ms** | 1.66 ms | **0.89** | + +These are +235% and +289% in throughput (`abbench`, 8 rounds, p = 0.000). The +host was at load 59 on 128 cores; NumPy ran in the same session on the same +pinned core. + +**Open defect: float32 `Log` is still 3× slower than NumPy** (`simd_log_f32`, +3.83 ULP in its source). The next step is the same treatment for log, and +the other architectures for exp. float64 `Log` is also 1.23× slower than +NumPy at 16 Ki on Zen 3 (85 µs against 69 µs), while float64 `Exp` is 16% +faster. ## SIMD coverage diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index e1edbe4..35aaa3a 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -54,6 +54,8 @@ func main() { f.Add(dotFMAKernel()) f.Add(gemmSSE2HalfKernel()) f.Add(amd64.FeatureProbe("hasFMA", amd64.FMA)) + f.Add(amd64.FeatureProbe("hasAVX2", amd64.AVX2)) + f.Add(exp32CRKernel()) f.Add(blocks8Kernel()) f.Add(bin32Kernel("add32SSE2", "ADDPS", "ADDSS")) f.Add(bin32Kernel("sub32SSE2", "SUBPS", "SUBSS")) @@ -368,6 +370,82 @@ func sqrt32Kernel() *emit.Function { return b.Func() } +// exp32CRKernel builds exp32CRAVX2(dst, src *float32, n int, k *expCRConsts) +// for n a positive multiple of 8: dst[i] = float32(expCR(clamp(src[i]))), the +// correctly rounded float32 exp, bit for bit the Go expCR (kernels/expcr.go, +// a transcription of Arm optimized-routines' cr_expf). +// +// Eight floats per iteration: clamped to [-512, 512] (exp of the clamped +// value rounds to the same float32 +Inf or 0 as the input's), widened to two +// groups of four doubles, and each group run through expCR's operations in +// the same order and with the same fusing: z = x*invln2 + shift, n = z - +// shift, r = x - n*ln2hi - n*ln2lo (VFNMADD231PD), the degree-5 polynomial, +// the scale assembled from the table entry z&63 (VPGATHERQQ) and the +// exponent bits z<<46, and s*y + s; then rounded once to float32. NaN lanes +// are restored from the input at the end (the clamp turned them into -512). +// +// The constants are 4-lane vectors in the Go block k (offsets below), used as +// memory operands, so all sixteen YMM registers stay free. Needs AVX2 (the +// gather) and FMA. +func exp32CRKernel() *emit.Function { + sig := amd64.Layout( + []string{"dst", "src", "n", "k"}, + []amd64.Type{amd64.Ptr, amd64.Ptr, amd64.Int64, amd64.Ptr}, + nil, nil, + ) + const ( + shift, invLn2, ln2Hi, ln2Lo = 0, 32, 64, 96 + c0, c1, c2, c3 = 128, 160, 192, 224 + idxMask, expMask = 256, 288 + lo, hi = 320, 352 + table = 384 + ) + b := amd64.NewFunc("exp32CRAVX2", sig, 0) + b.LoadArg("dst", "DI").LoadArg("src", "SI").LoadArg("n", "CX").LoadArg("k", "R8") + b.Raw(fmt.Sprintf("LEAQ %d(R8), R9", table)) // the 64 mantissas of 2^(i/64) + // half runs expCR on the four doubles in h, leaving the four float32 + // results in out. Temporaries Y4-Y10. + half := func(h, out string) { + b.Raw(fmt.Sprintf("VMOVUPD %d(R8), Y4", shift)) + b.Raw(fmt.Sprintf("VFMADD231PD %d(R8), %s, Y4", invLn2, h)) // z = x*invln2 + shift + b.Raw(fmt.Sprintf("VSUBPD %d(R8), Y4, Y5", shift)) // n = z - shift + b.Raw(fmt.Sprintf("VFNMADD231PD %d(R8), Y5, %s", ln2Hi, h)) // r = x - n*ln2hi + b.Raw(fmt.Sprintf("VFNMADD231PD %d(R8), Y5, %s", ln2Lo, h)) // r -= n*ln2lo + b.Raw(fmt.Sprintf("VMULPD %s, %s, Y6", h, h)) // r2 + b.Raw(fmt.Sprintf("VMOVUPD %d(R8), Y7", c0)) + b.Raw(fmt.Sprintf("VFMADD231PD %d(R8), %s, Y7", c1, h)) // p01 = r*c1 + c0 + b.Raw(fmt.Sprintf("VMOVUPD %d(R8), Y8", c2)) + b.Raw(fmt.Sprintf("VFMADD231PD %d(R8), %s, Y8", c3, h)) // p23 = r*c3 + c2 + b.Raw("VFMADD231PD Y8, Y6, Y7") // p04 = r2*p23 + p01 + b.Raw(fmt.Sprintf("VFMADD231PD Y7, Y6, %s", h)) // y = r2*p04 + r + b.Raw(fmt.Sprintf("VPAND %d(R8), Y4, Y5", idxMask)) // z & 63 + b.Raw("VPCMPEQQ Y9, Y9, Y9") // gather every lane + b.Raw("VPGATHERQQ Y9, (R9)(Y5*8), Y10") // mantissa of 2^(i/64) + b.Raw("VPSLLQ $46, Y4, Y4") + b.Raw(fmt.Sprintf("VPAND %d(R8), Y4, Y4", expMask)) // exponent and sign bits + b.Raw("VPOR Y10, Y4, Y4") // s + b.Raw(fmt.Sprintf("VFMADD231PD %s, Y4, Y4", h)) // s*y + s + b.Raw(fmt.Sprintf("VCVTPD2PSY Y4, %s", out)) // round once to float32 + } + b.Raw("eloop:") + b.Raw("VMOVUPS (SI), Y0") + b.Raw("VCMPPS $3, Y0, Y0, Y15") // unordered: the NaN lanes + b.Raw(fmt.Sprintf("VMAXPS %d(R8), Y0, Y1", lo)) + b.Raw(fmt.Sprintf("VMINPS %d(R8), Y1, Y1", hi)) + b.Raw("VCVTPS2PD X1, Y2") + b.Raw("VEXTRACTF128 $1, Y1, X3") + b.Raw("VCVTPS2PD X3, Y3") + half("Y2", "X11") + half("Y3", "X12") + b.Raw("VINSERTF128 $1, X12, Y11, Y11") + b.Raw("VBLENDVPS Y15, Y0, Y11, Y11") // NaN lanes: the input NaN + b.Raw("VMOVUPS Y11, (DI)") + b.Raw("ADDQ $32, SI").Raw("ADDQ $32, DI").Raw("SUBQ $8, CX").Raw("JNZ eloop") + b.Raw("VZEROUPPER") + b.Ret() + return b.Func() +} + // bin32Kernel builds NAME(dst, a, b *float32, n int): dst[i] = a[i] OP b[i], // 16 floats per iteration over four XMM registers with the packed op, then a // scalar tail with sca. Each lane is one IEEE float32 operation, so the result diff --git a/internal/kernels/expcr_amd64.go b/internal/kernels/expcr_amd64.go new file mode 100644 index 0000000..66f8342 --- /dev/null +++ b/internal/kernels/expcr_amd64.go @@ -0,0 +1,50 @@ +package kernels + +// expCRConsts is the constant block exp32CRAVX2 reads (asmgen/amd64/gen.go, +// exp32CRKernel): each constant as a 4-lane vector at the offset the kernel +// uses, then the clamp bounds as 8 floats, then the 64 table mantissas. +type expCRConsts struct { + shift, invLn2, ln2Hi, ln2Lo [4]float64 // offsets 0, 32, 64, 96 + c0, c1, c2, c3 [4]float64 // 128, 160, 192, 224 + idxMask, expMask [4]uint64 // 256, 288 + lo, hi [8]float32 // 320, 352 + table [64]uint64 // 384 +} + +var expCRK = func() *expCRConsts { + k := new(expCRConsts) + for i := range 4 { + k.shift[i], k.invLn2[i], k.ln2Hi[i], k.ln2Lo[i] = crShift, crInvLn2, crLn2Hi, crLn2Lo + k.c0[i], k.c1[i], k.c2[i], k.c3[i] = crC0, crC1, crC2, crC3 + k.idxMask[i], k.expMask[i] = 63, 0xfff0000000000000 + } + for i := range 8 { + k.lo[i], k.hi[i] = -512, 512 + } + k.table = crMant + return k +}() + +// useExp32AVX2 selects the AVX2 kernel; a var so tests can force the Go path. +var useExp32AVX2 = hasAVX2() && useFMA + +// exp32Fast is the correctly rounded float32 exp: blocks of 8 on the AVX2 +// kernel, the rest (and every element without AVX2+FMA) on exp32CR. Both +// compute expCR's operations in the same order, so they agree bit for bit. +func exp32Fast(dst, src []float32) { + src = src[:len(dst)] + n := 0 + if useExp32AVX2 { + n = len(dst) &^ 7 + if n > 0 { + exp32CRAVX2(&dst[0], &src[0], n, expCRK) + } + } + exp32CR(dst[n:], src[n:]) +} + +// hasAVX2 reports CPU and OS support for AVX2 (go-asmgen FeatureProbe). +func hasAVX2() bool + +//go:noescape +func exp32CRAVX2(dst, src *float32, n int, k *expCRConsts) diff --git a/internal/kernels/expcr_amd64_test.go b/internal/kernels/expcr_amd64_test.go new file mode 100644 index 0000000..8ee85e0 --- /dev/null +++ b/internal/kernels/expcr_amd64_test.go @@ -0,0 +1,59 @@ +package kernels + +import ( + "math" + "math/rand" + "testing" +) + +// TestExp32AVX2MatchesGo holds the AVX2 kernel to exp32CR bit for bit (NaN +// to NaN) on random inputs across the float32 range, the special values and +// the clamp and overflow edges, at every length 0..70 (blocks of 8 and the Go +// tail). +func TestExp32AVX2MatchesGo(t *testing.T) { + if !useExp32AVX2 { + t.Skip("no AVX2+FMA here") + } + r := rand.New(rand.NewSource(3)) + src := make([]float32, 0, 100000) + for _, v := range []float32{0, float32(math.Copysign(0, -1)), 1, -1, 88.72283, 88.72284, 89, -103.972, -103.973, -104, + 511.9, 512, 512.1, -512, -512.1, 1e30, -1e30, float32(math.Inf(1)), float32(math.Inf(-1)), + float32(math.NaN()), math.Float32frombits(0xffc00000), math.SmallestNonzeroFloat32} { + src = append(src, v) + } + for len(src) < cap(src) { + src = append(src, math.Float32frombits(r.Uint32())) + } + want := make([]float32, len(src)) + exp32CR(want, src) + got := make([]float32, len(src)) + exp32Fast(got, src) + for i := range src { + if math.Float32bits(got[i]) != math.Float32bits(want[i]) && !(got[i] != got[i] && want[i] != want[i]) { + t.Fatalf("exp(0x%08x): AVX2 0x%08x, Go 0x%08x", math.Float32bits(src[i]), math.Float32bits(got[i]), math.Float32bits(want[i])) + } + } + for n := 0; n <= 70; n++ { + d := make([]float32, n) + exp32Fast(d, src[:n]) + for i := range d { + if math.Float32bits(d[i]) != math.Float32bits(want[i]) && !(d[i] != d[i] && want[i] != want[i]) { + t.Fatalf("length %d [%d]: 0x%08x, want 0x%08x", n, i, math.Float32bits(d[i]), math.Float32bits(want[i])) + } + } + } +} + +// TestExp32AVX2StaysInside fences src and dst of the kernel. +func TestExp32AVX2StaysInside(t *testing.T) { + if !useExp32AVX2 { + t.Skip("no AVX2+FMA here") + } + for n := 8; n <= 64; n += 8 { + for _, s := range fences32(t, n) { + for _, d := range fences32(t, n) { + exp32CRAVX2(&d[0], &s[0], n, expCRK) + } + } + } +} diff --git a/internal/kernels/expcr_other.go b/internal/kernels/expcr_other.go new file mode 100644 index 0000000..ed70ad1 --- /dev/null +++ b/internal/kernels/expcr_other.go @@ -0,0 +1,7 @@ +//go:build !amd64 + +package kernels + +// exp32Fast is the correctly rounded float32 exp: expCR in Go on these +// architectures (no SIMD kernel yet). +func exp32Fast(dst, src []float32) { exp32CR(dst, src) } diff --git a/internal/kernels/sum_amd64.s b/internal/kernels/sum_amd64.s index 7c1bd88..de6f8f0 100644 --- a/internal/kernels/sum_amd64.s +++ b/internal/kernels/sum_amd64.s @@ -1692,6 +1692,95 @@ TEXT ·hasFMA(SB), NOSPLIT, $0-1 hasFMA_unsupported: RET +TEXT ·hasAVX2(SB), NOSPLIT, $0-1 + MOVB $0, ret+0(FP) + XORL AX, AX + CPUID + CMPL AX, $7 + JL hasAVX2_unsupported + MOVL $1, AX + CPUID + ANDL $0x18000000, CX // AVX and OSXSAVE + CMPL CX, $0x18000000 + JNE hasAVX2_unsupported + XORL CX, CX + XGETBV + ANDL $6, AX // XMM and YMM state enabled in XCR0 + CMPL AX, $6 + JNE hasAVX2_unsupported + MOVL $7, AX + XORL CX, CX + CPUID + TESTL $0x20, BX // AVX2 + JZ hasAVX2_unsupported + MOVB $1, ret+0(FP) +hasAVX2_unsupported: + RET + +TEXT ·exp32CRAVX2(SB), NOSPLIT, $0-32 + MOVQ dst+0(FP), DI + MOVQ src+8(FP), SI + MOVQ n+16(FP), CX + MOVQ k+24(FP), R8 + LEAQ 384(R8), R9 + eloop: + VMOVUPS (SI), Y0 + VCMPPS $3, Y0, Y0, Y15 + VMAXPS 320(R8), Y0, Y1 + VMINPS 352(R8), Y1, Y1 + VCVTPS2PD X1, Y2 + VEXTRACTF128 $1, Y1, X3 + VCVTPS2PD X3, Y3 + VMOVUPD 0(R8), Y4 + VFMADD231PD 32(R8), Y2, Y4 + VSUBPD 0(R8), Y4, Y5 + VFNMADD231PD 64(R8), Y5, Y2 + VFNMADD231PD 96(R8), Y5, Y2 + VMULPD Y2, Y2, Y6 + VMOVUPD 128(R8), Y7 + VFMADD231PD 160(R8), Y2, Y7 + VMOVUPD 192(R8), Y8 + VFMADD231PD 224(R8), Y2, Y8 + VFMADD231PD Y8, Y6, Y7 + VFMADD231PD Y7, Y6, Y2 + VPAND 256(R8), Y4, Y5 + VPCMPEQQ Y9, Y9, Y9 + VPGATHERQQ Y9, (R9)(Y5*8), Y10 + VPSLLQ $46, Y4, Y4 + VPAND 288(R8), Y4, Y4 + VPOR Y10, Y4, Y4 + VFMADD231PD Y2, Y4, Y4 + VCVTPD2PSY Y4, X11 + VMOVUPD 0(R8), Y4 + VFMADD231PD 32(R8), Y3, Y4 + VSUBPD 0(R8), Y4, Y5 + VFNMADD231PD 64(R8), Y5, Y3 + VFNMADD231PD 96(R8), Y5, Y3 + VMULPD Y3, Y3, Y6 + VMOVUPD 128(R8), Y7 + VFMADD231PD 160(R8), Y3, Y7 + VMOVUPD 192(R8), Y8 + VFMADD231PD 224(R8), Y3, Y8 + VFMADD231PD Y8, Y6, Y7 + VFMADD231PD Y7, Y6, Y3 + VPAND 256(R8), Y4, Y5 + VPCMPEQQ Y9, Y9, Y9 + VPGATHERQQ Y9, (R9)(Y5*8), Y10 + VPSLLQ $46, Y4, Y4 + VPAND 288(R8), Y4, Y4 + VPOR Y10, Y4, Y4 + VFMADD231PD Y3, Y4, Y4 + VCVTPD2PSY Y4, X12 + VINSERTF128 $1, X12, Y11, Y11 + VBLENDVPS Y15, Y0, Y11, Y11 + VMOVUPS Y11, (DI) + ADDQ $32, SI + ADDQ $32, DI + SUBQ $8, CX + JNZ eloop + VZEROUPPER + RET + TEXT ·blocks8SSE2(SB), NOSPLIT, $0-24 MOVQ r+0(FP), DI MOVQ a+8(FP), SI diff --git a/internal/kernels/via64.go b/internal/kernels/via64.go index 7214fa1..3ef320f 100644 --- a/internal/kernels/via64.go +++ b/internal/kernels/via64.go @@ -2,33 +2,26 @@ package kernels import "math" -// float32 transcendental functions on the float64 kernels. +// Correctly rounded float32 Exp and Log. // // float32 Exp and Log went through math.Exp and math.Log per element, 6-14x -// slower than NumPy's float32 SIMD on one Zen 3 core. The float64 kernels of -// this package (ports of Arm's optimized-routines, 0.504 ULP worst case in -// float64) are 3-4x faster than math's. A float32 element is widened (exact), -// computed in float64 and rounded once. The result is the correctly rounded -// float32 unless the exact value lies within the float64 kernel's error -// (about half a float64 ULP, 2^-29 of a float32 ULP) of a float32 rounding -// midpoint; it is within 0.5 + 2^-29 float32 ULP everywhere. Overflow and -// underflow follow from the rounding: exp(89) is finite in float64 and -// rounds to +Inf. +// slower than NumPy's float32 SIMD on one Zen 3 core. Both now compute in +// float64 and round once, and both are proven correctly rounded over every +// float32 input (via64_exhaustive_test.go, expcr_test.go): // -// The elements go through a 256-element stack buffer, widened, computed in -// place and rounded back, so nothing is allocated. Exp and Log are called -// directly, not through a func value, which would make the buffer escape. +// - Exp runs expCR (expcr.go), Arm's correctly rounded expf algorithm, on +// an AVX2 kernel where there is one (expcr_amd64.go), in Go elsewhere; +// - Log widens to this package's float64 Log (0.504 ULP) through a +// 256-element stack buffer, rounds once, and takes the 24 inputs whose +// float64 result lies next to a float32 rounding midpoint from a table. +// +// Nothing is allocated. Log is called directly, not through a func value, +// which would make the buffer escape. const via64Chunk = 256 -type via64Op uint8 - -const ( - via64Exp via64Op = iota - via64Log -) - -func via64(op via64Op, dst, src []float32) { +// log32 writes the correctly rounded log(src[i]) into dst[i]. +func log32(dst, src []float32) { var buf [via64Chunk]float64 src = src[:len(dst)] for i := 0; i < len(dst); i += via64Chunk { @@ -37,28 +30,17 @@ func via64(op via64Op, dst, src []float32) { for j := range b { b[j] = float64(s[j]) } - switch op { - case via64Exp: - Exp(b, b) - case via64Log: - Log(b, b) - } + Log(b, b) d := dst[i : i+len(b)] - if op == via64Log { - for j, v := range b { - f := float32(v) - // Within 8 float64 ULP of a float32 rounding midpoint (the - // 29 bits float32 drops are 0x10000000 +- 8): the double - // rounding may be wrong, so take the table's answer. - if math.Float64bits(v)&(1<<29-1)-(1<<28-8) < 16 { - f = log32Exception(s[j], f) - } - d[j] = f - } - continue - } for j, v := range b { - d[j] = float32(v) + f := float32(v) + // Within 8 float64 ULP of a float32 rounding midpoint (the 29 + // bits float32 drops are 0x10000000 +- 8): the double rounding + // may be wrong, so take the table's answer. + if math.Float64bits(v)&(1<<29-1)-(1<<28-8) < 16 { + f = log32Exception(s[j], f) + } + d[j] = f } } } @@ -100,23 +82,23 @@ func log32Exception(x, f float32) float32 { return f } -// Exp32 writes exp(src[i]) into dst[i] for float32 elements. -func Exp32(dst, src []float32) { via64(via64Exp, dst, src) } +// Exp32 writes the correctly rounded exp(src[i]) into dst[i]. +func Exp32(dst, src []float32) { exp32Fast(dst, src) } -// Log32 writes log(src[i]) into dst[i] for float32 elements. -func Log32(dst, src []float32) { via64(via64Log, dst, src) } +// Log32 writes the correctly rounded log(src[i]) into dst[i]. +func Log32(dst, src []float32) { log32(dst, src) } // Exp32P is Exp32, split across cores above ParThreshold. -func Exp32P(dst, src []float32) { via64P(via64Exp, dst, src) } +func Exp32P(dst, src []float32) { split32(exp32Fast, dst, src) } // Log32P is Log32, split across cores above ParThreshold. -func Log32P(dst, src []float32) { via64P(via64Log, dst, src) } +func Log32P(dst, src []float32) { split32(log32, dst, src) } -func via64P(op via64Op, dst, src []float32) { +func split32(f func(dst, src []float32), dst, src []float32) { n := len(dst) if n < ParThreshold { - via64(op, dst, src) + f(dst, src) return } - parallelFor(n, numWorkers(n), func(lo, hi int) { via64(op, dst[lo:hi], src[lo:hi]) }) + parallelFor(n, numWorkers(n), func(lo, hi int) { f(dst[lo:hi], src[lo:hi]) }) } diff --git a/internal/kernels/via64_exhaustive_test.go b/internal/kernels/via64_exhaustive_test.go index 8ba90f3..7731014 100644 --- a/internal/kernels/via64_exhaustive_test.go +++ b/internal/kernels/via64_exhaustive_test.go @@ -206,7 +206,8 @@ func exhaustiveStride() uint64 { } func TestExp32CorrectlyRounded(t *testing.T) { - runExhaustive(t, exhaustiveCase{"exp", Exp32, Exp, refExp, exp32Special, 2}, exhaustiveStride()) + // Exp32 rounds expCR (1.006 float64 ULP): the certifier's margin is 4. + runExhaustive(t, exhaustiveCase{"exp", Exp32, expCRSlice, refExp, exp32Special, 4}, exhaustiveStride()) } func TestLog32CorrectlyRounded(t *testing.T) { From 8875b7607a39f6f4775ca3694fc111a4efdee448 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 2 Nov 2026 11:53:03 +0100 Subject: [PATCH 3/4] style: gofmt the amd64 generator Co-Authored-By: Claude Opus 5.5 --- internal/kernels/asmgen/amd64/gen.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/kernels/asmgen/amd64/gen.go b/internal/kernels/asmgen/amd64/gen.go index 35aaa3a..f1bd76e 100644 --- a/internal/kernels/asmgen/amd64/gen.go +++ b/internal/kernels/asmgen/amd64/gen.go @@ -411,21 +411,21 @@ func exp32CRKernel() *emit.Function { b.Raw(fmt.Sprintf("VSUBPD %d(R8), Y4, Y5", shift)) // n = z - shift b.Raw(fmt.Sprintf("VFNMADD231PD %d(R8), Y5, %s", ln2Hi, h)) // r = x - n*ln2hi b.Raw(fmt.Sprintf("VFNMADD231PD %d(R8), Y5, %s", ln2Lo, h)) // r -= n*ln2lo - b.Raw(fmt.Sprintf("VMULPD %s, %s, Y6", h, h)) // r2 + b.Raw(fmt.Sprintf("VMULPD %s, %s, Y6", h, h)) // r2 b.Raw(fmt.Sprintf("VMOVUPD %d(R8), Y7", c0)) b.Raw(fmt.Sprintf("VFMADD231PD %d(R8), %s, Y7", c1, h)) // p01 = r*c1 + c0 b.Raw(fmt.Sprintf("VMOVUPD %d(R8), Y8", c2)) b.Raw(fmt.Sprintf("VFMADD231PD %d(R8), %s, Y8", c3, h)) // p23 = r*c3 + c2 - b.Raw("VFMADD231PD Y8, Y6, Y7") // p04 = r2*p23 + p01 - b.Raw(fmt.Sprintf("VFMADD231PD Y7, Y6, %s", h)) // y = r2*p04 + r - b.Raw(fmt.Sprintf("VPAND %d(R8), Y4, Y5", idxMask)) // z & 63 - b.Raw("VPCMPEQQ Y9, Y9, Y9") // gather every lane - b.Raw("VPGATHERQQ Y9, (R9)(Y5*8), Y10") // mantissa of 2^(i/64) + b.Raw("VFMADD231PD Y8, Y6, Y7") // p04 = r2*p23 + p01 + b.Raw(fmt.Sprintf("VFMADD231PD Y7, Y6, %s", h)) // y = r2*p04 + r + b.Raw(fmt.Sprintf("VPAND %d(R8), Y4, Y5", idxMask)) // z & 63 + b.Raw("VPCMPEQQ Y9, Y9, Y9") // gather every lane + b.Raw("VPGATHERQQ Y9, (R9)(Y5*8), Y10") // mantissa of 2^(i/64) b.Raw("VPSLLQ $46, Y4, Y4") b.Raw(fmt.Sprintf("VPAND %d(R8), Y4, Y4", expMask)) // exponent and sign bits b.Raw("VPOR Y10, Y4, Y4") // s - b.Raw(fmt.Sprintf("VFMADD231PD %s, Y4, Y4", h)) // s*y + s - b.Raw(fmt.Sprintf("VCVTPD2PSY Y4, %s", out)) // round once to float32 + b.Raw(fmt.Sprintf("VFMADD231PD %s, Y4, Y4", h)) // s*y + s + b.Raw(fmt.Sprintf("VCVTPD2PSY Y4, %s", out)) // round once to float32 } b.Raw("eloop:") b.Raw("VMOVUPS (SI), Y0") From 970ec5ae346b4b581c67ad24f2a494a678dacfb9 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 2 Nov 2026 12:06:13 +0100 Subject: [PATCH 4/4] test: the AVX2 exp fence test needs mmap: linux || darwin, like fence_test.go Co-Authored-By: Claude Opus 5.5 --- internal/kernels/expcr_amd64_test.go | 14 -------------- internal/kernels/expcr_fence_amd64_test.go | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 14 deletions(-) create mode 100644 internal/kernels/expcr_fence_amd64_test.go diff --git a/internal/kernels/expcr_amd64_test.go b/internal/kernels/expcr_amd64_test.go index 8ee85e0..bf8c2e3 100644 --- a/internal/kernels/expcr_amd64_test.go +++ b/internal/kernels/expcr_amd64_test.go @@ -43,17 +43,3 @@ func TestExp32AVX2MatchesGo(t *testing.T) { } } } - -// TestExp32AVX2StaysInside fences src and dst of the kernel. -func TestExp32AVX2StaysInside(t *testing.T) { - if !useExp32AVX2 { - t.Skip("no AVX2+FMA here") - } - for n := 8; n <= 64; n += 8 { - for _, s := range fences32(t, n) { - for _, d := range fences32(t, n) { - exp32CRAVX2(&d[0], &s[0], n, expCRK) - } - } - } -} diff --git a/internal/kernels/expcr_fence_amd64_test.go b/internal/kernels/expcr_fence_amd64_test.go new file mode 100644 index 0000000..6d6b959 --- /dev/null +++ b/internal/kernels/expcr_fence_amd64_test.go @@ -0,0 +1,20 @@ +//go:build linux || darwin + +package kernels + +import "testing" + +// TestExp32AVX2StaysInside fences src and dst of the AVX2 exp kernel +// (fences32 needs mmap/mprotect, hence the build tag of fence_test.go). +func TestExp32AVX2StaysInside(t *testing.T) { + if !useExp32AVX2 { + t.Skip("no AVX2+FMA here") + } + for n := 8; n <= 64; n += 8 { + for _, s := range fences32(t, n) { + for _, d := range fences32(t, n) { + exp32CRAVX2(&d[0], &s[0], n, expCRK) + } + } + } +}