diff --git a/docs/perf.md b/docs/perf.md index 380efb4..fa6bede 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -401,10 +401,28 @@ makes anyway), so a transposed, sliced, reversed or broadcast operand costs no extra pass. **Still behind OpenBLAS with 16 threads** (0.54–0.66× on large products, 0.3× -below 256²): per-core throughput in the parallel run is about a third of the -serial kernel's, so the next lever is the parallel schedule (a persistent -worker pool instead of a goroutine launch per round, and the BLIS jr-loop -split for short m), then an AVX-512 kernel for the hosts that have it. +below 256²). Where the parallel loss is, measured on the Zen 3: + +- It is not the hardware: 16 *independent* serial 512² GEMMs, one per core, + each ran at 41 GFLOP/s (656 in total), while the parallel 1024² GEMM ran its + kernel at 15–17 GFLOP/s per core. Per-core efficiency already drops from 40 + to ~30 GFLOP/s at 2 cores. +- It is not scheduling overhead: 83% of the parallel profile is the + micro-kernel itself, which stalls on the shared packed B panel. On this + part the last-level cache is split between core complexes, so the panel is + partly remote. +- Taller row bands (more reuse of each B micro-panel) do not help: interleaved + over 5 runs they were worse at 1024² and equal at 2048². +- **A 2-D tiling of C with private packing** (no barrier, nothing shared) was + built and measured, and is **not shipped**: ×1.24 at 1024² on the Zen 3, but + ×0.62 at 512² and Inner, and ×0.70–0.93 at every size on M4, whose cluster + L2 holds the shared panel well. A tile repacks (rows + cols)·k elements for + rows·cols·k multiply-adds, and a strided pack costs far more per element than + an FMA, so tiles under ~256×256 lose. + +The next lever is therefore a shared B panel per core complex (or per cache +domain), not a different split of C, then an AVX-512 kernel for the hosts that +have it. Correctness was checked on real hardware for five of the six 64-bit targets: amd64 (Zen 3, FMA path), arm64 (cfarm185), ppc64le (cfarm120), riscv64 (cfarm94) @@ -450,6 +468,37 @@ across cores. Below 64 elements per block the old path is kept. |----|--:|--:|--:| | before | 3.1 ms | 1.76 ms | 460 µs | | after | **0.84 ms** | **114–196 µs** | 460 µs | +## Exp (2026-10-04) + +`Exp` went through `Map(math.Exp)`, and `math.Exp` itself was the cost: 13.2 ns +per element on one Zen 3 core, 3.9 ns on M4 (the func-pointer call added only +15–20%). It is now a port of the double-precision exp of Arm's +optimized-routines (Szabolcs Nagy; glibc's exp since 2.28; MIT licence): x = +k·ln2/128 + r, 2^(k/128) from a 128-entry table as scale·(1+tail), exp(r)−1 from +a degree-5 polynomial. The common case is written out in the loop, because the +function is too large to inline and a call per element cost a third of the +time. + +- **Accuracy:** worst 0.504 ULP over 80 000 inputs across the whole finite + range, the region near 0, the overflow edge and the subnormal range, + measured against a 300-bit reference (`math/big`, ln 2 from its atanh + series), against Arm's documented 0.509. `math.Exp` measured 0.879 ULP on + the same inputs on arm64. The table was not trusted as copied: + `TestExpTable` re-derives all 256 entries from 2^(k/128) at 300 bits. +- **A Go bug it removes:** on amd64, Go 1.26's `math.Exp` returns **+Inf for + x ≥ 1023.5·ln 2 ≈ 709.436**, although the result is finite up to + ln(MaxFloat64) ≈ 709.7827 (NumPy: `exp(709.5)` = 1.3549863193146328e+308). + Its assembly rounds k = x·log2(e) to 1024 there, and the biased-exponent + check treats that as overflow although the reduced factor is below 1. Seen on + real Zen 3 hardware, not only Rosetta. go-ndarray's `Exp` inherited it until + now; `TestExpTopOfRange` pins the fix. +- Also just above ln(2^-1075), exp now rounds up to the smallest subnormal as + it should, where `math.Exp` returns 0. + +Speed, kernel only, 16 Ki elements: M4 30 µs (`math.Exp` 65 µs); one Zen 3 core +62 µs (`math.Exp` 215 µs, NumPy single-threaded 78 µs). Whole `Exp` on Zen 3, +16 cores: 1 Ki ×2.7, 16 Ki ×3, 256 Ki ×1.6, 4 Mi ×1.65; from 256 Ki on it is +2–5× NumPy. A SIMD version (AVX2 gathers for the table) is the next step. ## SIMD coverage diff --git a/internal/kernels/exp.go b/internal/kernels/exp.go new file mode 100644 index 0000000..e2511ec --- /dev/null +++ b/internal/kernels/exp.go @@ -0,0 +1,270 @@ +package kernels + +import "math" + +// Exp is a port of the double-precision exp of Arm's optimized-routines +// (math/exp.c and math/exp_data.c, by Szabolcs Nagy; also glibc's exp since +// 2.28), used under its MIT licence: +// +// Copyright (c) 2018-2025, Arm Limited. +// SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception +// +// exp(x) = 2^(k/N) * exp(r) with N = 128, x = k*ln2/N + r, |r| <= ln2/2N. +// 2^(k/N) comes from a 128-entry table as scale*(1+tail), and exp(r)-1 from a +// degree-5 polynomial; the result is scale + scale*(tail + exp(r)-1). Arm +// documents a worst case of 0.509 ULP (0.511 without FMA); TestExpULP measures +// this port against a 256-bit reference. It is branch-light and table-driven, +// unlike math.Exp, which is what makes it several times faster per element. + +const ( + expTableBits = 7 + expN = 1 << expTableBits + expInvLn2N = 0x1.71547652b82fep0 * expN + expNegLn2hiN = -0x1.62e42fefa0000p-8 + expNegLn2loN = -0x1.cf79abc9e3b3ap-47 + expShift = 0x1.8p52 + expC2 = 0x1.ffffffffffdbdp-2 + expC3 = 0x1.555555555543cp-3 + expC4 = 0x1.55555cf172b91p-5 + expC5 = 0x1.1111167a4d017p-7 +) + +// expTab holds, for k in [0, N): expTab[2k] = the bits of tail[k], and +// expTab[2k+1] = the bits of H[k] minus k<<45, where 2^(k/N) ~= H[k]*(1+tail[k]). +// Copied from exp_data.c (N == 128); TestExpTable re-derives every entry. +var expTab = [2 * expN]uint64{ + 0x0, 0x3ff0000000000000, + 0x3c9b3b4f1a88bf6e, 0x3feff63da9fb3335, + 0xbc7160139cd8dc5d, 0x3fefec9a3e778061, + 0xbc905e7a108766d1, 0x3fefe315e86e7f85, + 0x3c8cd2523567f613, 0x3fefd9b0d3158574, + 0xbc8bce8023f98efa, 0x3fefd06b29ddf6de, + 0x3c60f74e61e6c861, 0x3fefc74518759bc8, + 0x3c90a3e45b33d399, 0x3fefbe3ecac6f383, + 0x3c979aa65d837b6d, 0x3fefb5586cf9890f, + 0x3c8eb51a92fdeffc, 0x3fefac922b7247f7, + 0x3c3ebe3d702f9cd1, 0x3fefa3ec32d3d1a2, + 0xbc6a033489906e0b, 0x3fef9b66affed31b, + 0xbc9556522a2fbd0e, 0x3fef9301d0125b51, + 0xbc5080ef8c4eea55, 0x3fef8abdc06c31cc, + 0xbc91c923b9d5f416, 0x3fef829aaea92de0, + 0x3c80d3e3e95c55af, 0x3fef7a98c8a58e51, + 0xbc801b15eaa59348, 0x3fef72b83c7d517b, + 0xbc8f1ff055de323d, 0x3fef6af9388c8dea, + 0x3c8b898c3f1353bf, 0x3fef635beb6fcb75, + 0xbc96d99c7611eb26, 0x3fef5be084045cd4, + 0x3c9aecf73e3a2f60, 0x3fef54873168b9aa, + 0xbc8fe782cb86389d, 0x3fef4d5022fcd91d, + 0x3c8a6f4144a6c38d, 0x3fef463b88628cd6, + 0x3c807a05b0e4047d, 0x3fef3f49917ddc96, + 0x3c968efde3a8a894, 0x3fef387a6e756238, + 0x3c875e18f274487d, 0x3fef31ce4fb2a63f, + 0x3c80472b981fe7f2, 0x3fef2b4565e27cdd, + 0xbc96b87b3f71085e, 0x3fef24dfe1f56381, + 0x3c82f7e16d09ab31, 0x3fef1e9df51fdee1, + 0xbc3d219b1a6fbffa, 0x3fef187fd0dad990, + 0x3c8b3782720c0ab4, 0x3fef1285a6e4030b, + 0x3c6e149289cecb8f, 0x3fef0cafa93e2f56, + 0x3c834d754db0abb6, 0x3fef06fe0a31b715, + 0x3c864201e2ac744c, 0x3fef0170fc4cd831, + 0x3c8fdd395dd3f84a, 0x3feefc08b26416ff, + 0xbc86a3803b8e5b04, 0x3feef6c55f929ff1, + 0xbc924aedcc4b5068, 0x3feef1a7373aa9cb, + 0xbc9907f81b512d8e, 0x3feeecae6d05d866, + 0xbc71d1e83e9436d2, 0x3feee7db34e59ff7, + 0xbc991919b3ce1b15, 0x3feee32dc313a8e5, + 0x3c859f48a72a4c6d, 0x3feedea64c123422, + 0xbc9312607a28698a, 0x3feeda4504ac801c, + 0xbc58a78f4817895b, 0x3feed60a21f72e2a, + 0xbc7c2c9b67499a1b, 0x3feed1f5d950a897, + 0x3c4363ed60c2ac11, 0x3feece086061892d, + 0x3c9666093b0664ef, 0x3feeca41ed1d0057, + 0x3c6ecce1daa10379, 0x3feec6a2b5c13cd0, + 0x3c93ff8e3f0f1230, 0x3feec32af0d7d3de, + 0x3c7690cebb7aafb0, 0x3feebfdad5362a27, + 0x3c931dbdeb54e077, 0x3feebcb299fddd0d, + 0xbc8f94340071a38e, 0x3feeb9b2769d2ca7, + 0xbc87deccdc93a349, 0x3feeb6daa2cf6642, + 0xbc78dec6bd0f385f, 0x3feeb42b569d4f82, + 0xbc861246ec7b5cf6, 0x3feeb1a4ca5d920f, + 0x3c93350518fdd78e, 0x3feeaf4736b527da, + 0x3c7b98b72f8a9b05, 0x3feead12d497c7fd, + 0x3c9063e1e21c5409, 0x3feeab07dd485429, + 0x3c34c7855019c6ea, 0x3feea9268a5946b7, + 0x3c9432e62b64c035, 0x3feea76f15ad2148, + 0xbc8ce44a6199769f, 0x3feea5e1b976dc09, + 0xbc8c33c53bef4da8, 0x3feea47eb03a5585, + 0xbc845378892be9ae, 0x3feea34634ccc320, + 0xbc93cedd78565858, 0x3feea23882552225, + 0x3c5710aa807e1964, 0x3feea155d44ca973, + 0xbc93b3efbf5e2228, 0x3feea09e667f3bcd, + 0xbc6a12ad8734b982, 0x3feea012750bdabf, + 0xbc6367efb86da9ee, 0x3fee9fb23c651a2f, + 0xbc80dc3d54e08851, 0x3fee9f7df9519484, + 0xbc781f647e5a3ecf, 0x3fee9f75e8ec5f74, + 0xbc86ee4ac08b7db0, 0x3fee9f9a48a58174, + 0xbc8619321e55e68a, 0x3fee9feb564267c9, + 0x3c909ccb5e09d4d3, 0x3feea0694fde5d3f, + 0xbc7b32dcb94da51d, 0x3feea11473eb0187, + 0x3c94ecfd5467c06b, 0x3feea1ed0130c132, + 0x3c65ebe1abd66c55, 0x3feea2f336cf4e62, + 0xbc88a1c52fb3cf42, 0x3feea427543e1a12, + 0xbc9369b6f13b3734, 0x3feea589994cce13, + 0xbc805e843a19ff1e, 0x3feea71a4623c7ad, + 0xbc94d450d872576e, 0x3feea8d99b4492ed, + 0x3c90ad675b0e8a00, 0x3feeaac7d98a6699, + 0x3c8db72fc1f0eab4, 0x3feeace5422aa0db, + 0xbc65b6609cc5e7ff, 0x3feeaf3216b5448c, + 0x3c7bf68359f35f44, 0x3feeb1ae99157736, + 0xbc93091fa71e3d83, 0x3feeb45b0b91ffc6, + 0xbc5da9b88b6c1e29, 0x3feeb737b0cdc5e5, + 0xbc6c23f97c90b959, 0x3feeba44cbc8520f, + 0xbc92434322f4f9aa, 0x3feebd829fde4e50, + 0xbc85ca6cd7668e4b, 0x3feec0f170ca07ba, + 0x3c71affc2b91ce27, 0x3feec49182a3f090, + 0x3c6dd235e10a73bb, 0x3feec86319e32323, + 0xbc87c50422622263, 0x3feecc667b5de565, + 0x3c8b1c86e3e231d5, 0x3feed09bec4a2d33, + 0xbc91bbd1d3bcbb15, 0x3feed503b23e255d, + 0x3c90cc319cee31d2, 0x3feed99e1330b358, + 0x3c8469846e735ab3, 0x3feede6b5579fdbf, + 0xbc82dfcd978e9db4, 0x3feee36bbfd3f37a, + 0x3c8c1a7792cb3387, 0x3feee89f995ad3ad, + 0xbc907b8f4ad1d9fa, 0x3feeee07298db666, + 0xbc55c3d956dcaeba, 0x3feef3a2b84f15fb, + 0xbc90a40e3da6f640, 0x3feef9728de5593a, + 0xbc68d6f438ad9334, 0x3feeff76f2fb5e47, + 0xbc91eee26b588a35, 0x3fef05b030a1064a, + 0x3c74ffd70a5fddcd, 0x3fef0c1e904bc1d2, + 0xbc91bdfbfa9298ac, 0x3fef12c25bd71e09, + 0x3c736eae30af0cb3, 0x3fef199bdd85529c, + 0x3c8ee3325c9ffd94, 0x3fef20ab5fffd07a, + 0x3c84e08fd10959ac, 0x3fef27f12e57d14b, + 0x3c63cdaf384e1a67, 0x3fef2f6d9406e7b5, + 0x3c676b2c6c921968, 0x3fef3720dcef9069, + 0xbc808a1883ccb5d2, 0x3fef3f0b555dc3fa, + 0xbc8fad5d3ffffa6f, 0x3fef472d4a07897c, + 0xbc900dae3875a949, 0x3fef4f87080d89f2, + 0x3c74a385a63d07a7, 0x3fef5818dcfba487, + 0xbc82919e2040220f, 0x3fef60e316c98398, + 0x3c8e5a50d5c192ac, 0x3fef69e603db3285, + 0x3c843a59ac016b4b, 0x3fef7321f301b460, + 0xbc82d52107b43e1f, 0x3fef7c97337b9b5f, + 0xbc892ab93b470dc9, 0x3fef864614f5a129, + 0x3c74b604603a88d3, 0x3fef902ee78b3ff6, + 0x3c83c5ec519d7271, 0x3fef9a51fbc74c83, + 0xbc8ff7128fd391f0, 0x3fefa4afa2a490da, + 0xbc8dae98e223747d, 0x3fefaf482d8e67f1, + 0x3c8ec3bc41aa2008, 0x3fefba1bee615a27, + 0x3c842b94c3a9eb32, 0x3fefc52b376bba97, + 0x3c8a64a931d185ee, 0x3fefd0765b6e4540, + 0xbc8e37bae43be3ed, 0x3fefdbfdad9cbe14, + 0x3c77893b4d91cd9d, 0x3fefe7c1819e90d8, + 0x3c5305c14160cc89, 0x3feff3c22b8f71f1, +} + +// top12 is the sign and exponent bits of x. +func top12(x float64) uint32 { return uint32(math.Float64bits(x) >> 52) } + +// expSpecial finishes exp when scale*(1+tmp) may overflow or underflow: the +// exponent in sbits may have wrapped, k's sign (bit 31 of ki) tells which. +// The subnormal branch rounds y once before scaling it down, avoiding the +// double rounding that would cost up to 0.5 ULP more. Conversions to float64 +// keep the compiler from fusing the compensated sums into an FMA. +func expSpecial(tmp float64, sbits, ki uint64) float64 { + if ki&0x80000000 == 0 { + // k > 0: the exponent of scale might have overflowed by <= 460. + sbits -= 1009 << 52 + scale := math.Float64frombits(sbits) + return 0x1p1009 * float64(scale+float64(scale*tmp)) + } + // k < 0: care is needed in the subnormal range. + sbits += 1022 << 52 + scale := math.Float64frombits(sbits) + y := scale + float64(scale*tmp) + if y < 1.0 { + lo := scale - y + float64(scale*tmp) + hi := 1.0 + y + lo = 1.0 - hi + y + lo + y = float64(hi+lo) - 1.0 + if y == 0 { + y = 0 // never -0 + } + } + return 0x1p-1022 * y +} + +// exp1 is exp(x) for one element (exp_inline of exp.c with xtail = 0). +func exp1(x float64) float64 { + abstop := top12(x) & 0x7ff + if abstop-0x3c9 >= 0x408-0x3c9 { // |x| < 2^-54, or |x| >= 512, or not finite + if abstop-0x3c9 >= 0x80000000 { + return 1.0 + x // tiny x (and 0, a common input) + } + if abstop >= 0x409 { // |x| >= 1024, Inf or NaN + switch { + case x == math.Inf(-1): + return 0 + case abstop >= 0x7ff: + return 1.0 + x // NaN, +Inf + case math.Signbit(x): + return 0 // underflow + default: + return math.Inf(1) // overflow + } + } + abstop = 0 // large |x|: finished by expSpecial + } + // z - kd is in [-1, 1] in non-nearest rounding modes. + z := expInvLn2N * x + kd := float64(z + expShift) + ki := math.Float64bits(kd) + kd -= expShift + r := x + kd*expNegLn2hiN + kd*expNegLn2loN + idx := 2 * (ki % expN) + top := ki << (52 - expTableBits) + tail := math.Float64frombits(expTab[idx]) + sbits := expTab[idx+1] + top + r2 := r * r + tmp := tail + r + r2*(expC2+r*expC3) + r2*r2*(expC4+r*expC5) + if abstop == 0 { + return expSpecial(tmp, sbits, ki) + } + scale := math.Float64frombits(sbits) + return scale + scale*tmp +} + +// Exp writes exp(src[i]) into dst[i]. exp1 is too large to inline, and a +// call per element cost half the time, so the common case (2^-54 <= |x| < +// 512) is exp1's main path written out in the loop; everything else calls +// exp1. TestExpLoopMatchesExp1 holds the two to the same bits. +func Exp(dst, src []float64) { + dst = dst[:len(src)] + for i, x := range src { + if top12(x)&0x7ff-0x3c9 >= 0x408-0x3c9 { + dst[i] = exp1(x) + continue + } + z := expInvLn2N * x + kd := float64(z + expShift) + ki := math.Float64bits(kd) + kd -= expShift + r := x + kd*expNegLn2hiN + kd*expNegLn2loN + idx := 2 * (ki % expN) + tail := math.Float64frombits(expTab[idx]) + scale := math.Float64frombits(expTab[idx+1] + ki<<(52-expTableBits)) + r2 := r * r + tmp := tail + r + r2*(expC2+r*expC3) + r2*r2*(expC4+r*expC5) + dst[i] = scale + scale*tmp + } +} + +// ExpP is Exp, split across cores above ParThreshold. +func ExpP(dst, src []float64) { + n := len(dst) + if n < ParThreshold { + Exp(dst, src) + return + } + parallelFor(n, numWorkers(n), func(lo, hi int) { Exp(dst[lo:hi], src[lo:hi]) }) +} diff --git a/internal/kernels/exp_test.go b/internal/kernels/exp_test.go new file mode 100644 index 0000000..e48901d --- /dev/null +++ b/internal/kernels/exp_test.go @@ -0,0 +1,178 @@ +package kernels + +import ( + "math" + "math/big" + "math/rand" + "testing" +) + +const refPrec = 300 + +// bigLn2 is ln 2 to refPrec bits: 2*atanh(1/3) = 2*sum (1/3)^(2n+1)/(2n+1). +func bigLn2() *big.Float { + third := new(big.Float).SetPrec(refPrec).Quo(big.NewFloat(1).SetPrec(refPrec), big.NewFloat(3)) + ninth := new(big.Float).SetPrec(refPrec).Mul(third, third) + sum := new(big.Float).SetPrec(refPrec) + pow := new(big.Float).SetPrec(refPrec).Set(third) + for n := 0; n < 300; n++ { + term := new(big.Float).SetPrec(refPrec).Quo(pow, big.NewFloat(float64(2*n+1))) + sum.Add(sum, term) + pow.Mul(pow, ninth) + } + return sum.Mul(sum, big.NewFloat(2)) +} + +var ln2Big = bigLn2() + +// refExp is exp(x) to refPrec bits, independent of the code under test: +// x = k*ln2 + r with |r| <= ln2/2, exp(r) by its Taylor series, times 2^k. +func refExp(x float64) *big.Float { + bx := new(big.Float).SetPrec(refPrec).SetFloat64(x) + q := new(big.Float).SetPrec(refPrec).Quo(bx, ln2Big) + qf, _ := q.Float64() + k := math.Round(qf) + r := new(big.Float).SetPrec(refPrec).Mul(ln2Big, big.NewFloat(k)) + r.Sub(bx, r) + sum := new(big.Float).SetPrec(refPrec).SetInt64(1) + term := new(big.Float).SetPrec(refPrec).SetInt64(1) + for n := 1; n < 80; n++ { + term.Mul(term, r) + term.Quo(term, big.NewFloat(float64(n))) + sum.Add(sum, term) + } + return sum.SetMantExp(sum, int(k)) +} + +// ulpErr is |got - exact| in units of the float64 spacing at exact (2^-1074 +// in the subnormal range). +func ulpErr(got float64, exact *big.Float) float64 { + e := exact.MantExp(nil) - 1 // exact = m * 2^e, 1 <= m < 2 + ulpExp := max(e-52, -1074) + d := new(big.Float).SetPrec(refPrec).SetFloat64(got) + d.Sub(d, exact) + d.SetMantExp(d, -ulpExp) + f, _ := d.Float64() + return math.Abs(f) +} + +// TestExpTable re-derives every table entry from 2^(k/128) computed to 300 +// bits (2 square-rooted seven times, then raised to k): H[k] must be the double +// nearest 2^(k/N) and tail[k] the double nearest (2^(k/N) - H)/H. +func TestExpTable(t *testing.T) { + root := new(big.Float).SetPrec(refPrec).SetInt64(2) + for i := 0; i < expTableBits; i++ { + root.Sqrt(root) + } + e := new(big.Float).SetPrec(refPrec).SetInt64(1) + for k := 0; k < expN; k++ { + h := math.Float64frombits(expTab[2*k+1] + uint64(k)<<45) + if want, _ := e.Float64(); h != want { + t.Fatalf("H[%d] = %v, nearest double to 2^(k/N) is %v", k, h, want) + } + tail := new(big.Float).SetPrec(refPrec).SetFloat64(h) + tail.Sub(e, tail) + tail.Quo(tail, new(big.Float).SetFloat64(h)) + got := math.Float64frombits(expTab[2*k]) + if want, _ := tail.Float64(); got != want { + t.Fatalf("tail[%d] = %v, want %v", k, got, want) + } + e.Mul(e, root) + } +} + +// TestExpULP measures the worst error against the 256-bit reference over the +// whole finite domain, the region near 0, the overflow edge and the subnormal +// range, and requires it to stay within Arm's documented 0.511 ULP (+ margin). +func TestExpULP(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + var xs []float64 + n := 5000 + if testing.Short() { + n = 2000 + } + for i := 0; i < n; i++ { + xs = append(xs, + rng.Float64()*1454.78-745, // the whole finite range + (rng.Float64()*2-1)*math.Ldexp(1, -rng.Intn(60)), // near 0, log-uniform + 709+rng.Float64()*0.7827, // just below overflow + -745.13+rng.Float64()*37, // subnormal and near-subnormal results + ) + } + worst, worstX, goWorst := 0.0, 0.0, 0.0 + for _, x := range xs { + ref := refExp(x) + if e := ulpErr(exp1(x), ref); e > worst { + worst, worstX = e, x + } + goWorst = max(goWorst, ulpErr(math.Exp(x), ref)) + } + t.Logf("exp: worst %.4f ULP at x=%v over %d inputs (math.Exp: %.4f)", worst, worstX, len(xs), goWorst) + if worst > 0.52 { + t.Errorf("exp error %.4f ULP at x=%v exceeds 0.52", worst, worstX) + } +} + +// TestExpLoopMatchesExp1: Exp's inlined main path and exp1 give the same bits +// for every input, special or not. +func TestExpLoopMatchesExp1(t *testing.T) { + rng := rand.New(rand.NewSource(2)) + xs := []float64{0, 1e-300, math.Ldexp(1, -54), math.Ldexp(1, -55), 511.99, 512, -512, + 709.78, -745.13, 1000, math.Inf(1), math.Inf(-1), math.NaN()} + for i := 0; i < 20000; i++ { + xs = append(xs, rng.Float64()*1460-750, (rng.Float64()*2-1)*math.Ldexp(1, -rng.Intn(70))) + } + got := make([]float64, len(xs)) + Exp(got, xs) + for i, x := range xs { + if w := exp1(x); math.Float64bits(got[i]) != math.Float64bits(w) && !(math.IsNaN(w) && math.IsNaN(got[i])) { + t.Fatalf("Exp(%v) = %v, exp1 = %v", x, got[i], w) + } + } +} + +func TestExpSpecial(t *testing.T) { + for _, c := range []struct{ x, want float64 }{ + {0, 1}, {math.Copysign(0, -1), 1}, {1e-300, 1}, {-1e-20, 1}, + {math.Inf(1), math.Inf(1)}, {math.Inf(-1), 0}, + {709.8, math.Inf(1)}, {1000, math.Inf(1)}, {2000, math.Inf(1)}, + {-745.2, 0}, {-1000, 0}, {-2000, 0}, + {1, math.E}, + } { + if got := exp1(c.x); got != c.want { + t.Errorf("exp(%v) = %v, want %v", c.x, got, c.want) + } + } + // Against the reference: the subnormal range (just above ln(2^-1075) exp + // rounds up to the smallest subnormal; math.Exp gives 0), and the top of + // the finite range, where Go 1.26's amd64 math.Exp returns +Inf from + // x >= 1023.5*ln2 ~ 709.436 (its k = round(x*log2e) reaches 1024 and the + // biased exponent check calls that overflow, though fr*2^1024 is finite). + for _, x := range []float64{-745.13, -745.1332, -740, -720.5, -708.5, -708.39, + 709.436, 709.5, 709.78, 709.782712893383} { + want, _ := refExp(x).Float64() + if got := exp1(x); got != want { + t.Errorf("exp(%v) = %v, want %v", x, got, want) + } + } + if got := exp1(math.NaN()); !math.IsNaN(got) { + t.Errorf("exp(NaN) = %v", got) + } + dst := make([]float64, 3) + Exp(dst, []float64{0, 1, math.Inf(-1)}) + eqSlice(t, dst, []float64{1, math.E, 0}) +} + +// TestExpP: the parallel form equals the serial one element for element. +func TestExpP(t *testing.T) { + src := randVec(5000, 3) + for i := range src { + src[i] *= 40 + } + want, got := make([]float64, len(src)), make([]float64, len(src)) + Exp(want, src) + withThresholds(64, 1<<14, func() { ExpP(got, src) }) + eqSlice(t, got, want) + ExpP(got[:10], src[:10]) + eqSlice(t, got[:10], want[:10]) +} diff --git a/ufunc.go b/ufunc.go index 2d9546f..af2608b 100644 --- a/ufunc.go +++ b/ufunc.go @@ -47,7 +47,18 @@ func (a *Array) SqrtInto(out *Array) error { } // Exp returns the elementwise base-e exponential. -func (a *Array) Exp() *Array { return a.Map(math.Exp) } +// +// It runs a table-driven port of Arm's optimized-routines exp (0.51 ULP worst +// case, measured), split across cores: several times faster than math.Exp per +// element, and finite up to ln(MaxFloat64) on amd64, where Go 1.26's math.Exp +// returns +Inf from x ~ 709.436. +func (a *Array) Exp() *Array { + src := a.contiguousData() + dst := a.alloc(len(src), false) + kernels.ExpP(dst, src) + cp := append([]int(nil), a.shape...) + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws} +} // Log returns the elementwise natural logarithm. func (a *Array) Log() *Array { return a.Map(math.Log) } diff --git a/ufunc_test.go b/ufunc_test.go index 68949aa..3c73e2b 100644 --- a/ufunc_test.go +++ b/ufunc_test.go @@ -103,3 +103,16 @@ func TestComparisonBroadcastAndError(t *testing.T) { } } } + +// TestExpTopOfRange: exp is finite up to ln(MaxFloat64) on every arch. Go +// 1.26's amd64 math.Exp returns +Inf from x ~ 709.436, which Exp used to +// inherit; numpy gives 1.3549863193146328e+308 for 709.5. The input is a +// strided view, so the contiguous copy path is exercised too. +func TestExpTopOfRange(t *testing.T) { + x := mustArr(t, ok(FromData([]float64{709.5, 0, 709.78, 0, -1000, 0}, 6))) + v, err := x.Slice(Step(2)) + if err != nil { + t.Fatal(err) + } + wantData(t, v.Exp(), []int{3}, []float64{1.3549863193146328e+308, 1.7928227943945155e+308, 0}) +}