From 0a612f77560b88b8955726fa2472696631667dc5 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 12 Nov 2026 11:24:15 +0100 Subject: [PATCH 1/3] docs: accuracy contract (bound, oracle, exact) and TestAccuracyBound for Sum and MatMul Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +++ accuracy_bound_test.go | 155 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 accuracy_bound_test.go diff --git a/README.md b/README.md index 9754917..f3ba910 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,14 @@ NumPy's pairwise tree where NumPy cuts it and adds the subtrees back up the same tree, so the bits do not depend on the number of workers. The other architectures keep their own elementwise kernels here until floats has them. +## Accuracy contract + +The contract is **correct, and as fast as possible**. "Correct" means a stated error bound, checked against an exact oracle. Equality with NumPy's bits is not the contract: wherever it holds today, it is how a function was ported, and a faster default that stays within its bound may replace it. + +Bound: elementwise `+ − × ÷` and `Sqrt` are correctly rounded (one IEEE operation, ≤ 0.5 ULP). Float64 `Exp`/`Log` are within 0.51 ULP. Float32 `Exp`/`Log` are correctly rounded, proven over all 2³² inputs. `Max`/`Min` are exact. `Sum` satisfies |Ŝ − S| ≤ γ_d·Σ|xᵢ|, where d = 19 + ⌈log₂(n/128)⌉ is the rounding depth of NumPy's pairwise sum and γ_m = m·u/(1 − m·u), with u = 2⁻⁵³ for float64 and 2⁻²⁴ for float32. Every element of `MatMul`/`Dot` satisfies |Ĉᵢⱼ − Cᵢⱼ| ≤ γ_k·(|A|·|B|)ᵢⱼ (Higham, *Accuracy and Stability of Numerical Algorithms*, §3.1, §4.2). These two bounds hold for any summation order, with or without FMA. +Oracle: `math/big` at 1024 bits for sums and inner products (`TestAccuracyBoundSum`, `TestAccuracyBoundMatMul`), and a 300-bit reference with the exhaustive float32 proof for `Exp`/`Log`. +Exact: the defaults are still the exact ports, NumPy's pairwise order for `Sum` included, so results match NumPy bit for bit where they did before. When a faster default lands, that arithmetic stays available behind an explicit option. + ## Goroutines Operations on large arrays are spread over `GOMAXPROCS` goroutines. After the diff --git a/accuracy_bound_test.go b/accuracy_bound_test.go new file mode 100644 index 0000000..d340ad1 --- /dev/null +++ b/accuracy_bound_test.go @@ -0,0 +1,155 @@ +package ndarray_test + +import ( + "math" + "math/big" + "math/rand/v2" + "testing" + + "github.com/go-ndarray/ndarray" +) + +// The accuracy contract (README, "Accuracy contract") asserted against an exact +// oracle: math/big at 1024 bits, wide enough that every sum and dot product of +// these inputs is exact before its final rounding. The bounds are the rigorous +// worst-case bounds of the algorithm (Higham, "Accuracy and Stability of +// Numerical Algorithms", 2nd ed., §3.1 and §4.2), so they hold for any +// summation order and with or without FMA: a faster default that reorders or +// fuses is accepted as long as these tests pass. + +const oraclePrec = 1024 + +// gamma is Higham's γ_m = m·u/(1 − m·u), with u the unit roundoff. +func gamma(m int, u float64) float64 { return float64(m) * u / (1 - float64(m)*u) } + +// sumDepth is the longest chain of roundings in NumPy's pairwise sum, which +// Sum follows: blocks of at most 128 elements summed by eight accumulators +// (up to 16 serial additions each, then three levels combining them), and the +// blocks combined pairwise. Any reordering with a depth at most this one is +// within the same bound. +func sumDepth(n int) int { + d := 16 + 3 + for blocks := (n + 127) / 128; blocks > 1; blocks = (blocks + 1) / 2 { + d++ + } + return d +} + +func exactSum(xs []float64) (sum, abs float64) { + s, a := new(big.Float).SetPrec(oraclePrec), new(big.Float).SetPrec(oraclePrec) + for _, x := range xs { + s.Add(s, big.NewFloat(x)) + a.Add(a, big.NewFloat(math.Abs(x))) + } + sum, _ = s.Float64() + abs, _ = a.Float64() + return sum, abs +} + +// TestAccuracyBoundSum: |Sum(x) − Σx| ≤ γ_d·Σ|x|, d = sumDepth(n), in float64 +// and in float32 (whose sum accumulates in float32, as NumPy's does). +func TestAccuracyBoundSum(t *testing.T) { + r := rand.New(rand.NewPCG(1, 2)) + for _, n := range []int{1, 7, 128, 129, 1000, 65536, 1 << 20} { + xs := make([]float64, n) + x32 := make([]float32, n) + for i := range xs { + // Mixed signs and magnitudes, so cancellation is exercised. + // The range is kept to 2^±4 so that the bound stays below the + // smallest element: a dropped or doubled element fails it. + v := (r.Float64()*2 - 1) * math.Ldexp(1, r.IntN(8)-4) + if v == 0 { + v = 1 + } + x32[i] = float32(v) + xs[i] = v + } + a, err := ndarray.FromData(xs, n) + if err != nil { + t.Fatal(err) + } + want, abs := exactSum(xs) + if got, bound := a.Sum(), gamma(sumDepth(n), 0x1p-53)*abs; math.Abs(got-want) > bound { + t.Errorf("float64 n=%d: |Sum − exact| = %g > bound %g", n, math.Abs(got-want), bound) + } + + a32, err := ndarray.FromSlice(x32, n) + if err != nil { + t.Fatal(err) + } + xs32 := make([]float64, n) + for i, v := range x32 { + xs32[i] = float64(v) + } + want, abs = exactSum(xs32) + // One more rounding: the float32 result is returned as a float64. + if got, bound := a32.Sum(), gamma(sumDepth(n)+1, 0x1p-24)*abs; math.Abs(got-want) > bound { + t.Errorf("float32 n=%d: |Sum − exact| = %g > bound %g", n, math.Abs(got-want), bound) + } + } +} + +// TestAccuracyBoundMatMul: every element of C = A·B satisfies +// |Ĉij − Cij| ≤ γ_k·(|A|·|B|)ij, the bound of any inner-product order (blocked, +// packed, SIMD, with or without FMA), in float64 and float32. +func TestAccuracyBoundMatMul(t *testing.T) { + r := rand.New(rand.NewPCG(3, 4)) + for _, s := range [][3]int{{1, 1, 1}, {7, 13, 5}, {64, 64, 64}, {97, 300, 33}, {256, 256, 256}} { + m, k, n := s[0], s[1], s[2] + av, bv := make([]float64, m*k), make([]float64, k*n) + a32, b32 := make([]float32, m*k), make([]float32, k*n) + for i := range av { + a32[i] = float32(r.NormFloat64()) + av[i] = float64(a32[i]) + } + for i := range bv { + b32[i] = float32(r.NormFloat64()) + bv[i] = float64(b32[i]) + } + check := func(name string, got []float64, u float64, extra int) { + for i := 0; i < m; i++ { + for j := 0; j < n; j++ { + c, ab := new(big.Float).SetPrec(oraclePrec), new(big.Float).SetPrec(oraclePrec) + p := new(big.Float).SetPrec(oraclePrec) + for q := 0; q < k; q++ { + p.Mul(big.NewFloat(av[i*k+q]), big.NewFloat(bv[q*n+j])) + c.Add(c, p) + ab.Add(ab, p.Abs(p)) + } + want, _ := c.Float64() + abs, _ := ab.Float64() + if e, bound := math.Abs(got[i*n+j]-want), gamma(k+extra, u)*abs; e > bound { + t.Fatalf("%s %dx%dx%d C[%d,%d]: error %g > bound %g", name, m, k, n, i, j, e, bound) + } + } + } + } + + A, _ := ndarray.FromData(av, m, k) + B, _ := ndarray.FromData(bv, k, n) + C, err := A.MatMul(B) + if err != nil { + t.Fatal(err) + } + c64, _ := ndarray.Data[float64](C) + check("float64", c64, 0x1p-53, 0) + + A32, _ := ndarray.FromSlice(a32, m, k) + B32, _ := ndarray.FromSlice(b32, k, n) + C32, err := A32.MatMul(B32) + if err != nil { + t.Fatal(err) + } + d32, ok := ndarray.Data[float32](C32) + if !ok { + t.Fatalf("float32 MatMul returned %v, not float32", C32.DType()) + } + c32 := make([]float64, len(d32)) + for i, v := range d32 { + c32[i] = float64(v) + } + // A path that computes in float64 and rounds once to float32 is + // within γ_{k+1} of float32's unit roundoff too. + check("float32", c32, 0x1p-24, 1) + } +} From 09a6c6fb054259620b4cb8e53141f89b47bf8d7a Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 12 Nov 2026 11:27:01 +0100 Subject: [PATCH 2/3] review fixes: exact NumPy pairwise depth, big.Float comparison, Exp/Log figures as stated by floats Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +++-- accuracy_bound_test.go | 80 ++++++++++++++++++++++++++++-------------- 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index f3ba910..084de98 100644 --- a/README.md +++ b/README.md @@ -189,8 +189,12 @@ keep their own elementwise kernels here until floats has them. The contract is **correct, and as fast as possible**. "Correct" means a stated error bound, checked against an exact oracle. Equality with NumPy's bits is not the contract: wherever it holds today, it is how a function was ported, and a faster default that stays within its bound may replace it. -Bound: elementwise `+ − × ÷` and `Sqrt` are correctly rounded (one IEEE operation, ≤ 0.5 ULP). Float64 `Exp`/`Log` are within 0.51 ULP. Float32 `Exp`/`Log` are correctly rounded, proven over all 2³² inputs. `Max`/`Min` are exact. `Sum` satisfies |Ŝ − S| ≤ γ_d·Σ|xᵢ|, where d = 19 + ⌈log₂(n/128)⌉ is the rounding depth of NumPy's pairwise sum and γ_m = m·u/(1 − m·u), with u = 2⁻⁵³ for float64 and 2⁻²⁴ for float32. Every element of `MatMul`/`Dot` satisfies |Ĉᵢⱼ − Cᵢⱼ| ≤ γ_k·(|A|·|B|)ᵢⱼ (Higham, *Accuracy and Stability of Numerical Algorithms*, §3.1, §4.2). These two bounds hold for any summation order, with or without FMA. -Oracle: `math/big` at 1024 bits for sums and inner products (`TestAccuracyBoundSum`, `TestAccuracyBoundMatMul`), and a 300-bit reference with the exhaustive float32 proof for `Exp`/`Log`. +Bound: elementwise `+ − × ÷` and `Sqrt` are correctly rounded (one IEEE operation, ≤ 0.5 ULP). Float64 `Exp`/`Log` have a worst error of 0.511 / 0.519 ULP, the figures of Arm's optimized-routines, which go-simd/floats ports. Float32 `Exp`/`Log` are correctly rounded. The proof covers all 2³² inputs and lives in go-simd/floats; it runs on demand, and CI checks a sample. `Max`/`Min` are exact. + +`Sum` satisfies |Ŝ − S| ≤ γ_d·Σ|xᵢ|, with γ_m = m·u/(1 − m·u), u = 2⁻⁵³ for float64 and 2⁻²⁴ for float32. d is the rounding depth of NumPy's pairwise tree: at most 24 in a leaf of up to 128 elements, plus one per halving. The bound holds for any tree at most d deep, not for a serial loop, which needs γ_{n−1}. + +Every element of `MatMul`/`Dot` satisfies |Ĉᵢⱼ − Cᵢⱼ| ≤ γ_k·(|A|·|B|)ᵢⱼ, whatever the summation order, with or without FMA. Both bounds are from Higham, *Accuracy and Stability of Numerical Algorithms*, §3.1 and §4.2. +Oracle: `math/big` at 1024 bits for sums and inner products. `TestSumDepth`, `TestAccuracyBoundSum` and `TestAccuracyBoundMatMul` check them; the difference is taken in `big.Float`, so the oracle is never rounded. `Exp`/`Log` are checked against a 300-bit reference in go-simd/floats. Exact: the defaults are still the exact ports, NumPy's pairwise order for `Sum` included, so results match NumPy bit for bit where they did before. When a faster default lands, that arithmetic stays available behind an explicit option. ## Goroutines diff --git a/accuracy_bound_test.go b/accuracy_bound_test.go index d340ad1..058a7db 100644 --- a/accuracy_bound_test.go +++ b/accuracy_bound_test.go @@ -22,45 +22,71 @@ const oraclePrec = 1024 // gamma is Higham's γ_m = m·u/(1 − m·u), with u the unit roundoff. func gamma(m int, u float64) float64 { return float64(m) * u / (1 - float64(m)*u) } -// sumDepth is the longest chain of roundings in NumPy's pairwise sum, which -// Sum follows: blocks of at most 128 elements summed by eight accumulators -// (up to 16 serial additions each, then three levels combining them), and the -// blocks combined pairwise. Any reordering with a depth at most this one is -// within the same bound. +// sumDepth is the longest chain of roundings in NumPy's pairwise sum +// (numpy/_core/src/umath/loops_utils.h.src, pairwise_sum), which Sum follows, +// computed on the same tree rather than by a closed form: +// - n < 8: a serial loop, n−1 additions; +// - n ≤ 128: eight accumulators over the first n − n%8 elements (m/8 − 1 +// additions each), three levels combining them, then the n%8 remaining +// elements added one by one: up to 14 + 3 + 7 = 24 at n = 127; +// - otherwise: split at n2 = n/2 rounded down to a multiple of 8, one more +// addition joining the halves. +// +// Any other order whose tree is at most this deep is within the same bound. func sumDepth(n int) int { - d := 16 + 3 - for blocks := (n + 127) / 128; blocks > 1; blocks = (blocks + 1) / 2 { - d++ + switch { + case n < 8: + return max(n-1, 0) + case n <= 128: + m := n - n%8 + return (m/8 - 1) + 3 + n%8 + default: + n2 := n / 2 + n2 -= n2 % 8 + return 1 + max(sumDepth(n2), sumDepth(n-n2)) } - return d } -func exactSum(xs []float64) (sum, abs float64) { +// within reports whether |got − want| ≤ bound, with want the exact value: the +// difference is taken in big.Float, so the oracle is never rounded first. +func within(got float64, want *big.Float, bound float64) (bool, float64) { + d := new(big.Float).SetPrec(oraclePrec).Sub(big.NewFloat(got), want) + e, _ := d.Abs(d).Float64() + return e <= bound, e +} + +func exactSum(xs []float64) (sum *big.Float, abs float64) { s, a := new(big.Float).SetPrec(oraclePrec), new(big.Float).SetPrec(oraclePrec) for _, x := range xs { s.Add(s, big.NewFloat(x)) a.Add(a, big.NewFloat(math.Abs(x))) } - sum, _ = s.Float64() abs, _ = a.Float64() - return sum, abs + return s, abs +} + +func TestSumDepth(t *testing.T) { + for _, c := range []struct{ n, d int }{ + {1, 0}, {7, 6}, {8, 3}, {127, 24}, {128, 18}, {129, 1 + 11}, {247, 1 + 24}, // 129 = 64 + 65; 247 = 120 + 127 + } { + if got := sumDepth(c.n); got != c.d { + t.Errorf("sumDepth(%d) = %d, want %d", c.n, got, c.d) + } + } } // TestAccuracyBoundSum: |Sum(x) − Σx| ≤ γ_d·Σ|x|, d = sumDepth(n), in float64 -// and in float32 (whose sum accumulates in float32, as NumPy's does). +// and in float32 (whose sum accumulates in float32, as NumPy's does). The +// sizes include leaves of 121–127 elements (n = 127, 247), the deepest ones. func TestAccuracyBoundSum(t *testing.T) { r := rand.New(rand.NewPCG(1, 2)) - for _, n := range []int{1, 7, 128, 129, 1000, 65536, 1 << 20} { + for _, n := range []int{1, 7, 127, 128, 129, 247, 1000, 65536, 1 << 20} { xs := make([]float64, n) x32 := make([]float32, n) for i := range xs { - // Mixed signs and magnitudes, so cancellation is exercised. - // The range is kept to 2^±4 so that the bound stays below the - // smallest element: a dropped or doubled element fails it. + // Mixed signs and magnitudes in 2^±4, so cancellation is + // exercised and the bound stays far below a typical element. v := (r.Float64()*2 - 1) * math.Ldexp(1, r.IntN(8)-4) - if v == 0 { - v = 1 - } x32[i] = float32(v) xs[i] = v } @@ -69,8 +95,8 @@ func TestAccuracyBoundSum(t *testing.T) { t.Fatal(err) } want, abs := exactSum(xs) - if got, bound := a.Sum(), gamma(sumDepth(n), 0x1p-53)*abs; math.Abs(got-want) > bound { - t.Errorf("float64 n=%d: |Sum − exact| = %g > bound %g", n, math.Abs(got-want), bound) + if ok, e := within(a.Sum(), want, gamma(sumDepth(n), 0x1p-53)*abs); !ok { + t.Errorf("float64 n=%d: |Sum − exact| = %g > bound %g", n, e, gamma(sumDepth(n), 0x1p-53)*abs) } a32, err := ndarray.FromSlice(x32, n) @@ -82,9 +108,9 @@ func TestAccuracyBoundSum(t *testing.T) { xs32[i] = float64(v) } want, abs = exactSum(xs32) - // One more rounding: the float32 result is returned as a float64. - if got, bound := a32.Sum(), gamma(sumDepth(n)+1, 0x1p-24)*abs; math.Abs(got-want) > bound { - t.Errorf("float32 n=%d: |Sum − exact| = %g > bound %g", n, math.Abs(got-want), bound) + // float32 → float64 is exact, so the depth is the same. + if ok, e := within(a32.Sum(), want, gamma(sumDepth(n), 0x1p-24)*abs); !ok { + t.Errorf("float32 n=%d: |Sum − exact| = %g > bound %g", n, e, gamma(sumDepth(n), 0x1p-24)*abs) } } } @@ -116,9 +142,9 @@ func TestAccuracyBoundMatMul(t *testing.T) { c.Add(c, p) ab.Add(ab, p.Abs(p)) } - want, _ := c.Float64() abs, _ := ab.Float64() - if e, bound := math.Abs(got[i*n+j]-want), gamma(k+extra, u)*abs; e > bound { + if ok, e := within(got[i*n+j], c, gamma(k+extra, u)*abs); !ok { + bound := gamma(k+extra, u) * abs t.Fatalf("%s %dx%dx%d C[%d,%d]: error %g > bound %g", name, m, k, n, i, j, e, bound) } } From 983e22e041aaf66fe4e9aa1ab496028027626182 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Thu, 12 Nov 2026 11:27:37 +0100 Subject: [PATCH 3/3] test: limit the any-order sentence to MatMul Co-Authored-By: Claude Opus 5.5 --- accuracy_bound_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/accuracy_bound_test.go b/accuracy_bound_test.go index 058a7db..1de6285 100644 --- a/accuracy_bound_test.go +++ b/accuracy_bound_test.go @@ -13,9 +13,10 @@ import ( // oracle: math/big at 1024 bits, wide enough that every sum and dot product of // these inputs is exact before its final rounding. The bounds are the rigorous // worst-case bounds of the algorithm (Higham, "Accuracy and Stability of -// Numerical Algorithms", 2nd ed., §3.1 and §4.2), so they hold for any -// summation order and with or without FMA: a faster default that reorders or -// fuses is accepted as long as these tests pass. +// Numerical Algorithms", 2nd ed., §3.1 and §4.2). The MatMul bound holds for +// any summation order, with or without FMA; the Sum bound holds for any tree at +// most sumDepth(n) deep. A faster default that reorders or fuses is accepted as +// long as these tests pass. const oraclePrec = 1024