From b1fe76ba927901e8077a964d76ddb7ed731dbc25 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 18 Oct 2026 09:36:09 +0200 Subject: [PATCH] perf: narrow integer sums widen as they read, without an int64 copy A sum or product of bools or integers narrower than 64 bits accumulates in int64/uint64, as NumPy does, and reduceTyped got there by converting the whole array first: an extra pass and 8 bytes an element. Along a contiguous axis each element is now widened as it is read (foldWidened); the arithmetic, wrap-around included, is unchanged. POWER9, 1 thread, 6 interleaved runs, benchstat p=0.002: int32 Sum of 2^20 2.872 -> 1.548 ms (-46%), 2^14 -51%; float64, float32 and int64 ~ (controls). TestWidenedSums checks every narrow dtype, sums and products, wrap-around and a transposed view against widen-then-fold; a sabotaged fold (one element skipped) fails it 16 times. Coverage 100%, compiles for 386 and arm. Co-Authored-By: Claude Opus 5.5 --- arith.go | 9 ++++ docs/perf.md | 12 ++++-- sum_pairwise_test.go | 98 ++++++++++++++++++++++++++++++++++++++++++++ typed.go | 56 +++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 3 deletions(-) diff --git a/arith.go b/arith.go index 6a253b7..6044dac 100644 --- a/arith.go +++ b/arith.go @@ -330,6 +330,15 @@ func (a *Array) reduceTyped(axis int, keepdims bool, op reduceCode) (*Array, err return a.result(dst, shape), nil } if outer*inner > 0 { + if inner == 1 && dt != a.dtype { + // A sum or product of bools or narrow integers along a + // contiguous axis: fold the elements into the wide accumulator + // as they are read, rather than converting the whole array to + // it first (an extra pass and an allocation of 8 bytes an + // element). + foldWidened(op, dst, a.contiguousStore(), outer, axisLen) + return a.result(dst, shape), nil + } src := a.operandStore(dt, a.shape) if op == rSum || op == rProd { accumulateStore(op, dst, src, outer, axisLen, inner) diff --git a/docs/perf.md b/docs/perf.md index c9d1a5e..7203410 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -829,9 +829,15 @@ float32 sum is also more accurate: on 2¹⁶ values of mixed sign and magnitude its error against the exact sum (`math/big`) fell from 75 to 0.87, and 2²⁴ followed by 1000 ones no longer saturates at 2²⁴ (`sum_pairwise_test.go`). -Left: float32 has no SIMD kernel yet (still 7× float64 for `Sum`), and the -integer sums convert the whole array to int64 before adding (NumPy's -accumulator type), an extra pass and allocation. +The narrow integer sums still converted the whole array to int64 (NumPy's +accumulator) before adding: an extra pass and 8 bytes an element. Since +v0.14.0 a sum or product of bools or integers narrower than 64 bits along a +contiguous axis widens each element as it is read (`foldWidened`, same +arithmetic, wrap-around included): **int32 `Sum` of 2²⁰ elements 2.872 → +1.548 ms (−46%)**, 2¹⁴ −51%, with float64, float32 and int64 unchanged as +controls (POWER9, same protocol, p = 0.002). + +Left: float32 has no SIMD kernel yet (still 7× float64 for `Sum`). ## SIMD coverage diff --git a/sum_pairwise_test.go b/sum_pairwise_test.go index 372ad57..b94e915 100644 --- a/sum_pairwise_test.go +++ b/sum_pairwise_test.go @@ -86,6 +86,104 @@ func TestProdAlongANonContiguousAxis(t *testing.T) { } } +// TestWidenedSums: sums and products of bools and narrow integers along a +// contiguous axis fold each element into the 64-bit accumulator as it is +// read. The expectation widens each element first and folds with Go's +// arithmetic of the accumulator, wrap-around included; a transposed operand +// takes the gather path. +func TestWidenedSums(t *testing.T) { + const rows, cols = 3, 37 + check := func(name string, a *Array, wantSum, wantProd func(row []int) any) { + t.Helper() + for _, x := range []*Array{a, must(a.Transpose().Copy().Reshape(cols, rows)).Transpose()} { + s, p := must(x.SumAxis(1, false)), must(x.ProdAxis(1, false)) + for r := range rows { + idx := []int{r} + if gs, ws := s.itemAny(idx), wantSum([]int{r}); gs != ws { + t.Errorf("%s sum row %d = %v, want %v", name, r, gs, ws) + } + if gp, wp := p.itemAny(idx), wantProd([]int{r}); gp != wp { + t.Errorf("%s prod row %d = %v, want %v", name, r, gp, wp) + } + } + } + } + i32 := make([]int32, rows*cols) + u16 := make([]uint16, rows*cols) + b := make([]bool, rows*cols) + for i := range i32 { + i32[i] = int32(math.MaxInt32 - i*7919) // sums past 2^32, products that wrap + u16[i] = uint16(65535 - i*31) + b[i] = i%11 != 0 + } + b[cols+5] = true // row 1 all true except its i%11 == 0 slots + check("int32", must(FromSlice(i32, rows, cols)), func(r []int) any { + var acc int64 + for _, v := range i32[r[0]*cols : (r[0]+1)*cols] { + acc += int64(v) + } + return acc + }, func(r []int) any { + var acc int64 = 1 + for _, v := range i32[r[0]*cols : (r[0]+1)*cols] { + acc *= int64(v) + } + return acc + }) + check("uint16", must(FromSlice(u16, rows, cols)), func(r []int) any { + var acc uint64 + for _, v := range u16[r[0]*cols : (r[0]+1)*cols] { + acc += uint64(v) + } + return acc + }, func(r []int) any { + var acc uint64 = 1 + for _, v := range u16[r[0]*cols : (r[0]+1)*cols] { + acc *= uint64(v) + } + return acc + }) + check("bool", must(FromSlice(b, rows, cols)), func(r []int) any { + var acc int64 + for _, v := range b[r[0]*cols : (r[0]+1)*cols] { + if v { + acc++ + } + } + return acc + }, func(r []int) any { + var acc int64 = 1 + for _, v := range b[r[0]*cols : (r[0]+1)*cols] { + if !v { + acc = 0 + } + } + return acc + }) + for _, dt := range []DType{Int8, Int16, Uint8, Uint32} { + x := must(Arange(1, 10, 1)).AsType(dt) + if got := x.SumAll().itemAny([]int{}); got != map[bool]any{true: int64(45), false: uint64(45)}[dt.IsSigned()] { + t.Errorf("%v: SumAll = %v, want 45 in the accumulator type", dt, got) + } + if got := x.ProdAll().itemAny([]int{}); got != map[bool]any{true: int64(362880), false: uint64(362880)}[dt.IsSigned()] { + t.Errorf("%v: ProdAll = %v, want 362880", dt, got) + } + } +} + +// itemAny returns element idx of a in its Go type, for comparisons. +func (a *Array) itemAny(idx []int) any { + s := a.contiguousStore() + f := flatIndex(idx, a.shape) + switch st := s.(type) { + case []int64: + return st[f] + case []uint64: + return st[f] + } + return nil +} + // TestPairwiseSumBranches covers each length class of NumPy's scheme and // checks it against the exact sum of small integers (exact in float32), and // that a sum of only -0 stays -0. diff --git a/typed.go b/typed.go index f2956b8..9d91938 100644 --- a/typed.go +++ b/typed.go @@ -401,6 +401,62 @@ func foldRun[T numT](op reduceCode, s []T) T { return acc } +// foldWidened sums or multiplies each contiguous run of src (bools or +// integers narrower than 64 bits) into dst, NumPy's accumulator for them: +// []int64 for bools and signed integers, []uint64 for unsigned. Each element +// is widened as it is read; the arithmetic, wrap-around included, is that of +// converting first and folding after. +func foldWidened(op reduceCode, dst, src any, outer, axisLen int) { + switch s := src.(type) { + case []bool: + d := dst.([]int64) + for o := range outer { + var acc int64 = 1 // a product of bools is 1 unless one is false + if op == rSum { + acc = 0 + } + for _, v := range s[o*axisLen : (o+1)*axisLen] { + switch { + case op == rSum && v: + acc++ + case op == rProd && !v: + acc = 0 + } + } + d[o] = acc + } + case []int8: + widenRuns(op, dst.([]int64), s, outer, axisLen) + case []int16: + widenRuns(op, dst.([]int64), s, outer, axisLen) + case []int32: + widenRuns(op, dst.([]int64), s, outer, axisLen) + case []uint8: + widenRuns(op, dst.([]uint64), s, outer, axisLen) + case []uint16: + widenRuns(op, dst.([]uint64), s, outer, axisLen) + case []uint32: + widenRuns(op, dst.([]uint64), s, outer, axisLen) + } +} + +func widenRuns[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](op reduceCode, dst []D, src []S, outer, axisLen int) { + for o := range outer { + run := src[o*axisLen : (o+1)*axisLen] + acc := D(run[0]) + if op == rSum { + for _, v := range run[1:] { + acc += D(v) + } + } else { + for _, v := range run[1:] { + acc *= D(v) + } + } + dst[o] = acc + } +} + // pwBlock is NumPy's PW_BLOCKSIZE: runs up to this length are summed with // eight accumulators, longer ones are halved. const pwBlock = 128