Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions arith.go
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,15 @@ func (a *Array) reduceTyped(axis int, keepdims bool, op reduceCode) (*Array, err
return a.result(dst, shape), nil
}
if outer*inner > 0 {
if inner == 1 && dt != a.dtype {
// A sum or product of bools or narrow integers along a
// contiguous axis: fold the elements into the wide accumulator
// as they are read, rather than converting the whole array to
// it first (an extra pass and an allocation of 8 bytes an
// element).
foldWidened(op, dst, a.contiguousStore(), outer, axisLen)
return a.result(dst, shape), nil
}
src := a.operandStore(dt, a.shape)
if op == rSum || op == rProd {
accumulateStore(op, dst, src, outer, axisLen, inner)
Expand Down
12 changes: 9 additions & 3 deletions docs/perf.md
Original file line number Diff line number Diff line change
Expand Up @@ -829,9 +829,15 @@ float32 sum is also more accurate: on 2¹⁶ values of mixed sign and magnitude
its error against the exact sum (`math/big`) fell from 75 to 0.87, and 2²⁴
followed by 1000 ones no longer saturates at 2²⁴ (`sum_pairwise_test.go`).

Left: float32 has no SIMD kernel yet (still 7× float64 for `Sum`), and the
integer sums convert the whole array to int64 before adding (NumPy's
accumulator type), an extra pass and allocation.
The narrow integer sums still converted the whole array to int64 (NumPy's
accumulator) before adding: an extra pass and 8 bytes an element. Since
v0.14.0 a sum or product of bools or integers narrower than 64 bits along a
contiguous axis widens each element as it is read (`foldWidened`, same
arithmetic, wrap-around included): **int32 `Sum` of 2²⁰ elements 2.872 →
1.548 ms (−46%)**, 2¹⁴ −51%, with float64, float32 and int64 unchanged as
controls (POWER9, same protocol, p = 0.002).

Left: float32 has no SIMD kernel yet (still 7× float64 for `Sum`).

## SIMD coverage

Expand Down
98 changes: 98 additions & 0 deletions sum_pairwise_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,104 @@ func TestProdAlongANonContiguousAxis(t *testing.T) {
}
}

// TestWidenedSums: sums and products of bools and narrow integers along a
// contiguous axis fold each element into the 64-bit accumulator as it is
// read. The expectation widens each element first and folds with Go's
// arithmetic of the accumulator, wrap-around included; a transposed operand
// takes the gather path.
func TestWidenedSums(t *testing.T) {
const rows, cols = 3, 37
check := func(name string, a *Array, wantSum, wantProd func(row []int) any) {
t.Helper()
for _, x := range []*Array{a, must(a.Transpose().Copy().Reshape(cols, rows)).Transpose()} {
s, p := must(x.SumAxis(1, false)), must(x.ProdAxis(1, false))
for r := range rows {
idx := []int{r}
if gs, ws := s.itemAny(idx), wantSum([]int{r}); gs != ws {
t.Errorf("%s sum row %d = %v, want %v", name, r, gs, ws)
}
if gp, wp := p.itemAny(idx), wantProd([]int{r}); gp != wp {
t.Errorf("%s prod row %d = %v, want %v", name, r, gp, wp)
}
}
}
}
i32 := make([]int32, rows*cols)
u16 := make([]uint16, rows*cols)
b := make([]bool, rows*cols)
for i := range i32 {
i32[i] = int32(math.MaxInt32 - i*7919) // sums past 2^32, products that wrap
u16[i] = uint16(65535 - i*31)
b[i] = i%11 != 0
}
b[cols+5] = true // row 1 all true except its i%11 == 0 slots
check("int32", must(FromSlice(i32, rows, cols)), func(r []int) any {
var acc int64
for _, v := range i32[r[0]*cols : (r[0]+1)*cols] {
acc += int64(v)
}
return acc
}, func(r []int) any {
var acc int64 = 1
for _, v := range i32[r[0]*cols : (r[0]+1)*cols] {
acc *= int64(v)
}
return acc
})
check("uint16", must(FromSlice(u16, rows, cols)), func(r []int) any {
var acc uint64
for _, v := range u16[r[0]*cols : (r[0]+1)*cols] {
acc += uint64(v)
}
return acc
}, func(r []int) any {
var acc uint64 = 1
for _, v := range u16[r[0]*cols : (r[0]+1)*cols] {
acc *= uint64(v)
}
return acc
})
check("bool", must(FromSlice(b, rows, cols)), func(r []int) any {
var acc int64
for _, v := range b[r[0]*cols : (r[0]+1)*cols] {
if v {
acc++
}
}
return acc
}, func(r []int) any {
var acc int64 = 1
for _, v := range b[r[0]*cols : (r[0]+1)*cols] {
if !v {
acc = 0
}
}
return acc
})
for _, dt := range []DType{Int8, Int16, Uint8, Uint32} {
x := must(Arange(1, 10, 1)).AsType(dt)
if got := x.SumAll().itemAny([]int{}); got != map[bool]any{true: int64(45), false: uint64(45)}[dt.IsSigned()] {
t.Errorf("%v: SumAll = %v, want 45 in the accumulator type", dt, got)
}
if got := x.ProdAll().itemAny([]int{}); got != map[bool]any{true: int64(362880), false: uint64(362880)}[dt.IsSigned()] {
t.Errorf("%v: ProdAll = %v, want 362880", dt, got)
}
}
}

// itemAny returns element idx of a in its Go type, for comparisons.
func (a *Array) itemAny(idx []int) any {
s := a.contiguousStore()
f := flatIndex(idx, a.shape)
switch st := s.(type) {
case []int64:
return st[f]
case []uint64:
return st[f]
}
return nil
}

// TestPairwiseSumBranches covers each length class of NumPy's scheme and
// checks it against the exact sum of small integers (exact in float32), and
// that a sum of only -0 stays -0.
Expand Down
56 changes: 56 additions & 0 deletions typed.go
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,62 @@ func foldRun[T numT](op reduceCode, s []T) T {
return acc
}

// foldWidened sums or multiplies each contiguous run of src (bools or
// integers narrower than 64 bits) into dst, NumPy's accumulator for them:
// []int64 for bools and signed integers, []uint64 for unsigned. Each element
// is widened as it is read; the arithmetic, wrap-around included, is that of
// converting first and folding after.
func foldWidened(op reduceCode, dst, src any, outer, axisLen int) {
switch s := src.(type) {
case []bool:
d := dst.([]int64)
for o := range outer {
var acc int64 = 1 // a product of bools is 1 unless one is false
if op == rSum {
acc = 0
}
for _, v := range s[o*axisLen : (o+1)*axisLen] {
switch {
case op == rSum && v:
acc++
case op == rProd && !v:
acc = 0
}
}
d[o] = acc
}
case []int8:
widenRuns(op, dst.([]int64), s, outer, axisLen)
case []int16:
widenRuns(op, dst.([]int64), s, outer, axisLen)
case []int32:
widenRuns(op, dst.([]int64), s, outer, axisLen)
case []uint8:
widenRuns(op, dst.([]uint64), s, outer, axisLen)
case []uint16:
widenRuns(op, dst.([]uint64), s, outer, axisLen)
case []uint32:
widenRuns(op, dst.([]uint64), s, outer, axisLen)
}
}

func widenRuns[S int8 | int16 | int32 | uint8 | uint16 | uint32, D int64 | uint64](op reduceCode, dst []D, src []S, outer, axisLen int) {
for o := range outer {
run := src[o*axisLen : (o+1)*axisLen]
acc := D(run[0])
if op == rSum {
for _, v := range run[1:] {
acc += D(v)
}
} else {
for _, v := range run[1:] {
acc *= D(v)
}
}
dst[o] = acc
}
}

// pwBlock is NumPy's PW_BLOCKSIZE: runs up to this length are summed with
// eight accumulators, longer ones are halved.
const pwBlock = 128
Expand Down
Loading