diff --git a/broadcast_repeat_test.go b/broadcast_repeat_test.go new file mode 100644 index 0000000..bdb317f --- /dev/null +++ b/broadcast_repeat_test.go @@ -0,0 +1,83 @@ +package ndarray + +import ( + "testing" + + "github.com/go-ndarray/ndarray/internal/kernels" +) + +// TestBroadcastRepeat checks the streamed broadcast (a full operand and a +// repeated one, see repeatOperands) against an element-by-element oracle, for +// the non-commutative ops in both operand orders, several suffix shapes, a +// repeated operand too short to stream, and with the block loop forced onto +// several cores. +func TestBroadcastRepeat(t *testing.T) { + grid := func(seed int, shape ...int) *Array { + n := 1 + for _, d := range shape { + n *= d + } + d := make([]float64, n) + for i := range d { + d[i] = float64((i*13+seed)%29) - 14.5 + } + return mustArr(t, ok(FromData(d, shape...))) + } + cases := []struct { + name string + full, rep *Array + streamable bool + }{ + {"matrix, row", grid(1, 9, 70), grid(2, 1, 70), true}, + {"matrix, 1-D", grid(3, 5, 80), grid(4, 80), true}, + {"3-D, 2-D", grid(5, 4, 3, 30), grid(6, 3, 30), true}, + {"3-D, row of 1-D", grid(7, 2, 3, 100), grid(8, 1, 1, 100), true}, + {"short row", grid(9, 6, 10), grid(10, 1, 10), false}, + } + ops := []struct { + name string + f func(x, y *Array) (*Array, error) + g func(x, y float64) float64 + }{ + {"sub", (*Array).Sub, func(x, y float64) float64 { return x - y }}, + {"div", (*Array).Div, func(x, y float64) float64 { return x / y }}, + } + run := func() { + for _, c := range cases { + if _, _, _, got := repeatOperands(c.full, c.rep, c.full.shape); got != c.streamable { + t.Errorf("%s: streamable = %v, want %v", c.name, got, c.streamable) + } + for _, op := range ops { + for _, swap := range []bool{false, true} { + x, y := c.full, c.rep + if swap { + x, y = y, x + } + got, err := op.f(x, y) + if err != nil { + t.Fatal(err) + } + if !sameShape(got.shape, c.full.shape) { + t.Fatalf("%s %s: shape %v", c.name, op.name, got.shape) + } + m := c.rep.Size() + for i, v := range got.data { + fv, rv := c.full.data[i], c.rep.data[i%m] + want := op.g(fv, rv) + if swap { + want = op.g(rv, fv) + } + if v != want { + t.Fatalf("%s %s swap=%v [%d]: %v, want %v", c.name, op.name, swap, i, v, want) + } + } + } + } + } + } + run() + saved := kernels.ParThreshold + kernels.ParThreshold = 64 + defer func() { kernels.ParThreshold = saved }() + run() +} diff --git a/docs/perf.md b/docs/perf.md index 04f2d7a..380efb4 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -436,6 +436,20 @@ Zen 3, 16 cores pinned, minimum of three interleaved runs: | MaxAxis(1) | 601 µs | **93 µs** | 189 µs | 2.0× | On M4: MaxAxis(1) ×6–8, SumAxis(0) ×1.9. +## Broadcasting a row without materialising it (2026-10-04) + +`M + row` (1024×1024 plus 1×1024) lost to NumPy 0.26× on the Zen 3 even with a +`Workspace`: the broadcast path copied the row out to a full 8 MiB matrix +before adding. When one operand has the full shape and the other, leading 1s +dropped, is a suffix of it (a row, a 2-D mean subtracted from a 3-D stack), the +second simply repeats in blocks of its own length, so `binOp` now streams the +live slices block by block through the SIMD kernel (`kernels.RepeatP`), split +across cores. Below 64 elements per block the old path is kept. + +| BroadcastAdd 1024²+row, Zen 3 | heap | `Workspace` | NumPy 1 thread | +|----|--:|--:|--:| +| before | 3.1 ms | 1.76 ms | 460 µs | +| after | **0.84 ms** | **114–196 µs** | 460 µs | ## SIMD coverage diff --git a/internal/kernels/parallel.go b/internal/kernels/parallel.go index 5b4991d..835406b 100644 --- a/internal/kernels/parallel.go +++ b/internal/kernels/parallel.go @@ -83,6 +83,33 @@ func runBinaryP(k binaryKernel, dst, a, b []float64) { // IEEE-754 op — no reduction grouping), so the result matches the serial scalar // computation for every input. +// RepeatP computes dst = full OP rep, where rep is repeated along full in +// consecutive blocks of len(rep) — a broadcast of rep over the leading axes, +// such as a matrix plus a row. Each block is one call of the elementwise +// kernel k on live slices, so rep is never materialised to full's size; the +// blocks are split across cores above ParThreshold. With repFirst the operands +// are swapped (dst = rep OP full), for the non-commutative ops. len(dst) == +// len(full) is a multiple of len(rep) >= 1. +func RepeatP(k func(dst, a, b []float64), dst, full, rep []float64, repFirst bool) { + m := len(rep) + blocks := len(dst) / m + body := func(lo, hi int) { + for b := lo; b < hi; b++ { + s := b * m + if repFirst { + k(dst[s:s+m], rep, full[s:s+m]) + } else { + k(dst[s:s+m], full[s:s+m], rep) + } + } + } + if len(dst) < ParThreshold { + body(0, blocks) + return + } + parallelFor(blocks, min(numWorkers(len(dst)), blocks), body) +} + // AddP writes a[i]+b[i] into dst[i], parallelised above ParThreshold. func AddP(dst, a, b []float64) { runBinaryP(addBin, dst, a, b) } diff --git a/ndarray.go b/ndarray.go index 1cf47a9..14ea223 100644 --- a/ndarray.go +++ b/ndarray.go @@ -626,6 +626,12 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error if err != nil { return nil, err } + if full, rep, repFirst, ok := repeatOperands(a, b, shape); ok { + dst := a.alloc(prod(shape), false) + kernels.RepeatP(kernel, dst, full, rep, repFirst) + cp := append([]int(nil), shape...) + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil + } x := a.operandFor(shape) y := b.operandFor(shape) dst := a.alloc(prod(shape), false) @@ -634,6 +640,35 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil } +// repeatMin is the shortest repeated operand binOp streams block by block; +// below it a kernel call per block costs more than materialising. +const repeatMin = 64 + +// repeatOperands recognises the broadcast that is a plain repetition: one +// operand is contiguous with the full result shape, the other is contiguous +// and its shape, leading 1s dropped, is a suffix of the result shape (a matrix +// plus a row, a 3-D stack minus a 2-D mean). The second then repeats in +// consecutive blocks of its own length, and binOp can stream the live slices +// instead of materialising the broadcast (see kernels.RepeatP). +func repeatOperands(a, b *Array, shape []int) (full, rep []float64, repFirst, ok bool) { + isRep := func(r *Array) bool { + s := r.shape + for len(s) > 0 && s[0] == 1 { + s = s[1:] + } + return r.isContiguous() && r.Size() >= repeatMin && + len(s) <= len(shape) && sameShape(s, shape[len(shape)-len(s):]) + } + isFull := func(f *Array) bool { return f.isContiguous() && sameShape(f.shape, shape) } + switch { + case isFull(a) && isRep(b): + return a.data, b.data, false, true + case isFull(b) && isRep(a): + return b.data, a.data, true, true + } + return nil, nil, false, false +} + // Add returns the elementwise sum a+b with broadcasting. func (a *Array) Add(b *Array) (*Array, error) { return a.binOp(b, kernels.AddP) }