From 8f4e93321b4201d13403cba67e7adfd8b42c063b Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 15 Nov 2026 10:32:01 +0100 Subject: [PATCH 1/3] perf: stream scalar operands; fill by doubling copy (#118) A binary operation with a one-element operand (AddScalar ... DivScalar, x op Scalar(v), a 1x1 array against a matrix) materialised the scalar to the result's size before calling the kernel: two 800 kB buffers per call on 10^5 elements. binOp now recognises the case (scalarOperand) and kernels.ScalarP streams a block of 512 copies through the same SIMD kernel, like RepeatP does for repeated rows, so each element is computed by the same instruction and the result is the same bits. Full, Ones and FullOf (every dtype, through the generated fillStore) filled one element at a time. kernels.Fill, generic over the element type, writes one element and doubles the filled prefix with copy. cfarm420 (EPYC 7773X, Zen 3), one pinned core, GOMAXPROCS=1, load 2.2-2.4, three interleaved rounds, 10^5 float64: AddScalar 183-203 us -> 95-110 us, 1.6 MB -> 0.8 MB per call Full 131-135 us -> 84-104 us alloc_bench_test.go keeps the benchmarks that take the result's cost apart. A CPU profile of a bare make([]float64, 1e5) shows ~60% GC marking (assists) and ~23% zeroing; docs/perf.md says so and points to the *Into forms and Workspace. Refs #118 Co-Authored-By: Claude Opus 5.5 --- alloc_bench_test.go | 67 +++++++++++++++++++++++++++++++ docs/perf.md | 19 +++++++++ gen_dtypes.go | 8 +++- internal/kernels/parallel.go | 44 ++++++++++++++++++++ internal/kernels/scalar_test.go | 59 +++++++++++++++++++++++++++ ndarray.go | 25 ++++++++++-- scalar_operand_test.go | 69 ++++++++++++++++++++++++++++++++ zz_dtypes_gen.go | 71 +++++++++------------------------ 8 files changed, 304 insertions(+), 58 deletions(-) create mode 100644 alloc_bench_test.go create mode 100644 internal/kernels/scalar_test.go create mode 100644 scalar_operand_test.go diff --git a/alloc_bench_test.go b/alloc_bench_test.go new file mode 100644 index 0000000..1d9e364 --- /dev/null +++ b/alloc_bench_test.go @@ -0,0 +1,67 @@ +package ndarray + +import "testing" + +// The cost of a result, taken apart (#118): the allocation alone, the fill, +// and the operations built on them, at the size of the refbench table. + +const allocN = 100_000 + +var ( + sinkF []float64 + sinkA *Array +) + +func BenchmarkAllocMake(b *testing.B) { + for b.Loop() { + sinkF = make([]float64, allocN) + } +} + +func BenchmarkAllocZeros(b *testing.B) { + for b.Loop() { + sinkA, _ = Zeros(allocN) + } +} + +func BenchmarkAllocFull(b *testing.B) { + for b.Loop() { + sinkA, _ = Full(1.5, allocN) + } +} + +func BenchmarkFillLoop(b *testing.B) { + d := make([]float64, allocN) + for b.Loop() { + for i := range d { + d[i] = 1.5 + } + } + sinkF = d +} + +func BenchmarkFillCopy(b *testing.B) { + d := make([]float64, allocN) + for b.Loop() { + d[0] = 1.5 + for k := 1; k < len(d); k *= 2 { + copy(d[k:], d[:k]) + } + } + sinkF = d +} + +func BenchmarkAllocAdd(b *testing.B) { + x, _ := Full(1.25, allocN) + y, _ := Full(2.5, allocN) + for b.Loop() { + sinkA, _ = x.Add(y) + } +} + +func BenchmarkAllocAddScalar(b *testing.B) { + x, _ := Full(1.25, allocN) + for b.Loop() { + sinkA = x.AddScalar(1.5) + } +} diff --git a/docs/perf.md b/docs/perf.md index 17ac3f7..6bf805f 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -79,6 +79,25 @@ issue per row: Faster, for example: `SqrtInto` (0.51), `fft.FFT` (0.54), `Sum` (0.70), `Exp` (0.89). +### Since the table + +Fixed after v0.41.0, measured by `go test -bench` on the same core, not yet +by a new table run. On 10⁵ float64 at load 2.2–2.4, over three interleaved +rounds: +- The scalar forms (`AddScalar` … `DivScalar`, and any operation with a + one-element operand) no longer copy the scalar to the result's size. They + stream it through the same SIMD kernel, so the result is bit for bit the + same. `AddScalar` went from 183–203 µs to 95–110 µs, and from 1.6 MB to + 0.8 MB per call. +- `Full`, `Ones` and `FullOf` fill by doubling `copy` instead of storing one + element at a time: `Full` went from 131–135 µs to 84–104 µs. + +What remains of the allocation cause is Go's own cost. A CPU profile of a bare +`make([]float64, 100000)` shows about 60% garbage-collector marking (assists +in the allocating goroutine) and about 23% zeroing. NumPy frees each result +when its last reference goes and reuses that memory warm. In Go, the way to +avoid both costs is to not allocate: use the `*Into` forms or a `Workspace`. + The run was made from the release's own code. The test binary was cross-compiled from a clean v0.41.0 worktree, the tag was shipped to the host as a git bundle, and the binary ran from outside the clone so that the tree diff --git a/gen_dtypes.go b/gen_dtypes.go index ade494f..61ac2f2 100644 --- a/gen_dtypes.go +++ b/gen_dtypes.go @@ -87,7 +87,11 @@ func main() { p("") p("package ndarray") p("") - p(`import "strconv"`) + p(`import (`) + p(`"strconv"`) + p(``) + p(`"github.com/go-ndarray/ndarray/internal/kernels"`) + p(`)`) p("") p("// dtypeOfStore returns the dtype of a storage slice.") @@ -130,7 +134,7 @@ func main() { p("// storage slice of the same dtype.") p("func fillStore(dst, one any) {") p("switch d := dst.(type) {") - each("biucf", "case []$T: v := one.([]$T)[0]; for i := range d { d[i] = v }") + each("biucf", "case []$T: kernels.Fill(d, one.([]$T)[0])") p("}") p("}") p("") diff --git a/internal/kernels/parallel.go b/internal/kernels/parallel.go index 01010fe..dba06c5 100644 --- a/internal/kernels/parallel.go +++ b/internal/kernels/parallel.go @@ -179,6 +179,50 @@ func RepeatP(k func(dst, a, b []float64), dst, full, rep []float64, repFirst boo parallelFor(blocks, min(numWorkers(len(dst)), blocks), body) } +// scalarBlock is the length of the block ScalarP fills with the scalar: long +// enough that a kernel call per block costs little next to the block's work, +// short enough (4 KiB) to stay in L1 for the whole pass. +const scalarBlock = 512 + +// ScalarP computes dst = full OP s, where s is a single value broadcast over +// full (x + 1.5): it fills one block of scalarBlock copies of s and streams it +// through the elementwise kernel k like RepeatP, so s is never materialised to +// full's size. Each element goes through the same kernel as the materialised +// broadcast, so the result is the same, bit for bit. With sFirst the operands +// are swapped (dst = s OP full). len(dst) == len(full). +func ScalarP(k func(dst, a, b []float64), dst, full []float64, s float64, sFirst bool) { + if len(dst) == 0 { + return + } + // Heap, not stack: the parallel workers share it, so it would escape + // anyway, and a short array then pays for its own length only. + rep := make([]float64, min(scalarBlock, len(dst))) + Fill(rep, s) + whole := len(dst) / len(rep) * len(rep) + RepeatP(k, dst[:whole], full[:whole], rep, sFirst) + if tail := len(dst) - whole; tail > 0 { + if sFirst { + k(dst[whole:], rep[:tail], full[whole:]) + } else { + k(dst[whole:], full[whole:], rep[:tail]) + } + } +} + +// Fill sets every element of dst to v, for every element type. It writes one +// element, then doubles the filled prefix with copy, whose memmove stores +// whole vectors: 3.5× faster than an element-by-element loop on 10⁵ float64 +// (Zen 3, #118). +func Fill[T any](dst []T, v T) { + if len(dst) == 0 { + return + } + dst[0] = v + for k := 1; k < len(dst); k *= 2 { + copy(dst[k:], dst[:k]) + } +} + // AddP writes a[i]+b[i] into dst[i], parallelised above ParThreshold. func AddP(dst, a, b []float64) { runBinaryP(addBin, dst, a, b) } diff --git a/internal/kernels/scalar_test.go b/internal/kernels/scalar_test.go new file mode 100644 index 0000000..704441c --- /dev/null +++ b/internal/kernels/scalar_test.go @@ -0,0 +1,59 @@ +package kernels + +import ( + "math" + "testing" +) + +// TestScalarPSameBits: streaming the scalar block by block gives, bit for +// bit, what the kernel gives on the scalar materialised to full length, for +// both operand orders, at every block boundary and across ParThreshold. +func TestScalarPSameBits(t *testing.T) { + ks := map[string]func(dst, a, b []float64){"Add": AddP, "Sub": SubP, "Mul": MulP, "Div": DivP} + const s = 1.0 / 3 + for _, n := range []int{0, 1, 7, scalarBlock - 1, scalarBlock, scalarBlock + 1, 3*scalarBlock + 5, ParThreshold + 3} { + full := make([]float64, n) + for i := range full { + full[i] = math.Sin(float64(i)) * 7 + } + wide := make([]float64, n) + for i := range wide { + wide[i] = s + } + for name, k := range ks { + for _, sFirst := range []bool{false, true} { + want := make([]float64, n) + if sFirst { + k(want, wide, full) + } else { + k(want, full, wide) + } + got := make([]float64, n) + ScalarP(k, got, full, s, sFirst) + for i := range want { + if math.Float64bits(got[i]) != math.Float64bits(want[i]) { + t.Fatalf("%s n=%d sFirst=%v [%d]: got %v, want %v", name, n, sFirst, i, got[i], want[i]) + } + } + } + } + } +} + +// TestFill: every element is set, for lengths around the powers of two the +// doubling copy steps through, and nothing past len(dst) is touched. +func TestFill(t *testing.T) { + for _, n := range []int{0, 1, 2, 3, 63, 64, 65, 1000} { + buf := make([]float64, n+1) + buf[n] = -1 + Fill(buf[:n], 2.5) + for i := range n { + if buf[i] != 2.5 { + t.Fatalf("n=%d: [%d] = %v", n, i, buf[i]) + } + } + if buf[n] != -1 { + t.Fatalf("n=%d: wrote past the end", n) + } + } +} diff --git a/ndarray.go b/ndarray.go index a73b682..2b0dfce 100644 --- a/ndarray.go +++ b/ndarray.go @@ -125,9 +125,7 @@ func Full(v float64, shape ...int) (*Array, error) { if err != nil { return nil, err } - for i := range a.data { - a.data[i] = v - } + kernels.Fill(a.data, v) return a, nil } @@ -661,6 +659,12 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error if err != nil { return nil, err } + if full, s, sFirst, ok := scalarOperand(a, b, shape); ok { + dst := a.alloc(prod(shape), false) + kernels.ScalarP(kernel, dst, full, s, sFirst) + cp := append([]int(nil), shape...) + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil + } if full, rep, repFirst, ok := repeatOperands(a, b, shape); ok { dst := a.alloc(prod(shape), false) kernels.RepeatP(kernel, dst, full, rep, repFirst) @@ -675,6 +679,21 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil } +// scalarOperand recognises the broadcast of a single element (x + 1.5, or a +// 1×1 array against a matrix): one operand holds one element, the other is +// contiguous with the full result shape. binOp then streams the value through +// the kernel (kernels.ScalarP) instead of materialising it to the result's size. +func scalarOperand(a, b *Array, shape []int) (full []float64, s float64, sFirst, ok bool) { + isFull := func(f *Array) bool { return f.isContiguous() && sameShape(f.shape, shape) } + switch { + case b.Size() == 1 && isFull(a): + return a.data, b.data[b.offset], false, true + case a.Size() == 1 && isFull(b): + return b.data, a.data[a.offset], true, true + } + return nil, 0, false, false +} + // repeatMin is the shortest repeated operand binOp streams block by block; // below it a kernel call per block costs more than materialising. const repeatMin = 64 diff --git a/scalar_operand_test.go b/scalar_operand_test.go new file mode 100644 index 0000000..6593b6c --- /dev/null +++ b/scalar_operand_test.go @@ -0,0 +1,69 @@ +package ndarray + +import ( + "math" + "testing" +) + +// TestScalarOperandPaths: the operations that take binOp's single-element +// path (#118) — a scalar, a 1×1 array, a one-element view with an offset, on +// either side — give the same bits as the element-by-element definition. +func TestScalarOperandPaths(t *testing.T) { + const n = 1001 + x, _ := Linspace(-3, 5, n) + xs := x.data // Linspace is contiguous at offset 0 + mat, _ := x.Reshape(7, 143) + big, _ := Linspace(0, 10, 11) + view, _ := big.Slice(R(4, 5)) // the element 4.0, at offset 4 + oneByOne, _ := Full(4, 1, 1) + + ops := []struct { + name string + got func() (*Array, error) + want func(v float64) float64 + }{ + {"AddScalar", func() (*Array, error) { return x.AddScalar(0.1), nil }, func(v float64) float64 { return v + 0.1 }}, + {"SubScalar", func() (*Array, error) { return x.SubScalar(0.1), nil }, func(v float64) float64 { return v - 0.1 }}, + {"MulScalar", func() (*Array, error) { return x.MulScalar(0.1), nil }, func(v float64) float64 { return v * 0.1 }}, + {"DivScalar", func() (*Array, error) { return x.DivScalar(0.1), nil }, func(v float64) float64 { return v / 0.1 }}, + {"scalar-first Div", func() (*Array, error) { return scalarArray(0.1).Div(x) }, func(v float64) float64 { return 0.1 / v }}, + {"scalar-first Sub", func() (*Array, error) { return scalarArray(0.1).Sub(x) }, func(v float64) float64 { return 0.1 - v }}, + {"view at offset", func() (*Array, error) { return x.Sub(view) }, func(v float64) float64 { return v - 4 }}, + {"view first", func() (*Array, error) { return view.Div(x) }, func(v float64) float64 { return 4 / v }}, + {"1x1 against matrix", func() (*Array, error) { return mat.Mul(oneByOne) }, func(v float64) float64 { return v * 4 }}, + } + for _, op := range ops { + r, err := op.got() + if err != nil { + t.Fatalf("%s: %v", op.name, err) + } + if r.Size() != n { + t.Fatalf("%s: size %d, want %d", op.name, r.Size(), n) + } + for i, g := range r.data[r.offset : r.offset+n] { + if w := op.want(xs[i]); math.Float64bits(g) != math.Float64bits(w) { + t.Fatalf("%s [%d]: got %v, want %v", op.name, i, g, w) + } + } + } + if got := mat.MulScalar(2).Shape(); got[0] != 7 || got[1] != 143 { + t.Fatalf("shape %v, want [7 143]", got) + } +} + +// TestScalarOperandAllocs: a scalar operation allocates its result and a +// block of the scalar, never a copy of the scalar at the result's size. +func TestScalarOperandAllocs(t *testing.T) { + x, _ := Linspace(0, 1, 100_000) + var r *Array + bytes := testing.Benchmark(func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + r = x.AddScalar(1.5) + } + }).AllocedBytesPerOp() + if limit := int64(8*100_000 + 8*1024); bytes > limit { + t.Fatalf("AddScalar allocates %d B per call, want at most %d (result + one block)", bytes, limit) + } + _ = r +} diff --git a/zz_dtypes_gen.go b/zz_dtypes_gen.go index 65b9e0c..2701c43 100644 --- a/zz_dtypes_gen.go +++ b/zz_dtypes_gen.go @@ -2,7 +2,11 @@ package ndarray -import "strconv" +import ( + "strconv" + + "github.com/go-ndarray/ndarray/internal/kernels" +) // dtypeOfStore returns the dtype of a storage slice. func dtypeOfStore(s any) DType { @@ -139,70 +143,31 @@ func cloneStore(s any) any { func fillStore(dst, one any) { switch d := dst.(type) { case []bool: - v := one.([]bool)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]bool)[0]) case []int8: - v := one.([]int8)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]int8)[0]) case []int16: - v := one.([]int16)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]int16)[0]) case []int32: - v := one.([]int32)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]int32)[0]) case []int64: - v := one.([]int64)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]int64)[0]) case []uint8: - v := one.([]uint8)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]uint8)[0]) case []uint16: - v := one.([]uint16)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]uint16)[0]) case []uint32: - v := one.([]uint32)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]uint32)[0]) case []uint64: - v := one.([]uint64)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]uint64)[0]) case []float32: - v := one.([]float32)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]float32)[0]) case []float64: - v := one.([]float64)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]float64)[0]) case []complex64: - v := one.([]complex64)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]complex64)[0]) case []complex128: - v := one.([]complex128)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]complex128)[0]) } } From 046468415ce76fe1ffdfa6ca143a3eff1a12e00f Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 15 Nov 2026 10:35:08 +0100 Subject: [PATCH 2/3] perf: the *Into forms stream scalar operands too; docs scoped to what was measured (#118) Review of #121: binOpInto still copied a one-element operand to full size through operandFor; it now takes the scalarOperand path, out may alias an input as before. docs/perf.md names the cases the path covers (contiguous Float64 operand of the result's shape) and says FullOf was not timed. Refs #118 Co-Authored-By: Claude Opus 5.5 --- docs/perf.md | 9 ++++++--- ndarray.go | 4 ++++ scalar_operand_test.go | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/perf.md b/docs/perf.md index 6bf805f..e720f89 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -84,13 +84,16 @@ Faster, for example: `SqrtInto` (0.51), `fft.FFT` (0.54), `Sum` (0.70), `Exp` (0 Fixed after v0.41.0, measured by `go test -bench` on the same core, not yet by a new table run. On 10⁵ float64 at load 2.2–2.4, over three interleaved rounds: -- The scalar forms (`AddScalar` … `DivScalar`, and any operation with a - one-element operand) no longer copy the scalar to the result's size. They +- The scalar forms (`AddScalar` … `DivScalar`, `AddInto` … `DivInto`, and + any Float64 operation with a one-element operand against a contiguous + array of the result's shape) no longer copy the scalar to the result's + size. A strided operand, such as a transpose, still takes the general path. They stream it through the same SIMD kernel, so the result is bit for bit the same. `AddScalar` went from 183–203 µs to 95–110 µs, and from 1.6 MB to 0.8 MB per call. - `Full`, `Ones` and `FullOf` fill by doubling `copy` instead of storing one - element at a time: `Full` went from 131–135 µs to 84–104 µs. + element at a time. `Full` went from 131–135 µs to 84–104 µs. `Ones` calls + `Full`. `FullOf`, which fills the other dtypes, was not timed. What remains of the allocation cause is Go's own cost. A CPU profile of a bare `make([]float64, 100000)` shows about 60% garbage-collector marking (assists diff --git a/ndarray.go b/ndarray.go index 2b0dfce..34b3936 100644 --- a/ndarray.go +++ b/ndarray.go @@ -757,6 +757,10 @@ func (a *Array) binOpInto(out, b *Array, kernel func(dst, x, y []float64)) error return fmt.Errorf("%w: out shape %v != result shape %v", ErrBroadcast, out.shape, shape) } + if full, s, sFirst, ok := scalarOperand(a, b, shape); ok { + kernels.ScalarP(kernel, out.data, full, s, sFirst) + return nil + } x := a.operandFor(shape) y := b.operandFor(shape) kernel(out.data, x, y) diff --git a/scalar_operand_test.go b/scalar_operand_test.go index 6593b6c..7b64357 100644 --- a/scalar_operand_test.go +++ b/scalar_operand_test.go @@ -31,6 +31,14 @@ func TestScalarOperandPaths(t *testing.T) { {"view at offset", func() (*Array, error) { return x.Sub(view) }, func(v float64) float64 { return v - 4 }}, {"view first", func() (*Array, error) { return view.Div(x) }, func(v float64) float64 { return 4 / v }}, {"1x1 against matrix", func() (*Array, error) { return mat.Mul(oneByOne) }, func(v float64) float64 { return v * 4 }}, + {"DivInto scalar first", func() (*Array, error) { + out, _ := New(n) + return out, scalarArray(0.1).DivInto(out, x) + }, func(v float64) float64 { return 0.1 / v }}, + {"SubInto aliasing x", func() (*Array, error) { + y := x.Copy() + return y, y.SubInto(y, scalarArray(0.1)) + }, func(v float64) float64 { return v - 0.1 }}, } for _, op := range ops { r, err := op.got() @@ -66,4 +74,14 @@ func TestScalarOperandAllocs(t *testing.T) { t.Fatalf("AddScalar allocates %d B per call, want at most %d (result + one block)", bytes, limit) } _ = r + out, _ := New(100_000) + intoBytes := testing.Benchmark(func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = x.AddInto(out, scalarArray(1.5)) + } + }).AllocedBytesPerOp() + if intoBytes > 8*1024 { + t.Fatalf("AddInto with a scalar allocates %d B per call, want at most one block", intoBytes) + } } From 8d6da342e00f9341491a41d298608d8e82b1007d Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 15 Nov 2026 10:36:30 +0100 Subject: [PATCH 3/3] test: ScalarP in place, across ParThreshold (#118) Review of #121: the aliasing case (out == x in AddInto) was tested only below ParThreshold. TestScalarPSameBits now also runs every kernel and order with dst == full, at every block boundary and at ParThreshold + 3, where the blocks are split across workers. Refs #118 Co-Authored-By: Claude Opus 5.5 --- internal/kernels/scalar_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/kernels/scalar_test.go b/internal/kernels/scalar_test.go index 704441c..23aa294 100644 --- a/internal/kernels/scalar_test.go +++ b/internal/kernels/scalar_test.go @@ -30,10 +30,16 @@ func TestScalarPSameBits(t *testing.T) { } got := make([]float64, n) ScalarP(k, got, full, s, sFirst) + // In place, as AddInto(out = x) does: dst is full itself. + inPlace := append([]float64(nil), full...) + ScalarP(k, inPlace, inPlace, s, sFirst) for i := range want { if math.Float64bits(got[i]) != math.Float64bits(want[i]) { t.Fatalf("%s n=%d sFirst=%v [%d]: got %v, want %v", name, n, sFirst, i, got[i], want[i]) } + if math.Float64bits(inPlace[i]) != math.Float64bits(want[i]) { + t.Fatalf("%s n=%d sFirst=%v in place [%d]: got %v, want %v", name, n, sFirst, i, inPlace[i], want[i]) + } } } }