diff --git a/alloc_bench_test.go b/alloc_bench_test.go new file mode 100644 index 0000000..1d9e364 --- /dev/null +++ b/alloc_bench_test.go @@ -0,0 +1,67 @@ +package ndarray + +import "testing" + +// The cost of a result, taken apart (#118): the allocation alone, the fill, +// and the operations built on them, at the size of the refbench table. + +const allocN = 100_000 + +var ( + sinkF []float64 + sinkA *Array +) + +func BenchmarkAllocMake(b *testing.B) { + for b.Loop() { + sinkF = make([]float64, allocN) + } +} + +func BenchmarkAllocZeros(b *testing.B) { + for b.Loop() { + sinkA, _ = Zeros(allocN) + } +} + +func BenchmarkAllocFull(b *testing.B) { + for b.Loop() { + sinkA, _ = Full(1.5, allocN) + } +} + +func BenchmarkFillLoop(b *testing.B) { + d := make([]float64, allocN) + for b.Loop() { + for i := range d { + d[i] = 1.5 + } + } + sinkF = d +} + +func BenchmarkFillCopy(b *testing.B) { + d := make([]float64, allocN) + for b.Loop() { + d[0] = 1.5 + for k := 1; k < len(d); k *= 2 { + copy(d[k:], d[:k]) + } + } + sinkF = d +} + +func BenchmarkAllocAdd(b *testing.B) { + x, _ := Full(1.25, allocN) + y, _ := Full(2.5, allocN) + for b.Loop() { + sinkA, _ = x.Add(y) + } +} + +func BenchmarkAllocAddScalar(b *testing.B) { + x, _ := Full(1.25, allocN) + for b.Loop() { + sinkA = x.AddScalar(1.5) + } +} diff --git a/docs/perf.md b/docs/perf.md index 17ac3f7..e720f89 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -79,6 +79,28 @@ issue per row: Faster, for example: `SqrtInto` (0.51), `fft.FFT` (0.54), `Sum` (0.70), `Exp` (0.89). +### Since the table + +Fixed after v0.41.0, measured by `go test -bench` on the same core, not yet +by a new table run. On 10⁵ float64 at load 2.2–2.4, over three interleaved +rounds: +- The scalar forms (`AddScalar` … `DivScalar`, `AddInto` … `DivInto`, and + any Float64 operation with a one-element operand against a contiguous + array of the result's shape) no longer copy the scalar to the result's + size. A strided operand, such as a transpose, still takes the general path. They + stream it through the same SIMD kernel, so the result is bit for bit the + same. `AddScalar` went from 183–203 µs to 95–110 µs, and from 1.6 MB to + 0.8 MB per call. +- `Full`, `Ones` and `FullOf` fill by doubling `copy` instead of storing one + element at a time. `Full` went from 131–135 µs to 84–104 µs. `Ones` calls + `Full`. `FullOf`, which fills the other dtypes, was not timed. + +What remains of the allocation cause is Go's own cost. A CPU profile of a bare +`make([]float64, 100000)` shows about 60% garbage-collector marking (assists +in the allocating goroutine) and about 23% zeroing. NumPy frees each result +when its last reference goes and reuses that memory warm. In Go, the way to +avoid both costs is to not allocate: use the `*Into` forms or a `Workspace`. + The run was made from the release's own code. The test binary was cross-compiled from a clean v0.41.0 worktree, the tag was shipped to the host as a git bundle, and the binary ran from outside the clone so that the tree diff --git a/gen_dtypes.go b/gen_dtypes.go index ade494f..61ac2f2 100644 --- a/gen_dtypes.go +++ b/gen_dtypes.go @@ -87,7 +87,11 @@ func main() { p("") p("package ndarray") p("") - p(`import "strconv"`) + p(`import (`) + p(`"strconv"`) + p(``) + p(`"github.com/go-ndarray/ndarray/internal/kernels"`) + p(`)`) p("") p("// dtypeOfStore returns the dtype of a storage slice.") @@ -130,7 +134,7 @@ func main() { p("// storage slice of the same dtype.") p("func fillStore(dst, one any) {") p("switch d := dst.(type) {") - each("biucf", "case []$T: v := one.([]$T)[0]; for i := range d { d[i] = v }") + each("biucf", "case []$T: kernels.Fill(d, one.([]$T)[0])") p("}") p("}") p("") diff --git a/internal/kernels/parallel.go b/internal/kernels/parallel.go index 01010fe..dba06c5 100644 --- a/internal/kernels/parallel.go +++ b/internal/kernels/parallel.go @@ -179,6 +179,50 @@ func RepeatP(k func(dst, a, b []float64), dst, full, rep []float64, repFirst boo parallelFor(blocks, min(numWorkers(len(dst)), blocks), body) } +// scalarBlock is the length of the block ScalarP fills with the scalar: long +// enough that a kernel call per block costs little next to the block's work, +// short enough (4 KiB) to stay in L1 for the whole pass. +const scalarBlock = 512 + +// ScalarP computes dst = full OP s, where s is a single value broadcast over +// full (x + 1.5): it fills one block of scalarBlock copies of s and streams it +// through the elementwise kernel k like RepeatP, so s is never materialised to +// full's size. Each element goes through the same kernel as the materialised +// broadcast, so the result is the same, bit for bit. With sFirst the operands +// are swapped (dst = s OP full). len(dst) == len(full). +func ScalarP(k func(dst, a, b []float64), dst, full []float64, s float64, sFirst bool) { + if len(dst) == 0 { + return + } + // Heap, not stack: the parallel workers share it, so it would escape + // anyway, and a short array then pays for its own length only. + rep := make([]float64, min(scalarBlock, len(dst))) + Fill(rep, s) + whole := len(dst) / len(rep) * len(rep) + RepeatP(k, dst[:whole], full[:whole], rep, sFirst) + if tail := len(dst) - whole; tail > 0 { + if sFirst { + k(dst[whole:], rep[:tail], full[whole:]) + } else { + k(dst[whole:], full[whole:], rep[:tail]) + } + } +} + +// Fill sets every element of dst to v, for every element type. It writes one +// element, then doubles the filled prefix with copy, whose memmove stores +// whole vectors: 3.5× faster than an element-by-element loop on 10⁵ float64 +// (Zen 3, #118). +func Fill[T any](dst []T, v T) { + if len(dst) == 0 { + return + } + dst[0] = v + for k := 1; k < len(dst); k *= 2 { + copy(dst[k:], dst[:k]) + } +} + // AddP writes a[i]+b[i] into dst[i], parallelised above ParThreshold. func AddP(dst, a, b []float64) { runBinaryP(addBin, dst, a, b) } diff --git a/internal/kernels/scalar_test.go b/internal/kernels/scalar_test.go new file mode 100644 index 0000000..23aa294 --- /dev/null +++ b/internal/kernels/scalar_test.go @@ -0,0 +1,65 @@ +package kernels + +import ( + "math" + "testing" +) + +// TestScalarPSameBits: streaming the scalar block by block gives, bit for +// bit, what the kernel gives on the scalar materialised to full length, for +// both operand orders, at every block boundary and across ParThreshold. +func TestScalarPSameBits(t *testing.T) { + ks := map[string]func(dst, a, b []float64){"Add": AddP, "Sub": SubP, "Mul": MulP, "Div": DivP} + const s = 1.0 / 3 + for _, n := range []int{0, 1, 7, scalarBlock - 1, scalarBlock, scalarBlock + 1, 3*scalarBlock + 5, ParThreshold + 3} { + full := make([]float64, n) + for i := range full { + full[i] = math.Sin(float64(i)) * 7 + } + wide := make([]float64, n) + for i := range wide { + wide[i] = s + } + for name, k := range ks { + for _, sFirst := range []bool{false, true} { + want := make([]float64, n) + if sFirst { + k(want, wide, full) + } else { + k(want, full, wide) + } + got := make([]float64, n) + ScalarP(k, got, full, s, sFirst) + // In place, as AddInto(out = x) does: dst is full itself. + inPlace := append([]float64(nil), full...) + ScalarP(k, inPlace, inPlace, s, sFirst) + for i := range want { + if math.Float64bits(got[i]) != math.Float64bits(want[i]) { + t.Fatalf("%s n=%d sFirst=%v [%d]: got %v, want %v", name, n, sFirst, i, got[i], want[i]) + } + if math.Float64bits(inPlace[i]) != math.Float64bits(want[i]) { + t.Fatalf("%s n=%d sFirst=%v in place [%d]: got %v, want %v", name, n, sFirst, i, inPlace[i], want[i]) + } + } + } + } + } +} + +// TestFill: every element is set, for lengths around the powers of two the +// doubling copy steps through, and nothing past len(dst) is touched. +func TestFill(t *testing.T) { + for _, n := range []int{0, 1, 2, 3, 63, 64, 65, 1000} { + buf := make([]float64, n+1) + buf[n] = -1 + Fill(buf[:n], 2.5) + for i := range n { + if buf[i] != 2.5 { + t.Fatalf("n=%d: [%d] = %v", n, i, buf[i]) + } + } + if buf[n] != -1 { + t.Fatalf("n=%d: wrote past the end", n) + } + } +} diff --git a/ndarray.go b/ndarray.go index a73b682..34b3936 100644 --- a/ndarray.go +++ b/ndarray.go @@ -125,9 +125,7 @@ func Full(v float64, shape ...int) (*Array, error) { if err != nil { return nil, err } - for i := range a.data { - a.data[i] = v - } + kernels.Fill(a.data, v) return a, nil } @@ -661,6 +659,12 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error if err != nil { return nil, err } + if full, s, sFirst, ok := scalarOperand(a, b, shape); ok { + dst := a.alloc(prod(shape), false) + kernels.ScalarP(kernel, dst, full, s, sFirst) + cp := append([]int(nil), shape...) + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil + } if full, rep, repFirst, ok := repeatOperands(a, b, shape); ok { dst := a.alloc(prod(shape), false) kernels.RepeatP(kernel, dst, full, rep, repFirst) @@ -675,6 +679,21 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil } +// scalarOperand recognises the broadcast of a single element (x + 1.5, or a +// 1×1 array against a matrix): one operand holds one element, the other is +// contiguous with the full result shape. binOp then streams the value through +// the kernel (kernels.ScalarP) instead of materialising it to the result's size. +func scalarOperand(a, b *Array, shape []int) (full []float64, s float64, sFirst, ok bool) { + isFull := func(f *Array) bool { return f.isContiguous() && sameShape(f.shape, shape) } + switch { + case b.Size() == 1 && isFull(a): + return a.data, b.data[b.offset], false, true + case a.Size() == 1 && isFull(b): + return b.data, a.data[a.offset], true, true + } + return nil, 0, false, false +} + // repeatMin is the shortest repeated operand binOp streams block by block; // below it a kernel call per block costs more than materialising. const repeatMin = 64 @@ -738,6 +757,10 @@ func (a *Array) binOpInto(out, b *Array, kernel func(dst, x, y []float64)) error return fmt.Errorf("%w: out shape %v != result shape %v", ErrBroadcast, out.shape, shape) } + if full, s, sFirst, ok := scalarOperand(a, b, shape); ok { + kernels.ScalarP(kernel, out.data, full, s, sFirst) + return nil + } x := a.operandFor(shape) y := b.operandFor(shape) kernel(out.data, x, y) diff --git a/scalar_operand_test.go b/scalar_operand_test.go new file mode 100644 index 0000000..7b64357 --- /dev/null +++ b/scalar_operand_test.go @@ -0,0 +1,87 @@ +package ndarray + +import ( + "math" + "testing" +) + +// TestScalarOperandPaths: the operations that take binOp's single-element +// path (#118) — a scalar, a 1×1 array, a one-element view with an offset, on +// either side — give the same bits as the element-by-element definition. +func TestScalarOperandPaths(t *testing.T) { + const n = 1001 + x, _ := Linspace(-3, 5, n) + xs := x.data // Linspace is contiguous at offset 0 + mat, _ := x.Reshape(7, 143) + big, _ := Linspace(0, 10, 11) + view, _ := big.Slice(R(4, 5)) // the element 4.0, at offset 4 + oneByOne, _ := Full(4, 1, 1) + + ops := []struct { + name string + got func() (*Array, error) + want func(v float64) float64 + }{ + {"AddScalar", func() (*Array, error) { return x.AddScalar(0.1), nil }, func(v float64) float64 { return v + 0.1 }}, + {"SubScalar", func() (*Array, error) { return x.SubScalar(0.1), nil }, func(v float64) float64 { return v - 0.1 }}, + {"MulScalar", func() (*Array, error) { return x.MulScalar(0.1), nil }, func(v float64) float64 { return v * 0.1 }}, + {"DivScalar", func() (*Array, error) { return x.DivScalar(0.1), nil }, func(v float64) float64 { return v / 0.1 }}, + {"scalar-first Div", func() (*Array, error) { return scalarArray(0.1).Div(x) }, func(v float64) float64 { return 0.1 / v }}, + {"scalar-first Sub", func() (*Array, error) { return scalarArray(0.1).Sub(x) }, func(v float64) float64 { return 0.1 - v }}, + {"view at offset", func() (*Array, error) { return x.Sub(view) }, func(v float64) float64 { return v - 4 }}, + {"view first", func() (*Array, error) { return view.Div(x) }, func(v float64) float64 { return 4 / v }}, + {"1x1 against matrix", func() (*Array, error) { return mat.Mul(oneByOne) }, func(v float64) float64 { return v * 4 }}, + {"DivInto scalar first", func() (*Array, error) { + out, _ := New(n) + return out, scalarArray(0.1).DivInto(out, x) + }, func(v float64) float64 { return 0.1 / v }}, + {"SubInto aliasing x", func() (*Array, error) { + y := x.Copy() + return y, y.SubInto(y, scalarArray(0.1)) + }, func(v float64) float64 { return v - 0.1 }}, + } + for _, op := range ops { + r, err := op.got() + if err != nil { + t.Fatalf("%s: %v", op.name, err) + } + if r.Size() != n { + t.Fatalf("%s: size %d, want %d", op.name, r.Size(), n) + } + for i, g := range r.data[r.offset : r.offset+n] { + if w := op.want(xs[i]); math.Float64bits(g) != math.Float64bits(w) { + t.Fatalf("%s [%d]: got %v, want %v", op.name, i, g, w) + } + } + } + if got := mat.MulScalar(2).Shape(); got[0] != 7 || got[1] != 143 { + t.Fatalf("shape %v, want [7 143]", got) + } +} + +// TestScalarOperandAllocs: a scalar operation allocates its result and a +// block of the scalar, never a copy of the scalar at the result's size. +func TestScalarOperandAllocs(t *testing.T) { + x, _ := Linspace(0, 1, 100_000) + var r *Array + bytes := testing.Benchmark(func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + r = x.AddScalar(1.5) + } + }).AllocedBytesPerOp() + if limit := int64(8*100_000 + 8*1024); bytes > limit { + t.Fatalf("AddScalar allocates %d B per call, want at most %d (result + one block)", bytes, limit) + } + _ = r + out, _ := New(100_000) + intoBytes := testing.Benchmark(func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + _ = x.AddInto(out, scalarArray(1.5)) + } + }).AllocedBytesPerOp() + if intoBytes > 8*1024 { + t.Fatalf("AddInto with a scalar allocates %d B per call, want at most one block", intoBytes) + } +} diff --git a/zz_dtypes_gen.go b/zz_dtypes_gen.go index 65b9e0c..2701c43 100644 --- a/zz_dtypes_gen.go +++ b/zz_dtypes_gen.go @@ -2,7 +2,11 @@ package ndarray -import "strconv" +import ( + "strconv" + + "github.com/go-ndarray/ndarray/internal/kernels" +) // dtypeOfStore returns the dtype of a storage slice. func dtypeOfStore(s any) DType { @@ -139,70 +143,31 @@ func cloneStore(s any) any { func fillStore(dst, one any) { switch d := dst.(type) { case []bool: - v := one.([]bool)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]bool)[0]) case []int8: - v := one.([]int8)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]int8)[0]) case []int16: - v := one.([]int16)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]int16)[0]) case []int32: - v := one.([]int32)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]int32)[0]) case []int64: - v := one.([]int64)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]int64)[0]) case []uint8: - v := one.([]uint8)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]uint8)[0]) case []uint16: - v := one.([]uint16)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]uint16)[0]) case []uint32: - v := one.([]uint32)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]uint32)[0]) case []uint64: - v := one.([]uint64)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]uint64)[0]) case []float32: - v := one.([]float32)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]float32)[0]) case []float64: - v := one.([]float64)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]float64)[0]) case []complex64: - v := one.([]complex64)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]complex64)[0]) case []complex128: - v := one.([]complex128)[0] - for i := range d { - d[i] = v - } + kernels.Fill(d, one.([]complex128)[0]) } }