From 741e7214e8c6539486cd32ba2cbf58288bd7cafc Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 4 Oct 2026 15:11:12 +0200 Subject: [PATCH] feat: Workspace, an arena for the results of array operations in loops Every operation returns a new array; in a loop those are garbage the collector must chase, while NumPy frees its temporaries the moment their reference count drops. On a Zen 3 (16 cores) that made x + y lose to NumPy 0.13-0.25x at every size up to 256 Ki elements, although AddInto wins. Measured on M4: a GC cycle every ~22 calls on a benchmark's tiny heap, and with a realistic 256 MiB-2 GiB live heap still half the overhead (zeroing memory the kernel overwrites, cold memory). GOGC=off only trades the cycles for page faults, and a library must not set GOGC anyway. Workspace is ND4J's answer to the same problem: ws.Use(x) binds an input, every result computed from a bound array is carved from the arena (and stays bound, so a chain stays in it), and ws.Reset() recycles the pass. A pass that needed several blocks is folded into one, so after the first pass nothing is allocated. Sites whose kernel writes every element skip the zeroing; those that accumulate (GEMM, vec.mat, axis reductions) clear their slot. Detach takes a result out. Allocation is serialized, so a workspace can be shared between goroutines. Without Use nothing changes: a nil workspace allocates from the heap as before. Zen 3, 16 cores pinned, vs NumPy 2.5.3 single-threaded: x + y 1 024 2.6 us -> 0.38 us (2.1x NumPy) x + y 256 Ki 498 us -> 67 us (2.2x NumPy) sqrt(x*y+x) 1 024 12.2 us -> 2.6 us (1.4x NumPy) sqrt(x*y+x) 256 Ki 1.18 ms -> 0.39 ms (7.5x NumPy) Tests: every allocating operation is compared bit for bit with its heap result, also on passes whose recycled memory was first filled with NaN (planting a missing zeroing in the GEMM makes it fail); block folding, alignment and capacity, Detach surviving Reset, and concurrent allocation under -race (removing the lock makes it fail). ExampleWorkspace runs the README's loop. Co-Authored-By: Claude Opus 5.5 --- README.md | 30 ++++++ benchmarks/bench_numpy.py | 1 + benchmarks/bench_test.go | 51 +++++++++ docs/perf.md | 47 +++++++- indexing.go | 8 +- linalg.go | 18 ++-- manipulation.go | 11 +- ndarray.go | 26 +++-- reductions.go | 20 ++-- slicing.go | 2 +- ufunc.go | 4 +- workspace.go | 127 ++++++++++++++++++++++ workspace_example_test.go | 24 +++++ workspace_test.go | 221 ++++++++++++++++++++++++++++++++++++++ 14 files changed, 548 insertions(+), 42 deletions(-) create mode 100644 workspace.go create mode 100644 workspace_example_test.go create mode 100644 workspace_test.go diff --git a/README.md b/README.md index 13fe942..307bf4f 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,36 @@ `MaskSelect` (`a[mask]`), `Nonzero` (`flatnonzero`), and `Take`. - **Manipulation** — `Concatenate`/`Stack`/`VStack`/`HStack`. - **Linear algebra** — `MatMul`/`Dot`/`Inner`/`Outer`. +- **Memory reuse** — `Workspace`, an arena for loops: bind the inputs with + `ws.Use`, compute as usual, `ws.Reset()` at the end of each pass. Results stop + costing the garbage collector anything (see below). + +### Loops: `Workspace` + +Every operation returns a new array, and in a loop those results are garbage +the collector must chase. NumPy frees its temporaries the moment their +reference count drops; Go cannot. On a Zen 3 that made `x + y` on 1 024 +elements 3× slower than NumPy, and a chain like `sqrt(x*y + x)` 3.4× slower. +A `Workspace` gives the results of one pass a single arena that `Reset` hands +back: + +```go +ws := ndarray.NewWorkspace() +for step := 0; step < steps; step++ { + x := ws.Use(state) // results computed from x come from ws + p, _ := x.Mul(w) + q, _ := p.Add(bias) + state = q.Sqrt().Detach() // the one result that outlives the pass + ws.Reset() // everything else is recycled +} +``` + +After the first pass nothing is allocated, nothing is zeroed that the kernel +overwrites anyway, and the memory stays warm in cache: `x + y` on 1 024 +elements goes from 2.6 µs to 0.38 µs (2.1× NumPy), `sqrt(x*y + x)` on 256 Ki +from 1.18 ms to 0.39 ms (7.5× NumPy). ⚠ An array allocated from the workspace +is invalid after `Reset`; `Detach` is how a result leaves it. The `*Into` +methods (`AddInto`, `SqrtInto`, …) remain the way to reuse one named buffer. The numeric inner loops are kept behind a narrow kernel API. Behind it, **large** elementwise ops, reductions and matmul run **multicore** (across diff --git a/benchmarks/bench_numpy.py b/benchmarks/bench_numpy.py index 18708c9..a4be05f 100644 --- a/benchmarks/bench_numpy.py +++ b/benchmarks/bench_numpy.py @@ -59,6 +59,7 @@ def main(): ("AddInto", "np.add(x, y, out=z)"), ("MulInto", "np.multiply(x, y, out=z)"), ("SqrtInto", "np.sqrt(x, out=z)"), + ("Chain", "np.sqrt(x * y + x)"), ("Sum", "x.sum()"), ("Mean", "x.mean()"), ("Max", "x.max()"), diff --git a/benchmarks/bench_test.go b/benchmarks/bench_test.go index e28d853..15bab9a 100644 --- a/benchmarks/bench_test.go +++ b/benchmarks/bench_test.go @@ -106,6 +106,57 @@ func BenchmarkAddInto(b *testing.B) { } } +// BenchmarkAddWS is Add with its operands bound to a Workspace that is Reset +// every iteration: the allocating form, minus the garbage collector and the +// zeroing (see Workspace). Compare with BenchmarkAdd and BenchmarkAddInto. +func BenchmarkAddWS(b *testing.B) { + for _, n := range elemSizes { + ws := nd.NewWorkspace() + x, y := ws.Use(vec(n)), vec(n) + b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { + b.SetBytes(int64(n * 8)) + for i := 0; i < b.N; i++ { + _, _ = x.Add(y) + ws.Reset() + } + }) + } +} + +// BenchmarkChain is sqrt(x*y + x) written the plain way (three allocating +// operations, two temporaries), the shape of real array code; numpy's row +// "Chain" is np.sqrt(x * y + x). +func BenchmarkChain(b *testing.B) { + for _, n := range elemSizes { + x, y := vec(n), vec(n) + b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { + b.SetBytes(int64(n * 8)) + for i := 0; i < b.N; i++ { + p, _ := x.Mul(y) + q, _ := p.Add(x) + _ = q.Sqrt() + } + }) + } +} + +// BenchmarkChainWS is BenchmarkChain in a Workspace reset every iteration. +func BenchmarkChainWS(b *testing.B) { + for _, n := range elemSizes { + ws := nd.NewWorkspace() + x, y := ws.Use(vec(n)), vec(n) + b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { + b.SetBytes(int64(n * 8)) + for i := 0; i < b.N; i++ { + p, _ := x.Mul(y) + q, _ := p.Add(x) + _ = q.Sqrt() + ws.Reset() + } + }) + } +} + func BenchmarkMulInto(b *testing.B) { for _, n := range elemSizes { x, y := vec(n), vec(n) diff --git a/docs/perf.md b/docs/perf.md index 600dcce..116e529 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -123,7 +123,7 @@ with no result array, so they have no separate *into* row.) path is the supported way to hit/beat NumPy there (and at large n even the alloc form wins, because the kernel time then dwarfs the one allocation). (Sum and Max are the exceptions — their reduction kernels need no result allocation and win even - serially.) + serially.) **Superseded for loops by `Workspace`, see below.** ### NaN convention for Max / Min (and the SIMD max kernel) @@ -313,6 +313,51 @@ The SIMD kernels are also validated against the scalar oracle per-arch in CI: | `MatMul` vs tuned BLAS (OpenBLAS) | **FIXED — parity at n=1024 (~0.99×, ~203 GFLOP/s); 0.97× at n=512** | by-lane FMLA micro-kernel (`FMLA Vd.2D,Vn.2D,Vm.D[i]` via `WORD`) closed the prior 0.76× gap. Only small n=256 trails (0.67×) on per-call overhead, not throughput — see above | | other `Map` ufuncs (`Exp`, `Log`, `Sin`…) | NumPy ~parity (libm-bound) | a packed `VEXP`/`VLOG` is libm-accuracy work; the math, not the dispatch, dominates here | +## Allocation: `Workspace` (2026-10-04) + +On real amd64 hardware the allocating forms were far worse than on the M4: on +a Zen 3 (16 cores), `x + y` lost to NumPy 0.13–0.25× at every size up to 256 Ki +elements, while `AddInto` won. Slicing copies, `Concatenate` and broadcasting +adds (whose results also allocate) lost 5–9×. Measured causes, on M4: + +- **The garbage collector.** A benchmark's live heap is tiny, so a 128 KiB + result per call triggered a GC cycle every ~22 calls (912 cycles in 20 000). + With 256 MiB–2 GiB of live heap, which is closer to a real program, the + overhead halves: 8.3 → 3.6 µs at 16 Ki, against 2.4 µs for `AddInto`. +- **The rest:** `make` zeroes memory the kernel overwrites anyway, and fresh + memory is cold or page-faulted. `GOGC=off` does not help: it trades the + cycles for page faults (9.4 µs). + +A library must not set `GOGC`, and Go has no unzeroed allocation. What NumPy's +reference counting gets for free is scoped reuse. ND4J, the Java ndarray +library, has the same problem and answers it with *workspaces*: an arena that +results of one pass come from, recycled at the end of the pass. `Workspace` is +that. `ws.Use(x)` binds an input, every result computed from a bound array is +carved from the arena (64-byte aligned, capped so an `append` cannot spill), +results stay bound so a chain stays in it, and `Reset` recycles everything, +folding a pass that needed several blocks into one block. Sites whose kernel +writes every element (elementwise, ufuncs, copies, broadcasts, concatenation, +`Where`, `Clip`, scans, `Take`, `Outer`, mat·vec) skip the zeroing. Those that +accumulate (GEMM, vec·mat, axis reductions) clear their slot. + +Zen 3, 16 cores pinned, NumPy 2.5.3 single-threaded, median of two runs: + +| op | n | heap | **Workspace** | `*Into` | NumPy | Workspace vs NumPy | +|----|--:|--:|--:|--:|--:|:--:| +| Add | 1 024 | 2.6 µs | **0.38 µs** | 0.31 µs | 0.82 µs | 2.1× | +| Add | 16 Ki | 35 µs | **4.6 µs** | 4.3 µs | 5.1 µs | 1.1× | +| Add | 256 Ki | 498 µs | **67 µs** | 75 µs | 150 µs | 2.2× | +| Add | 4 Mi | 8.0 ms | **2.2 ms** | 2.2 ms | 6.3 ms | 2.9× | +| sqrt(x·y+x) | 1 024 | 12.2 µs | **2.6 µs** | — | 3.6 µs | 1.4× | +| sqrt(x·y+x) | 16 Ki | 112 µs | **33 µs** | — | 34 µs | 1.0× | +| sqrt(x·y+x) | 256 Ki | 1.18 ms | **0.39 ms** | — | 2.97 ms | 7.5× | +| sqrt(x·y+x) | 4 Mi | 23.4 ms | **9.1 ms** | — | 19.2 ms | 2.1× | + +On M4 the same chain gains ×5 at 1 024 and ×1.6 at 4 Mi. Correctness: every +operation is checked bit for bit against its heap result, including on passes +whose recycled memory was first filled with NaN (a site that skipped zeroing it +needed would leak the NaN; planting that bug in the GEMM made the test fail). + ## amd64 on real hardware (2026-10-04) Every figure above was taken on Apple silicon. This round measured amd64 for diff --git a/indexing.go b/indexing.go index 4fa2fa3..9eac0c0 100644 --- a/indexing.go +++ b/indexing.go @@ -32,7 +32,7 @@ func (a *Array) MaskSelect(mask *Array) (*Array, error) { out = append(out, av[i]) } } - return &Array{data: out, shape: []int{len(out)}, strides: []int{1}}, nil + return &Array{data: out, shape: []int{len(out)}, strides: []int{1}, ws: a.ws}, nil } // Nonzero returns a 1-D array of the flat (row-major) indices at which a is @@ -46,7 +46,7 @@ func (a *Array) Nonzero() *Array { out = append(out, float64(i)) } } - return &Array{data: out, shape: []int{len(out)}, strides: []int{1}} + return &Array{data: out, shape: []int{len(out)}, strides: []int{1}, ws: a.ws} } // Take returns a 1-D array gathering a's flattened (row-major) elements at the @@ -55,7 +55,7 @@ func (a *Array) Nonzero() *Array { func (a *Array) Take(indices ...int) (*Array, error) { flat := a.contiguousData() n := len(flat) - out := make([]float64, len(indices)) + out := a.alloc(len(indices), false) for k, idx := range indices { j := idx if j < 0 { @@ -67,5 +67,5 @@ func (a *Array) Take(indices ...int) (*Array, error) { } out[k] = flat[j] } - return &Array{data: out, shape: []int{len(out)}, strides: []int{1}}, nil + return &Array{data: out, shape: []int{len(out)}, strides: []int{1}, ws: a.ws}, nil } diff --git a/linalg.go b/linalg.go index a64119f..c1137dd 100644 --- a/linalg.go +++ b/linalg.go @@ -15,10 +15,10 @@ var ErrLinalg = fmt.Errorf("ndarray: incompatible shapes for linear algebra") // array. Either operand may be any 2-D view: the GEMM packs it through its // strides (matView), so nothing is materialised first. func matmul2D(a, b *Array, m, k, n int) *Array { - dst := make([]float64, m*n) + dst := a.ws.alloc(m*n, true) // MatMulStridedP accumulates into dst kernels.MatMulStridedP(dst, a.matView(), b.matView(), m, k, n) shape := []int{m, n} - return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape)} + return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape), ws: a.ws} } // matView is a 2-D array as the GEMM's strided operand view. The packing @@ -60,7 +60,7 @@ func (a *Array) Dot(b *Array) (*Array, error) { ErrLinalg, a.shape[0], b.shape[0]) } s := kernels.Dot1DP(a.contiguousData(), b.contiguousData()) - return &Array{data: []float64{s}, shape: []int{}, strides: []int{}}, nil + return &Array{data: []float64{s}, shape: []int{}, strides: []int{}, ws: a.ws}, nil case na == 2 && nb == 2: return a.MatMul(b) @@ -73,9 +73,9 @@ func (a *Array) Dot(b *Array) (*Array, error) { ErrLinalg, a.shape, b.shape) } m, k := a.shape[0], a.shape[1] - dst := make([]float64, m) + dst := a.alloc(m, false) kernels.MatVecP(dst, a.contiguousData(), b.contiguousData(), m, k) - return &Array{data: dst, shape: []int{m}, strides: []int{1}}, nil + return &Array{data: dst, shape: []int{m}, strides: []int{1}, ws: a.ws}, nil case na == 1 && nb == 2: // (k,) · (k x n) = (n,): cache-friendly row-streaming accumulation, no @@ -85,9 +85,9 @@ func (a *Array) Dot(b *Array) (*Array, error) { ErrLinalg, a.shape, b.shape) } k, n := b.shape[0], b.shape[1] - dst := make([]float64, n) + dst := a.alloc(n, true) // VecMatP accumulates into dst kernels.VecMatP(dst, a.contiguousData(), b.contiguousData(), k, n) - return &Array{data: dst, shape: []int{n}, strides: []int{1}}, nil + return &Array{data: dst, shape: []int{n}, strides: []int{1}, ws: a.ws}, nil default: return nil, fmt.Errorf("%w: Dot supports 1-D and 2-D operands, got %d-D and %d-D", @@ -126,7 +126,7 @@ func (a *Array) Outer(b *Array) *Array { u := a.contiguousData() v := b.contiguousData() m, n := len(u), len(v) - dst := make([]float64, m*n) + dst := a.alloc(m*n, false) for i := 0; i < m; i++ { ui := u[i] row := dst[i*n : i*n+n] @@ -135,5 +135,5 @@ func (a *Array) Outer(b *Array) *Array { } } shape := []int{m, n} - return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape)} + return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape), ws: a.ws} } diff --git a/manipulation.go b/manipulation.go index 1bf8f3c..6eada76 100644 --- a/manipulation.go +++ b/manipulation.go @@ -8,7 +8,7 @@ import "fmt" // always-copy contract). func (a *Array) Flatten() *Array { data := a.materialize() - return &Array{data: data, shape: []int{len(data)}, strides: []int{1}} + return &Array{data: data, shape: []int{len(data)}, strides: []int{1}, ws: a.ws} } // ExpandDims returns a view with a new length-1 axis inserted at the given @@ -31,7 +31,7 @@ func (a *Array) ExpandDims(axis int) (*Array, error) { // The inserted axis has length 1; its stride is irrelevant, use 0. strides = append(strides, 0) strides = append(strides, a.strides[axis:]...) - return &Array{data: a.data, shape: shape, strides: strides, offset: a.offset}, nil + return &Array{data: a.data, shape: shape, strides: strides, offset: a.offset, ws: a.ws}, nil } // Squeeze returns a view with length-1 axes removed. With no axes given, every @@ -70,7 +70,7 @@ func (a *Array) Squeeze(axes ...int) (*Array, error) { strides = append(strides, a.strides[i]) } } - return &Array{data: a.data, shape: shape, strides: strides, offset: a.offset}, nil + return &Array{data: a.data, shape: shape, strides: strides, offset: a.offset, ws: a.ws}, nil } // Concatenate joins the given arrays along an existing axis, matching @@ -115,7 +115,7 @@ func concatInto(arrays []*Array, ax int, out []int) *Array { inner := prod(out[ax+1:]) outer := prod(out[:ax]) axisTotal := out[ax] - data := make([]float64, outer*axisTotal*inner) + data := wsOf(arrays...).alloc(outer*axisTotal*inner, false) // every slab is copied in // Column offset (in the joined axis) where the next array's slab begins. colBase := 0 for _, arr := range arrays { @@ -131,7 +131,7 @@ func concatInto(arrays []*Array, ax int, out []int) *Array { } colBase += aLen } - return &Array{data: data, shape: out, strides: rowMajorStrides(out)} + return &Array{data: data, shape: out, strides: rowMajorStrides(out), ws: wsOf(arrays...)} } // Stack joins the given arrays along a new axis, matching numpy.stack. Every @@ -191,6 +191,7 @@ func atLeast2D(a *Array) *Array { shape: []int{1, a.shape[0]}, strides: []int{a.shape[0] * a.strides[0], a.strides[0]}, offset: a.offset, + ws: a.ws, } } return a diff --git a/ndarray.go b/ndarray.go index 147e603..7a6123b 100644 --- a/ndarray.go +++ b/ndarray.go @@ -27,6 +27,7 @@ type Array struct { shape []int strides []int offset int + ws *Workspace // where results computed from this array are allocated; nil = heap } // ErrShapeMismatch is returned when a requested shape is invalid or @@ -275,7 +276,7 @@ func (a *Array) forEach(f func(linear, pos int)) { // general per-element gather. func (a *Array) materialize() []float64 { n := a.Size() - out := make([]float64, n) + out := a.alloc(n, false) // every element is written below if n == 0 { return out } @@ -336,6 +337,7 @@ func (a *Array) Copy() *Array { data: a.materialize(), shape: cp, strides: rowMajorStrides(cp), + ws: a.ws, } } @@ -392,12 +394,14 @@ func (a *Array) Reshape(shape ...int) (*Array, error) { data: a.data, shape: cp, strides: rowMajorStrides(cp), + ws: a.ws, }, nil } return &Array{ data: a.materialize(), shape: cp, strides: rowMajorStrides(cp), + ws: a.ws, }, nil } @@ -409,6 +413,7 @@ func (a *Array) Ravel() *Array { data: data, shape: []int{len(data)}, strides: []int{1}, + ws: a.ws, } } @@ -427,6 +432,7 @@ func (a *Array) Transpose() *Array { shape: shape, strides: strides, offset: a.offset, + ws: a.ws, } } @@ -505,7 +511,7 @@ func (a *Array) broadcastTo(shape []int) []float64 { estrides[i] = a.strides[j] } } - out := make([]float64, prod(shape)) + out := a.alloc(prod(shape), false) // every element is written below if len(out) == 0 { return out } @@ -612,9 +618,9 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error // this is the minimal-overhead single-core path that matches NumPy's C loop // at small n. Strided/broadcasting operands fall through to the general path. if a.isContiguous() && b.isContiguous() && sameShape(a.shape, b.shape) { - dst := make([]float64, len(a.data)) + dst := a.alloc(len(a.data), false) kernel(dst, a.data, b.data) - return &Array{data: dst, shape: a.shape, strides: a.strides}, nil + return &Array{data: dst, shape: a.shape, strides: a.strides, ws: a.ws}, nil } shape, err := broadcastShape(a.shape, b.shape) if err != nil { @@ -622,10 +628,10 @@ func (a *Array) binOp(b *Array, kernel func(dst, x, y []float64)) (*Array, error } x := a.operandFor(shape) y := b.operandFor(shape) - dst := make([]float64, prod(shape)) + dst := a.alloc(prod(shape), false) kernel(dst, x, y) cp := append([]int(nil), shape...) - return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp)}, nil + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws}, nil } // Add returns the elementwise sum a+b with broadcasting. @@ -707,10 +713,10 @@ func (a *Array) DivScalar(v float64) *Array { r, _ := a.Div(scalarArray(v)); ret // must be safe to call concurrently (the package's math ufuncs are). func (a *Array) Map(f func(float64) float64) *Array { src := a.contiguousData() - dst := make([]float64, len(src)) + dst := a.alloc(len(src), false) kernels.MapP(dst, src, f) cp := append([]int(nil), a.shape...) - return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp)} + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws} } // contiguousData returns the receiver's elements as a contiguous row-major @@ -831,10 +837,10 @@ func (a *Array) reduceAxis( // row-major (the common case), so a contiguous array reaches the kernel with // zero copy; only strided views pay a materialise. The kernel only reads src. src := a.contiguousData() - dst := make([]float64, outer*inner) + dst := a.alloc(outer*inner, true) kernels.RunAxisP(kernel, dst, src, outer, axisLen, inner) shape := a.reduceShape(axis, keepdims) - return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape)}, nil + return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape), ws: a.ws}, nil } // SumAxis returns the sum along the given axis. With keepdims the reduced axis diff --git a/reductions.go b/reductions.go index d6ce6d3..826175c 100644 --- a/reductions.go +++ b/reductions.go @@ -55,10 +55,10 @@ func (a *Array) scanAxis( } outer, axisLen, inner := a.reduceLayout(axis) src := a.materialize() - dst := make([]float64, len(src)) + dst := a.alloc(len(src), false) kernel(dst, src, outer, axisLen, inner) shape := append([]int(nil), a.shape...) - return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape)}, nil + return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape), ws: a.ws}, nil } // CumSum returns the cumulative sum along the given axis, matching @@ -78,18 +78,18 @@ func (a *Array) CumProd(axis int) (*Array, error) { // order), matching numpy.cumsum with no axis. func (a *Array) CumSumFlat() *Array { src := a.materialize() - dst := make([]float64, len(src)) + dst := a.alloc(len(src), false) kernels.CumSumAxis(dst, src, 1, len(src), 1) - return &Array{data: dst, shape: []int{len(dst)}, strides: []int{1}} + return &Array{data: dst, shape: []int{len(dst)}, strides: []int{1}, ws: a.ws} } // CumProdFlat returns the cumulative product over the flattened array (1-D, // row-major order), matching numpy.cumprod with no axis. func (a *Array) CumProdFlat() *Array { src := a.materialize() - dst := make([]float64, len(src)) + dst := a.alloc(len(src), false) kernels.CumProdAxis(dst, src, 1, len(src), 1) - return &Array{data: dst, shape: []int{len(dst)}, strides: []int{1}} + return &Array{data: dst, shape: []int{len(dst)}, strides: []int{1}, ws: a.ws} } // Clip returns a new array with every element limited to the range [lo, hi], @@ -99,10 +99,10 @@ func (a *Array) Clip(lo, hi float64) (*Array, error) { return nil, fmt.Errorf("%w: clip bounds lo=%g > hi=%g", ErrShapeMismatch, lo, hi) } src := a.contiguousData() - dst := make([]float64, len(src)) + dst := a.alloc(len(src), false) kernels.Clip(dst, src, lo, hi) shape := append([]int(nil), a.shape...) - return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape)}, nil + return &Array{data: dst, shape: shape, strides: rowMajorStrides(shape), ws: a.ws}, nil } // Where returns an array selecting from t where cond is truthy (non-zero) and @@ -121,8 +121,8 @@ func Where(cond, t, f *Array) (*Array, error) { c := cond.operandFor(shape) tv := t.operandFor(shape) fv := f.operandFor(shape) - dst := make([]float64, prod(shape)) + dst := wsOf(cond, t, f).alloc(prod(shape), false) kernels.Where(dst, c, tv, fv) cp := append([]int(nil), shape...) - return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp)}, nil + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: wsOf(cond, t, f)}, nil } diff --git a/slicing.go b/slicing.go index 78ba38c..6138b76 100644 --- a/slicing.go +++ b/slicing.go @@ -174,5 +174,5 @@ func (a *Array) Slice(idx ...Index) (*Array, error) { shape = append(shape, count) strides = append(strides, step*a.strides[axis]) } - return &Array{data: a.data, shape: shape, strides: strides, offset: offset}, nil + return &Array{data: a.data, shape: shape, strides: strides, offset: offset, ws: a.ws}, nil } diff --git a/ufunc.go b/ufunc.go index bef1d85..2d9546f 100644 --- a/ufunc.go +++ b/ufunc.go @@ -21,10 +21,10 @@ import ( // math.Sqrt loop (sqrt(-x)=NaN, sqrt(+Inf)=+Inf, matching NumPy). func (a *Array) Sqrt() *Array { src := a.contiguousData() - dst := make([]float64, len(src)) + dst := a.alloc(len(src), false) kernels.SqrtP(dst, src) cp := append([]int(nil), a.shape...) - return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp)} + return &Array{data: dst, shape: cp, strides: rowMajorStrides(cp), ws: a.ws} } // SqrtInto writes the elementwise square root of a into the caller-provided diff --git a/workspace.go b/workspace.go new file mode 100644 index 0000000..fad3b12 --- /dev/null +++ b/workspace.go @@ -0,0 +1,127 @@ +package ndarray + +import "sync" + +// Workspace is an arena for the results of array operations, for loops that +// compute the same shapes over and over. Bind an input with Use; every result +// computed from a bound array (and every result computed from those) takes its +// memory from the workspace instead of the Go heap, and Reset hands all of it +// back at once: +// +// ws := ndarray.NewWorkspace() +// for step := 0; step < n; step++ { +// x := ws.Use(state) +// y, _ := x.Mul(w) // from the workspace +// z, _ := y.Add(bias) // from the workspace +// state = z.Detach() // the one result that must outlive the step +// ws.Reset() +// } +// +// After the first pass the arena holds the peak footprint of one pass in a +// single block, and later passes allocate nothing: no garbage-collector +// cycles, no zeroing of memory the kernel overwrites anyway, and the same +// cache-warm bytes every time. This is the workspace pattern of ND4J, the +// Java ndarray library, and the scoped counterpart of NumPy's reference-count +// free-list, which a garbage-collected language does not have. The *Into +// methods (AddInto, SqrtInto, ...) remain the way to reuse one named buffer. +// +// ⚠ Reset invalidates every array allocated from the workspace since the +// previous Reset, and every view of one: their memory is reused by the next +// results, so reading them afterwards returns other values. Detach (a heap +// copy, unbound) is how a result leaves the workspace. Arrays bound with Use +// but not allocated from it (the inputs) are not affected. +// +// A Workspace may be used from several goroutines; allocation is serialized. +// The zero value is an empty workspace ready to use. +type Workspace struct { + mu sync.Mutex + chunks [][]float64 // backing blocks; chunks[cur], the last, is being carved + cur int // index of the block being carved + off int // elements of chunks[cur] already handed out + used int // elements handed out since the last Reset, all blocks +} + +// minChunk is the smallest block the arena asks the heap for (1 MiB), so a +// pass of small results does not grow it one result at a time. +const minChunk = 1 << 17 + +// NewWorkspace returns an empty workspace. It holds no memory until the first +// allocation. +func NewWorkspace() *Workspace { return &Workspace{} } + +// Use returns a view of a (sharing its data) that is bound to w: the results +// of operations on it are allocated from w. a itself is not modified, and its +// data stays where it is. +func (w *Workspace) Use(a *Array) *Array { + v := *a + v.ws = w + return &v +} + +// Reset makes all memory allocated from w since the last Reset available +// again (see the warning on Workspace). If the pass needed more than one +// block, they are replaced by a single block of the pass's total, so a +// repeated pass settles into one contiguous block. +func (w *Workspace) Reset() { + w.mu.Lock() + defer w.mu.Unlock() + if w.cur > 0 { + w.chunks = [][]float64{make([]float64, w.used)} + } + w.cur, w.off, w.used = 0, 0, 0 +} + +// alloc returns n elements from the workspace. Each allocation starts on a +// 64-byte boundary of its block (8 elements), so a result never shares a +// cache line with its neighbour, and has cap n, so an append can never write +// into the next one. Unless zero is set, the elements hold whatever the +// previous pass left there; callers pass zero=false only when the kernel +// writes every element. +func (w *Workspace) alloc(n int, zero bool) []float64 { + if w == nil { + return make([]float64, n) + } + w.mu.Lock() + defer w.mu.Unlock() + need := (n + 7) &^ 7 + if len(w.chunks) == 0 || w.off+need > len(w.chunks[w.cur]) { + // Start a new block, at least double the last: the blocks only ever + // grow within a pass, and Reset folds them back into one. + size := max(need, minChunk) + if len(w.chunks) > 0 { + size = max(size, 2*len(w.chunks[w.cur])) + } + w.chunks = append(w.chunks, make([]float64, size)) + w.cur, w.off = len(w.chunks)-1, 0 + } + s := w.chunks[w.cur][w.off : w.off+n : w.off+n] + w.off += need + w.used += need + if zero { + clear(s) + } + return s +} + +// alloc returns n elements for a result computed from a: from a's workspace +// when it is bound to one, else from the heap (always zeroed there). +func (a *Array) alloc(n int, zero bool) []float64 { return a.ws.alloc(n, zero) } + +// Detach returns a heap copy of a that is not bound to any workspace: the way +// a result computed in a workspace outlives its Reset. +func (a *Array) Detach() *Array { + u := *a + u.ws = nil + return u.Copy() +} + +// wsOf returns the workspace of the first bound array among xs, or nil: the +// rule for functions of several arrays (Where, Concatenate, ...). +func wsOf(xs ...*Array) *Workspace { + for _, x := range xs { + if x.ws != nil { + return x.ws + } + } + return nil +} diff --git a/workspace_example_test.go b/workspace_example_test.go new file mode 100644 index 0000000..9e9e921 --- /dev/null +++ b/workspace_example_test.go @@ -0,0 +1,24 @@ +package ndarray_test + +import ( + "fmt" + + "github.com/go-ndarray/ndarray" +) + +func ExampleWorkspace() { + state, _ := ndarray.FromData([]float64{1, 4, 9}, 3) + w, _ := ndarray.FromData([]float64{2, 2, 2}, 3) + bias, _ := ndarray.FromData([]float64{0, 1, 2}, 3) + + ws := ndarray.NewWorkspace() + for step := 0; step < 3; step++ { + x := ws.Use(state) // results computed from x come from ws + p, _ := x.Mul(w) + q, _ := p.Add(bias) + state = q.Sqrt().Detach() // the one result that outlives the pass + ws.Reset() // everything else is recycled + } + fmt.Printf("%.4f %.4f %.4f\n", state.At(0), state.At(1), state.At(2)) + // Output: 1.8340 2.5083 2.9354 +} diff --git a/workspace_test.go b/workspace_test.go new file mode 100644 index 0000000..571fc14 --- /dev/null +++ b/workspace_test.go @@ -0,0 +1,221 @@ +package ndarray + +import ( + "math" + "sync" + "testing" +) + +// wsPass runs one pass of operations covering every allocation site that can +// draw from a workspace, starting from inputs bound to ws (nil = heap), and +// returns the results' data in order. +func wsPass(t *testing.T, ws *Workspace) [][]float64 { + t.Helper() + bind := func(a *Array) *Array { + if ws == nil { + return a + } + return ws.Use(a) + } + mk := func(seed float64, shape ...int) *Array { + n := 1 + for _, d := range shape { + n *= d + } + d := make([]float64, n) + for i := range d { + d[i] = math.Mod(float64(i)*0.37+seed, 7) - 3 + } + return bind(mustArr(t, ok(FromData(d, shape...)))) + } + a, b, row := mk(1, 6, 5), mk(2, 6, 5), mk(3, 1, 5) + v, m := mk(4, 5), mk(5, 5, 4) + var out []*Array + add := func(x *Array, err error) *Array { + t.Helper() + if err != nil { + t.Fatal(err) + } + out = append(out, x) + return x + } + s := add(a.Add(b)) // same-shape fast path + add(s.Mul(row)) // broadcast path (broadcastTo) + add(a.Sub(b)) // + add(a.Abs().Sqrt(), nil) // Map + Sqrt + add(a.Transpose().Copy(), nil) // materialize, strided gather + add(a.MatMul(m)) // GEMM, needs zeroed dst + add(a.Inner(b)) // GEMM on a transposed view + add(a.Dot(v)) // MatVec + add(v.Dot(m)) // VecMat, needs zeroed dst + add(v.Dot(v)) // scalar + add(v.Outer(v), nil) // + add(a.SumAxis(0, false)) // axis reduction + add(a.MaxAxis(1, true)) // + add(a.CumSum(1)) // scan + add(a.CumSumFlat(), nil) // + add(a.CumProdFlat(), nil) // + add(a.Clip(-1, 1)) // + gt := mustArr(t, ok(a.Greater(b))) + add(Where(gt, a, b)) // + add(Concatenate([]*Array{a, b}, 0)) + add(a.Take(0, 3, -1)) // + add(a.Reshape(5, 6)) // view + add(a.Ravel(), nil) // + add(a.Flatten(), nil) // + add(a.MaskSelect(gt)) // + res := make([][]float64, len(out)) + for i, x := range out { + res[i] = x.Copy().Detach().data + } + return res +} + +// TestWorkspaceMatchesHeap: every result computed in a workspace is the same, +// bit for bit, as on the heap — on the first pass (fresh blocks) and on a pass +// after Reset whose recycled memory was first poisoned with NaN. An operation +// that relied on zeroed memory while asking for unzeroed would leak the NaN. +func TestWorkspaceMatchesHeap(t *testing.T) { + want := wsPass(t, nil) + ws := NewWorkspace() + check := func(pass string, got [][]float64) { + t.Helper() + for i := range want { + if len(got[i]) != len(want[i]) { + t.Fatalf("%s result %d: len %d, want %d", pass, i, len(got[i]), len(want[i])) + } + for j := range want[i] { + if math.Float64bits(got[i][j]) != math.Float64bits(want[i][j]) { + t.Fatalf("%s result %d [%d]: %v, want %v", pass, i, j, got[i][j], want[i][j]) + } + } + } + } + check("first pass", wsPass(t, ws)) + for pass := 0; pass < 3; pass++ { + ws.Reset() + for _, c := range ws.chunks { + for i := range c { + c[i] = math.NaN() + } + } + check("poisoned pass", wsPass(t, ws)) + } + if len(ws.chunks) != 1 { + t.Errorf("after repeated passes the workspace holds %d blocks, want 1", len(ws.chunks)) + } +} + +// TestWorkspaceSettlesIntoOneBlock: a pass that outgrows the first block makes +// several; Reset folds them into one sized to the pass, and the next identical +// pass fits in it without growing. +func TestWorkspaceSettlesIntoOneBlock(t *testing.T) { + var ws Workspace // the zero value is usable + pass := func() { + for i := 0; i < 5; i++ { + ws.alloc(minChunk/2+3, false) + } + ws.alloc(5*minChunk, true) // larger than double the last block + } + pass() + if len(ws.chunks) < 2 { + t.Fatalf("pass made %d blocks, want several", len(ws.chunks)) + } + ws.Reset() + if len(ws.chunks) != 1 { + t.Fatalf("Reset left %d blocks, want 1", len(ws.chunks)) + } + size := len(ws.chunks[0]) + pass() + if len(ws.chunks) != 1 || len(ws.chunks[0]) != size { + t.Errorf("second pass grew the workspace: %d blocks, first %d (was %d)", + len(ws.chunks), len(ws.chunks[0]), size) + } + ws.Reset() // single-block pass: nothing to fold + if len(ws.chunks) != 1 || len(ws.chunks[0]) != size { + t.Errorf("Reset after a one-block pass changed the block") + } +} + +// TestWorkspaceAllocShape: every allocation starts 64-byte aligned within its +// block, has exactly the requested length and capacity (an append cannot +// spill into the neighbour), and zero=true clears recycled memory. +func TestWorkspaceAllocShape(t *testing.T) { + ws := NewWorkspace() + a := ws.alloc(3, false) + b := ws.alloc(5, false) + if len(a) != 3 || cap(a) != 3 || len(b) != 5 || cap(b) != 5 { + t.Fatalf("len/cap a=%d/%d b=%d/%d", len(a), cap(a), len(b), cap(b)) + } + if &ws.chunks[0][8] != &b[0] { + t.Errorf("second allocation does not start at element 8 of the block") + } + for i := range b { + b[i] = 7 + } + ws.Reset() + ws.alloc(3, false) + if c := ws.alloc(5, true); c[0] != 0 || c[4] != 0 { + t.Errorf("zero=true returned %v", c) + } + if h := (*Workspace)(nil).alloc(4, false); len(h) != 4 { + t.Errorf("nil workspace alloc len %d", len(h)) + } +} + +// TestWorkspaceDetachAndUse: Detach is a heap copy that survives Reset; Use +// leaves its argument unbound and shares its data; views stay bound. +func TestWorkspaceDetachAndUse(t *testing.T) { + ws := NewWorkspace() + x := mustArr(t, ok(FromData([]float64{1, 2, 3, 4}, 2, 2))) + bx := ws.Use(x) + if x.ws != nil || bx.ws != ws || &bx.data[0] != &x.data[0] { + t.Fatal("Use must bind a view sharing x's data and leave x unbound") + } + if bx.Transpose().ws != ws { + t.Error("a view of a bound array is not bound") + } + y := mustArr(t, ok(bx.Add(bx))) + kept := y.Detach() + if kept.ws != nil { + t.Error("Detach returned a bound array") + } + ws.Reset() + for i := range ws.chunks[0] { + ws.chunks[0][i] = -1 + } + wantData(t, kept, []int{2, 2}, []float64{2, 4, 6, 8}) + wantData(t, x, []int{2, 2}, []float64{1, 2, 3, 4}) +} + +// TestWorkspaceConcurrent: allocation from several goroutines is serialized, +// so concurrent operations on arrays bound to one workspace never overlap. +func TestWorkspaceConcurrent(t *testing.T) { + ws := NewWorkspace() + x := ws.Use(mustArr(t, ok(Arange(0, 1000, 1)))) + var wg sync.WaitGroup + results := make([]*Array, 8) + for g := range results { + wg.Add(1) + go func() { + defer wg.Done() + r, err := x.Add(x) + if err != nil { + t.Error(err) + return + } + for i := range r.data { + r.data[i] += float64(g) + } + results[g] = r + }() + } + wg.Wait() + for g, r := range results { + for i, v := range r.data { + if v != float64(2*i+g) { + t.Fatalf("goroutine %d [%d] = %v: results overlap", g, i, v) + } + } + } +}