diff --git a/src/state.jl b/src/state.jl index 8adce9f3..82c3cbeb 100644 --- a/src/state.jl +++ b/src/state.jl @@ -18,7 +18,6 @@ function checkpoint!(pool::AdaptiveArrayPool) pool._current_depth += 1 push!(pool._touched_type_masks, UInt16(0)) push!(pool._touched_has_others, false) - push!(pool._touched_others_checkpoints, length(pool._touched_others)) _runtime_check(pool) && push!(pool._others_ptr_bounds_checkpoints, length(pool._others_ptr_bounds)) depth = pool._current_depth @@ -53,7 +52,6 @@ Also updates _current_depth and bitmask state for type touch tracking. # _typed_lazy_rewind! iterates pool.others even if _acquire_impl! # (which bypasses _record_type_touch!) is the only acquire path. push!(pool._touched_has_others, _fixed_slot_bit(T) == UInt16(0)) - push!(pool._touched_others_checkpoints, length(pool._touched_others)) _runtime_check(pool) && push!(pool._others_ptr_bounds_checkpoints, length(pool._others_ptr_bounds)) if _fixed_slot_bit(T) == UInt16(0) # Fallback type: checkpoint + record in the touched-others stack so the @@ -97,26 +95,33 @@ compile-time unrolling. Increments _current_depth once for all types. pool._current_depth += 1 push!(pool._touched_type_masks, UInt16(0)) push!(pool._touched_has_others, $has_any_fallback) - push!(pool._touched_others_checkpoints, length(pool._touched_others)) _runtime_check(pool) && push!(pool._others_ptr_bounds_checkpoints, length(pool._others_ptr_bounds)) $(checkpoint_exprs...) nothing end end -# Internal helper for checkpoint (works for any AbstractTypedPool) -@inline function _checkpoint_typed_pool!(tp::AbstractTypedPool, depth::Int) - - # Guard: skip if already checkpointed at this depth (prevents double-push - # when get_typed_pool! auto-checkpoints a new fallback type and then - # checkpoint!(pool, types...) calls _checkpoint_typed_pool! for the same type). - if @inbounds(tp._checkpoint_depths[end]) != depth - push!(tp._checkpoint_n_active, tp.n_active) - push!(tp._checkpoint_depths, depth) +# T-independent checkpoint core. `st` is a PoolCheckpointState (CPU) or a +# flat-field typed pool acting as its own state carrier (GPU backends) — both +# expose the same three properties. +@inline function _checkpoint_state_core!(st, depth::Int) + # Guard: skip if already checkpointed at this depth (prevents double-push). + if @inbounds(st._checkpoint_depths[end]) != depth + push!(st._checkpoint_n_active, st.n_active) + push!(st._checkpoint_depths, depth) end return nothing end +# Map a typed pool to its checkpoint-state carrier: CPU pools carry a dedicated +# PoolCheckpointState; other backends (GPU) keep flat fields and act as their own. +@inline _cp_state(tp::AbstractTypedPool) = tp +@inline _cp_state(tp::TypedPool) = getfield(tp, :state) +@inline _cp_state(tp::BitTypedPool) = getfield(tp, :state) + +@inline _checkpoint_typed_pool!(tp::AbstractTypedPool, depth::Int) = + _checkpoint_state_core!(_cp_state(tp), depth) + """ _lazy_checkpoint!(pool::AdaptiveArrayPool) @@ -141,7 +146,6 @@ Performance: ~2ns vs ~540ns for full `checkpoint!`. # _LAZY_MODE_BIT = lazy mode flag (bits 0–7 are fixed-slot type bits) push!(pool._touched_type_masks, _LAZY_MODE_BIT) push!(pool._touched_has_others, false) - push!(pool._touched_others_checkpoints, length(pool._touched_others)) _runtime_check(pool) && push!(pool._others_ptr_bounds_checkpoints, length(pool._others_ptr_bounds)) return nothing end @@ -184,7 +188,7 @@ function rewind!(pool::AdaptiveArrayPool{S}) where {S} for tp in pool._others_values _rewind_typed_pool!(tp, cur_depth, S) end - _truncate_touched_others!(pool) + _truncate_touched_others!(pool, cur_depth) if S >= 1 && length(pool._others_ptr_bounds_checkpoints) > 1 resize!(pool._others_ptr_bounds, pop!(pool._others_ptr_bounds_checkpoints)) @@ -349,45 +353,31 @@ end # Internal: Rewind with Orphan Cleanup # ============================================================================== -# Internal helper for rewind with orphan cleanup (works for any AbstractTypedPool) -# Uses 1-based sentinel pattern: no isempty checks needed (sentinel [0] guarantees non-empty) -# -# S parameter: runtime check level (0=off, 1=on). When called from AdaptiveArrayPool{S} -# callers, S is a compile-time constant → `S >= 1` dead-code-eliminates at S=0. -@inline function _rewind_typed_pool!(tp::AbstractTypedPool, current_depth::Int, S::Int) - - # 1. Orphaned Checkpoints Cleanup - # If there are checkpoints from deeper scopes (depth > current), pop them first. - # This happens when a nested scope did full checkpoint but typed rewind, - # leaving orphaned checkpoints that must be cleaned before finding current state. - while @inbounds tp._checkpoint_depths[end] > current_depth - pop!(tp._checkpoint_depths) - pop!(tp._checkpoint_n_active) +# T-independent rewind core: orphan cleanup + Case A/B restore. Returns the +# pre-rewind n_active so the (S >= 1) caller can decide whether to invalidate. +@inline function _rewind_state_core!(st, current_depth::Int) + # 1. Orphaned checkpoints from deeper scopes + while @inbounds(st._checkpoint_depths[end]) > current_depth + pop!(st._checkpoint_depths) + pop!(st._checkpoint_n_active) end - - # Capture n_active before restore (compiler eliminates dead variable at S=0) - _old_n_active = tp.n_active - - # 2. Normal Rewind Logic (Sentinel Pattern) - # Now the stack top is guaranteed to be at depth <= current depth. - if @inbounds tp._checkpoint_depths[end] == current_depth - # Checkpointed at current depth: pop and restore - pop!(tp._checkpoint_depths) - tp.n_active = pop!(tp._checkpoint_n_active) + old_n_active = st.n_active + # 2. Case A (pop) / Case B (restore from parent top) + if @inbounds(st._checkpoint_depths[end]) == current_depth + pop!(st._checkpoint_depths) + st.n_active = pop!(st._checkpoint_n_active) else - # No checkpoint at current depth (this type was excluded from typed checkpoint) - # MUST restore n_active from parent checkpoint value! - # - Untracked acquire may have modified n_active - # - If sentinel (_checkpoint_n_active=[0]), restores to n_active=0 - tp.n_active = @inbounds tp._checkpoint_n_active[end] + st.n_active = @inbounds st._checkpoint_n_active[end] end + return old_n_active +end - # 3. Safety: invalidate released slots (Level 1+) - # At S=0: `0 >= 1` is false → entire branch eliminated (dead code) +@inline function _rewind_typed_pool!(tp::AbstractTypedPool, current_depth::Int, S::Int) + _old_n_active = _rewind_state_core!(_cp_state(tp), current_depth) + # 3. Safety: invalidate released slots (S >= 1; DCE'd at S = 0) if S >= 1 && _old_n_active > tp.n_active _invalidate_released_slots!(tp, _old_n_active, S) end - return nothing end @@ -395,36 +385,49 @@ end # Touched-Others Stack (per-scope selective fallback checkpoint/rewind) # ============================================================================== -# Rewind and remove the fallback typed pools first-touched in the current scope. -# The current depth's segment of `_touched_others` is (base+1):end, where base is -# the saved length pushed by the matching checkpoint variant. O(touched this scope). +# Rewind and remove this depth's touched-fallback entries. At S = 0 the loop runs +# entirely on concrete PoolCheckpointState objects — zero dynamic dispatch; at +# S >= 1 it routes through the typed pools so released slots get invalidated. +# Fixed-only scopes exit via one isempty/top-depth check. @inline function _drain_touched_others!(pool::AdaptiveArrayPool{S}, d::Int) where {S} - base = pop!(pool._touched_others_checkpoints) - stack = pool._touched_others - for i in (base + 1):length(stack) - _rewind_typed_pool!(@inbounds(stack[i]), d, S) + depths = pool._touched_others_depths + states = pool._touched_others_states + while !isempty(depths) && @inbounds(depths[end]) == d + pop!(depths) + st = pop!(states) + if S >= 1 + _rewind_typed_pool!(pop!(pool._touched_others_pools), d, S) + else + _rewind_state_core!(st, d) + end end - length(stack) > base && resize!(stack, base) return nothing end # Truncate-only variant for full rewind!(pool): its _others_values sweep already # rewound every fallback pool, so draining again would double-pop checkpoints. -@inline function _truncate_touched_others!(pool::AdaptiveArrayPool) - resize!(pool._touched_others, pop!(pool._touched_others_checkpoints)) +@inline function _truncate_touched_others!(pool::AdaptiveArrayPool{S}, d::Int) where {S} + depths = pool._touched_others_depths + while !isempty(depths) && @inbounds(depths[end]) == d + pop!(depths) + pop!(pool._touched_others_states) + S >= 1 && pop!(pool._touched_others_pools) + end return nothing end -# First-touch producer: checkpoint a fallback pool at `depth` and record it in the -# touched-others stack, exactly once per depth. The `_checkpoint_depths[end] != depth` -# guard makes this idempotent across the three producer paths (typed checkpoint!, -# _record_type_touch!, get_typed_pool! registration) and skips pools already +# First-touch producer: checkpoint a fallback pool at `depth` and push one +# depth-tagged entry, exactly once per (pool, depth). The checkpoint-depth guard +# keeps it idempotent across the three producer paths and skips pools already # eagerly checkpointed by a full checkpoint!(pool) at this depth. @inline function _touch_fallback_pool!(pool::AdaptiveArrayPool, tp::AbstractTypedPool, depth::Int) - if @inbounds(tp._checkpoint_depths[end]) != depth - push!(tp._checkpoint_n_active, tp.n_active) - push!(tp._checkpoint_depths, depth) - push!(pool._touched_others, tp) + st = _cp_state(tp)::PoolCheckpointState + if @inbounds(st._checkpoint_depths[end]) != depth + push!(st._checkpoint_n_active, st.n_active) + push!(st._checkpoint_depths, depth) + push!(pool._touched_others_states, st) + push!(pool._touched_others_depths, depth) + _runtime_check(pool) && push!(pool._touched_others_pools, tp) end return nothing end @@ -587,15 +590,19 @@ function Base.empty!(pool::AdaptiveArrayPool) empty!(pool.others) empty!(pool._others_values) + # Memo points into the registry being cleared — drop it with the registry. + pool._lookup_memo_type = nothing + pool._lookup_memo_tp = nothing + # Reset pre-collected pointer bounds empty!(pool._others_ptr_bounds) empty!(pool._others_ptr_bounds_checkpoints) push!(pool._others_ptr_bounds_checkpoints, 0) # Sentinel # Reset touched-others tracking (transient scope state) - empty!(pool._touched_others) - empty!(pool._touched_others_checkpoints) - push!(pool._touched_others_checkpoints, 0) # Sentinel + empty!(pool._touched_others_states) + empty!(pool._touched_others_depths) + empty!(pool._touched_others_pools) # Reset type touch tracking state (1-based sentinel pattern) pool._current_depth = 1 # 1 = global scope (sentinel) @@ -694,9 +701,9 @@ function reset!(pool::AdaptiveArrayPool{S}) where {S} push!(pool._others_ptr_bounds_checkpoints, 0) # Sentinel # Reset touched-others tracking (transient scope state) - empty!(pool._touched_others) - empty!(pool._touched_others_checkpoints) - push!(pool._touched_others_checkpoints, 0) # Sentinel + empty!(pool._touched_others_states) + empty!(pool._touched_others_depths) + empty!(pool._touched_others_pools) # Reset type touch tracking state (1-based sentinel pattern) pool._current_depth = 1 # 1 = global scope (sentinel) diff --git a/src/types.jl b/src/types.jl index 7f12c5a6..1d6eb0e9 100644 --- a/src/types.jl +++ b/src/types.jl @@ -81,6 +81,27 @@ pooling_enabled(::DisabledPool) = false # 1-Based Sentinel Pattern: Arrays start with sentinel values to eliminate # isempty() checks in hot paths. See docstrings for details. +""" + PoolCheckpointState + +Concrete, type-parameter-free checkpoint bookkeeping shared by all CPU typed pools. + +`checkpoint!`/`rewind!` at `S = 0` only ever touch these three fields — none of the +`T`-dependent storage — so extracting them lets the touched-others stack hold +`PoolCheckpointState` objects directly and drain with zero dynamic dispatch +(`Vector{PoolCheckpointState}` is concrete; a `Vector{Any}` of `TypedPool{T}` is not). + +1-based sentinel pattern: both stacks start at `[0]` (depth 0 = no checkpoint), so +`[end]` access never needs an isempty guard. +""" +mutable struct PoolCheckpointState + n_active::Int + _checkpoint_n_active::Vector{Int} # Saved n_active at each checkpoint + _checkpoint_depths::Vector{Int} # Depth of each checkpoint +end + +PoolCheckpointState() = PoolCheckpointState(0, [0], [0]) + """ TypedPool{T} <: AbstractTypedPool{T, Vector{T}} @@ -121,10 +142,11 @@ mutable struct TypedPool{T} <: AbstractTypedPool{T, Vector{T}} # capacity high-water mark, not the current extent. slot_extents::Vector{Int} - # --- State Management (1-based sentinel pattern) --- - n_active::Int - _checkpoint_n_active::Vector{Int} # Saved n_active at each checkpoint - _checkpoint_depths::Vector{Int} # Depth of each checkpoint + # Checkpoint bookkeeping, extracted into a concrete shared struct (see + # PoolCheckpointState). `const`: the reference never changes after construction, + # so the compiler can hoist the pointer load on hot paths. Accessed as + # tp.n_active / tp._checkpoint_* via property forwarding below. + const state::PoolCheckpointState # --- Auto-trim telemetry --- # Peak `n_active` reached since the last auto-trim — the recent working-set width. @@ -140,10 +162,7 @@ TypedPool{T}() where {T} = TypedPool{T}( Union{Nothing, Vector{Any}}[], # Per-slot current logical extent (parallel to `vectors`) Int[], - # State Management (1-based sentinel pattern: guaranteed non-empty) - 0, # n_active - [0], # _checkpoint_n_active: sentinel (n_active=0 at depth=0) - [0], # _checkpoint_depths: sentinel (depth=0 = no checkpoint) + PoolCheckpointState(), # state 0, # _am_peak_n_active: no usage observed yet ) @@ -244,10 +263,11 @@ mutable struct BitTypedPool <: AbstractTypedPool{Bool, BitVector} # --- N-D Wrapper Cache (setfield!-based reuse) --- arr_wrappers::Vector{Union{Nothing, Vector{Any}}} # index=N (dimensionality), value=per-slot BitArray{N} - # --- State Management (1-based sentinel pattern) --- - n_active::Int - _checkpoint_n_active::Vector{Int} - _checkpoint_depths::Vector{Int} + # Checkpoint bookkeeping, extracted into a concrete shared struct (see + # PoolCheckpointState). `const`: the reference never changes after construction, + # so the compiler can hoist the pointer load on hot paths. Accessed as + # tp.n_active / tp._checkpoint_* via property forwarding below. + const state::PoolCheckpointState # --- Auto-trim telemetry (parallel to TypedPool; see its docstring) --- _am_peak_n_active::Int @@ -258,13 +278,34 @@ BitTypedPool() = BitTypedPool( BitVector[], # N-D Wrapper Cache Union{Nothing, Vector{Any}}[], - # State Management (1-based sentinel pattern) - 0, # n_active - [0], # _checkpoint_n_active: sentinel - [0], # _checkpoint_depths: sentinel + PoolCheckpointState(), # state 0, # _am_peak_n_active ) +# ============================================================================== +# Checkpoint-State Property Forwarding +# ============================================================================== +# The three checkpoint fields moved into `state`, but ~76 call sites across src/ +# (plus the GPU extensions' shared duck-typed functions and the test suite) +# address them as `tp.n_active` etc. Forward exactly those three names; every +# other name stays a plain field. The Symbol comparisons are constant-folded for +# literal-symbol accesses (the only kind in this codebase). + +@inline function Base.getproperty(tp::Union{TypedPool, BitTypedPool}, f::Symbol) + f === :n_active && return getfield(tp, :state).n_active + f === :_checkpoint_n_active && return getfield(tp, :state)._checkpoint_n_active + f === :_checkpoint_depths && return getfield(tp, :state)._checkpoint_depths + return getfield(tp, f) +end + +@inline function Base.setproperty!(tp::Union{TypedPool, BitTypedPool}, f::Symbol, v) + f === :n_active && return setfield!(getfield(tp, :state), :n_active, convert(Int, v)) + return setfield!(tp, f, convert(fieldtype(typeof(tp), f), v)) # moved vector fields are never reassigned; loud error if tried +end + +Base.propertynames(tp::Union{TypedPool, BitTypedPool}) = + (fieldnames(typeof(tp))..., :n_active, :_checkpoint_n_active, :_checkpoint_depths) + # ============================================================================== # Fixed Slot Configuration # ============================================================================== @@ -390,13 +431,26 @@ mutable struct AdaptiveArrayPool{S} <: AbstractArrayPool _others_ptr_bounds::Vector{UInt} # flat [ptr1,end1,ptr2,end2,...] _others_ptr_bounds_checkpoints::Vector{Int} # per-depth: saved length of bounds vector - # Touched-others tracking (per-scope selective fallback rewind). - # Flat stack of fallback typed pools first-touched at each open depth, plus - # per-depth saved lengths — same pattern as _others_ptr_bounds(+_checkpoints). - # Rewind drains only the current depth's segment: O(touched fallback types - # this scope) instead of O(all registered fallback pools). - _touched_others::Vector{Any} - _touched_others_checkpoints::Vector{Int} + # Touched-others tracking (depth-tagged, concrete). + # One entry per (fallback pool, depth) first-touch, in three parallel stacks. + # Checkpoint variants push NOTHING — a fixed-only scope pays a single isempty + # check at rewind. Producers push on first touch; rewind pops while the top + # entry's depth tag matches the current depth. Tags are monotone non-decreasing + # bottom-to-top (entries are only ever pushed at the current depth). + # `_touched_others_pools` is populated only when S >= 1: the S=0 drain needs + # just the concrete states (zero dispatch); slot invalidation needs the pools. + _touched_others_states::Vector{PoolCheckpointState} + _touched_others_depths::Vector{Int} + _touched_others_pools::Vector{Any} + + # Last-lookup memo for the fallback registry. The public acquire! path resolves + # the same fallback type twice per call (touch record + impl); this turns the + # second — and, steady-state, both — into one pointer compare (~1 ns vs ~7 ns + # IdDict lookup). Pure cache of others[T]: set on every slow-path lookup, + # cleared by empty!/_make_pool. reset!/trim!/compact! keep pool identities, so + # the memo stays valid through them. Task-local pool → single-owner, no races. + _lookup_memo_type::Any # DataType of the memoized entry, or nothing + _lookup_memo_tp::Any # its TypedPool # Borrow registry (S = 1 only) _pending_callsite::String # "" = no pending; set by macro before acquire @@ -428,8 +482,11 @@ function AdaptiveArrayPool{S}() where {S} Any[], # _others_values: empty cache UInt[], # _others_ptr_bounds: no bounds Int[0], # _others_ptr_bounds_checkpoints: sentinel - Any[], # _touched_others: no fallback touches yet - Int[0], # _touched_others_checkpoints: sentinel + PoolCheckpointState[], # _touched_others_states: no fallback touches yet + Int[], # _touched_others_depths: no fallback touches yet + Any[], # _touched_others_pools: no fallback touches yet + nothing, # _lookup_memo_type: no memoized lookup yet + nothing, # _lookup_memo_tp: no memoized lookup yet "", # _pending_callsite: no pending "", # _pending_return_site: no pending nothing, # _borrow_log: lazily created at S=1 @@ -462,39 +519,11 @@ _make_pool(runtime_check::Bool) = _make_pool(Int(runtime_check)) return AdaptiveArrayPool{1}() end -""" - _make_pool(level, old::AdaptiveArrayPool) -> AdaptiveArrayPool - -Create a new pool, transferring cached arrays and scope state from `old`. -Only reference copies — no memory allocation for the underlying buffers. - -Transferred: all TypedPool/BitTypedPool slots, `others`, depth & touch tracking. -Reset: `_pending_callsite/return_site` (transient macro state), - `_borrow_log` (created fresh when S >= 1). -""" -_make_pool(runtime_check::Bool, old::AdaptiveArrayPool) = _make_pool(Int(runtime_check), old) -@noinline function _make_pool(level::Int, old::AdaptiveArrayPool) - _new(::Val{S}) where {S} = AdaptiveArrayPool{S}( - old.float64, old.float32, old.int64, old.int32, - old.complexf64, old.complexf32, old.bool, old.bits, - old.others, - old._current_depth, - old._touched_type_masks, - old._touched_has_others, - old._others_values, - old._others_ptr_bounds, - old._others_ptr_bounds_checkpoints, - old._touched_others, - old._touched_others_checkpoints, - "", # _pending_callsite: reset - "", # _pending_return_site: reset - S >= 1 ? IdDict{Any, String}() : nothing, # _borrow_log - false, # _compact_requested: reset on migration - false, # _trim_requested: reset on migration - ) - level == 0 && return _new(Val(0)) - return _new(Val(1)) -end +# NOTE: a former 2-arg `_make_pool(level, old)` overload migrated a pool across S +# while transferring caches by reference. Removed (no production callers): the +# touched-others stack's shape is S-dependent (pools entries exist only at S >= 1), +# so a mid-scope migration would desync the next rewind. If runtime S switching is +# ever needed, reintroduce it with an explicit global-scope guard. # ============================================================================== # Type Dispatch (Zero-cost for Fixed Slots) @@ -517,25 +546,35 @@ const _FIXED_SLOT_TYPES = Union{Float64, Float32, Int64, Int32, ComplexF64, Comp # Slow Path: rare types via IdDict @inline function get_typed_pool!(p::AdaptiveArrayPool, ::Type{T}) where {T} + # Memo fast path: same type as the previous slow-path lookup. + p._lookup_memo_type === T && return p._lookup_memo_tp::TypedPool{T} tp = get(p.others, T, nothing) - tp !== nothing && return tp::TypedPool{T} + if tp !== nothing + tp = tp::TypedPool{T} + p._lookup_memo_type = T + p._lookup_memo_tp = tp + return tp + end # New type — create, register in IdDict + values cache, and auto-checkpoint new_tp = TypedPool{T}() p.others[T] = new_tp push!(p._others_values, new_tp) + p._lookup_memo_type = T + p._lookup_memo_tp = new_tp # If inside a checkpoint scope (_current_depth > 1 means inside @with_pool), # auto-checkpoint the new pool to prevent issues on rewind if p._current_depth > 1 - push!(new_tp._checkpoint_n_active, 0) # n_active starts at 0 - push!(new_tp._checkpoint_depths, p._current_depth) + st = getfield(new_tp, :state) + push!(st._checkpoint_n_active, 0) # n_active starts at 0 + push!(st._checkpoint_depths, p._current_depth) + push!(p._touched_others_states, st) + push!(p._touched_others_depths, p._current_depth) + _runtime_check(p) && push!(p._touched_others_pools, new_tp) # Signal that a fallback type was touched so lazy/typed-lazy rewind # iterates pool.others. Without this, _acquire_impl! (which bypasses # _record_type_touch!) would leave has_others=false, causing the # rewind to skip pool.others entirely and leak this new type's n_active. @inbounds p._touched_has_others[p._current_depth] = true - # Record in the touched-others stack so the selective rewind visits this - # brand-new pool (its checkpoint was just pushed above). - push!(p._touched_others, new_tp) end return new_tp end diff --git a/test/runtests.jl b/test/runtests.jl index 7bf101da..85091b62 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -57,6 +57,7 @@ else include("test_allocation.jl") include("test_fallback_reclamation.jl") include("test_touched_others.jl") + include("test_pool_checkpoint_state.jl") include("test_scope_depth_validation.jl") else include("test_aqua.jl") diff --git a/test/test_pool_checkpoint_state.jl b/test/test_pool_checkpoint_state.jl new file mode 100644 index 00000000..374696e6 --- /dev/null +++ b/test/test_pool_checkpoint_state.jl @@ -0,0 +1,83 @@ +# PoolCheckpointState extraction: checkpoint bookkeeping lives in a concrete, +# type-parameter-free struct; property forwarding keeps tp.n_active etc. working. +using Test +using AdaptiveArrayPools +using AdaptiveArrayPools: TypedPool, BitTypedPool, PoolCheckpointState + +@testset "PoolCheckpointState: extraction & forwarding" begin + tp = TypedPool{Float64}() + st = tp.state + @test st isa PoolCheckpointState + @test st.n_active == 0 + @test st._checkpoint_n_active == [0] + @test st._checkpoint_depths == [0] + + # forwarding reads the SAME objects (identity, not copies) + @test tp._checkpoint_n_active === st._checkpoint_n_active + @test tp._checkpoint_depths === st._checkpoint_depths + + # forwarded write goes to the state + tp.n_active = 3 + @test st.n_active == 3 + @test tp.n_active == 3 + tp.n_active = 0 + + # fallback setproperty! keeps default auto-convert semantics + tp._am_peak_n_active = Int32(2) + @test tp._am_peak_n_active === 2 + tp._am_peak_n_active = 0 + + # non-forwarded fields still direct + @test tp.vectors isa Vector{Vector{Float64}} + @test tp._am_peak_n_active == 0 + + btp = BitTypedPool() + @test btp.state isa PoolCheckpointState + @test btp._checkpoint_depths === btp.state._checkpoint_depths + btp.n_active = 2 + @test btp.state.n_active == 2 + + # `state` is const — rebinding must throw + @test_throws Exception (tp.state = PoolCheckpointState()) +end + +@testset "PoolCheckpointState: end-to-end through public API" begin + pool = AdaptiveArrayPool() + v = acquire!(pool, Float64, 8) + @test pool.float64.state.n_active == 1 + reset!(pool) + @test pool.float64.state.n_active == 0 + @test pool.float64.state._checkpoint_depths == [0] +end + +using AdaptiveArrayPools: _cp_state, _checkpoint_state_core!, _rewind_state_core! + +@testset "state cores: checkpoint/rewind on bare PoolCheckpointState" begin + st = PoolCheckpointState() + _checkpoint_state_core!(st, 2) + @test st._checkpoint_depths == [0, 2] + _checkpoint_state_core!(st, 2) # same-depth guard: no double push + @test st._checkpoint_depths == [0, 2] + st.n_active = 5 + @test _rewind_state_core!(st, 2) == 5 # returns pre-rewind n_active + @test st.n_active == 0 # Case A restore + @test st._checkpoint_depths == [0] + + # Case B: no checkpoint at depth → restore from stack top + st.n_active = 7 + @test _rewind_state_core!(st, 3) == 7 + @test st.n_active == 0 # sentinel top + + # orphan cleanup: stale deeper entries popped first + _checkpoint_state_core!(st, 4) + st.n_active = 9 + @test _rewind_state_core!(st, 2) == 9 # pops orphan depth-4 entry, Case B + @test st._checkpoint_depths == [0] + @test st.n_active == 0 + + # _cp_state mapping + tp = TypedPool{Int32}() + @test _cp_state(tp) === tp.state + btp = BitTypedPool() + @test _cp_state(btp) === btp.state +end diff --git a/test/test_task_local_pool.jl b/test/test_task_local_pool.jl index 31d5b193..3490c832 100644 --- a/test/test_task_local_pool.jl +++ b/test/test_task_local_pool.jl @@ -231,16 +231,16 @@ @test pool1 isa AdaptiveArrayPool{1} @test pool1._borrow_log === nothing # lazily initialized on first borrow - # _make_pool(Bool, old) preserves cached arrays - pool0_with_data = AdaptiveArrayPools._make_pool(false) - AdaptiveArrayPools._lazy_checkpoint!(pool0_with_data) - acquire!(pool0_with_data, Float64, 10) - AdaptiveArrayPools._lazy_rewind!(pool0_with_data) - - old_f64 = pool0_with_data.float64 - pool1_from_old = AdaptiveArrayPools._make_pool(true, pool0_with_data) - @test pool1_from_old isa AdaptiveArrayPool{1} - @test pool1_from_old.float64 === old_f64 # same TypedPool reference + # The former 2-arg _make_pool(level, old) S-migration overload was removed + # from the modern tree: the touched-others stack's shape is S-dependent, so + # cross-S transfer of open scope state would desync the drain (no production + # callers existed). The legacy tree (Julia < 1.12) has no depth-tagged stack, + # so its overload is harmless and retained. + @static if VERSION >= v"1.12-" + @test !hasmethod(AdaptiveArrayPools._make_pool, Tuple{Int, AdaptiveArrayPool}) + else + @test hasmethod(AdaptiveArrayPools._make_pool, Tuple{Int, AdaptiveArrayPool}) + end end @testset "Pool growth warning at 512 arrays" begin diff --git a/test/test_touched_others.jl b/test/test_touched_others.jl index 544e11f0..39be8fe0 100644 --- a/test/test_touched_others.jl +++ b/test/test_touched_others.jl @@ -1,8 +1,9 @@ # Tests for the touched-others stack: per-scope selective fallback checkpoint/rewind. -# Invariant: a fallback pool has a checkpoint entry at depth d ⟺ it is in the depth-d -# segment of pool._touched_others — except under full checkpoint!(pool), whose eager -# sweep pairs with full rewind!(pool)'s sweep (segment stays empty, truncate-only). -# See the docstrings of _touch_fallback_pool!/_drain_touched_others! in src/state.jl. +# Invariant: a fallback pool has an entry tagged with depth d in the depth-tagged +# stack (states/depths, kept in lockstep) ⟺ it was first touched at depth d — except +# under full checkpoint!(pool), whose eager sweep pairs with full rewind!(pool)'s +# sweep (stack stays empty, truncate-only). +# See the docstrings of the touch/drain helpers in src/state.jl. using Test using AdaptiveArrayPools @@ -10,6 +11,16 @@ using AdaptiveArrayPools: get_typed_pool!, checkpoint!, rewind!, _lazy_checkpoint!, _lazy_rewind!, _typed_lazy_checkpoint!, _typed_lazy_rewind!, _tracked_mask_for_types, _can_use_typed_path, get_task_local_pool, @with_pool +# Depth-tagged stack shape: states/depths always in lockstep; pools populated +# only in runtime-check builds. +function _to_stack_shape_ok(pool) + ok = length(pool._touched_others_states) == length(pool._touched_others_depths) + expected_pools = AdaptiveArrayPools.RUNTIME_CHECK >= 1 ? + length(pool._touched_others_states) : 0 + return ok && length(pool._touched_others_pools) == expected_pools +end +_to_stack_len(pool) = length(pool._touched_others_states) + # Distinct isbits fallback types (not fixed slots) struct TOFooA x::Float64 @@ -23,21 +34,21 @@ end @testset "touched-others: fields & lifecycle" begin pool = AdaptiveArrayPool() - @test pool._touched_others == Any[] - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_states) && isempty(pool._touched_others_depths) && isempty(pool._touched_others_pools) + @test isempty(pool._touched_others_depths) # reset! clears transient scope state, keeps registry acquire!(pool, TOFooA, 4) reset!(pool) - @test isempty(pool._touched_others) - @test pool._touched_others_checkpoints == [0] + @test _to_stack_len(pool) == 0 + @test isempty(pool._touched_others_depths) @test haskey(pool.others, TOFooA) # registry kept # empty! clears everything acquire!(pool, TOFooA, 4) empty!(pool) - @test isempty(pool._touched_others) - @test pool._touched_others_checkpoints == [0] + @test _to_stack_len(pool) == 0 + @test isempty(pool._touched_others_depths) @test !haskey(pool.others, TOFooA) end @@ -46,38 +57,38 @@ end # lazy pair _lazy_checkpoint!(pool) - @test pool._touched_others_checkpoints == [0, 0] + @test isempty(pool._touched_others_depths) _lazy_rewind!(pool) - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_depths) # typed single pair (fixed-slot type) checkpoint!(pool, Float64) - @test length(pool._touched_others_checkpoints) == 2 + @test isempty(pool._touched_others_depths) rewind!(pool, Float64) - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_depths) # typed multi pair checkpoint!(pool, Float64, Int64) rewind!(pool, Float64, Int64) - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_depths) # full pair checkpoint!(pool) rewind!(pool) - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_depths) # typed-lazy pair _typed_lazy_checkpoint!(pool, Float64) _typed_lazy_rewind!(pool, _tracked_mask_for_types(Float64)) - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_depths) # nesting _lazy_checkpoint!(pool) checkpoint!(pool, Float64) - @test length(pool._touched_others_checkpoints) == 3 + @test isempty(pool._touched_others_depths) rewind!(pool, Float64) _lazy_rewind!(pool) - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_depths) end @testset "touched-others: no eager checkpoint on lazy entry (pollution regression)" begin @@ -96,17 +107,17 @@ end v = acquire!(pool, TOFooA, 8) @test tpA._checkpoint_depths[end] == 2 # first-touch checkpoint at depth 2 - @test length(pool._touched_others) == 1 - @test pool._touched_others[end] === tpA + @test _to_stack_len(pool) == 1 && _to_stack_shape_ok(pool) + @test pool._touched_others_states[end] === tpA.state @test tpA.n_active == 1 # Re-acquire same type: no duplicate stack entry acquire!(pool, TOFooA, 8) - @test length(pool._touched_others) == 1 + @test _to_stack_len(pool) == 1 && _to_stack_shape_ok(pool) _lazy_rewind!(pool) @test tpA.n_active == 0 - @test isempty(pool._touched_others) + @test _to_stack_len(pool) == 0 @test tpB._checkpoint_depths == [0] # still never visited @test tpC._checkpoint_depths == [0] end @@ -123,7 +134,7 @@ end acquire!(pool, TOFooA, 4) acquire!(pool, TOFooA, 4) @test tpA.n_active == 3 - @test length(pool._touched_others) == 2 # one entry per depth + @test _to_stack_len(pool) == 2 && _to_stack_shape_ok(pool) # one entry per depth _lazy_rewind!(pool) # exit depth 3 @test tpA.n_active == 1 @@ -132,6 +143,33 @@ end @test tpA.n_active == 0 end +@testset "touched-others: depth tags are exact and monotone" begin + pool = AdaptiveArrayPool() + tpA = get_typed_pool!(pool, TOFooA) + + _lazy_checkpoint!(pool) # depth 2 + @test isempty(pool._touched_others_depths) # entry pushes nothing + acquire!(pool, TOFooA, 4) + @test pool._touched_others_depths == [2] + @test pool._touched_others_states[end] === tpA.state + + _lazy_checkpoint!(pool) # depth 3 + acquire!(pool, TOFooB, 4) + acquire!(pool, TOFooA, 4) # same type, new depth → new entry + @test pool._touched_others_depths == [2, 3, 3] + @test issorted(pool._touched_others_depths) # monotone invariant + @test _to_stack_shape_ok(pool) + + _lazy_rewind!(pool) # drains ONLY the ==3 entries + @test pool._touched_others_depths == [2] + @test tpA.n_active == 1 + @test get_typed_pool!(pool, TOFooB).n_active == 0 + + _lazy_rewind!(pool) + @test isempty(pool._touched_others_depths) + @test tpA.n_active == 0 +end + @testset "touched-others: nested scope NOT touching outer's fallback" begin pool = AdaptiveArrayPool() tpA = get_typed_pool!(pool, TOFooA) @@ -140,7 +178,7 @@ end acquire!(pool, TOFooA, 4) _lazy_checkpoint!(pool) # depth 3: does not touch TOFooA zeros!(pool, 16) # Float64 fixed-slot work only - @test length(pool._touched_others) == 1 # no new fallback entry + @test _to_stack_len(pool) == 1 && _to_stack_shape_ok(pool) # no new fallback entry _lazy_rewind!(pool) @test tpA.n_active == 1 # outer's array untouched _lazy_rewind!(pool) @@ -155,7 +193,7 @@ end zeros!(pool, 8) # tracked fixed-slot work acquire!(pool, TOFooB, 4) # untracked helper-style fallback touch @test pool._touched_has_others[end] == true - @test pool._touched_others[end] === tpB + @test pool._touched_others_states[end] === tpB.state @test !_can_use_typed_path(pool, _tracked_mask_for_types(Float64)) _typed_lazy_rewind!(pool, _tracked_mask_for_types(Float64)) @test tpB.n_active == 0 @@ -166,13 +204,13 @@ end checkpoint!(pool, TOFooA) # fallback T tracked by macro tpA = get_typed_pool!(pool, TOFooA) - @test pool._touched_others[end] === tpA # pushed at checkpoint + @test pool._touched_others_states[end] === tpA.state # pushed at checkpoint acquire!(pool, TOFooA, 4) # public-API acquire: no double push - @test count(tp -> tp === tpA, pool._touched_others) == 1 + @test count(st -> st === tpA.state, pool._touched_others_states) == 1 # macro exit path for has_others=true is _typed_lazy_rewind! _typed_lazy_rewind!(pool, _tracked_mask_for_types(TOFooA)) @test tpA.n_active == 0 - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_depths) end @testset "touched-others: new type registered mid-scope" begin @@ -180,8 +218,8 @@ end _lazy_checkpoint!(pool) acquire!(pool, TOFooC, 4) # first-ever registration, in-scope tpC = get_typed_pool!(pool, TOFooC) - @test pool._touched_others[end] === tpC - @test count(tp -> tp === tpC, pool._touched_others) == 1 + @test pool._touched_others_states[end] === tpC.state + @test count(st -> st === tpC.state, pool._touched_others_states) == 1 _lazy_rewind!(pool) @test tpC.n_active == 0 end @@ -194,10 +232,10 @@ end checkpoint!(pool) # eager: checkpoints ALL others @test tpA._checkpoint_depths[end] == 2 acquire!(pool, TOFooA, 4) - @test isempty(pool._touched_others) # guard saw existing depth-2 entry + @test _to_stack_len(pool) == 0 # guard saw existing depth-2 entry rewind!(pool) @test tpA.n_active == 0 - @test pool._touched_others_checkpoints == [0] + @test isempty(pool._touched_others_depths) end @testset "touched-others: similar! records fallback touch" begin @@ -206,7 +244,7 @@ end _lazy_checkpoint!(pool) similar!(pool, src) tpA = get_typed_pool!(pool, TOFooA) - @test pool._touched_others[end] === tpA + @test pool._touched_others_states[end] === tpA.state _lazy_rewind!(pool) @test tpA.n_active == 0 end @@ -230,7 +268,7 @@ end @test f_leaf(8) == 8 tl = get_task_local_pool() @test get_typed_pool!(tl, TOFooA).n_active == 0 - @test isempty(tl._touched_others) + @test isempty(tl._touched_others_states) # Inner scope throws, outer catches: outer exit must clean up leaked state function f_outer() @@ -250,8 +288,8 @@ end @test tl._current_depth == 1 @test get_typed_pool!(tl, TOFooB).n_active == 0 @test get_typed_pool!(tl, TOFooC).n_active == 0 - @test isempty(tl._touched_others) - @test tl._touched_others_checkpoints == [0] + @test isempty(tl._touched_others_states) + @test isempty(tl._touched_others_depths) empty!(tl) # leave the task-local pool clean for other test files end @@ -283,3 +321,24 @@ end @test @allocated(_to_macro_roundtrip(32)) == 0 empty!(get_task_local_pool()) end + +@testset "fallback lookup memo: identity and invalidation" begin + pool = AdaptiveArrayPool() + tp1 = get_typed_pool!(pool, TOFooA) + @test get_typed_pool!(pool, TOFooA) === tp1 # repeat lookup: same pool + @test get_typed_pool!(pool, TOFooB) !== tp1 # different type: different pool + @test get_typed_pool!(pool, TOFooA) === tp1 # alternating types stay correct + + reset!(pool) # keeps registry → memo may stay + @test get_typed_pool!(pool, TOFooA) === tp1 + + empty!(pool) # kills registry → memo MUST die + tp2 = get_typed_pool!(pool, TOFooA) + @test tp2 !== tp1 # stale-memo regression guard + @test tp2 === pool.others[TOFooA] + + # end-to-end: acquire after empty! must use the fresh pool + v = acquire!(pool, TOFooA, 4) + @test tp2.n_active == 1 && tp1.n_active == 0 + reset!(pool) +end