diff --git a/ALGORITHMS.md b/ALGORITHMS.md index 593ec67a..efbd80f2 100644 --- a/ALGORITHMS.md +++ b/ALGORITHMS.md @@ -158,3 +158,57 @@ al. 2017 · Pimentel et al. · Brage et al. 2004 · Keytel et al. 2005 · Tudor- 2019 · Hopkins 2004 · Mahalanobis 1936 · Killick et al. 2012 · Wittmann & Roenneberg 2006 · Pietilä et al. 2018 · Malik et al. + + + +| Batch change | Method and limits | +|---|---| +| Leave long RR artifact runs as gaps | Lipponen & Tarvainen 2019; Peltola 2012; The 32-beat cap is an internal ESTIMATE rule with low confidence, not a published threshold. Longer runs split the series; each side is corrected independently. The synthetic 33-beat case keeps 200 NN beats and drops 33. | + + + +| Batch change | Method and limits | +|---|---| +| Reduce spectral error on epoch timestamps | Lomb 1976; Scargle 1982; Press & Rybicki 1989; Evaluate trigonometric arguments relative to the first sample. The mathematical spectrum is time-shift invariant; floating-point band powers and threshold decisions can change for all Lomb-Scargle callers. | + + + +| Batch change | Method and limits | +|---|---| +| Abstain when a beat has no finite time | Task Force 1996 HRV; Welch 1967 spectra; input-validity guard is an implementation rule; HRV frequency/time, nocturnal and session RMSSD, RSA and CVHR return absent metrics for non-finite beat times. This prevents endless segment walks and invalid integer conversions. Finite-input formulas are unchanged. | + + + +| Batch change | Method and limits | +|---|---| +| Refuse unidentifiable cosinor fits | Nelson et al. 1979, Methods for cosinor-rhythmometry; Welford 1962 centered co-moments; Use centered least squares and a relative determinant gate of 1e-12 of the diagonal product. This is a numerical policy. Nearly collinear phase columns abstain, and ordinary fit parameters can move in their last digits. Upstream zero-amplitude significance gating is retained. | + + + +| Batch change | Method and limits | +|---|---| +| Retain small TRIMP contributions in the total | Morton, Fitz-Clarke & Banister 1990; Neumaier 1974 compensated summation; Compensate the per-minute sum. The reserve and sex-specific weighting are unchanged; the synthetic total moves in its last digits. | + + + +| Batch change | Method and limits | +|---|---| +| Evaluate training-load decay with affine arithmetic | Banister et al. 1975; Morton et al. 1990; Neumaier 1974 prefix summation; Evaluate the same EWMA recurrence as scale times seed plus offset. This matches the arithmetic used by the later incremental implementation. Last digits change; 42/7-day defaults and bounded-prefix priming remain implementation choices. | + + + +| Batch change | Method and limits | +|---|---| +| Count repeated zone intervals before summing | Karvonen 1957 supports reserve zones only; interval credit is an internal ESTIMATE rule; Group equal duration credits by zone. Cadence validation, median capping and tail credit are retained. Grouped multiplication changes the last digits of zone seconds. | + + + +| Batch change | Method and limits | +|---|---| +| Compute circadian variance with centered moments | Witting 1990; van Someren 1999; Welford 1962 centered variance; IS and IV may move in their last digits. Batch circular profile sums and first-winner tie selection are unchanged on the rebased head. Chunked merges can reorder profile sums and move near-tie L5/M10 start epochs; that stream-specific behavior belongs with the aggregators. | + + + +| Batch change | Method and limits | +|---|---| +| Compute HRV moments without subtracting a large mean | Task Force 1996; Welford 1962 centered sums and products; Use origin-shifted moments for NN levels and five-minute bins, and centered lag co-moments for ACF. Last digits and decisions within rounding of jitter gates can change. Existing respiratory-line exemptions, coverage gates and absent-value rules remain. | diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..be0e6d9d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,78 @@ +# Changelog + +## Split #99 output changes + +Edge must bump `kAlgoVersion` when adopting these changed batch outputs. + +### Review input and gap validity + +HRV time, nocturnal and session RMSSD abstain on non-finite NN intervals before +segmentation. Cosinor rejects non-finite or non-positive periods before computing +its design. CTL/ATL abstains on unknown/non-finite daily load; explicit observed +zero-load rest days still decay. These are input-validity policies, not new +physiological models. Capped RR runs now report `BeatClass.gap`, appended after +existing enum values, rather than claiming every omitted beat is long/short. +HRV also abstains when retained beat times regress, including when RR +accumulation disagrees with the recorded time at a capped-gap reset. Recorded +segment anchors are preserved; timestamps are never shifted to hide the conflict. +Frequency HRV, RSA and CVHR use the same regressing-clock refusal. Time HRV +also refuses supplied clock vectors whose length does not match the NN vector; +NN-only calls remain supported. Cosinor refuses non-finite design/response +moments, including overflow from a positive subnormal period or finite responses. +Other finite-input formulas, corrected NN and cap accounting are unchanged. + + +### Leave long RR artifact runs as gaps + +The 32-beat cap is an internal ESTIMATE rule with low confidence, not a published threshold. Longer runs split the series; each side is corrected independently. The synthetic 33-beat case keeps 200 NN beats and drops 33. + +Method: Lipponen & Tarvainen 2019; Peltola 2012. Reproduce with `dart run tool/reproduce_outputs.dart`. + +### Reduce spectral error on epoch timestamps + +Evaluate trigonometric arguments relative to the first sample. The mathematical spectrum is time-shift invariant; floating-point band powers and threshold decisions can change for all Lomb-Scargle callers. + +Method: Lomb 1976; Scargle 1982; Press & Rybicki 1989. Reproduce with `dart run tool/reproduce_outputs.dart`. + +### Abstain when a beat has no finite time + +HRV frequency/time, nocturnal and session RMSSD, RSA and CVHR return absent metrics for non-finite beat times. This prevents endless segment walks and invalid integer conversions. Finite-input formulas are unchanged. + +Method: Task Force 1996 HRV; Welch 1967 spectra; input-validity guard is an implementation rule. Reproduce with `dart run tool/reproduce_outputs.dart`. + +### Refuse unidentifiable cosinor fits + +Use centered least squares and a relative determinant gate of 1e-12 of the diagonal product. This is a numerical policy. Nearly collinear phase columns abstain, and ordinary fit parameters can move in their last digits. Upstream zero-amplitude significance gating is retained. + +Method: Nelson et al. 1979, Methods for cosinor-rhythmometry; Welford 1962 centered co-moments. Reproduce with `dart run tool/reproduce_outputs.dart`. + +### Retain small TRIMP contributions in the total + +Compensate the per-minute sum. The reserve and sex-specific weighting are unchanged; the synthetic total moves in its last digits. + +Method: Morton, Fitz-Clarke & Banister 1990; Neumaier 1974 compensated summation. Reproduce with `dart run tool/reproduce_outputs.dart`. + +### Evaluate training-load decay with affine arithmetic + +Evaluate the same EWMA recurrence as scale times seed plus offset. This matches the arithmetic used by the later incremental implementation. Last digits change; 42/7-day defaults and bounded-prefix priming remain implementation choices. + +Method: Banister et al. 1975; Morton et al. 1990; Neumaier 1974 prefix summation. Reproduce with `dart run tool/reproduce_outputs.dart`. + +### Count repeated zone intervals before summing + +Group equal duration credits by zone. Cadence validation, median capping and tail credit are retained. Grouped multiplication changes the last digits of zone seconds. + +Method: Karvonen 1957 supports reserve zones only; interval credit is an internal ESTIMATE rule. Reproduce with `dart run tool/reproduce_outputs.dart`. + +### Compute circadian variance with centered moments + +IS and IV may move in their last digits. Batch circular profile sums and first-winner tie selection are unchanged on the rebased head. Chunked merges can reorder profile sums and move near-tie L5/M10 start epochs; that stream-specific behavior belongs with the aggregators. + +Method: Witting 1990; van Someren 1999; Welford 1962 centered variance. Reproduce with `dart run tool/reproduce_outputs.dart`. + +### Compute HRV moments without subtracting a large mean + +Use origin-shifted moments for NN levels and five-minute bins, and centered lag co-moments for ACF. Last digits and decisions within rounding of jitter gates can change. Existing respiratory-line exemptions, coverage gates and absent-value rules remain. + +Method: Task Force 1996; Welford 1962 centered sums and products. Reproduce with `dart run tool/reproduce_outputs.dart`. + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82abddbe..447291f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,14 @@ There is **one** sleep source (`segmentSleep`) and **one** headline readiness ## Tests +On Linux, `nix develop` supplies the pinned Flutter 3.41.6 / Dart 3.11.4 +toolchain used by Edge, plus Git, Clang, CMake, Ninja, and pkg-config. +The interactive shell enters an FHS environment for native build tools. +For a command without an interactive shell, use `nix develop -c dart test`; +`nix develop -c analytics-fhs ` also supplies the FHS environment. + ```bash +nix develop dart pub get dart analyze dart test diff --git a/docs/AGGREGATOR_DESIGN.md b/docs/AGGREGATOR_DESIGN.md new file mode 100644 index 00000000..2beab003 --- /dev/null +++ b/docs/AGGREGATOR_DESIGN.md @@ -0,0 +1,234 @@ +# Aggregators: mergeable summaries without a rigid monoid + +Goal: make it cheap to turn an analytics calculation into something that can +be computed in chunks (checkpoint/resume, segment recompute, parallel work) +and merged, with one generic law check instead of hand-written chunking tests. + +## Why not a plain monoid + +A textbook monoid on summary values asks for three things that cost speed or +honesty here: + +1. Immutable values combined one element at a time. Folding 86,400 samples a + day through `combine(summary, singleton(x))` allocated per sample: about + 180,000 µs against a 141 µs hand loop + (edge.research/experiments/dart-monoid/REPORT.md). +2. Structural equality of states. Two states can differ (float rounding + order, a field `finish` never reads) and still give the same metric. +3. One context-free operation. Several calculations depend on context known + before any data arrives: session bounds, a bin origin, heart-rate anchors. + +## The contract + +An `Aggregator` is a dictionary of four operations over slices of a +column batch `B`: + +| op | role | +|---|---| +| `S empty()` | fresh identity state | +| `S addSlice(S s, B batch, int start, int end)` | fold rows `[start, end)`; may mutate `s` and return it | +| `S combine(S left, S right)` | merge two adjacent chunks, `left` first; may mutate and return `left`; consumes both | +| `R finish(S s)` | state to result; must not mutate `s` | + +`Batch` is a run of rows stored column-wise (`length`, `copyRange`, +`join`). Missing doubles are NaN, so filtering happens inside tight loops +instead of through per-row closures. `ListBatch` wraps a plain list for +calculations over objects (hypnogram segments); `DoubleColumn` wraps one +`Float64List`. A module defines its own batch when it needs several columns +(`Beats`, `Epochs`, `TimedValues`). + +What this relaxes, each one on purpose: + +- **Slices, not elements.** The unit of `add` is a slice. Generic dispatch + then costs once per slice and the per-row work is a plain loop. +- **Mutation and consumption.** `addSlice` and `combine` reuse their inputs. + The laws are stated on fresh folds, so nothing reuses a consumed state. +- **Equality after `finish`.** Laws compare finished results with a + caller-supplied equality (exact for integer outputs, a tolerance for + floats). State layout is free. +- **Order matters.** `combine` joins a chunk with the one that follows it, so + a state can carry bounded edge context across a seam. No commutativity. +- **Context-indexed instances.** An aggregator may be built from context + (`HypnogramAggregator(sessionStart, sessionEnd)`, + `hrvTimeAggregator(originMs: …)`, `banisterTrimpAggregator(restingHr: …)`). + The laws hold per instance. + +## Laws + +`checkLaws(agg, cases, equals: …)` (lib/src/aggregate/laws.dart) checks, all +through `finish` and `equals`: + +1. left identity: `combine(empty, fold xs) ≈ fold xs` +2. right identity: `combine(fold xs, empty) ≈ fold xs` +3. split: for split points k (every k for short cases), + `combine(fold [0,k), fold [k,n)) ≈ fold xs` +4. associativity on three adjacent slices +5. singletons: left-combining one-row folds ≈ `fold xs` +6. slicing: one state fed random consecutive sub-slices ≈ `fold xs` (the + `addSlice`-after-data path, separate from `combine`) +7. finishPure: finishing a state twice gives the same result, and the state + keeps working afterwards +8. roundTrip (persistable aggregators only): see Checkpoints + +Law 3 is what the app needs: any chunking gives the same metric. The harness +tests itself against deliberately broken aggregators +(test/aggregate/laws_harness_test.dart), so a lawful verdict means something. +In tests, `expectLawful(agg, cases, equals: …)` is the one-liner. + +## Checkpoints + +A state can be stored and resumed. `encodeState` / `decodeState` turn a +state into JSON and back; `persistable` says whether an aggregator supports +it (a composite is persistable when all its children are). Doubles round-trip +exactly; NaN and ±Infinity are written as the strings `"NaN"`, `"Infinity"`, +`"-Infinity"`. Decoders throw `FormatException` on anything malformed. + +`encodeCheckpoint(agg, state, kind:, version:)` wraps a state with a kind and +version. `decodeCheckpoint` returns null, and never guesses, when the kind or +version differs or the payload is damaged; the caller then recomputes from +raw data. Bump the version whenever a module's state layout or arithmetic +changes. + +When an aggregator is persistable, `checkLaws` adds the law `roundTrip`: a state that went through JSON combines and continues exactly +like the original. So every `expectConversion` call also checks persistence. + +## Building blocks + +All lawful, so anything assembled from them is lawful too: + +- counts and sums: `Count`, `CountFinite`, `CountWhere`, `Sum` + (Neumaier-compensated), `SumSq`, `SumOf(col, f)` (sum of `f(x)`, NaN skips) +- extremes and edges: `Min`, `Max`, `First`, `Last` +- moments: `Moments` (wraps `RunningMoments`) +- composition: `both(a, b)` (mutable pair state, no per-row allocation), + `.map(f)` on the result +- grouping: `GroupBy(key, inner)` (consecutive equal keys reach `inner` as one + slice; per-key order kept) +- suffixes: `LastRows(k)` (the last k rows of the whole input, kept + verbatim) +- prefixes: `Prefix(k, head, tail)` (the first k rows of the whole input go + to `head`, kept verbatim, at most k rows; every later row goes to `tail`) +- recurrences: `Ewma(col, λ)` finishes as an `Affine` map (see below) +- neighbour context: `Lookback(k, inner)`. `inner` processes every length-k + window inside each slice it is given. `Lookback` keeps the first and last + k−1 rows and hands `inner` the seam rows on `combine` and on + `addSlice`-after-data, so each window is counted exactly once under any + chunking. `Lookback(2)` gives successive differences, `Lookback(3)` lag + pairs, and `Lookback(epochsPerDay + 1)` the SRI day-to-day comparison. + +Where no block fits, a small hand-written inner (an `Aggregator` with a tight +loop) is fine. It is still checked by the same laws. + +## Two ways to make an "unmergeable" calculation mergeable + +Earlier experiments filed some calculations as not mergeable. Two +techniques moved them across: + +- **Summarise a function, not a value.** An EWMA step + `c ← (1 − λ)c + λx` is an affine map of the running value. Affine maps + compose associatively, so a chunk's summary is the composed map + `(scale, offset)`, and `finish` applies it to the seed. CTL/ATL/TSB is + `both(Count, Prefix(prime, mean, both(Ewma, Ewma)))`. +- **Summarise a distribution exactly when its support is small.** + Time-in-zone credits each sample with `min(gap, m)`, where `m` is the + median gap of the whole stream. The state keeps exact histograms of gap + values, which are bounded by the number of distinct gaps (a handful on a + 1 Hz stream), and the median and crediting move to `finish`. The median is + bit-identical to the batch code. + +## Converting a module: recipe + +1. Pick the batch: an existing one, or a small `Batch` subclass with one typed + column per input. +2. List the sufficient statistics `finish` needs. If they're sums, counts, + extremes or moments, use the blocks; if they need neighbours, wrap them in + `Lookback(k)`; if they're per bin or per day, wrap them in `GroupBy`. +3. Write `finish` (`.map`) from the old function's formulas, including its + absent cases and notes. +4. Tests: one `expectConversion(build, cases, reference)` call checks parity + with the old function on every evaluation path (whole, chunked, + right-nested, copies, sliced, singletons) and every law, including + `roundTrip` once the aggregator is persistable. Add hand-worked value and + absent-case tests as usual. +5. Integer outputs: rewire the old function in place to run the aggregator, + and keep a frozen copy of the old body in the test. Float outputs: keep + the aggregator additive beside the old function, so no published value + moves and no kAlgoVersion bump is needed. + +## Results so far + +| module | how | parity | shape | +|---|---|---|---| +| hypnogram metrics | rewired in place | exact | hand inner over `ListBatch`; wake spans kept as a list (clipped at finish) | +| HRV time domain | additive | 1e-9 | `Lookback(2)`, `Lookback(3)`, `Moments`, `GroupBy` over `Beats` | +| Phillips SRI | rewired in place | exact | `Lookback(epochsPerDay + 1)` + `GroupBy` per day pair over `Epochs` | +| Banister TRIMP | additive | 1e-12 | `both(Count, SumOf)` | +| cosinor | additive | 1e-9 (R² 1e-7) | one inner computing cos/sin once per row; centred co-moments | +| time in zone | additive | 1e-12 | `Lookback(2)` + exact gap histograms; median at finish | +| CTL/ATL/TSB | additive | 1e-9 | `Count`, `Prefix`, `Sum`, two `Ewma` maps | +| readiness lnRMSSD | rewired in place | exact | `Count` + `LastRows(windowDays + 1)` | +| PRSA DC/AC | additive | 1e-9 | `Count` + `Lookback(2l + 1)` with a profile-sum inner | +| circadian IS/IV/M10/L5/RA | additive | 1e-12 int data, 1e-9 real | `Moments`, `GroupBy`, `Lookback(2)` + one 15-line inner | + +Every module above is persistable and resumes from a mid-stream checkpoint. +The circadian conversion, done by following the recipe, took a subagent +about nine tool calls: a 134-line aggregator and a 166-line test. + +Speed, AOT, Dart 3.11.4 on this box (`tool/bench_aggregate.dart`): + +| 30,000 beats, HRV time domain | µs | +|---|---:| +| `hrvTime()` (current batch code) | 4,956 | +| hand loop, same statistics | 639 | +| composed, one slice | 1,281 | +| composed, 24 slices + combine | 1,209 | +| composed, one-row slices | 10,106 | +| per-element composed (earlier API) | ~24,800 | + +The composed version is about 2× a hand loop and about 4× faster than the +current batch function. Chunking costs nothing extra. (It was 1.7× before the +review fixes switched the lag-1 ACF to centred co-moments, which cost one +division per row and removed a cancellation bug.) + +## Proposed next step: corrected HRV end to end + +Every HRV aggregator here starts from already-corrected NN intervals. RR +artifact correction is the upstream blocker the earlier experiments named. +The package already has a streaming `RrCorrector` +(lib/src/onehz/foundations/rr_correction_stream.dart): a left-to-right fold +whose output for a beat is final about 2H beats (H = windowBeats ~/ 2) plus +the current artifact run plus two normal beats behind the newest beat, and +whose checkpoint holds that window, not the series. + +That suggests a mergeable "sandwich" summary for raw RR: + +- raw head: the first beats of a chunk whose correction still depends on the + chunk before it (window, carried class, left normal pair); +- settled middle: corrected beats, folded straight into a downstream + aggregator (HRV time, PRSA, …); +- raw tail: the last beats whose correction still depends on the chunk after. + +`combine` re-runs correction on `left.tail ++ right.head` only, feeds the +newly settled beats to the downstream aggregator, and keeps the remainder as +the new head/tail. That is `Lookback` with a transducer in front of the +inner aggregator, and it is bounded only if artifact runs are bounded. So it +needs a cap on run length (a longer run becomes a gap: honest absence), which +changes output and needs a kAlgoVersion bump. Owner decision. + +## Known limits + +- `Lookback` copies k−1 rows per seam. With large k and tiny slices this + dominates: SRI over 7 days at 1,440 epochs/day takes 0.2 ms as one slice, + 0.5 ms in hourly slices, 17.7 ms in one-epoch slices. Feed slices of at + least k rows, or give `Lookback` an appendable edge buffer if a caller + can't. +- Still batch-only: exact medians and quantiles over floats, pairwise + statistics (sample entropy), Viterbi backtrack, and anything whose context + is unbounded (current RR artifact correction). These need a bounded variant + behind a kAlgoVersion bump, or stay batch. +- Still hard: `nocturnalRhr` slides a TIME window whose qualifying rule + depends on the stream-wide median cadence. A mergeable version would keep, + per possible valid-sample count, the lowest window mean (bounded by the + window length) plus a time-based lookback block, which doesn't exist yet. +- Float results are equal within a tolerance, not bit-identical, across + chunkings. That is why float modules stay additive. diff --git a/docs/OUTPUT_REPRODUCTION.md b/docs/OUTPUT_REPRODUCTION.md new file mode 100644 index 00000000..606088c2 --- /dev/null +++ b/docs/OUTPUT_REPRODUCTION.md @@ -0,0 +1,83 @@ +# Reproducing the #99 batch output changes + +This branch replaces the batch-output portion of #99. The aggregator branch stacks +on this branch. Edge must bump `kAlgoVersion` when adopting these changed outputs. + +Run `dart pub get` with network access on a fresh clone, then +`dart run tool/reproduce_outputs.dart`. Once dependencies are cached, +`dart pub get --offline` also works. +`--json` prints unrounded machine-readable results. The script calls batch APIs +only. Its input recipes are in `test/fixtures/output_changes.json`, a small +synthetic fixture. It contains no personal measurements. Formula parameters +expand to beat sequences in memory, including 20,000 nearly collinear cosinor +samples. The fixture's `old` and `new` values are recorded observations, not +values substituted into library results. + +Old values were measured on `3a7ca34`, new values on `f8e7d3c` with #98 merged. +The output branch reproduces the new values. Each output commit pins its own +unrounded result. Tests contain no NaN list-equality expectations. + +| Calculation | up/main `3a7ca34` | Split output branch | +|---|---|---| +| RR kept / dropped / corrected | `[199, 34, 0]` | `[200, 33, 0]` | +| Lomb-Scargle PSD at 0.137 Hz | `32627.260306094682` | `32627.260527635714` | +| Cosinor mesor / amplitude / R² / p | `[60.20591440999805, 1.9937577719142656, 0.9902626829318173, 2.900200339561918e-94]` | `[60.205914409998044, 1.9937577719142632, 0.9902626829318162, 2.9002003395774255e-94]` | +| Near-singular cosinor present | `True` | `False` | +| HRV RMSSD / SDNN / SDANN / SDNN index / ACF1 | `[3.615265234174367, 21.52663924532639, 1.067383333873884, 21.55214616158443, 0.9849552962327551]` | `[3.615265234174367, 21.52663924532641, 1.0673833338741983, 21.55214616158443, 0.984955296232754]` | +| HRV with infinite final timestamp present | `throws` | `False` | +| Banister TRIMP | `32.71197112260296` | `32.711971122602975` | +| CTL / ATL / TSB | `[98.49620732954641, 103.12182624417179, -4.625618914625377]` | `[98.49620732954642, 103.12182624417177, -4.625618914625349]` | +| Zone seconds, zones 1 through 5 | `[0.0, 120.01199999999982, 0.0, 0.0, 0.0]` | `[0.0, 120.01199999999977, 0.0, 0.0, 0.0]` | +| Circadian IS / IV / M10 / L5 / start epochs | `[1.0, 0.331267605633807, 3.505000000000011, 1.3600000000000079, 14, 0]` | `[1.0, 0.3312676056338072, 3.505000000000011, 1.3600000000000079, 14, 0]` | +| Gap-aware batch irregular SD1 / pNN70 | `[22.360679774997894, 83.33333333333333]` | `[22.360679774997894, 83.33333333333333]` | + +The synthetic RR run contains 100 normal 900 ms beats, 33 artifacts at 250 ms, +and 100 normal beats. The over-cap run becomes a gap. At 32 artifacts or fewer, +the capped and uncapped results agree. The cap is an internal ESTIMATE policy, +not a published physiological threshold. + +The cosinor phase columns in the near-singular case cover only about 20 +microhours around hour 3. Main accepts this numerically unidentifiable design; +the relative determinant gate abstains. The threshold is a numerical policy. +Nelson's significance test and nullable acrophase from #101 remain intact. + +The invalid-timestamp table uses HRV because main throws promptly. The focused +guard test also calls frequency HRV, RSA, CVHR, nocturnal RMSSD and session +RMSSD with NaN and both infinities. Running old frequency HRV/RSA with an infinite +last time is unsafe because its segment walk may never terminate. The script +does not call those old endless paths. + +Main already excludes timestamp gaps in the batch irregular-rhythm screen. +The unchanged row above is a control. #99 adds the corresponding stream policy, +previous-beat timestamp checkpointing and old-checkpoint refusal in the +aggregator branch. Diagnostics and their tests also stay there. + +Main and the rebased #99 head use the same circular profile sums and strict +first-winner comparisons for batch L5/M10. Their starts agree here and on the +near-tie controls. Centered variance changes IS/IV last digits. Chunked merges +can reorder profile sums, so near-tie starts may move in the aggregator branch. +The original #99 description's broad near-tie warning should be read in that +context. These checked fixtures and tolerances are not universal error bounds. + +To independently recompute the old column, create a detached worktree at +`3a7ca34`, copy `tool/reproduce_outputs.dart` and +`test/fixtures/output_changes.json` into the same paths there, run +`dart pub get --offline`, then run the tool with `--json`. The table printer +labels its recorded baseline separately from the currently computed result. + +Methods and limits are beside the code and in `ALGORITHMS.md`: Lipponen & +Tarvainen 2019 and Peltola 2012 for RR correction; Lomb 1976, Scargle 1982 and +Press & Rybicki 1989 for spectra; Nelson et al. 1979 for cosinor; Task Force +1996 and Welford 1962 for HRV moments; Morton, Fitz-Clarke & Banister 1990 and +Banister et al. 1975 for training load; Neumaier 1974 for compensated sums; +Witting 1990 and van Someren 1999 for circadian indices. Karvonen 1957 supports +reserve zones only. Gap credit, tail credit and the RR cap remain internal rules. + +`--corpus-json` expands twelve deterministic cases, varying RR run lengths, +spectral origins and frequencies, jitter frequencies, timestamp gaps and record +lengths. `test/fixtures/output_batch_parity.json` freezes the output branch's +unrounded public results before the aggregator conversion. Both branches run +`batch_parity_test.dart` against these fixed results. All twelve cases also +matched the rebased #99 head exactly in the split audit. This supplements the +existing law tests and frozen pre-#99 references; it does not establish parity +for every possible input. Both reproduction fixtures together are about 10 KB. diff --git a/lib/aggregate.dart b/lib/aggregate.dart new file mode 100644 index 00000000..d6cae607 --- /dev/null +++ b/lib/aggregate.dart @@ -0,0 +1,17 @@ +/// Mergeable summaries and the building blocks used to compose them. +library aggregate; + +export 'src/aggregate/aggregator.dart'; +export 'src/aggregate/checkpoint.dart'; +export 'src/aggregate/laws.dart'; +export 'src/aggregate/primitives.dart'; + + + + + + + + + + diff --git a/lib/onehz.dart b/lib/onehz.dart index 308538be..ea5d9a40 100644 --- a/lib/onehz.dart +++ b/lib/onehz.dart @@ -18,6 +18,7 @@ export 'src/onehz/device.dart'; // Layer 0: foundations. export 'src/onehz/foundations/rr_correction.dart'; + export 'src/onehz/foundations/baseline.dart'; export 'src/onehz/foundations/ewma_baselines.dart'; export 'src/onehz/foundations/fusion.dart'; @@ -33,6 +34,8 @@ export 'src/onehz/clinical/cosinor.dart'; export 'src/onehz/clinical/load_trimp.dart'; export 'src/onehz/clinical/stress_si.dart'; export 'src/onehz/clinical/irregular_rhythm.dart'; + + export 'src/onehz/clinical/cardiac_coherence.dart'; export 'src/onehz/clinical/vo2max.dart'; @@ -43,3 +46,10 @@ export 'src/onehz/motion/motion.dart'; export 'src/onehz/workout/workout.dart'; export 'src/onehz/wellness/wellness.dart'; export 'src/onehz/human/human.dart'; + + + +// Calculation-group registry (design 06): the groups this package declares. + + +export 'src/onehz/incremental_core.dart'; diff --git a/lib/src/aggregate/aggregator.dart b/lib/src/aggregate/aggregator.dart new file mode 100644 index 00000000..7763f979 --- /dev/null +++ b/lib/src/aggregate/aggregator.dart @@ -0,0 +1,271 @@ +import 'dart:typed_data'; + +import 'json_support.dart'; + +/// A run of rows stored column-wise. +abstract class Batch> { + int get length; + B copyRange(int start, int end); + B join(B other); + + /// A JSON-encodable copy of the rows. Throws [UnsupportedError] unless the + /// batch type implements it. + Object? toJson() => + throw UnsupportedError('$runtimeType does not support toJson'); +} + +/// How a [ListBatch] writes and reads one row. [decode] must throw +/// [FormatException] on a malformed row. +class RowCodec { + const RowCodec(this.encode, this.decode); + + final Object? Function(A row) encode; + final A Function(Object? json) decode; + + /// Reads a coded batch from the JSON list written by [ListBatch.toJson]. + ListBatch batchFromJson(Object? json) => + ListBatch.coded([for (final row in decodeList(json)) decode(row)], this); +} + +/// A batch backed by a normal Dart list. +class ListBatch extends Batch> { + ListBatch(this.rows) : _codec = null; + + /// A batch whose rows can be written with [codec] (see [toJson]). + ListBatch.coded(this.rows, RowCodec codec) : _codec = codec; + + final List rows; + final RowCodec? _codec; + + @override + int get length => rows.length; + + @override + ListBatch copyRange(int start, int end) => + ListBatch._(rows.sublist(start, end), _codec); + + @override + ListBatch join(ListBatch other) => + ListBatch._([...rows, ...other.rows], _codec ?? other._codec); + + ListBatch._(this.rows, this._codec); + + /// The encoded rows. Throws [UnsupportedError] for a batch without a codec. + @override + Object? toJson() { + final codec = _codec; + if (codec == null) { + throw UnsupportedError('ListBatch has no row codec; use ListBatch.coded'); + } + return [for (final row in rows) codec.encode(row)]; + } +} + +/// A batch containing one double column. NaN represents a missing row. +class DoubleColumn extends Batch { + DoubleColumn(this.values); + + final Float64List values; + + @override + int get length => values.length; + + @override + DoubleColumn copyRange(int start, int end) { + final copied = Float64List(end - start) + ..setRange(0, end - start, values, start); + return DoubleColumn(copied); + } + + @override + DoubleColumn join(DoubleColumn other) { + final joined = Float64List(length + other.length); + joined.setRange(0, length, values); + joined.setRange(length, joined.length, other.values); + return DoubleColumn(joined); + } + + /// The values as a JSON list; NaN and infinities become strings. + @override + Object? toJson() => encodeDoubles(values); + + factory DoubleColumn.fromJson(Object? json) => DoubleColumn(decodeDoubles(json)); +} + +/// A mergeable summary over slices of a [Batch]. +abstract class Aggregator, R> { + const Aggregator(); + + S empty(); + S addSlice(S state, B batch, int start, int end); + S combine(S left, S right); + R finish(S state); + + /// Whether [encodeState] and [decodeState] are implemented. A composite is + /// persistable only when every child is. + bool get persistable => false; + + /// A JSON-encodable form of [state] (null, bool, num, String, List, or a + /// Map with String keys). Throws [UnsupportedError] unless [persistable]. + Object? encodeState(S state) => + throw UnsupportedError('$runtimeType is not persistable'); + + /// Inverse of [encodeState]. Throws [FormatException] on malformed input, + /// and [UnsupportedError] unless [persistable]. + S decodeState(Object? json) => + throw UnsupportedError('$runtimeType is not persistable'); +} + +/// Convenience operations and result adapters for [Aggregator]. +extension AggregatorOps, R> on Aggregator { + S fold(B batch, [int start = 0, int? end]) { + final actualEnd = end ?? batch.length; + return addSlice(empty(), batch, start, actualEnd); + } + + R run(B batch) => finish(fold(batch)); + + Aggregator map(R2 Function(R result) transform) => + _MappedAggregator(this, transform); +} + +/// Element-world convenience for aggregators over [ListBatch]. +extension ListAggregatorOps on Aggregator, R> { + R runList(List rows) => run(ListBatch(rows)); +} + +class _MappedAggregator, R, R2> + extends Aggregator { + const _MappedAggregator(this.inner, this.transform); + + final Aggregator inner; + final R2 Function(R result) transform; + + @override + S empty() => inner.empty(); + + @override + S addSlice(S state, B batch, int start, int end) => + inner.addSlice(state, batch, start, end); + + @override + S combine(S left, S right) => inner.combine(left, right); + + @override + R2 finish(S state) => transform(inner.finish(state)); + + @override + bool get persistable => inner.persistable; + + @override + Object? encodeState(S state) => inner.encodeState(state); + + @override + S decodeState(Object? json) => inner.decodeState(json); +} + +/// Mutable state for [both]. +class Pair { + Pair(this.first, this.second); + + A first; + B second; +} + +/// Runs two aggregators over each same slice. +Aggregator, B, (R1, R2)> + both, R1, R2>( + Aggregator first, + Aggregator second, +) => + _BothAggregator(first, second); + +/// Adds module-specific consistency checks to a persistable checkpoint. +/// +/// [validate] runs after [inner] has decoded each child state. It receives the +/// encoded state so a composed module can compare child values whose concrete +/// state types are private to their aggregators. Encoding and hot paths are +/// otherwise unchanged. +Aggregator validateCheckpoint, R>( + Aggregator inner, + void Function(Object? json) validate, +) => + _CheckpointValidatedAggregator(inner, validate); + +class _CheckpointValidatedAggregator, R> + extends Aggregator { + const _CheckpointValidatedAggregator(this.inner, this.validate); + + final Aggregator inner; + final void Function(Object? json) validate; + + @override + S empty() => inner.empty(); + + @override + S addSlice(S state, B batch, int start, int end) => + inner.addSlice(state, batch, start, end); + + @override + S combine(S left, S right) => inner.combine(left, right); + + @override + R finish(S state) => inner.finish(state); + + @override + bool get persistable => inner.persistable; + + @override + Object? encodeState(S state) => inner.encodeState(state); + + @override + S decodeState(Object? json) { + final state = inner.decodeState(json); + validate(json); + return state; + } +} + +class _BothAggregator, R1, R2> + extends Aggregator, B, (R1, R2)> { + const _BothAggregator(this.first, this.second); + + final Aggregator first; + final Aggregator second; + + @override + Pair empty() => Pair(first.empty(), second.empty()); + + @override + Pair addSlice(Pair state, B batch, int start, int end) { + state.first = first.addSlice(state.first, batch, start, end); + state.second = second.addSlice(state.second, batch, start, end); + return state; + } + + @override + Pair combine(Pair left, Pair right) { + left.first = first.combine(left.first, right.first); + left.second = second.combine(left.second, right.second); + return left; + } + + @override + (R1, R2) finish(Pair state) => + (first.finish(state.first), second.finish(state.second)); + + @override + bool get persistable => first.persistable && second.persistable; + + @override + Object? encodeState(Pair state) => + [first.encodeState(state.first), second.encodeState(state.second)]; + + @override + Pair decodeState(Object? json) { + final list = decodeList(json); + if (list.length != 2) throw const FormatException('Expected two states'); + return Pair( + first.decodeState(list[0]), second.decodeState(list[1])); + } +} diff --git a/lib/src/aggregate/checkpoint.dart b/lib/src/aggregate/checkpoint.dart new file mode 100644 index 00000000..b27dbb76 --- /dev/null +++ b/lib/src/aggregate/checkpoint.dart @@ -0,0 +1,39 @@ +import 'aggregator.dart'; + +/// Wraps the JSON form of [state] in a versioned envelope: +/// {'kind', 'version', 'state'}. Throws [UnsupportedError] if [agg] is not +/// persistable. +Map encodeCheckpoint( + Aggregator agg, + S state, { + required String kind, + required int version, +}) { + if (!agg.persistable) { + throw UnsupportedError('${agg.runtimeType} is not persistable'); + } + return {'kind': kind, 'version': version, 'state': agg.encodeState(state)}; +} + +/// Restores a state written by [encodeCheckpoint]. Returns null, never a +/// guess, when the kind or version differ or the envelope or state is +/// malformed; the caller then recomputes from raw data. +S? decodeCheckpoint( + Aggregator agg, + Object? json, { + required String kind, + required int version, +}) { + if (json is! Map || + json['kind'] != kind || + json['version'] is! int || + json['version'] != version || + !json.containsKey('state')) { + return null; + } + try { + return agg.decodeState(json['state']); + } on FormatException { + return null; + } +} diff --git a/lib/src/aggregate/json_support.dart b/lib/src/aggregate/json_support.dart new file mode 100644 index 00000000..911521b3 --- /dev/null +++ b/lib/src/aggregate/json_support.dart @@ -0,0 +1,64 @@ +// JSON helpers shared by the aggregate persistence codecs. Not exported. +// +// Non-finite doubles are written as the strings 'NaN', 'Infinity' and +// '-Infinity' (the same strings incremental_core.dart reads). Every reader +// throws FormatException, never TypeError or RangeError. + +import 'dart:typed_data'; + +/// A double as a JSON value: finite stays a number, the rest become strings. +Object encodeDouble(double value) { + if (value.isFinite) return value; + if (value.isNaN) return 'NaN'; + return value > 0 ? 'Infinity' : '-Infinity'; +} + +/// Inverse of [encodeDouble]; an int is accepted for a double. +double decodeDouble(Object? value) { + if (value is num) return value.toDouble(); + if (value == 'NaN') return double.nan; + if (value == 'Infinity') return double.infinity; + if (value == '-Infinity') return double.negativeInfinity; + throw const FormatException('Invalid number'); +} + +double? decodeDoubleOrNull(Object? value) => + value == null ? null : decodeDouble(value); + +/// A non-negative int. +int decodeCount(Object? value) { + if (value is! int || value < 0) throw const FormatException('Invalid count'); + return value; +} + +List decodeList(Object? value) { + if (value is! List) throw const FormatException('Expected a list'); + return value; +} + +/// A JSON object with string keys. +Map decodeMap(Object? value) { + if (value is! Map) throw const FormatException('Expected an object'); + for (final key in value.keys) { + if (key is! String) throw const FormatException('Expected string keys'); + } + return value.cast(); +} + +/// The value at [key]; the key must be present (null is a value). +Object? field(Map map, String key) { + if (!map.containsKey(key)) throw FormatException('Missing field $key'); + return map[key]; +} + +List encodeDoubles(Float64List values) => + [for (final value in values) encodeDouble(value)]; + +Float64List decodeDoubles(Object? value) { + final list = decodeList(value); + final result = Float64List(list.length); + for (var i = 0; i < list.length; i++) { + result[i] = decodeDouble(list[i]); + } + return result; +} diff --git a/lib/src/aggregate/laws.dart b/lib/src/aggregate/laws.dart new file mode 100644 index 00000000..ba92b232 --- /dev/null +++ b/lib/src/aggregate/laws.dart @@ -0,0 +1,343 @@ +import 'dart:convert'; +import 'dart:math' as math; + +import 'aggregator.dart'; + +typedef ResultEquals = bool Function(R left, R right); + +class LawViolation { + const LawViolation(this.law, this.detail); + final String law; + final String detail; +} + +/// Checks aggregator merge and slicing laws on each batch in [cases]. +List checkLaws, R>( + Aggregator aggregator, + List cases, { + ResultEquals? equals, + int maxSplits = 64, + int seed = 0, +}) { + final equal = equals ?? (R left, R right) => left == right; + final violations = []; + + R whole(B batch) => aggregator.finish(aggregator.fold(batch)); + R joined(B batch, int split) => aggregator.finish(aggregator.combine( + aggregator.fold(batch, 0, split), + aggregator.fold(batch, split, batch.length))); + + for (var caseIndex = 0; caseIndex < cases.length; caseIndex++) { + final batch = cases[caseIndex]; + final detail = 'case $caseIndex: $batch'; + final expected = whole(batch); + final leftIdentity = aggregator.finish( + aggregator.combine(aggregator.empty(), aggregator.fold(batch))); + if (!equal(leftIdentity, expected)) { + violations.add(LawViolation('leftIdentity', detail)); + } + final rightIdentity = aggregator.finish( + aggregator.combine(aggregator.fold(batch), aggregator.empty())); + if (!equal(rightIdentity, expected)) { + violations.add(LawViolation('rightIdentity', detail)); + } + + final splits = _splitPoints(batch.length, maxSplits, seed + caseIndex); + for (final split in splits) { + if (!equal(joined(batch, split), expected)) { + violations.add(LawViolation('split', '$detail; split $split')); + } + } + for (var i = 0; i < splits.length; i++) { + for (var j = i; j < splits.length; j++) { + final first = splits[i]; + final second = splits[j]; + final left = aggregator.finish(aggregator.combine( + aggregator.combine(aggregator.fold(batch, 0, first), + aggregator.fold(batch, first, second)), + aggregator.fold(batch, second, batch.length))); + final right = aggregator.finish(aggregator.combine( + aggregator.fold(batch, 0, first), + aggregator.combine(aggregator.fold(batch, first, second), + aggregator.fold(batch, second, batch.length)))); + if (!equal(left, right)) { + violations.add( + LawViolation('associativity', '$detail; splits $first, $second')); + } + } + } + + var singletonState = aggregator.empty(); + for (var i = 0; i < batch.length; i++) { + singletonState = aggregator.combine( + singletonState, aggregator.fold(batch, i, i + 1)); + } + if (!equal(aggregator.finish(singletonState), expected)) { + violations.add(LawViolation('singletons', detail)); + } + + var sliced = aggregator.empty(); + final slicePoints = _splitPoints(batch.length, maxSplits, seed ^ caseIndex); + for (var i = 0; i + 1 < slicePoints.length; i++) { + sliced = aggregator.addSlice(sliced, batch, slicePoints[i], slicePoints[i + 1]); + } + if (!equal(aggregator.finish(sliced), expected)) { + violations.add(LawViolation('slicing', detail)); + } + + // finish must not consume or otherwise close a state. Keep separate + // states for the add and combine continuations: combine is allowed to + // consume either input. + final continuationStart = batch.length ~/ 2; + var afterFinish = aggregator.fold(batch, 0, continuationStart); + final firstFinish = aggregator.finish(afterFinish); + final secondFinish = aggregator.finish(afterFinish); + var finishIsPure = equal(firstFinish, secondFinish); + + afterFinish = aggregator.addSlice( + afterFinish, batch, continuationStart, batch.length); + var addReference = aggregator.fold(batch, 0, continuationStart); + addReference = aggregator.addSlice( + addReference, batch, continuationStart, batch.length); + if (!equal(aggregator.finish(afterFinish), aggregator.finish(addReference))) { + finishIsPure = false; + } + + var combinedAfterFinish = aggregator.fold(batch, 0, continuationStart); + aggregator.finish(combinedAfterFinish); + aggregator.finish(combinedAfterFinish); + combinedAfterFinish = aggregator.combine( + combinedAfterFinish, + aggregator.fold(batch, continuationStart, batch.length)); + var combineReference = aggregator.fold(batch, 0, continuationStart); + combineReference = aggregator.combine( + combineReference, + aggregator.fold(batch, continuationStart, batch.length)); + if (!equal(aggregator.finish(combinedAfterFinish), + aggregator.finish(combineReference))) { + finishIsPure = false; + } + if (!finishIsPure) { + violations.add(LawViolation('finishPure', detail)); + } + + if (aggregator.persistable) { + // A state written to JSON text and read back must behave like the + // original: alone, on either side of a merge, and when extended. + S roundTrip(S state) => aggregator.decodeState( + jsonDecode(jsonEncode(aggregator.encodeState(state)))); + void check(String what, R Function() actual) { + try { + if (!equal(actual(), expected)) { + violations.add(LawViolation('roundTrip', '$detail; $what')); + } + } catch (error) { + violations.add(LawViolation('roundTrip', '$detail; $what: $error')); + } + } + + check('whole state', () => aggregator.finish(roundTrip(aggregator.fold(batch)))); + for (final split in splits) { + check( + 'combine after split $split (restored left)', + () => aggregator.finish(aggregator.combine( + roundTrip(aggregator.fold(batch, 0, split)), + aggregator.fold(batch, split, batch.length)))); + check( + 'combine after split $split (restored right)', + () => aggregator.finish(aggregator.combine( + aggregator.fold(batch, 0, split), + roundTrip(aggregator.fold(batch, split, batch.length))))); + check( + 'combine after split $split (restored both)', + () => aggregator.finish(aggregator.combine( + roundTrip(aggregator.fold(batch, 0, split)), + roundTrip(aggregator.fold(batch, split, batch.length))))); + check( + 'addSlice after split $split', + () => aggregator.finish(aggregator.addSlice( + roundTrip(aggregator.fold(batch, 0, split)), + batch, + split, + batch.length))); + } + } + } + return violations; +} + +class ConversionFailure { + const ConversionFailure(this.path, this.caseIndex, this.detail); + final String path; + final int caseIndex; + final String detail; +} + +/// Checks that an aggregator reproduces [reference] on every evaluation path. +/// +/// Paths: 'whole', 'chunked' (left-to-right combine of ranges of the original +/// batch), 'chunkedRight' (same bounds, right-nested combine), 'copies' (each +/// chunk cut with copyRange first), 'sliced' (one state, consecutive slices) +/// and 'singletons'. With [laws], checkLaws violations appear as 'law:'. +/// Deterministic per [seed]. +List checkConversion, R>( + Aggregator Function(B input) build, + List cases, + R Function(B input) reference, { + ResultEquals? equals, + int chunkings = 8, + int seed = 0, + bool laws = true, +}) { + final equal = equals ?? (R left, R right) => left == right; + final failures = []; + + for (var caseIndex = 0; caseIndex < cases.length; caseIndex++) { + final batch = cases[caseIndex]; + final aggregator = build(batch); + final expected = reference(batch); + final seen = {}; + + void check(String path, R Function() actual, [String where = '']) { + if (seen.contains(path)) return; + final got = actual(); + if (equal(got, expected)) return; + seen.add(path); + failures.add(ConversionFailure(path, caseIndex, + 'case $caseIndex: $batch$where; got $got, expected $expected')); + } + + check('whole', () => aggregator.finish(aggregator.fold(batch))); + + final rng = _Lcg(seed * 7919 + caseIndex); + for (var c = 0; c < chunkings; c++) { + final bounds = _chunkBounds(batch.length, rng); + final where = '; bounds $bounds'; + List states({bool copy = false}) => [ + for (var i = 0; i + 1 < bounds.length; i++) + copy + ? aggregator.fold(batch.copyRange(bounds[i], bounds[i + 1])) + : aggregator.fold(batch, bounds[i], bounds[i + 1]), + ]; + S leftToRight(List parts) { + var acc = parts.first; + for (var i = 1; i < parts.length; i++) { + acc = aggregator.combine(acc, parts[i]); + } + return acc; + } + + check('chunked', () => aggregator.finish(leftToRight(states())), where); + check('chunkedRight', () { + final parts = states(); + var acc = parts.last; + for (var i = parts.length - 2; i >= 0; i--) { + acc = aggregator.combine(parts[i], acc); + } + return aggregator.finish(acc); + }, where); + check('copies', + () => aggregator.finish(leftToRight(states(copy: true))), where); + check('sliced', () { + var state = aggregator.empty(); + for (var i = 0; i + 1 < bounds.length; i++) { + state = aggregator.addSlice(state, batch, bounds[i], bounds[i + 1]); + } + return aggregator.finish(state); + }, where); + } + + check('singletons', () { + var state = aggregator.empty(); + for (var i = 0; i < batch.length; i++) { + state = aggregator.combine(state, aggregator.fold(batch, i, i + 1)); + } + return aggregator.finish(state); + }); + + if (laws) { + for (final v in checkLaws(aggregator, [batch], + equals: equals, seed: seed + caseIndex)) { + failures.add(ConversionFailure( + 'law:${v.law}', caseIndex, v.detail.replaceFirst('case 0', 'case $caseIndex'))); + } + } + } + return failures; +} + +class _Lcg { + _Lcg(int seed) : _state = seed & 0x7fffffff; + int _state; + int next(int bound) { + _state = (_state * 1103515245 + 12345) & 0x7fffffff; + return (_state >> 8) % bound; + } +} + +/// Sorted chunk bounds from 0 to [length]: always cuts at 1 and length - 1 and +/// a duplicated cut (an empty chunk), plus a few random cuts. +List _chunkBounds(int length, _Lcg rng) { + final cuts = [ + 0, + length, + math.min(1, length), + math.max(0, length - 1), + for (var i = rng.next(4); i > 0; i--) rng.next(length + 1), + ]; + cuts.add(cuts[2 + rng.next(cuts.length - 2)]); + return cuts..sort(); +} + +List _splitPoints(int length, int maxSplits, int seed) { + if (length == 0) return [0]; + final wanted = math.max(2, maxSplits); + if (length + 1 <= wanted) return [for (var i = 0; i <= length; i++) i]; + final points = {0, length}; + var state = seed & 0x7fffffff; + while (points.length < wanted) { + state = (state * 1103515245 + 12345) & 0x7fffffff; + points.add(1 + state % (length - 1)); + } + return points.toList()..sort(); +} + +/// Compares the listed fields pairwise; numbers use the tolerances and lists +/// compare element by element with the same rules. +ResultEquals fieldsEqual( + List Function(R value) fields, { + double relTol = 0, + double absTol = 0, +}) => + (left, right) { + final a = fields(left); + final b = fields(right); + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_fieldEqual(a[i], b[i], relTol, absTol)) return false; + } + return true; + }; + +bool _fieldEqual(Object? left, Object? right, double relTol, double absTol) { + // Converting two ints to double loses distinctions above 2^53. + if (left is int && right is int) return left == right; + if (left is List || right is List) { + if (left is! List || right is! List || left.length != right.length) { + return false; + } + for (var i = 0; i < left.length; i++) { + if (!_fieldEqual(left[i], right[i], relTol, absTol)) return false; + } + return true; + } + if (left == right) return true; + if (left == null || right == null) return false; + if (left is! num || right is! num) return false; + if (left.isNaN || right.isNaN) return left.isNaN && right.isNaN; + if (!left.isFinite || !right.isFinite) return false; + final a = left.toDouble(); + final b = right.toDouble(); + final tolerance = absTol + relTol * math.max(a.abs(), b.abs()); + return (a - b).abs() <= tolerance; +} diff --git a/lib/src/aggregate/primitives.dart b/lib/src/aggregate/primitives.dart new file mode 100644 index 00000000..d2ddb26b --- /dev/null +++ b/lib/src/aggregate/primitives.dart @@ -0,0 +1,885 @@ +// Mergeable sums, centered moments and affine EWMA composition reformulate +// the corresponding batch arithmetic. Composition is algebraically equal; +// floating-point evaluation order can differ. Count/min/max and structural +// wrappers are definitional operations, not additional physiological estimators. + +import 'dart:typed_data'; + +import '../onehz/incremental_core.dart' show RunningMoments; + +import 'aggregator.dart'; +import 'json_support.dart'; + +/// Counts every row, including rows whose column value is NaN. +class Count> extends Aggregator { + const Count(); + @override + int empty() => 0; + @override + int addSlice(int state, B batch, int start, int end) => state + end - start; + @override + int combine(int left, int right) => left + right; + @override + int finish(int state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(int state) => state; + @override + int decodeState(Object? json) => decodeCount(json); +} + +/// Counts non-missing values in a double column. +class CountFinite> extends Aggregator { + const CountFinite(this.column); + final Float64List Function(B batch) column; + @override + int empty() => 0; + @override + int addSlice(int state, B batch, int start, int end) { + final values = column(batch); + var count = state; + for (var i = start; i < end; i++) { + if (!values[i].isNaN) count++; + } + return count; + } + @override + int combine(int left, int right) => left + right; + @override + int finish(int state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(int state) => state; + @override + int decodeState(Object? json) => decodeCount(json); +} + +/// Counts non-missing values in a double column which pass [test]. +class CountWhere> extends Aggregator { + const CountWhere(this.column, this.test); + final Float64List Function(B batch) column; + final bool Function(double value) test; + @override + int empty() => 0; + @override + int addSlice(int state, B batch, int start, int end) { + final values = column(batch); + var count = state; + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN && test(value)) count++; + } + return count; + } + @override + int combine(int left, int right) => left + right; + @override + int finish(int state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(int state) => state; + @override + int decodeState(Object? json) => decodeCount(json); +} + +/// Neumaier-compensated sum of non-missing double column values. +/// +/// Chunk-invariant results are supported while all partial sums are finite. +/// Once a partial sum overflows, IEEE-754 arithmetic can make the result +/// depend on chunking. +class Sum> extends Aggregator<_SumState, B, double> { + const Sum(this.column); + final Float64List Function(B batch) column; + @override + _SumState empty() => _SumState(); + @override + _SumState addSlice(_SumState state, B batch, int start, int end) { + final values = column(batch); + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN) state.add(value); + } + return state; + } + @override + _SumState combine(_SumState left, _SumState right) { + left.add(right.sum); + left.add(right.correction); + return left; + } + @override + double finish(_SumState state) => state.sum + state.correction; + @override + bool get persistable => true; + + @override + Object? encodeState(_SumState state) => state.toJson(); + + @override + _SumState decodeState(Object? json) => _SumState.fromJson(json); +} + +/// Neumaier-compensated sum of a function of non-missing double column values. +/// +/// A NaN source value is skipped without calling [transform]. A NaN result from +/// [transform] is also skipped. +/// Chunk-invariant results are supported while all partial sums are finite. +/// Once a partial sum overflows, IEEE-754 arithmetic can make the result +/// depend on chunking. +class SumOf> extends Aggregator<_SumState, B, double> { + const SumOf(this.column, this.transform); + + final Float64List Function(B batch) column; + final double Function(double value) transform; + + @override + _SumState empty() => _SumState(); + + @override + _SumState addSlice(_SumState state, B batch, int start, int end) { + final values = column(batch); + for (var i = start; i < end; i++) { + final value = values[i]; + if (value.isNaN) continue; + final transformed = transform(value); + if (!transformed.isNaN) state.add(transformed); + } + return state; + } + + @override + _SumState combine(_SumState left, _SumState right) { + left.add(right.sum); + left.add(right.correction); + return left; + } + + @override + double finish(_SumState state) => state.sum + state.correction; + @override + bool get persistable => true; + + @override + Object? encodeState(_SumState state) => state.toJson(); + + @override + _SumState decodeState(Object? json) => _SumState.fromJson(json); +} + +class _SumState { + _SumState(); + + factory _SumState.fromJson(Object? json) { + final map = decodeMap(json); + return _SumState() + ..sum = decodeDouble(field(map, 'sum')) + ..correction = decodeDouble(field(map, 'correction')); + } + + double sum = 0.0; + double correction = 0.0; + + Map toJson() => + {'sum': encodeDouble(sum), 'correction': encodeDouble(correction)}; + + void add(double value) { + final next = sum + value; + if (!next.isFinite) { + sum = next; + return; + } + if (sum.abs() >= value.abs()) { + correction += (sum - next) + value; + } else { + correction += (value - next) + sum; + } + sum = next; + } +} + +/// Sum of squares of non-missing double column values. +class SumSq> extends Aggregator { + const SumSq(this.column); + final Float64List Function(B batch) column; + @override + double empty() => 0.0; + @override + double addSlice(double state, B batch, int start, int end) { + final values = column(batch); + var sum = state; + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN) sum += value * value; + } + return sum; + } + @override + double combine(double left, double right) => left + right; + @override + double finish(double state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(double state) => encodeDouble(state); + @override + double decodeState(Object? json) => decodeDouble(json); +} + +/// Smallest non-missing value in a double column. +class Min> extends Aggregator { + const Min(this.column); + final Float64List Function(B batch) column; + @override + double? empty() => null; + @override + double? addSlice(double? state, B batch, int start, int end) { + final values = column(batch); + var result = state; + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN && (result == null || value < result)) result = value; + } + return result; + } + @override + double? combine(double? left, double? right) { + if (left == null || right == null) return left ?? right; + return right < left ? right : left; + } + @override + double? finish(double? state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(double? state) => + state == null ? null : encodeDouble(state); + @override + double? decodeState(Object? json) => decodeDoubleOrNull(json); +} + +/// Largest non-missing value in a double column. +class Max> extends Aggregator { + const Max(this.column); + final Float64List Function(B batch) column; + @override + double? empty() => null; + @override + double? addSlice(double? state, B batch, int start, int end) { + final values = column(batch); + var result = state; + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN && (result == null || value > result)) result = value; + } + return result; + } + @override + double? combine(double? left, double? right) { + if (left == null || right == null) return left ?? right; + return right > left ? right : left; + } + @override + double? finish(double? state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(double? state) => + state == null ? null : encodeDouble(state); + @override + double? decodeState(Object? json) => decodeDoubleOrNull(json); +} + +// Delegates updates/removal to Welford 1962 and partition merging to Chan, +// Golub & LeVeque 1983; full references and clamp details: RunningMoments. +/// Centered moments of non-missing double column values. +class Moments> + extends Aggregator { + const Moments(this.column); + final Float64List Function(B batch) column; + @override + RunningMoments empty() => RunningMoments(); + @override + RunningMoments addSlice(RunningMoments state, B batch, int start, int end) { + final values = column(batch); + for (var i = start; i < end; i++) { + final value = values[i]; + if (!value.isNaN) state.add(value); + } + return state; + } + @override + RunningMoments combine(RunningMoments left, RunningMoments right) { + left.merge(right); + return left; + } + @override + RunningMoments finish(RunningMoments state) => state; + @override + bool get persistable => true; + @override + Object? encodeState(RunningMoments state) => state.toJson(); + @override + RunningMoments decodeState(Object? json) => + RunningMoments.fromJson(decodeMap(json)); +} + +class _EdgeState> { + B? value; +} + +Object? _encodeEdge>( + _EdgeState state, B Function(Object?)? batchFromJson) { + if (batchFromJson == null) { + throw UnsupportedError('Pass batchFromJson to persist this block'); + } + return {'value': state.value?.toJson()}; +} + +_EdgeState _decodeEdge>( + Object? json, B Function(Object?)? batchFromJson) { + if (batchFromJson == null) { + throw UnsupportedError('Pass batchFromJson to persist this block'); + } + final raw = field(decodeMap(json), 'value'); + final state = _EdgeState(); + if (raw != null) { + final row = batchFromJson(raw); + if (row.length != 1) throw const FormatException('Expected one row'); + state.value = row; + } + return state; +} + +/// The first row as a one-row copy, or null for no rows. +class First> extends Aggregator<_EdgeState, B, B?> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]). + const First({this.batchFromJson}); + final B Function(Object? json)? batchFromJson; + @override + _EdgeState empty() => _EdgeState(); + @override + _EdgeState addSlice(_EdgeState state, B batch, int start, int end) { + if (state.value == null && start < end) { + state.value = batch.copyRange(start, start + 1); + } + return state; + } + @override + _EdgeState combine(_EdgeState left, _EdgeState right) { + if (left.value == null) left.value = right.value; + return left; + } + @override + B? finish(_EdgeState state) => state.value; + @override + bool get persistable => batchFromJson != null; + @override + Object? encodeState(_EdgeState state) => _encodeEdge(state, batchFromJson); + @override + _EdgeState decodeState(Object? json) => _decodeEdge(json, batchFromJson); +} + +/// The last row as a one-row copy, or null for no rows. +class Last> extends Aggregator<_EdgeState, B, B?> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]). + const Last({this.batchFromJson}); + final B Function(Object? json)? batchFromJson; + @override + _EdgeState empty() => _EdgeState(); + @override + _EdgeState addSlice(_EdgeState state, B batch, int start, int end) { + if (start < end) state.value = batch.copyRange(end - 1, end); + return state; + } + @override + _EdgeState combine(_EdgeState left, _EdgeState right) { + if (right.value != null) left.value = right.value; + return left; + } + @override + B? finish(_EdgeState state) => state.value; + @override + bool get persistable => batchFromJson != null; + @override + Object? encodeState(_EdgeState state) => _encodeEdge(state, batchFromJson); + @override + _EdgeState decodeState(Object? json) => _decodeEdge(json, batchFromJson); +} + +class _RowsState> { + B? rows; +} + +/// The last [k] rows of the whole input as a copy (fewer when the input is +/// shorter), or null for no rows. +class LastRows> extends Aggregator<_RowsState, B, B?> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]). + const LastRows(this.k, {this.batchFromJson}) : assert(k >= 1); + final int k; + final B Function(Object? json)? batchFromJson; + + B _tail(B rows) => + rows.length <= k ? rows : rows.copyRange(rows.length - k, rows.length); + + @override + _RowsState empty() => _RowsState(); + @override + _RowsState addSlice(_RowsState state, B batch, int start, int end) { + if (start >= end) return state; + final from = end - start > k ? end - k : start; + final piece = batch.copyRange(from, end); + final rows = state.rows; + state.rows = rows == null ? piece : _tail(rows.join(piece)); + return state; + } + @override + _RowsState combine(_RowsState left, _RowsState right) { + final l = left.rows; + final r = right.rows; + if (r == null) return left; + left.rows = l == null ? r : _tail(l.join(r)); + return left; + } + @override + B? finish(_RowsState state) => state.rows; + @override + bool get persistable => batchFromJson != null; + @override + Object? encodeState(_RowsState state) { + if (batchFromJson == null) { + throw UnsupportedError('Pass batchFromJson to persist this block'); + } + return {'rows': state.rows?.toJson()}; + } + @override + _RowsState decodeState(Object? json) { + final fromJson = batchFromJson; + if (fromJson == null) { + throw UnsupportedError('Pass batchFromJson to persist this block'); + } + final raw = field(decodeMap(json), 'rows'); + final state = _RowsState(); + if (raw != null) { + final rows = fromJson(raw); + if (rows.length < 1 || rows.length > k) { + throw const FormatException('Row count out of range'); + } + state.rows = rows; + } + return state; + } +} + +/// Routes consecutive equal keys to [inner] as one slice per run. +class GroupBy, S, R> + extends Aggregator, B, Map> { + const GroupBy(this.key, this.inner); + final int Function(B batch, int row) key; + final Aggregator inner; + @override + Map empty() => {}; + @override + Map addSlice(Map state, B batch, int start, int end) { + if (start == end) return state; + var runStart = start; + var runKey = key(batch, start); + for (var i = start + 1; i < end; i++) { + final nextKey = key(batch, i); + if (nextKey != runKey) { + _addRun(state, runKey, batch, runStart, i); + runStart = i; + runKey = nextKey; + } + } + _addRun(state, runKey, batch, runStart, end); + return state; + } + void _addRun(Map state, int runKey, B batch, int start, int end) { + final previous = state.containsKey(runKey) ? state[runKey] as S : inner.empty(); + state[runKey] = inner.addSlice(previous, batch, start, end); + } + @override + Map combine(Map left, Map right) { + for (final entry in right.entries) { + if (left.containsKey(entry.key)) { + left[entry.key] = inner.combine(left[entry.key] as S, entry.value); + } else { + left[entry.key] = entry.value; + } + } + return left; + } + @override + Map finish(Map state) => { + for (final entry in state.entries) entry.key: inner.finish(entry.value), + }; + @override + bool get persistable => inner.persistable; + + @override + Object? encodeState(Map state) => { + for (final entry in state.entries) + entry.key.toString(): inner.encodeState(entry.value), + }; + + @override + Map decodeState(Object? json) { + final map = decodeMap(json); + final result = {}; + for (final entry in map.entries) { + final key = int.tryParse(entry.key); + if (key == null || key.toString() != entry.key) { + throw FormatException('Invalid group key ${entry.key}'); + } + result[key] = inner.decodeState(entry.value); + } + return result; + } +} + +/// Makes an inner length-[k] window aggregator invariant to chunk boundaries. +class Lookback, S, R> + extends Aggregator<_LookbackState, B, R> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]) and [inner] is persistable. + Lookback(this.k, this.inner, {this.batchFromJson}) : assert(k >= 1); + final int k; + final Aggregator inner; + final B Function(Object? json)? batchFromJson; + int get _edge => k - 1; + @override + _LookbackState empty() => _LookbackState(inner.empty()); + @override + _LookbackState addSlice( + _LookbackState state, B batch, int start, int end) { + if (start == end) return state; + if (state.length > 0 && _edge > 0) { + final seam = state.last!.join(batch.copyRange(start, _prefixEnd(start, end))); + state.innerState = inner.addSlice(state.innerState, seam, 0, seam.length); + } + state.innerState = inner.addSlice(state.innerState, batch, start, end); + _appendEdges(state, batch, start, end); + return state; + } + int _prefixEnd(int start, int end) => + start + (end - start < _edge ? end - start : _edge); + void _appendEdges(_LookbackState state, B batch, int start, int end) { + final sliceLength = end - start; + if (_edge == 0) { + state.length += sliceLength; + return; + } + if (state.length == 0) { + state.first = batch.copyRange(start, _prefixEnd(start, end)); + state.last = batch.copyRange( + end - (sliceLength < _edge ? sliceLength : _edge), end); + } else { + if (state.length < _edge) { + final needed = _edge - state.length; + final take = sliceLength < needed ? sliceLength : needed; + if (take > 0) state.first = state.first!.join(batch.copyRange(start, start + take)); + } + if (sliceLength >= _edge) { + state.last = batch.copyRange(end - _edge, end); + } else { + final joined = state.last!.join(batch.copyRange(start, end)); + final keepStart = joined.length > _edge ? joined.length - _edge : 0; + state.last = joined.copyRange(keepStart, joined.length); + } + } + state.length += sliceLength; + } + @override + _LookbackState combine( + _LookbackState left, _LookbackState right) { + if (left.length > 0 && right.length > 0 && _edge > 0) { + final seam = left.last!.join(right.first!); + left.innerState = inner.addSlice(left.innerState, seam, 0, seam.length); + } + left.innerState = inner.combine(left.innerState, right.innerState); + _combineEdges(left, right); + return left; + } + void _combineEdges(_LookbackState left, _LookbackState right) { + if (right.length == 0) return; + if (left.length == 0) { + left.length = right.length; + left.first = right.first; + left.last = right.last; + return; + } + if (_edge > 0) { + if (left.length < _edge) { + final needed = _edge - left.length; + final take = right.length < needed ? right.length : needed; + if (take > 0) left.first = left.first!.join(right.first!.copyRange(0, take)); + } + if (right.length >= _edge) { + left.last = right.last; + } else { + final joined = left.last!.join(right.last!); + final keepStart = joined.length > _edge ? joined.length - _edge : 0; + left.last = joined.copyRange(keepStart, joined.length); + } + } + left.length += right.length; + } + @override + R finish(_LookbackState state) => inner.finish(state.innerState); + + @override + bool get persistable => batchFromJson != null && inner.persistable; + + @override + Object? encodeState(_LookbackState state) { + _requireBatchCodec(); + return { + 'length': state.length, + 'first': state.first?.toJson(), + 'last': state.last?.toJson(), + 'inner': inner.encodeState(state.innerState), + }; + } + + @override + _LookbackState decodeState(Object? json) { + final fromJson = _requireBatchCodec(); + final map = decodeMap(json); + final length = decodeCount(field(map, 'length')); + // Edges hold the first/last min(length, k - 1) rows. + final held = length < _edge ? length : _edge; + B? edge(String name) { + final raw = field(map, name); + if (raw == null) { + if (held != 0) throw FormatException('Missing $name rows'); + return null; + } + if (held == 0) throw FormatException('Unexpected $name rows'); + final rows = fromJson(raw); + if (rows.length != held) { + throw FormatException('Expected $held $name rows'); + } + return rows; + } + + return _LookbackState(inner.decodeState(field(map, 'inner'))) + ..length = length + ..first = edge('first') + ..last = edge('last'); + } + + B Function(Object?) _requireBatchCodec() => + batchFromJson ?? + (throw UnsupportedError('Pass batchFromJson to persist Lookback')); +} + +class _LookbackState, S> { + _LookbackState(this.innerState); + S innerState; + int length = 0; + B? first; + B? last; +} + +/// An affine map x -> scale * x + offset. Composition is associative, so a +/// run of affine steps can be summarised by one map and merged in any grouping. +class Affine { + const Affine(this.scale, this.offset); + + final double scale; + final double offset; + + double apply(double x) => scale * x + offset; + + /// The map that applies this one first, then [next]. + Affine then(Affine next) => Affine( + next.scale * scale, next.scale * offset + next.offset); +} + +class _AffineState { + double scale = 1; + double offset = 0; +} + +// Algebraically equal to the EWMA recurrence: f_x(c)=a*c+lambda*x, a=1-lambda; +// f_y(f_x(c))=a*a*c+a*lambda*x+lambda*y. Floating-point order may differ. +/// An exponentially weighted moving average, as the composed affine map of the +/// running value: each row x applies c -> (1 - lambda) * c + lambda * x. +/// +/// NaN rows are skipped (they act as the identity map), and no rows finish as +/// the identity. ±Infinity input is unsupported because scale can underflow to +/// zero across a seam and yield `0*Infinity`. Apply the result to the seed +/// value to get the final average. +class Ewma> extends Aggregator<_AffineState, B, Affine> { + const Ewma(this.column, this.lambda); + + final Float64List Function(B batch) column; + final double lambda; + + @override + _AffineState empty() => _AffineState(); + + @override + _AffineState addSlice(_AffineState state, B batch, int start, int end) { + final values = column(batch); + final keep = 1 - lambda; + var scale = state.scale; + var offset = state.offset; + for (var i = start; i < end; i++) { + final x = values[i]; + if (x.isNaN) continue; + scale *= keep; + offset = keep * offset + lambda * x; + } + state.scale = scale; + state.offset = offset; + return state; + } + + @override + _AffineState combine(_AffineState left, _AffineState right) { + // Right's map runs after left's. + left.offset = right.scale * left.offset + right.offset; + left.scale = right.scale * left.scale; + return left; + } + + @override + Affine finish(_AffineState state) => Affine(state.scale, state.offset); + @override + bool get persistable => true; + + @override + Object? encodeState(_AffineState state) => { + 'scale': encodeDouble(state.scale), + 'offset': encodeDouble(state.offset), + }; + + @override + _AffineState decodeState(Object? json) { + final map = decodeMap(json); + return _AffineState() + ..scale = decodeDouble(field(map, 'scale')) + ..offset = decodeDouble(field(map, 'offset')); + } +} + +class _PrefixState, S2> { + _PrefixState(this.tail); + + /// The first rows of the input seen so far (at most k), null before any. + B? rows; + int held = 0; + S2 tail; +} + +/// Sends the first [k] rows of the whole input to [head] and every later row +/// to [tail], in order, however the input is chunked. +/// +/// The state keeps the prefix rows (a copy, at most k) and the tail state. +/// The head result is computed at finish by running [head] over the stored +/// rows, so [head] never needs a working combine. With fewer than k rows in +/// total, head sees all of them and the tail sees none. +class Prefix, S1, R1, S2, R2> + extends Aggregator<_PrefixState, B, (R1, R2)> { + /// Persistable only when [batchFromJson] is given (the inverse of + /// [Batch.toJson] for [B]) and [tail] is persistable. [head] keeps no state, + /// so it does not matter. + const Prefix(this.k, this.head, this.tail, {this.batchFromJson}); + + final int k; + final Aggregator head; + final Aggregator tail; + final B Function(Object? json)? batchFromJson; + + @override + _PrefixState empty() => _PrefixState(tail.empty()); + + /// Appends [piece] to the stored prefix rows. + void _hold(_PrefixState state, B piece) { + final rows = state.rows; + state.rows = rows == null ? piece : rows.join(piece); + state.held += piece.length; + } + + @override + _PrefixState addSlice( + _PrefixState state, B batch, int start, int end) { + final room = k - state.held; + final take = room < end - start ? room : end - start; + if (take > 0) _hold(state, batch.copyRange(start, start + take)); + if (start + take < end) { + state.tail = tail.addSlice(state.tail, batch, start + take, end); + } + return state; + } + + @override + _PrefixState combine( + _PrefixState left, _PrefixState right) { + final rightRows = right.rows; + var rest = right.tail; + if (rightRows != null) { + // Right's leading rows fill whatever room left's prefix still has; the + // remainder are tail rows that come before right's own tail. + final room = k - left.held; + final take = room < right.held ? room : right.held; + if (take > 0) _hold(left, rightRows.copyRange(0, take)); + if (take < right.held) { + final extra = + tail.addSlice(tail.empty(), rightRows, take, right.held); + rest = tail.combine(extra, rest); + } + } + left.tail = tail.combine(left.tail, rest); + return left; + } + + @override + (R1, R2) finish(_PrefixState state) { + final rows = state.rows; + final headState = rows == null + ? head.empty() + : head.addSlice(head.empty(), rows, 0, rows.length); + return (head.finish(headState), tail.finish(state.tail)); + } + + @override + bool get persistable => batchFromJson != null && tail.persistable; + + @override + Object? encodeState(_PrefixState state) { + _requireBatchCodec(); + return { + 'rows': state.rows?.toJson(), + 'tail': tail.encodeState(state.tail), + }; + } + + @override + _PrefixState decodeState(Object? json) { + final fromJson = _requireBatchCodec(); + final map = decodeMap(json); + final raw = field(map, 'rows'); + final result = _PrefixState(tail.decodeState(field(map, 'tail'))); + if (raw != null) { + final rows = fromJson(raw); + if (rows.length < 1 || rows.length > k) { + throw const FormatException('Invalid prefix row count'); + } + result.rows = rows; + result.held = rows.length; + } + return result; + } + + B Function(Object?) _requireBatchCodec() => + batchFromJson ?? + (throw UnsupportedError('Pass batchFromJson to persist Prefix')); +} diff --git a/lib/src/onehz/clinical/cosinor.dart b/lib/src/onehz/clinical/cosinor.dart index 6d4ddfab..eaa8f5b6 100644 --- a/lib/src/onehz/clinical/cosinor.dart +++ b/lib/src/onehz/clinical/cosinor.dart @@ -60,14 +60,6 @@ class CosinorFit { /// reported. const double cosinorAcrophaseAlpha = 0.05; -/// Upper tail of F(2, d): P(F > f) = (1 + 2f/d)^(−d/2), exact for 2 numerator -/// degrees of freedom. -double _f2Survival(double f, int d) { - if (f.isNaN) return 1.0; - if (f.isInfinite) return 0.0; - if (f <= 0) return 1.0; - return math.pow(1 + 2 * f / d, -d / 2).toDouble(); -} /// Minimum samples for a cosinor fit. The model has 3 free parameters /// (M, β, γ); with n = 4 there is a single residual degree of freedom, so R² @@ -86,74 +78,119 @@ Metric cosinor( List y, { double periodHours = 24, int minPoints = cosinorMinPoints, +}) { + if (!periodHours.isFinite || periodHours <= 0) { + return const Metric.absent( + tier: Tier.high, + inputs_used: ['signal_timeseries'], + note: 'invalid cosinor period', + ); + } + if (tHours.length != y.length) { + return const Metric.absent(tier: Tier.high, + inputs_used: ['signal_timeseries'], note: 'too few points for cosinor'); + } + final state = _CosinorState(); + final w = 2 * math.pi / periodHours; + for (var i = 0; i < y.length; i++) { + state.addCentered(math.cos(w * tHours[i]), math.sin(w * tHours[i]), y[i]); + } + return _finishCosinor(state, periodHours: periodHours, minPoints: minPoints); +} + +// Nelson et al. (1979), Methods for cosinor-rhythmometry. +// Centered least squares changes rounding. Relative 1e-12 gate is a +// numerical identifiability policy, not a physiological cutoff. +class _CosinorState { + var n = 0; + var meanC = 0.0; + var meanS = 0.0; + var meanY = 0.0; + var mCC = 0.0; + var mSS = 0.0; + var mYY = 0.0; + var mCS = 0.0; + var mCY = 0.0; + var mSY = 0.0; + + void addCentered(double c, double s, double y) { + final nextN = n + 1; + final dc = c - meanC; + final ds = s - meanS; + final dy = y - meanY; + final nextMeanC = meanC + dc / nextN; + final nextMeanS = meanS + ds / nextN; + final nextMeanY = meanY + dy / nextN; + mCC += dc * (c - nextMeanC); + mSS += ds * (s - nextMeanS); + mYY += dy * (y - nextMeanY); + mCS += dc * (s - nextMeanS); + mCY += dc * (y - nextMeanY); + mSY += ds * (y - nextMeanY); + meanC = nextMeanC; + meanS = nextMeanS; + meanY = nextMeanY; + n = nextN; + } + +} + +Metric _finishCosinor( + _CosinorState state, { + required double periodHours, + required int minPoints, }) { const inputs = ['signal_timeseries']; - final n = y.length; - if (n < math.max(4, minPoints) || tHours.length != n || periodHours <= 0) { + final n = state.n; + if (n < math.max(4, minPoints) || periodHours <= 0) { return const Metric.absent( tier: Tier.high, inputs_used: inputs, note: 'too few points for cosinor', ); } - final w = 2 * math.pi / periodHours; - final c = [for (final t in tHours) math.cos(w * t)]; - final s = [for (final t in tHours) math.sin(w * t)]; - - // Normal equations for [M, β, γ] of y = M + β·c + γ·s. - // Build 3×3 symmetric system. - var sC = 0.0, sS = 0.0, sCC = 0.0, sSS = 0.0, sCS = 0.0; - var sY = 0.0, sYC = 0.0, sYS = 0.0; - for (var i = 0; i < n; i++) { - sC += c[i]; - sS += s[i]; - sCC += c[i] * c[i]; - sSS += s[i] * s[i]; - sCS += c[i] * s[i]; - sY += y[i]; - sYC += y[i] * c[i]; - sYS += y[i] * s[i]; + // A finite period or response can still overflow the design arithmetic. + // Reject invalid moments before NaN can bypass the identifiability gate. + if (![state.meanC, state.meanS, state.meanY, state.mCC, state.mSS, + state.mYY, state.mCS, state.mCY, state.mSY].every((v) => v.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite cosinor design', + ); } - // Matrix A = [[n, sC, sS],[sC, sCC, sCS],[sS, sCS, sSS]], b=[sY,sYC,sYS]. - final a = [ - [n.toDouble(), sC, sS], - [sC, sCC, sCS], - [sS, sCS, sSS], - ]; - final b = [sY, sYC, sYS]; - final sol = _solve3(a, b); - if (sol == null) { + final determinant = state.mCC * state.mSS - state.mCS * state.mCS; + // The determinant is relative to the diagonal co-moments. A design needs + // at least 1e-12 of that scale after removing its means to identify both + // cosine coefficients; below that, numerical noise decides the fit. + if (determinant <= 1e-12 * state.mCC * state.mSS) { return const Metric.absent( tier: Tier.high, inputs_used: inputs, note: 'singular cosinor design (insufficient phase coverage)', ); } - final mesor = sol[0]; - final beta = sol[1]; - final gamma = sol[2]; + final beta = + (state.mCY * state.mSS - state.mSY * state.mCS) / determinant; + final gamma = + (state.mSY * state.mCC - state.mCY * state.mCS) / determinant; + final mesor = state.meanY - beta * state.meanC - gamma * state.meanS; final amplitude = math.sqrt(beta * beta + gamma * gamma); - final acro = math.atan2(-gamma, beta); // peak phase - // Convert acrophase to clock-hours within the period (0..period). - var phaseHours = -acro / w; // time of peak relative to origin + final acro = math.atan2(-gamma, beta); + final w = 2 * math.pi / periodHours; + var phaseHours = -acro / w; phaseHours = phaseHours % periodHours; if (phaseHours < 0) phaseHours += periodHours; - // R². - final yMean = sY / n; - var ssTot = 0.0, ssRes = 0.0; - for (var i = 0; i < n; i++) { - final fit = mesor + beta * c[i] + gamma * s[i]; - ssTot += (y[i] - yMean) * (y[i] - yMean); - ssRes += (y[i] - fit) * (y[i] - fit); - } + final ssTot = state.mYY; + final ssRes = state.mYY - + 2 * beta * state.mCY - + 2 * gamma * state.mSY + + beta * beta * state.mCC + + gamma * gamma * state.mSS + + 2 * beta * gamma * state.mCS; final double r2 = ssTot == 0 ? 0.0 : (1 - ssRes / ssTot).clamp(0, 1); - // Adjusted for the 3 fitted parameters (M, β, γ). Confidence MUST come from - // the adjusted value: the raw R² of a 3-parameter fit is upward-biased - // (E[R²] = 2/(n−1) under the null), so a handful of noise points used to - // score confidence 0.95 at tier HIGH. final double r2Adj = (1 - (1 - r2) * (n - 1) / (n - 3)).clamp(0, 1); - // Zero-amplitude F-test (Nelson 1979): the acrophase is only defined once // the test rejects. A flat or rhythm-free series still yields some // atan2(−γ, β) from residual noise or float rounding. @@ -169,7 +206,6 @@ Metric cosinor( } final phaseDefined = ampP < cosinorAcrophaseAlpha; - final conf = r2Adj.clamp(0.1, 0.95); return Metric( value: CosinorFit( mesor: mesor, @@ -181,7 +217,7 @@ Metric cosinor( r2Adj: r2Adj, amplitudeP: ampP, ), - confidence: conf, + confidence: r2Adj.clamp(0.1, 0.95), tier: Tier.high, inputs_used: inputs, note: phaseDefined @@ -192,34 +228,12 @@ Metric cosinor( ); } -/// Solve a 3×3 linear system by Gaussian elimination with partial pivoting. -/// Returns null if singular. -List? _solve3(List> a, List b) { - final m = [ - [a[0][0], a[0][1], a[0][2], b[0]], - [a[1][0], a[1][1], a[1][2], b[1]], - [a[2][0], a[2][1], a[2][2], b[2]], - ]; - for (var col = 0; col < 3; col++) { - var piv = col; - for (var r = col + 1; r < 3; r++) { - if (m[r][col].abs() > m[piv][col].abs()) piv = r; - } - if (m[piv][col].abs() < 1e-12) return null; - final tmp = m[col]; - m[col] = m[piv]; - m[piv] = tmp; - final d = m[col][col]; - for (var j = col; j < 4; j++) { - m[col][j] /= d; - } - for (var r = 0; r < 3; r++) { - if (r == col) continue; - final f = m[r][col]; - for (var j = col; j < 4; j++) { - m[r][j] -= f * m[col][j]; - } - } - } - return [m[0][3], m[1][3], m[2][3]]; + +/// Upper tail of F(2, d): P(F > f) = (1 + 2f/d)^(−d/2), exact for 2 numerator +/// degrees of freedom. +double _f2Survival(double f, int d) { + if (f.isNaN) return 1.0; + if (f.isInfinite) return 0.0; + if (f <= 0) return 1.0; + return math.pow(1 + 2 * f / d, -d / 2).toDouble(); } diff --git a/lib/src/onehz/clinical/hrv_freq.dart b/lib/src/onehz/clinical/hrv_freq.dart index d7592486..1cc57995 100644 --- a/lib/src/onehz/clinical/hrv_freq.dart +++ b/lib/src/onehz/clinical/hrv_freq.dart @@ -84,6 +84,20 @@ Metric hrvFreq( // Times in seconds for Hz output; tachogram value = NN (ms). final tSec = [for (final t in nnTimesMs) t / 1000.0]; + if (tSec.any((t) => !t.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite beat time', + ); + } + if (hasRegressingBeatTimes(nnTimesMs)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'regressing beat time', + ); + } final spanSec = tSec.last - tSec.first; if (spanSec <= 0) { return const Metric.absent( diff --git a/lib/src/onehz/clinical/hrv_time.dart b/lib/src/onehz/clinical/hrv_time.dart index d2b35104..e34f0477 100644 --- a/lib/src/onehz/clinical/hrv_time.dart +++ b/lib/src/onehz/clinical/hrv_time.dart @@ -17,6 +17,7 @@ import 'dart:math' as math; import '../types.dart'; import '../util.dart'; +import 'rmssd_gate.dart'; /// Lag-1 ACF floor for the NN successive-difference series, below which RMSSD /// and pNN50 are REFUSED (SDNN / SDANN survive it and are the honest lead). @@ -121,82 +122,13 @@ const double kNnDiffNoiseShareCeiling = 0.7; /// clear of the floor, or diffs on a beat-time grid coarse against their size /// (see [_onCoarseLattice]). double? nnDiffNoiseShare(List> diffRuns, {int minSegments = 20}) { - const n = 64, kLo = 10, kHi = n ~/ 2; // kLo/n ≈ 0.15 cycles/beat - final w = [ - for (var i = 0; i < n; i++) 0.5 - 0.5 * math.cos(2 * math.pi * i / (n - 1)) - ]; - var w2 = 0.0; - for (final x in w) { - w2 += x * x; - } - final cosT = [ - for (var k = kLo; k <= kHi; k++) - [for (var i = 0; i < n; i++) math.cos(2 * math.pi * k * i / n)] - ]; - final sinT = [ - for (var k = kLo; k <= kHi; k++) - [for (var i = 0; i < n; i++) math.sin(2 * math.pi * k * i / n)] - ]; - final psd = List.filled(kHi + 1, 0.0); - var segs = 0, nd = 0; - var ssd = 0.0, wsd = 0.0, wsum = 0.0, pSum = 0.0, p2Sum = 0.0; + final sums = NoiseShareSums(); for (final r in diffRuns) { - for (final d in r) { - ssd += d * d; - nd++; - } - // Integrate the run back to RR levels (up to a constant the mean removes). - final x = List.filled(r.length + 1, 0.0); - for (var i = 0; i < r.length; i++) { - x[i + 1] = x[i] + r[i]; - } - final cov = List.filled(x.length, 0.0); // Σ w² per sample - for (var s = 0; s + n <= x.length; s += n ~/ 2) { - var m = 0.0; - for (var i = 0; i < n; i++) { - m += x[s + i]; - cov[s + i] += w[i] * w[i]; - } - m /= n; - final v = [for (var i = 0; i < n; i++) (x[s + i] - m) * w[i]]; - var p = 0.0; - for (var k = kLo; k <= kHi; k++) { - final c = cosT[k - kLo], sn = sinT[k - kLo]; - var re = 0.0, im = 0.0; - for (var i = 0; i < n; i++) { - re += v[i] * c[i]; - im -= v[i] * sn[i]; - } - psd[k] += re * re + im * im; - p += re * re + im * im; - } - pSum += p; - p2Sum += p * p; - segs++; - } - for (var i = 0; i < r.length; i++) { - final c = (cov[i] + cov[i + 1]) / 2; - wsd += c * r[i] * r[i]; - wsum += c; - } + sums.addRun(r); } - if (segs < minSegments || ssd == 0 || wsd == 0) return null; - if (_onCoarseLattice(diffRuns, ssd / nd)) return null; - // One loud stretch dominates the average: count segments by power, not number. - if (pSum * pSum / p2Sum < 0.75 * minSegments) return null; - final band = psd.sublist(kLo); - var peak = 0; - for (var k = 1; k < band.length; k++) { - if (band[k] > band[peak]) peak = k; - } - if (peak + kLo >= kHi - 1) return null; // ~0.477–0.5 band, see above - if (band.last > 0.2 * band[peak]) return null; // hump on Nyquist, see above - final med = median(band)!; - if (band[peak] < 4.5 * med) return null; // no line, no exemption - final floor = med / segs / w2; - final vetted = wsd / wsum, all = ssd / nd; - // 1 − (structured share of RMSSD²); equals 2σ²/mean(d²) when stationary. - return 1 - (vetted - 2 * floor) / math.max(vetted, all); + return noiseShareFrom(sums, + lattice: () => _onCoarseLattice(diffRuns, sums.ssd / sums.nd), + minSegments: minSegments); } /// True when every successive difference is a whole multiple of one step q and @@ -208,22 +140,8 @@ double? nnDiffNoiseShare(List> diffRuns, {int minSegments = 20}) { /// sets RMSSD. // ponytail: exact lattice only; a grid re-rounded to whole ms (7.8 ms → 7/8) // breaks the common step and is not caught. -bool _onCoarseLattice(List> diffRuns, double msd) { - var q = double.infinity; - for (final r in diffRuns) { - for (final d in r) { - if (d.abs() > 1e-6 && d.abs() < q) q = d.abs(); - } - } - if (q.isInfinite || msd >= 2 * q * q) return false; - for (final r in diffRuns) { - for (final d in r) { - final k = d.abs() / q; - if ((k - k.roundToDouble()).abs() > 1e-3) return false; - } - } - return true; -} +bool _onCoarseLattice(List> diffRuns, double msd) => + coarseLattice([for (final r in diffRuns) for (final d in r) d.abs()], msd); /// The one RMSSD jitter verdict every path shares: ACF1 below the floor AND /// the spectrum does not show a respiratory line on a low white floor. @@ -256,48 +174,6 @@ List _clearedWindows(double? nightAcf1, List rmssds, ]; } -final _hann128 = [ - for (var i = 0; i < 128; i++) 0.5 - 0.5 * math.cos(2 * math.pi * i / 127) -]; -final _hann128Sq = _hann128.fold(0.0, (a, w) => a + w * w); - -/// Welch power (128-beat Hann, 50 % overlap, averaged over segments) of one -/// window's RR, rebuilt from its difference runs, at cycles-per-beat -/// frequencies [f]. Null when no run holds a full segment. -List? _windowPsd(List> diffRuns, List f) { - const n = 128; - final p = List.filled(f.length, 0.0); - var segs = 0; - for (final r in diffRuns) { - final x = List.filled(r.length + 1, 0.0); - for (var i = 0; i < r.length; i++) { - x[i + 1] = x[i] + r[i]; - } - for (var s = 0; s + n <= x.length; s += n ~/ 2) { - var m = 0.0; - for (var i = 0; i < n; i++) { - m += x[s + i]; - } - m /= n; - for (var j = 0; j < f.length; j++) { - final c = math.cos(2 * math.pi * f[j]), sn = math.sin(2 * math.pi * f[j]); - var cr = 1.0, ci = 0.0, re = 0.0, im = 0.0; - for (var i = 0; i < n; i++) { - final v = (x[s + i] - m) * _hann128[i]; - re += v * cr; - im += v * ci; - final t = cr * c - ci * sn; - ci = cr * sn + ci * c; - cr = t; - } - p[j] += re * re + im * im; - } - segs++; - } - } - return segs == 0 ? null : [for (final v in p) v / segs]; -} - /// Last resort for a night the jitter gate refused: the windows whose /// breathing line holds still in Hz while the heart rate moves. /// @@ -324,115 +200,29 @@ List? _steadyBreathingWindows( nd++; } } - if (nd == 0 || _onCoarseLattice(all, ssd / nd)) return null; - final ref = median([for (final m in meanRrMs) if (m > 0) m]); - if (ref == null) return null; - // 0.15–0.5 cycles/beat at the night's median beat. - final fc = [for (var j = 38; j <= 128; j++) j / 256]; - final byBeat = List.filled(fc.length, 0.0); - final byHz = List.filled(fc.length, 0.0); - final cover = List.filled(fc.length, 0); - final peakHz = {}; // window -> its own peak, cycles per ms - final psd = >{}; // window -> its cycles-per-beat PSD - for (var w = 0; w < winRuns.length; w++) { - if (meanRrMs[w] <= 0) continue; - final pb = _windowPsd(winRuns[w], fc); - if (pb == null) continue; - final norm = median(pb)!; - if (norm <= 0) continue; - final k = meanRrMs[w] / ref; - final fh = [for (final f in fc) f * k]; - final ph = _windowPsd(winRuns[w], fh)!; - var best = -1; - for (var j = 0; j < fc.length; j++) { - byBeat[j] += pb[j] / norm; - if (fh[j] > 0.5) continue; - byHz[j] += ph[j] / norm; - cover[j]++; - if (best < 0 || ph[j] > ph[best]) best = j; - } - if (best >= 0) peakHz[w] = fc[best] / ref; - psd[w] = pb; - } - final nw = peakHz.length; - if (nw < 12) return null; - final bins = [for (var j = 0; j < fc.length; j++) if (cover[j] == nw) j]; - if (bins.length < 20) return null; - final hz = [for (final j in bins) byHz[j]]; - final bt = [for (final j in bins) byBeat[j]]; - var pk = 0; - for (var i = 1; i < hz.length; i++) { - if (hz[i] > hz[pk]) pk = i; - } - if (pk == 0 || pk == hz.length - 1) return null; - final f0 = fc[bins[pk]] / ref; - final brpm = f0 * 60000; - if (brpm < 8 || brpm > 30) return null; - // Same bins, same normalisation: the line must stand taller aligned in Hz - // than the beat pooling stands anywhere within ±12 bins of it. - var near = 0.0; - for (var i = math.max(0, pk - 12); i <= math.min(bt.length - 1, pk + 12); i++) { - near = math.max(near, bt[i]); - } - // The pooled noise ripple shrinks as 1/√windows, so the bar does too. - final promHz = hz[pk] / median(hz)!; - if (promHz < math.max(2.5, 1.5 + 7 / math.sqrt(nw))) return null; - if (promHz < 1.1 * near / median(bt)!) return null; - // Alternation lives at Nyquist on the beat axis, outside these bins. - if (byBeat.last / median(bt)! >= 0.5 * promHz) return null; - final keep = [ - for (final e in peakHz.entries) - if (math.log(e.value / f0).abs() <= 0.15) e.key - ]; - if (keep.length < 6) return null; - // The line proves breathing is there, not that it carries RMSSD: beat-time - // jitter loud enough to fail the gate would still be most of the number. - // Same ceiling as [nnDiffNoiseShare], over the kept windows. Jitter σ² has - // an RR spectrum σ²·(2 − 2cos ω), flat once divided by that shape, and its - // differences carry 6σ²; physiology only adds power on top. Breathing - // wanders across the night and its line spreads with it, so each window - // drops the bins of every rate the kept windows peaked at (plus a Hann - // main lobe), mapped onto its own beats. σ² is the floor of what is left, - // pooled over windows and smoothed over four resolution cells. - var lo = double.infinity, hi = 0.0; - for (final w in keep) { - lo = math.min(lo, peakHz[w]!); - hi = math.max(hi, peakHz[w]!); - } - var sq = 0.0, nAll = 0; - final acc = List.filled(fc.length, 0.0); - final wt = List.filled(fc.length, 0.0); - for (final w in keep) { - var n = 0; - for (final r in winRuns[w]) { - for (final d in r) { - sq += d * d; - n++; + return steadyBreathingFrom( + windows: winRuns.length, + meanRrMs: meanRrMs, + psdAt: (w, f) => windowPsd(winRuns[w], f), + diffs: (w) { + var n = 0; + for (final r in winRuns[w]) { + n += r.length; } - } - nAll += n; - final a = lo * meanRrMs[w] - 3 / 128, b = hi * meanRrMs[w] + 3 / 128; - for (var j = 0; j < fc.length; j++) { - if (fc[j] > a && fc[j] < b) continue; - acc[j] += n * psd[w]![j] / (2 - 2 * math.cos(2 * math.pi * fc[j])); - wt[j] += n; - } - } - var floor = double.infinity; - for (var j = 8; j < fc.length - 8; j++) { - var s = 0.0, m = 0; - for (var i = j - 8; i <= j + 8; i++) { - if (wt[i] < nAll / 2) break; - s += acc[i] / wt[i]; - m++; - } - if (m == 17) floor = math.min(floor, s / 17); - } - if (floor == double.infinity) return null; - // The lowest stretch of a noisy curve reads below its mean; 1.1 puts the - // floor back at these pooled sizes, so jitter alone is not undercounted. - final noise = 1.1 * 6 * floor / _hann128Sq * nAll; - return noise < kNnDiffNoiseShareCeiling * sq ? keep : null; + return n; + }, + addSquares: (w, acc) { + for (final r in winRuns[w]) { + for (final d in r) { + acc += d * d; + } + } + return acc; + }, + allSsd: ssd, + allNd: nd, + lattice: (msd) => _onCoarseLattice(all, msd), + ); } /// The window RMSSDs a windowed headline publishes, or null for none: the @@ -624,6 +414,32 @@ Metric hrvTime( RrCoverage? coverage, }) { const inputs = ['rr_cleaned']; + // An unknown interval cannot support a finite HRV estimate. Refuse the + // input before segmentation or moments rather than discarding evidence. + if (nnTimesMs != null && nnTimesMs.length != nnMs.length) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'beat time length mismatch', + ); + } + // A reset may expose disagreement between accumulated RR and recorded + // sensor time. Do not turn overlapping segments into successive beats. + if (nnTimesMs != null && nnTimesMs.length == nnMs.length && + hasRegressingBeatTimes(nnTimesMs)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'regressing beat time', + ); + } + if (nnMs.any((value) => !value.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite NN interval', + ); + } if (nnMs.length < 2) { return const Metric.absent( tier: Tier.high, @@ -632,6 +448,15 @@ Metric hrvTime( ); } + if (nnTimesMs != null && + nnTimesMs.length == nnMs.length && + nnTimesMs.any((t) => !t.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite beat time', + ); + } // RMSSD / pNN50: root mean square of SUCCESSIVE differences — successive in // TIME, not merely adjacent in the compacted list. correctRr drops multi-beat // artifact runs while advancing its clock across them, so nn[i-1] and nn[i] @@ -672,7 +497,29 @@ Metric hrvTime( // are far less contaminated (jitter share 1–27 % against RMSSD's 11–100 % on // the audit corpus) — they keep publishing, which is what the header has // always advised. - final acf1 = nnDiffAcf1(runs); + // Welford (1962) centered sums/products, Task Force (1996) HRV. + // Accumulate difference moments and lagged co-moments in sample order. + var dn = 0, ln = 0; + var dm = 0.0, d2 = 0.0, ma = 0.0, mb = 0.0, co = 0.0; + for (final r in runs) { + for (var i = 0; i < r.length; i++) { + final d = r[i]; + dn++; + final delta = d - dm; + dm += delta / dn; + d2 += delta * (d - dm); + if (i > 0) { + ln++; + final da = r[i - 1] - ma; + final db = d - mb; + ma += da / ln; + mb += db / ln; + co += da * (d - mb); + } + } + } + final double? acf1 = dn >= 30 && d2 > 0 + ? (co + ln * (ma - dm) * (mb - dm)) / d2 : null; final jittery = _jitterRefused(acf1, runs); // More beat-time than elapsed: the stream itself is wrong, so neither the // differences nor any breathing line in them describe one heart. @@ -686,15 +533,16 @@ Metric hrvTime( rmssd = _breathingRmssd(nnMs, nnTimesMs); } final byBreathing = jittery && rmssd != null; - final sdnn = stddev(nnMs); + final sdnn = _centeredMoments(nnMs).sd; double? sdann, sdnnIndex; if (gapAware) { final seg = _fiveMinSegments(nnMs, nnTimesMs); if (seg.length >= 2) { - final means = [for (final s in seg) mean(s)!]; - sdann = stddev(means); - final sds = [for (final s in seg) stddev(s)].whereType().toList(); + final moments = [for (final s in seg) _centeredMoments(s)]; + final means = [for (final m in moments) m.mean!]; + sdann = _centeredMoments(means).sd; + final sds = [for (final m in moments) m.sd].whereType().toList(); sdnnIndex = sds.isEmpty ? null : mean(sds); } } @@ -804,6 +652,24 @@ Metric nocturnalRmssd( RrCoverage? coverage, }) { const inputs = ['rr_cleaned', 'beat_times']; + // An unknown interval cannot support a finite HRV estimate. Refuse the + // input before segmentation or moments rather than discarding evidence. + // A reset may expose disagreement between accumulated RR and recorded + // sensor time. Do not turn overlapping segments into successive beats. + if (hasRegressingBeatTimes(nnTimesMs)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'regressing beat time', + ); + } + if (nnMs.any((value) => !value.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite NN interval', + ); + } if (coverage != null && coverage.overCounted) { return Metric.absent( tier: Tier.high, @@ -818,6 +684,13 @@ Metric nocturnalRmssd( note: 'too few NN intervals for windowed nocturnal RMSSD', ); } + if (nnTimesMs.any((t) => !t.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite beat time', + ); + } final t0 = nnTimesMs.first; // Bucket beat INDICES by window index, so each window keeps its beat times and // the successive differences below can skip the ones that straddle a dropped @@ -884,7 +757,29 @@ Metric nocturnalRmssd( meanRr.add(mean([for (final i in seg) nnMs[i]])!); rmssds.add(math.sqrt(ssd / nd)); } - final acf1 = nnDiffAcf1(runs); + // Welford (1962) centered sums/products, Task Force (1996) HRV. + // Accumulate difference moments and lagged co-moments in sample order. + var dn = 0, ln = 0; + var dm = 0.0, d2 = 0.0, ma = 0.0, mb = 0.0, co = 0.0; + for (final r in runs) { + for (var i = 0; i < r.length; i++) { + final d = r[i]; + dn++; + final delta = d - dm; + dm += delta / dn; + d2 += delta * (d - dm); + if (i > 0) { + ln++; + final da = r[i - 1] - ma; + final db = d - mb; + ma += da / ln; + mb += db / ln; + co += da * (d - mb); + } + } + } + final double? acf1 = dn >= 30 && d2 > 0 + ? (co + ln * (ma - dm) * (mb - dm)) / d2 : null; final refused = _jitterRefused(acf1, runs); if (rmssds.isEmpty && !refused) { return const Metric.absent( @@ -1030,6 +925,24 @@ Metric sleepSessionRmssdDetail( int minDiffsPerWindow = kMinDiffsPerRmssdWindow, }) { const inputs = ['rr_sleep_window']; + // An unknown interval cannot support a finite HRV estimate. Refuse the + // input before segmentation or moments rather than discarding evidence. + // A reset may expose disagreement between accumulated RR and recorded + // sensor time. Do not turn overlapping segments into successive beats. + if (hasRegressingBeatTimes(rrTsMs)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'regressing beat time', + ); + } + if (rrMs.any((value) => !value.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite NN interval', + ); + } if (startSec <= 0 || endSec <= startSec || rrMs.isEmpty || @@ -1042,6 +955,13 @@ Metric sleepSessionRmssdDetail( ); } + if (rrTsMs.any((t) => !t.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite beat time', + ); + } final buckets = >{}; final bucketsTs = >{}; final inRr = []; @@ -1143,7 +1063,29 @@ Metric sleepSessionRmssdDetail( // THE HEADLINE nightly RMSSD (→ ln_rmssd → readiness). When the differences // are noise, the honest output is no headline, not a plausible one — the // readiness composite already treats a null HRV driver as absent. - final acf1 = nnDiffAcf1(runs); + // Welford (1962) centered sums/products, Task Force (1996) HRV. + // Accumulate difference moments and lagged co-moments in sample order. + var dn = 0, ln = 0; + var dm = 0.0, d2 = 0.0, ma = 0.0, mb = 0.0, co = 0.0; + for (final r in runs) { + for (var i = 0; i < r.length; i++) { + final d = r[i]; + dn++; + final delta = d - dm; + dm += delta / dn; + d2 += delta * (d - dm); + if (i > 0) { + ln++; + final da = r[i - 1] - ma; + final db = d - mb; + ma += da / ln; + mb += db / ln; + co += da * (d - mb); + } + } + } + final double? acf1 = dn >= 30 && d2 > 0 + ? (co + ln * (ma - dm) * (mb - dm)) / d2 : null; final refused = _jitterRefused(acf1, runs); if (rmssds.isEmpty && !refused) { return Metric.absent( @@ -1282,3 +1224,22 @@ List> _cleanWindowRuns(List rr, List ts) { if (run.isNotEmpty) runs.add(run); return runs; } + +// Welford, B. P. (1962), Technometrics 4(3):419-420. +// Origin-shifted batch recurrence; skips NaN just as #99's Moments does. +({double? mean, double? sd}) _centeredMoments(List values) { + var n = 0; + var origin = 0.0, offset = 0.0, m2 = 0.0; + for (final value in values) { + if (value.isNaN) continue; + if (!value.isFinite) throw ArgumentError.value(value, 'value', 'Must be finite'); + if (n == 0) { origin = value; n = 1; continue; } + final centered = value - origin; + final delta = centered - offset; + n++; + offset += delta / n; + m2 += delta * (centered - offset); + } + return (mean: n == 0 ? null : origin + offset, + sd: n < 2 ? null : math.sqrt(m2 / (n - 1))); +} diff --git a/lib/src/onehz/clinical/load_trimp.dart b/lib/src/onehz/clinical/load_trimp.dart index 3c89ca4f..53187efa 100644 --- a/lib/src/onehz/clinical/load_trimp.dart +++ b/lib/src/onehz/clinical/load_trimp.dart @@ -63,7 +63,8 @@ Metric banisterTrimp( ); } final reserve = maxHr - restingHr; - var trimp = 0.0; + // Neumaier (1974), Rundungsfehleranalyse einiger Verfahren zur Summation. + var trimp = 0.0, correction = 0.0; for (final hr in hrPerMin) { if (!hr.isFinite || hr <= 0) continue; // off-skin + non-finite guard var hrr = (hr - restingHr) / reserve; @@ -71,10 +72,16 @@ Metric banisterTrimp( if (hrr > 1) hrr = 1; // ONE Banister implementation for the whole package: the sex-specific // weighting factor y lives in [StrainScorer.banisterY]. 1 minute each. - trimp += 1.0 * hrr * StrainScorer.banisterY(hrr, female: sex == Sex.female); + final value = hrr * StrainScorer.banisterY(hrr, female: sex == Sex.female); + final next = trimp + value; + if (next.isFinite) { + correction += trimp.abs() >= value.abs() + ? (trimp - next) + value : (value - next) + trimp; + } + trimp = next; } return Metric( - value: trimp, + value: trimp + correction, confidence: 0.6, tier: Tier.estimate, inputs_used: inputs, @@ -829,7 +836,8 @@ const int ctlAtlMinDays = 14; /// CTL/ATL/TSB from a time-ordered daily-TRIMP series (oldest→newest). /// EWMA with time constants 42 d (CTL) and 7 d (ATL): λ = 1 − e^(−1/τ). -/// A missing day contributes a 0-load impulse (rest day) — the EWMA decays. +/// An observed rest day is 0 load and decays the EWMA. Unknown/non-finite +/// daily load abstains; it is never substituted with a rest day. /// /// SEEDING (Banister 1975 impulse-response; the load before the record started /// is UNKNOWN): both accumulators used to be seeded at `dailyTrimp.first`, @@ -846,6 +854,13 @@ Metric ctlAtlTsb(List dailyTrimp, int minDays = ctlAtlMinDays, int primeDays = 7}) { const inputs = ['daily_trimp']; + if (dailyTrimp.any((value) => !value.isFinite)) { + return const Metric.absent( + tier: Tier.estimate, + inputs_used: inputs, + note: 'non-finite daily TRIMP', + ); + } if (dailyTrimp.length < minDays) { return Metric.absent( tier: Tier.estimate, @@ -856,13 +871,31 @@ Metric ctlAtlTsb(List dailyTrimp, final lc = 1 - math.exp(-1 / ctlDays); final la = 1 - math.exp(-1 / atlDays); final prime = math.min(math.max(primeDays, 1), dailyTrimp.length); - final seed = mean(dailyTrimp.sublist(0, prime))!; - var ctl = seed; - var atl = seed; - for (var i = prime; i < dailyTrimp.length; i++) { - ctl = ctl + lc * (dailyTrimp[i] - ctl); - atl = atl + la * (dailyTrimp[i] - atl); + // Neumaier (1974) compensation for the bounded-prefix mean. + var sum = 0.0, correction = 0.0; + for (final value in dailyTrimp.take(prime)) { + final next = sum + value; + if (next.isFinite) { + correction += sum.abs() >= value.abs() + ? (sum - next) + value : (value - next) + sum; + } + sum = next; + } + final seed = (sum + correction) / prime; + // Banister et al. (1975) recurrence evaluated as an affine map. + // f(x) = scale*x + offset, with keep = 1-lambda. + double evaluate(double lambda) { + var scale = 1.0, offset = 0.0; + final keep = 1 - lambda; + for (var i = prime; i < dailyTrimp.length; i++) { + final value = dailyTrimp[i]; + scale *= keep; + offset = keep * offset + lambda * value; + } + return scale * seed + offset; } + final ctl = evaluate(lc); + final atl = evaluate(la); final conf = (dailyTrimp.length / 42.0).clamp(0.3, 0.85); return Metric( value: LoadState(ctl, atl, ctl - atl), diff --git a/lib/src/onehz/clinical/rmssd_gate.dart b/lib/src/onehz/clinical/rmssd_gate.dart new file mode 100644 index 00000000..6c8972df --- /dev/null +++ b/lib/src/onehz/clinical/rmssd_gate.dart @@ -0,0 +1,319 @@ +// Rejection and breathing-line exemption thresholds are an internal rule. +// Welch 1967 spectral estimation; Hirsch & Bishop 1981 provides the +// breathing/RSA basis. This is hrvTime()'s existing internal jitter policy, +// ESTIMATE with low confidence, not a published classifier. Shared formulas +// preserve the calculation; reordered sums can change decisions within rounding +// of a threshold. +// CLINICAL — the spectral half of the RMSSD jitter gate, as pieces. +// +// `hrvTime`, `nocturnalRmssd` and `sleepSessionWindowedRmssd` (hrv_time.dart) +// read the successive-difference runs themselves when the lag-1 ACF is under +// [kNnDiffAcf1Floor]: a respiratory line on a low white floor exempts the night +// ([nnDiffNoiseShare]), and a refused night may still publish from the 5-min +// windows on a breathing line steady in Hz. Both reduce to sums over Welch +// segments, so this file holds them as SUMS plus the decisions over those +// sums, written once: the batch fills the sums from the runs in hand, and +// `hrvTimeAggregator` fills the same sums from mergeable per-chunk state. +// +// Not exported from `onehz.dart`. + +import 'dart:math' as math; + +import '../util.dart'; +import 'hrv_time.dart' show kNnDiffNoiseShareCeiling; + +/// Welch segment length (beats) of the noise-share spectrum, the first and last +/// bins of its 0.15–0.5 cycles/beat band. +const int kNoiseSegment = 64, kNoiseLoBin = 10, kNoiseHiBin = kNoiseSegment ~/ 2; + +/// The segment's Hann window (symmetric, n − 1 in the denominator). +final List noiseHann = [ + for (var i = 0; i < kNoiseSegment; i++) + 0.5 - 0.5 * math.cos(2 * math.pi * i / (kNoiseSegment - 1)) +]; + +final double _noiseHannSq = () { + var w2 = 0.0; + for (final x in noiseHann) { + w2 += x * x; + } + return w2; +}(); + +final List> _cosT = [ + for (var k = kNoiseLoBin; k <= kNoiseHiBin; k++) + [for (var i = 0; i < kNoiseSegment; i++) math.cos(2 * math.pi * k * i / kNoiseSegment)] +]; +final List> _sinT = [ + for (var k = kNoiseLoBin; k <= kNoiseHiBin; k++) + [for (var i = 0; i < kNoiseSegment; i++) math.sin(2 * math.pi * k * i / kNoiseSegment)] +]; + +/// Adds the Welch segment `x[s .. s + 63]` (RR levels integrated from the +/// differences) to [psd] (bins [kNoiseLoBin]..[kNoiseHiBin]) and returns its +/// band power. +double addNoiseSegment(List x, int s, List psd) { + const n = kNoiseSegment; + var m = 0.0; + for (var i = 0; i < n; i++) { + m += x[s + i]; + } + m /= n; + final v = [for (var i = 0; i < n; i++) (x[s + i] - m) * noiseHann[i]]; + var p = 0.0; + for (var k = kNoiseLoBin; k <= kNoiseHiBin; k++) { + final c = _cosT[k - kNoiseLoBin], sn = _sinT[k - kNoiseLoBin]; + var re = 0.0, im = 0.0; + for (var i = 0; i < n; i++) { + re += v[i] * c[i]; + im -= v[i] * sn[i]; + } + psd[k] += re * re + im * im; + p += re * re + im * im; + } + return p; +} + +/// The sums [nnDiffNoiseShare] decides on. +class NoiseShareSums { + final List psd = List.filled(kNoiseHiBin + 1, 0.0); + int segs = 0, nd = 0; + double ssd = 0.0, wsd = 0.0, wsum = 0.0, pSum = 0.0, p2Sum = 0.0; + + /// One contiguous run of differences, in the batch's own order. + void addRun(List r) { + const n = kNoiseSegment; + for (final d in r) { + ssd += d * d; + nd++; + } + // Integrate the run back to RR levels (up to a constant the mean removes). + final x = List.filled(r.length + 1, 0.0); + for (var i = 0; i < r.length; i++) { + x[i + 1] = x[i] + r[i]; + } + final cov = List.filled(x.length, 0.0); // Σ w² per sample + for (var s = 0; s + n <= x.length; s += n ~/ 2) { + for (var i = 0; i < n; i++) { + cov[s + i] += noiseHann[i] * noiseHann[i]; + } + final p = addNoiseSegment(x, s, psd); + pSum += p; + p2Sum += p * p; + segs++; + } + for (var i = 0; i < r.length; i++) { + final c = (cov[i] + cov[i + 1]) / 2; + wsd += c * r[i] * r[i]; + wsum += c; + } + } +} + +/// The white-noise share from [s] (see [nnDiffNoiseShare] in hrv_time.dart for +/// what it measures and every refusal). [lattice] says whether the differences +/// sit on a coarse beat-time grid ([coarseLattice]); it is only asked when +/// the counts allow a verdict. +double? noiseShareFrom(NoiseShareSums s, + {required bool Function() lattice, int minSegments = 20}) { + final segs = s.segs; + if (segs < minSegments || s.ssd == 0 || s.wsd == 0) return null; + if (lattice()) return null; + // One loud stretch dominates the average: count segments by power, not number. + if (s.pSum * s.pSum / s.p2Sum < 0.75 * minSegments) return null; + final band = s.psd.sublist(kNoiseLoBin); + var peak = 0; + for (var k = 1; k < band.length; k++) { + if (band[k] > band[peak]) peak = k; + } + if (peak + kNoiseLoBin >= kNoiseHiBin - 1) return null; // ~0.477–0.5 band + if (band.last > 0.2 * band[peak]) return null; // hump on Nyquist + final med = median(band)!; + if (band[peak] < 4.5 * med) return null; // no line, no exemption + final floor = med / segs / _noiseHannSq; + final vetted = s.wsd / s.wsum, all = s.ssd / s.nd; + // 1 − (structured share of RMSSD²); equals 2σ²/mean(d²) when stationary. + return 1 - (vetted - 2 * floor) / math.max(vetted, all); +} + +/// True when every successive difference is a whole multiple of one step q +/// (the smallest |d| over 1e-6) and the mean squared difference [msd] is under +/// 2q². [magnitudes] are the |d| values (repeats change nothing), in any +/// order. See hrv_time.dart `_onCoarseLattice` for why. +bool coarseLattice(Iterable magnitudes, double msd) { + var q = double.infinity; + for (final a in magnitudes) { + if (a > 1e-6 && a < q) q = a; + } + if (q.isInfinite || msd >= 2 * q * q) return false; + for (final a in magnitudes) { + final k = a / q; + if ((k - k.roundToDouble()).abs() > 1e-3) return false; + } + return true; +} + +/// The breathing-line test's Hann window (128 beats). +final List hann128 = [ + for (var i = 0; i < 128; i++) 0.5 - 0.5 * math.cos(2 * math.pi * i / 127) +]; +final double _hann128Sq = hann128.fold(0.0, (a, w) => a + w * w); + +/// The cycles-per-beat axis the breathing-line test pools on: 0.15–0.5 +/// cycles/beat at the night's median beat. +final List breathingAxis = [for (var j = 38; j <= 128; j++) j / 256]; + +/// Welch power (128-beat Hann, 50 % overlap, averaged over segments) of one +/// window's RR, rebuilt from its difference runs, at cycles-per-beat +/// frequencies [f]. Null when no run holds a full segment. +List? windowPsd(List> diffRuns, List f) { + const n = 128; + final p = List.filled(f.length, 0.0); + var segs = 0; + for (final r in diffRuns) { + final x = List.filled(r.length + 1, 0.0); + for (var i = 0; i < r.length; i++) { + x[i + 1] = x[i] + r[i]; + } + for (var s = 0; s + n <= x.length; s += n ~/ 2) { + var m = 0.0; + for (var i = 0; i < n; i++) { + m += x[s + i]; + } + m /= n; + for (var j = 0; j < f.length; j++) { + final c = math.cos(2 * math.pi * f[j]), sn = math.sin(2 * math.pi * f[j]); + var cr = 1.0, ci = 0.0, re = 0.0, im = 0.0; + for (var i = 0; i < n; i++) { + final v = (x[s + i] - m) * hann128[i]; + re += v * cr; + im += v * ci; + final t = cr * c - ci * sn; + ci = cr * sn + ci * c; + cr = t; + } + p[j] += re * re + im * im; + } + segs++; + } + } + return segs == 0 ? null : [for (final v in p) v / segs]; +} + +/// The breathing-line test over [windows] 5-min windows, each read through +/// callbacks so a caller can hold its windows as runs or as sums: +/// [psdAt] is the window's [windowPsd] at the given frequencies (null without +/// a full segment), [diffs] its difference count, [addSquares] adds its +/// squared differences to a running total (in the caller's order), [meanRrMs] +/// its mean RR. [allSsd] / [allNd] are the squared differences and their count +/// over every window, and [lattice] is [coarseLattice] over every window's +/// differences at the mean square it is given. Returns the indices of the +/// windows whose own peak sits on the line, or null. See hrv_time.dart +/// `_steadyBreathingWindows` for the method. +List? steadyBreathingFrom({ + required int windows, + required List meanRrMs, + required List? Function(int w, List f) psdAt, + required int Function(int w) diffs, + required double Function(int w, double acc) addSquares, + required double allSsd, + required int allNd, + required bool Function(double msd) lattice, +}) { + if (allNd == 0 || lattice(allSsd / allNd)) return null; + final ref = median([for (final m in meanRrMs) if (m > 0) m]); + if (ref == null) return null; + final fc = breathingAxis; + final byBeat = List.filled(fc.length, 0.0); + final byHz = List.filled(fc.length, 0.0); + final cover = List.filled(fc.length, 0); + final peakHz = {}; // window -> its own peak, cycles per ms + final psd = >{}; // window -> its cycles-per-beat PSD + for (var w = 0; w < windows; w++) { + if (meanRrMs[w] <= 0) continue; + final pb = psdAt(w, fc); + if (pb == null) continue; + final norm = median(pb)!; + if (norm <= 0) continue; + final k = meanRrMs[w] / ref; + final fh = [for (final f in fc) f * k]; + final ph = psdAt(w, fh)!; + var best = -1; + for (var j = 0; j < fc.length; j++) { + byBeat[j] += pb[j] / norm; + if (fh[j] > 0.5) continue; + byHz[j] += ph[j] / norm; + cover[j]++; + if (best < 0 || ph[j] > ph[best]) best = j; + } + if (best >= 0) peakHz[w] = fc[best] / ref; + psd[w] = pb; + } + final nw = peakHz.length; + if (nw < 12) return null; + final bins = [for (var j = 0; j < fc.length; j++) if (cover[j] == nw) j]; + if (bins.length < 20) return null; + final hz = [for (final j in bins) byHz[j]]; + final bt = [for (final j in bins) byBeat[j]]; + var pk = 0; + for (var i = 1; i < hz.length; i++) { + if (hz[i] > hz[pk]) pk = i; + } + if (pk == 0 || pk == hz.length - 1) return null; + final f0 = fc[bins[pk]] / ref; + final brpm = f0 * 60000; + if (brpm < 8 || brpm > 30) return null; + // Same bins, same normalisation: the line must stand taller aligned in Hz + // than the beat pooling stands anywhere within ±12 bins of it. + var near = 0.0; + for (var i = math.max(0, pk - 12); i <= math.min(bt.length - 1, pk + 12); i++) { + near = math.max(near, bt[i]); + } + // The pooled noise ripple shrinks as 1/√windows, so the bar does too. + final promHz = hz[pk] / median(hz)!; + if (promHz < math.max(2.5, 1.5 + 7 / math.sqrt(nw))) return null; + if (promHz < 1.1 * near / median(bt)!) return null; + // Alternation lives at Nyquist on the beat axis, outside these bins. + if (byBeat.last / median(bt)! >= 0.5 * promHz) return null; + final keep = [ + for (final e in peakHz.entries) + if (math.log(e.value / f0).abs() <= 0.15) e.key + ]; + if (keep.length < 6) return null; + // Jitter that fails the gate would still be most of the number: same + // ceiling as the noise share, over the kept windows (see hrv_time.dart). + var lo = double.infinity, hi = 0.0; + for (final w in keep) { + lo = math.min(lo, peakHz[w]!); + hi = math.max(hi, peakHz[w]!); + } + var sq = 0.0, nAll = 0; + final acc = List.filled(fc.length, 0.0); + final wt = List.filled(fc.length, 0.0); + for (final w in keep) { + sq = addSquares(w, sq); + final n = diffs(w); + nAll += n; + final a = lo * meanRrMs[w] - 3 / 128, b = hi * meanRrMs[w] + 3 / 128; + for (var j = 0; j < fc.length; j++) { + if (fc[j] > a && fc[j] < b) continue; + acc[j] += n * psd[w]![j] / (2 - 2 * math.cos(2 * math.pi * fc[j])); + wt[j] += n; + } + } + var floor = double.infinity; + for (var j = 8; j < fc.length - 8; j++) { + var s = 0.0, m = 0; + for (var i = j - 8; i <= j + 8; i++) { + if (wt[i] < nAll / 2) break; + s += acc[i] / wt[i]; + m++; + } + if (m == 17) floor = math.min(floor, s / 17); + } + if (floor == double.infinity) return null; + // The lowest stretch of a noisy curve reads below its mean; 1.1 puts the + // floor back at these pooled sizes, so jitter alone is not undercounted. + final noise = 1.1 * 6 * floor / _hann128Sq * nAll; + return noise < kNnDiffNoiseShareCeiling * sq ? keep : null; +} diff --git a/lib/src/onehz/foundations/rr_correction.dart b/lib/src/onehz/foundations/rr_correction.dart index 89840130..a8e1738f 100644 --- a/lib/src/onehz/foundations/rr_correction.dart +++ b/lib/src/onehz/foundations/rr_correction.dart @@ -1,3 +1,11 @@ +// Lipponen & Tarvainen 2019 classification with Peltola 2012 correction policy. +// The unpublished artifact-run cap is an internal rule, ESTIMATE with low +// confidence: +// over-cap runs become gaps and each side is re-corrected. Without a cap split +// the result equals uncapped correction. Shared kernel arithmetic and raw seam +// context preserve capped batch NN/classes; settled stream plus provisional tail +// equals capped batch. RrSandwich preserves source-beat absolute timestamps; +// uncapped arbitrarily long artifact runs have no exactness guarantee. // FOUNDATION — RR artifact correction. // // Lipponen & Tarvainen 2019 ("A robust algorithm for heart rate variability @@ -13,11 +21,22 @@ // This is PRV. Output = cleaned NN series + per-beat artifact mask + clean // fraction. Multi-beat gaps are never silently bridged. -import 'dart:math' as math; -import '../util.dart'; +import 'rr_correction_kernel.dart'; + +/// Longest artifact run (beats) the corrector will carry. A longer run is not a +/// correction problem but an absence: it breaks the series into independent +/// segments and its beats are left out (a gap). 32 beats leaves no run on the +/// owner's 18 days of data over the cap (longest seen: 16) and bounds the +/// streaming state; see combined-bump/brief.md. +const int kMaxArtifactRun = 32; /// Artifact label for a single beat. -enum BeatClass { normal, ectopic, longShort, missed, extra } +enum BeatClass { + normal, ectopic, longShort, missed, extra, + /// An over-cap artifact run, without a claim about its individual types. + /// Appended to preserve the indices of existing checkpoint labels. + gap, +} class RrCorrectionResult { /// Cleaned NN intervals (ms). Isolated artifacts spline-corrected; multi-beat @@ -34,7 +53,16 @@ class RrCorrectionResult { /// spliced out; pass the timestamps. final List nnTimesMs; + /// For each NN, the index of the input beat it came from: the beat itself, + /// or, for a spline-corrected beat, that beat's own index. Strictly + /// increasing; the input beats not listed here were dropped. Lets a caller + /// pair each NN with that beat's own (absolute) timestamp. Empty on a + /// hand-built result that did not track it. + final List nnSourceIndex; + /// Per-input-beat classification (same length as the input rr). + /// Over-cap runs use [BeatClass.gap]; other artifact labels retain their + /// inferred classification. A gap makes no physiological type claim. final List classes; /// Fraction of input beats classified normal (0..1). @@ -53,6 +81,7 @@ class RrCorrectionResult { required this.cleanFraction, required this.droppedCount, required this.correctedCount, + this.nnSourceIndex = const [], }); } @@ -74,6 +103,10 @@ class RrCorrectionResult { /// `rec_ts*1000` and therefore whole-second — and re-anchors to the real /// timestamp whenever the wall step exceeds the interval by more than /// [reanchorGapMs], i.e. at a dropout. +/// +/// [maxArtifactRun] caps a run of artifact beats: a longer run splits the series +/// (output = correctRr(before) ++ correctRr(after), times absolute, the run's +/// beats absent). null = uncapped, the pre-cap behaviour. RrCorrectionResult correctRr( List rrMs, { List? rrTsMs, @@ -81,7 +114,55 @@ RrCorrectionResult correctRr( int windowBeats = 91, double minThresholdMs = 100, double reanchorGapMs = 1000, + int? maxArtifactRun = kMaxArtifactRun, }) { + if (maxArtifactRun != null && maxArtifactRun < 1) { + throw ArgumentError.value(maxArtifactRun, 'maxArtifactRun'); + } + if (windowBeats < 1) throw ArgumentError.value(windowBeats, 'windowBeats'); + if (maxArtifactRun != null) { + // The overwhelmingly common path has no run over the cap. Reuse the + // original vectorized correction verbatim, including its clock, and only + // evaluate split windows when a run actually requires a cut. + var hardRun = 0, hardGap = false; + for (final rr in rrMs) { + hardRun = rr < 300 || rr > 2000 ? hardRun + 1 : 0; + if (hardRun > maxArtifactRun) { hardGap = true; break; } + } + if (!hardGap) { + final ordinary = correctRr(rrMs, rrTsMs: rrTsMs, alpha: alpha, + windowBeats: windowBeats, minThresholdMs: minThresholdMs, + reanchorGapMs: reanchorGapMs, maxArtifactRun: null); + var run = 0, gap = false; + for (final cls in ordinary.classes) { + run = cls == BeatClass.normal ? 0 : run + 1; + if (run > maxArtifactRun) { gap = true; break; } + } + if (!gap) return ordinary; + } + final w = RrCapWindow.evaluate(rrMs, + ts: rrTsMs, cap: maxArtifactRun, alpha: alpha, + windowBeats: windowBeats, minThresholdMs: minThresholdMs, + reanchorGapMs: reanchorGapMs); + final nn = [], times = [], source = []; + var normal = 0, dropped = 0, corrected = 0; + for (var i = 0; i < rrMs.length; i++) { + if (w.classes[i] == BeatClass.normal) normal++; + final v = w.values[i]; + if (v == null) { + dropped++; + } else { + nn.add(v); + times.add(w.times[i]); + source.add(i); + if (w.classes[i] != BeatClass.normal) corrected++; + } + } + return RrCorrectionResult(nn: nn, nnTimesMs: times, classes: w.classes, + cleanFraction: rrMs.isEmpty ? 0 : normal / rrMs.length, + droppedCount: dropped, correctedCount: corrected, + nnSourceIndex: source); + } final n = rrMs.length; // Beat-end time for EVERY input beat, kept or not. One array, built once, so // every branch below reads a clock instead of advancing its own. @@ -105,10 +186,12 @@ RrCorrectionResult correctRr( ]; final nn = []; final times = []; + final source = []; for (var i = 0; i < n; i++) { if (classes[i] == BeatClass.normal) { nn.add(rrMs[i]); times.add(tAll[i]); + source.add(i); } } return RrCorrectionResult( @@ -118,6 +201,7 @@ RrCorrectionResult correctRr( cleanFraction: nn.length / n, droppedCount: n - nn.length, correctedCount: 0, + nnSourceIndex: source, ); } @@ -133,32 +217,13 @@ RrCorrectionResult correctRr( // medRR: rr minus local median (for missed/extra long-range tests). final med = _slidingMedian(rrMs, windowBeats); - final mRR = List.generate(n, (i) { - final d = rrMs[i] - med[i]; - return d < 0 ? d * 2 : d; // paper asymmetry weight - }); + final mRR = List.generate(n, (i) => medianDeviation(rrMs[i], med[i])); final th2 = _slidingThreshold(mRR, windowBeats, alpha, minThresholdMs); - final classes = List.filled(n, BeatClass.normal); - for (var i = 0; i < n; i++) { - final hardLong = rrMs[i] > 2000; - final hardShort = rrMs[i] < 300; - final bigJump = dRR[i].abs() > th1[i]; - final bigDev = mRR[i].abs() > th2[i]; - if (hardLong || (bigDev && mRR[i] > 0)) { - // Long interval: likely a MISSED beat (interval ~ multiple of normal). - classes[i] = (med[i] > 0 && rrMs[i] > 1.5 * med[i]) - ? BeatClass.missed - : BeatClass.longShort; - } else if (hardShort || (bigDev && mRR[i] < 0)) { - // Short interval: likely an EXTRA (spurious) beat. - classes[i] = (med[i] > 0 && rrMs[i] < 0.6 * med[i]) - ? BeatClass.extra - : BeatClass.longShort; - } else if (bigJump) { - classes[i] = BeatClass.ectopic; - } - } + final classes = [ + for (var i = 0; i < n; i++) + classifyBeat(rrMs[i], dRR[i], th1[i], med[i], mRR[i], th2[i]) + ]; // Compensatory-pair reconciliation. A single ectopic/extra beat shows up as // TWO successive dRR spikes of opposite sign (the bad beat, then the @@ -169,17 +234,13 @@ RrCorrectionResult correctRr( // single isolated artifact rather than a spurious 2-beat run. for (var k = 1; k < n; k++) { if (classes[k] != BeatClass.ectopic) continue; - final prevBad = classes[k - 1] == BeatClass.extra || - classes[k - 1] == BeatClass.missed || - classes[k - 1] == BeatClass.ectopic || - classes[k - 1] == BeatClass.longShort; - if (!prevBad) continue; - final oppositeSign = dRR[k] * dRR[k - 1] < 0; - final valueNormal = med[k] > 0 && - rrMs[k] >= 300 && - rrMs[k] <= 2000 && - (rrMs[k] - med[k]).abs() <= 0.2 * med[k]; - if (oppositeSign && valueNormal) { + final prevBad = classes[k - 1] != BeatClass.normal; + if (isRecoveryBeat( + prevIsArtifact: prevBad, + rr: rrMs[k], + dRR: dRR[k], + dRRPrev: dRR[k - 1], + med: med[k])) { classes[k] = BeatClass.normal; } } @@ -189,6 +250,7 @@ RrCorrectionResult correctRr( final isArtifact = [for (final c in classes) c != BeatClass.normal]; final nn = []; final times = []; + final source = []; var dropped = 0; var corrected = 0; var i = 0; @@ -196,6 +258,7 @@ RrCorrectionResult correctRr( if (!isArtifact[i]) { nn.add(rrMs[i]); times.add(tAll[i]); + source.add(i); i++; continue; } @@ -216,6 +279,7 @@ RrCorrectionResult correctRr( // — 1.6 % of a night at one missed beat in sixty. nn.add(corr); times.add(tAll[i]); + source.add(i); corrected++; } else { dropped++; // no anchors -> honest drop; the clock already elapsed @@ -241,6 +305,7 @@ RrCorrectionResult correctRr( cleanFraction: normalCount / n, droppedCount: dropped, correctedCount: corrected, + nnSourceIndex: source, ); } @@ -306,42 +371,63 @@ List _slidingThreshold( final n = x.length; final out = List.filled(n, 0); final half = win ~/ 2; + final w = _SortedWindow(x, half); for (var i = 0; i < n; i++) { - final lo = math.max(0, i - half); - final hi = math.min(n - 1, i + half); - final seg = []; - for (var k = lo; k <= hi; k++) { - seg.add(x[k]); - } - final q1 = percentile(seg, 25) ?? 0; - final q3 = percentile(seg, 75) ?? 0; - final qd = (q3 - q1) / 2; - // Floor keeps a gross outlier detectable on (near-)quantized clean data - // where the QD genuinely collapses to 0 (constant RR). On any series with - // real beat-to-beat variability α·QD dominates the floor, so the floor - // never governs a physiological signal. - out[i] = math.max(alpha * qd, floor); + w.advanceTo(i); + out[i] = thresholdOfSorted(w.a, alpha, floor); } return out; } +/// Median of the window around each beat EXCLUDING the beat itself (falls back +/// to the beat when the window holds nothing else). Same value as sorting the +/// window-minus-self per beat, but the window is kept sorted as it slides and +/// the self element is skipped by index arithmetic instead of copied out. List _slidingMedian(List x, int win) { final n = x.length; final out = List.filled(n, 0); final half = win ~/ 2; + final w = _SortedWindow(x, half); for (var i = 0; i < n; i++) { - final lo = math.max(0, i - half); - final hi = math.min(n - 1, i + half); - final seg = []; - for (var k = lo; k <= hi; k++) { - if (k == i) continue; - seg.add(x[k]); - } - out[i] = median(seg) ?? x[i]; + w.advanceTo(i); + out[i] = medianExcluding(w.a, x[i]) ?? x[i]; } return out; } +/// Ascending-sorted copy of `x[max(0,i-half) .. min(n-1,i+half)]`, maintained +/// incrementally as [advanceTo] steps i forward by one (O(log w) search plus a +/// w-element memmove per beat instead of an O(w log w) sort). +class _SortedWindow { + final List _x; + final int _half; + final _set = SortedMultiset(); + List get a => _set.a; + int _lo = 0; + int _hi = -1; // inclusive; -1 = empty + + _SortedWindow(this._x, this._half); + + void advanceTo(int i) { + final n = _x.length; + final newLo = i - _half < 0 ? 0 : i - _half; + final newHi = i + _half > n - 1 ? n - 1 : i + _half; + if (_hi < _lo) { + // First call: build once. + _set.resetFrom(_x.getRange(newLo, newHi + 1)); + } else { + while (_lo < newLo) { + _set.remove(_x[_lo++]); + } + while (_hi < newHi) { + _set.add(_x[++_hi]); + } + } + _lo = newLo; + _hi = newHi; + } +} + /// Catmull-Rom cubic interpolation at the artifact index using the nearest two /// NORMAL beats on each side. Returns null if anchors are unavailable. double? _splineCorrect(List rr, List isArtifact, int idx) { @@ -353,19 +439,5 @@ double? _splineCorrect(List rr, List isArtifact, int idx) { for (var k = idx + 1; k < rr.length && right.length < 2; k++) { if (!isArtifact[k]) right.add(rr[k]); } - if (left.isEmpty || right.isEmpty) return null; - final p1 = left.last; - final p2 = right.first; - final p0 = left.length >= 2 ? left.first : p1; - final p3 = right.length >= 2 ? right.last : p2; - // Catmull-Rom at t=0.5 between p1 and p2. - const t = 0.5; - final t2 = t * t; - final t3 = t2 * t; - final v = 0.5 * - ((2 * p1) + - (-p0 + p2) * t + - (2 * p0 - 5 * p1 + 4 * p2 - p3) * t2 + - (-p0 + 3 * p1 - 3 * p2 + p3) * t3); - return v; + return splineMid(left, right); } diff --git a/lib/src/onehz/foundations/rr_correction_kernel.dart b/lib/src/onehz/foundations/rr_correction_kernel.dart new file mode 100644 index 00000000..c0eeb7cf --- /dev/null +++ b/lib/src/onehz/foundations/rr_correction_kernel.dart @@ -0,0 +1,376 @@ +// Lipponen & Tarvainen 2019 classification with Peltola 2012 correction policy. +// The unpublished artifact-run cap is an internal rule, ESTIMATE with low +// confidence: +// over-cap runs become gaps and each side is re-corrected. Without a cap split +// the result equals uncapped correction. Shared kernel arithmetic and raw seam +// context preserve capped batch NN/classes; settled stream plus provisional tail +// equals capped batch. RrSandwich preserves source-beat absolute timestamps; +// uncapped arbitrarily long artifact runs have no exactness guarantee. +// FOUNDATION — the per-beat pieces of Lipponen–Tarvainen correction that both +// `correctRr` (whole series) and `RrCorrector` (streaming) run. +// +// One copy of the sorted window, the threshold, the leave-one-out median, the +// class decision, the compensatory-pair rule and the spline, so the batch and +// the stream cannot drift apart: the stream's contract is bit-identity with +// `correctRr`, and sharing the arithmetic is what keeps that cheap to hold. +// +// Not exported from `onehz.dart` — internal to foundations/. + +import 'dart:math' as math; +import '../util.dart'; +import 'rr_correction.dart' show BeatClass; + +/// First index whose element is not less than [v] under `compareTo` — the same +/// total order `List.sort()` uses (-0.0 < 0.0, NaN last), so a window +/// kept in this order matches the one a per-beat sort produced. +int lowerBound(List a, double v) { + var lo = 0; + var hi = a.length; + while (lo < hi) { + final m = (lo + hi) >> 1; + if (a[m].compareTo(v) < 0) { + lo = m + 1; + } else { + hi = m; + } + } + return lo; +} + +/// An ascending multiset of doubles: O(log w) search plus a w-element memmove +/// per add / remove, instead of an O(w log w) sort per beat. +class SortedMultiset { + /// The values, ascending. Read-only to callers. + final List a = []; + + void add(double v) => a.insert(lowerBound(a, v), v); + + /// Removes one occurrence of [v], which must be present. + void remove(double v) => a.removeAt(lowerBound(a, v)); + + void resetFrom(Iterable xs) { + a + ..clear() + ..addAll(xs) + ..sort(); + } +} + +/// `alpha × QD` of an ascending window, floored: QD = (Q3 − Q1)/2. +/// +/// Floor keeps a gross outlier detectable on (near-)quantized clean data where +/// the QD genuinely collapses to 0 (constant RR). On any series with real +/// beat-to-beat variability α·QD dominates the floor. +double thresholdOfSorted(List sorted, double alpha, double floor) { + final q1 = percentileSorted(sorted, 25) ?? 0; + final q3 = percentileSorted(sorted, 75) ?? 0; + final qd = (q3 - q1) / 2; + return math.max(alpha * qd, floor); +} + +/// `median(sorted minus one occurrence of v)` — [percentile]'s linear +/// interpolation at p = 50 over the sorted list with v's slot skipped, in the +/// same operation order so the result is bit-identical. Null if nothing is left. +double? medianExcluding(List sorted, double v) { + final len = sorted.length - 1; + if (len <= 0) return null; + final skip = lowerBound(sorted, v); + double at(int j) => j < skip ? sorted[j] : sorted[j + 1]; + if (len == 1) return at(0); + final rank = (50.0 / 100) * (len - 1); + final lo = rank.floor(); + final hi = rank.ceil(); + if (lo == hi) return at(lo); + final frac = rank - lo; + return at(lo) + (at(hi) - at(lo)) * frac; +} + +/// The paper's asymmetric deviation from the local median: below-median +/// deviations count double. +double medianDeviation(double rr, double med) { + final d = rr - med; + return d < 0 ? d * 2 : d; +} + +/// First-pass class of one beat from its dRR, mRR and the two thresholds. +BeatClass classifyBeat( + double rr, double dRR, double th1, double med, double mRR, double th2) { + final hardLong = rr > 2000; + final hardShort = rr < 300; + final bigJump = dRR.abs() > th1; + final bigDev = mRR.abs() > th2; + if (hardLong || (bigDev && mRR > 0)) { + // Long interval: likely a MISSED beat (interval ~ multiple of normal). + return (med > 0 && rr > 1.5 * med) ? BeatClass.missed : BeatClass.longShort; + } else if (hardShort || (bigDev && mRR < 0)) { + // Short interval: likely an EXTRA (spurious) beat. + return (med > 0 && rr < 0.6 * med) ? BeatClass.extra : BeatClass.longShort; + } else if (bigJump) { + return BeatClass.ectopic; + } + return BeatClass.normal; +} + +/// Compensatory-pair rule: beat k was flagged ONLY by its dRR jump, the beat +/// before it was an artifact, the two dRR spikes have opposite sign and k's own +/// value is normal and close to the local median — k is just the recovery from +/// the previous beat's event, so it is demoted to normal. +bool isRecoveryBeat({ + required bool prevIsArtifact, + required double rr, + required double dRR, + required double dRRPrev, + required double med, +}) { + if (!prevIsArtifact) return false; + final oppositeSign = dRR * dRRPrev < 0; + final valueNormal = + med > 0 && rr >= 300 && rr <= 2000 && (rr - med).abs() <= 0.2 * med; + return oppositeSign && valueNormal; +} + +/// Catmull-Rom at t = 0.5 between the nearest NORMAL beats either side of an +/// isolated artifact. [left] / [right] hold up to two values each, nearest to +/// the artifact LAST in [left] and FIRST in [right]. Null without an anchor on +/// both sides. +double? splineMid(List left, List right) { + if (left.isEmpty || right.isEmpty) return null; + final p1 = left.last; + final p2 = right.first; + final p0 = left.length >= 2 ? left.first : p1; + final p3 = right.length >= 2 ? right.last : p2; + const t = 0.5; + final t2 = t * t; + final t3 = t2 * t; + return 0.5 * + ((2 * p1) + + (-p0 + p2) * t + + (2 * p0 - 5 * p1 + 4 * p2 - p3) * t2 + + (-p0 + 3 * p1 - 3 * p2 + p3) * t3); +} + +/// Internal bounded-window evaluator shared by the capped batch and stream. +/// [from] has its complete left window in [rr], or is the segment's first beat. +/// Only classes at and after [from] are evaluated; the preceding class and +/// normal anchors are carried explicitly, so truncating old context cannot +/// change recovery or spline decisions. +class RrCapWindow { + final List classes; + final List values; + final List times; + final List gaps; + final List hardGaps; + RrCapWindow._(int n) + : classes = List.filled(n, BeatClass.normal), + values = List.filled(n, null), + times = List.filled(n, 0), + gaps = List.filled(n, false), + hardGaps = List.filled(n, false); + + static RrCapWindow evaluate( + List rr, { + List? ts, + required int cap, + double alpha = 5.2, + int windowBeats = 91, + double minThresholdMs = 100, + double reanchorGapMs = 1000, + int from = 0, + BeatClass previous = BeatClass.normal, + List leftNormals = const [], + bool leadingGap = false, + bool leadingHardGap = false, + int segmentStart = 0, + double clock = 0, + double? origin, + }) { + final out = RrCapWindow._(rr.length); + if (rr.isEmpty) return out; + final wall = ts != null && ts.length == rr.length; + final base = origin ?? (wall ? ts[0] - rr[0] : 0.0); + final h = windowBeats ~/ 2; + + List classify(int a, int b, int start, BeatClass prev) { + final n = b - a; + if (n < 3) { + return [ + for (var i = start; i < b; i++) + rr[i] >= 300 && rr[i] <= 2000 + ? BeatClass.normal + : BeatClass.longShort + ]; + } + final med = List.filled(n, 0); + final mrr = List.filled(n, 0); + final th1 = List.filled(n, 0); + final wd = SortedMultiset(), wr = SortedMultiset(); + var lo = a, hi = a; + double d(int i) => i == a ? 0 : rr[i] - rr[i - 1]; + for (var i = a; i < b; i++) { + final nextLo = math.max(a, i - h), nextHi = math.min(b, i + h + 1); + while (hi < nextHi) { + wd.add(d(hi)); + wr.add(rr[hi++]); + } + while (lo < nextLo) { + wd.remove(d(lo)); + wr.remove(rr[lo++]); + } + med[i - a] = medianExcluding(wr.a, rr[i]) ?? rr[i]; + mrr[i - a] = medianDeviation(rr[i], med[i - a]); + th1[i - a] = thresholdOfSorted(wd.a, alpha, minThresholdMs); + } + final wm = SortedMultiset(); + lo = a; + hi = a; + final cl = []; + for (var i = start; i < b; i++) { + final nextLo = math.max(a, i - h), nextHi = math.min(b, i + h + 1); + if (i == start) lo = nextLo; + if (i == start) hi = nextLo; + while (hi < nextHi) wm.add(mrr[hi++ - a]); + while (lo < nextLo) wm.remove(mrr[lo++ - a]); + var c = classifyBeat(rr[i], d(i), th1[i - a], med[i - a], mrr[i - a], + thresholdOfSorted(wm.a, alpha, minThresholdMs)); + if (i > a && + c == BeatClass.ectopic && + isRecoveryBeat( + prevIsArtifact: prev != BeatClass.normal, + rr: rr[i], + dRR: d(i), + dRRPrev: d(i - 1), + med: med[i - a])) { + c = BeatClass.normal; + } + cl.add(c); + prev = c; + } + return cl; + } + + // A stack avoids recursion depth proportional to the number of gaps. + final pending = <(int, int, int, BeatClass, List, bool, bool)>[ + ( + segmentStart, + rr.length, + from, + previous, + leftNormals, + leadingGap, + leadingHardGap + ) + ]; + while (pending.isNotEmpty) { + final (a, b, start, prev, anchors, inGap, hardLeading) = + pending.removeLast(); + if (start >= b) continue; + if (inGap && hardLeading && rr[start] >= 300 && rr[start] <= 2000) { + pending + .add((start, b, start, BeatClass.normal, const [], false, false)); + continue; + } + var hardCut = false; + var cutA = -1, cutB = -1; + // Physiologically impossible intervals are artifacts independently of + // every dispersion window. Cut these first: otherwise their medians can + // absorb valid beats on either side into the detected run itself. + for (var i = start; i < b;) { + if (rr[i] >= 300 && rr[i] <= 2000) { + i++; + continue; + } + var j = i + 1; + while (j < b && (rr[j] < 300 || rr[j] > 2000)) { + j++; + } + if (j - i > cap || (inGap && i == start)) { + cutA = i; + cutB = j; + hardCut = true; + break; + } + i = j; + } + final cl = cutA >= 0 ? [] : classify(a, b, start, prev); + if (cutA < 0 && inGap && cl.first == BeatClass.normal) { + pending + .add((start, b, start, BeatClass.normal, const [], false, false)); + continue; + } + if (cutA < 0) { + for (var i = start; i < b;) { + if (cl[i - start] == BeatClass.normal) { + i++; + continue; + } + var j = i + 1; + while (j < b && cl[j - start] != BeatClass.normal) { + j++; + } + if (j - i > cap || (inGap && i == start)) { + cutA = i; + cutB = j; + break; + } + i = j; + } + } + if (cutA >= 0) { + for (var i = cutA; i < cutB; i++) { + out.classes[i] = BeatClass.gap; + out.gaps[i] = true; + out.hardGaps[i] = hardCut || (inGap && hardLeading && cutA == start); + } + // Cut BEFORE classifying either side, including their dispersion + // windows. The gap's labels are canonical, never provisional medians. + pending.add((cutB, b, cutB, BeatClass.normal, const [], false, false)); + pending.add((a, cutA, start, prev, anchors, inGap, hardLeading)); + continue; + } + for (var i = start; i < b; i++) { + out.classes[i] = cl[i - start]; + } + final normals = List.of(anchors); + for (var i = start; i < b;) { + if (out.classes[i] == BeatClass.normal) { + out.values[i] = rr[i]; + normals.add(rr[i]); + if (normals.length > 2) normals.removeAt(0); + i++; + continue; + } + var j = i + 1; + while (j < b && out.classes[j] != BeatClass.normal) { + j++; + } + if (j == i + 1) { + final right = []; + for (var k = j; k < b && right.length < 2; k++) { + if (out.classes[k] == BeatClass.normal) right.add(rr[k]); + } + out.values[i] = splineMid(normals, right); + } + i = j; + } + } + var cur = clock; + var gap = leadingGap; + for (var i = from; i < rr.length; i++) { + final reset = gap && !out.gaps[i]; + final dropout = + wall && i > 0 && ts[i] - ts[i - 1] - rr[i] > reanchorGapMs; + final anchored = wall ? ts[i] - base : 0.0; + // Independent segments restart on recorded time, even if invalid RR + // accumulated ahead of it. A monotonic clamp would invent timestamps. + // HRV consumers refuse any regression in the retained beat clock. + cur = wall && reset + ? anchored + : dropout && anchored > cur + ? anchored + : cur + rr[i]; + out.times[i] = cur; + gap = out.gaps[i]; + } + return out; + } +} diff --git a/lib/src/onehz/incremental_core.dart b/lib/src/onehz/incremental_core.dart new file mode 100644 index 00000000..08e78d25 --- /dev/null +++ b/lib/src/onehz/incremental_core.dart @@ -0,0 +1,636 @@ +import 'dart:math' as math; + +import '../aggregate/json_support.dart' show decodeDouble, encodeDouble; +import 'util.dart'; + +/// Removable, mergeable centered moments, with an origin to retain precision +/// when a series has a large offset and a small spread. +// Welford, B. P. (1962). Note on a method for calculating corrected sums of +// squares and products. Technometrics 4(3):419-420. +// add uses Welford's recurrence; remove algebraically reverses that update, +// with a nonnegative M2 clamp. Partition merging follows +// Chan, T. F., Golub, G. H., LeVeque, R. J. (1983). Algorithms for computing +// the sample variance: analysis and recommendations. The American Statistician +// 37(3):242-247. +// Removable/mergeable centered-moment recurrence. Algebraically equivalent to +// batch moments; floating-point order differs. The long-run drift fixture checks +// 1e-9, not bit identity. +class RunningMoments { + int _count = 0; + double _origin = 0, _meanOffset = 0, _m2 = 0; + + RunningMoments(); + + factory RunningMoments.fromJson(Map json) { + _checkVersion(json, 'RunningMoments'); + final result = RunningMoments(); + result._count = _readCount(json['count']); + result._origin = _readFinite(json['origin']); + result._meanOffset = _readFinite(json['meanOffset']); + result._m2 = _readM2(json['m2']); + if (result._m2 < 0 || + (result._count == 0 && + (result._origin != 0 || + result._meanOffset != 0 || + result._m2 != 0)) || + (result._count == 1 && result._m2 != 0)) { + throw const FormatException('Invalid centered moments'); + } + return result; + } + + int get count => _count; + double? get mean => _count == 0 ? null : _origin + _meanOffset; + double? get sampleSd => _count < 2 ? null : math.sqrt(_m2 / (_count - 1)); + double? get populationSd => _count == 0 ? null : math.sqrt(_m2 / _count); + + void add(double x) { + if (!x.isFinite) throw ArgumentError.value(x, 'x', 'Must be finite'); + if (_count == 0) { + _origin = x; + _count = 1; + return; + } + final centered = x - _origin; + final delta = centered - _meanOffset; + _count++; + _meanOffset += delta / _count; + _m2 += delta * (centered - _meanOffset); + } + + /// The caller must remove a value represented by this summary. + void remove(double x) { + if (!x.isFinite) throw ArgumentError.value(x, 'x', 'Must be finite'); + if (_count == 0) throw StateError('Cannot remove from empty moments'); + if (_count == 1) { + _count = 0; + _origin = _meanOffset = _m2 = 0; + return; + } + final centered = x - _origin; + final oldMean = _meanOffset; + _count--; + _meanOffset += (_meanOffset - centered) / _count; + _m2 = math.max(0, _m2 - (centered - oldMean) * (centered - _meanOffset)); + if (_count == 1) _m2 = 0; + } + + void merge(RunningMoments other) { + if (other._count == 0) return; + if (_count == 0) { + _count = other._count; + _origin = other._origin; + _meanOffset = other._meanOffset; + _m2 = other._m2; + return; + } + final otherCount = other._count; + final combinedCount = _count + otherCount; + final delta = (other._origin - _origin) + other._meanOffset - _meanOffset; + _m2 += other._m2 + delta * delta * _count * otherCount / combinedCount; + _meanOffset += delta * otherCount / combinedCount; + _count = combinedCount; + } + + Map toJson() => { + 'version': 1, + 'type': 'RunningMoments', + 'count': _count, + 'origin': encodeDouble(_origin), + 'meanOffset': encodeDouble(_meanOffset), + 'm2': encodeDouble(_m2), + }; +} + +/// Exact order statistics of integer-valued data, kept as a sorted +/// `value -> count` list. Heart rate is whole bpm with a few dozen distinct +/// values a night, so the list stays tiny however many samples it holds. Add, +/// remove and merge are exact; [percentile] returns the same double +/// `percentileSorted` returns for the expanded sorted list, not an estimate. +/// +/// Only for data that really is integral. Values that are nearly all distinct +/// (accelerometer magnitudes) gain nothing: the list is as long as the data. +// Hyndman, R. J., Fan, Y. (1996). Sample quantiles in statistical packages. +// The American Statistician 50(4):361-365. +// percentileSorted implements type 7: zero-based rank = (n - 1) * p / 100, +// linearly interpolating adjacent ranks; this histogram preserves that rule. +// Exact histogram reformulation of percentileSorted(): counts identify identical +// expanded-list ranks with the same interpolation. No new estimator. +class IntHistogram { + static const _maxInt64 = 0x7fffffffffffffff; + + List _values = []; + List _counts = []; + int _count = 0; + + IntHistogram(); + + factory IntHistogram.fromJson(Map json) { + _checkVersion(json, 'IntHistogram'); + final values = json['values'], counts = json['counts']; + if (values is! List || counts is! List || values.length != counts.length) { + throw const FormatException('Invalid histogram bins'); + } + final result = IntHistogram(); + for (var i = 0; i < values.length; i++) { + final v = values[i], c = counts[i]; + if (v is! int || c is! int || c < 1 || (i > 0 && v <= values[i - 1])) { + throw const FormatException('Invalid histogram bins'); + } + if (result._count > _maxInt64 - c) { + throw const FormatException('Invalid histogram counts'); + } + result._values.add(v); + result._counts.add(c); + result._count += c; + } + return result; + } + + /// Number of values held, counting repeats. + int get count => _count; + + /// Number of different values held. + int get distinct => _values.length; + + /// Linear-interpolated median, or null when empty. + double? get median => percentile(50); + + /// Index of [value], or the insertion point as `-(point + 1)`. + int _find(int value) { + var lo = 0, hi = _values.length; + while (lo < hi) { + final mid = (lo + hi) >> 1; + if (_values[mid] < value) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo < _values.length && _values[lo] == value ? lo : -(lo + 1); + } + + /// Adds [times] copies of [value], which must be a finite whole number. + /// Throws [StateError] if the total count would overflow int64. + void add(num value, [int times = 1]) { + if (!value.isFinite || value != value.truncate()) { + throw ArgumentError.value(value, 'value', 'Must be a whole number'); + } + if (times < 1) throw ArgumentError.value(times, 'times', 'Must be positive'); + if (_count > _maxInt64 - times) { + throw StateError('Histogram count would overflow int64'); + } + final v = value.toInt(); + final at = _find(v); + if (at >= 0) { + _counts[at] += times; + } else { + _values.insert(-at - 1, v); + _counts.insert(-at - 1, times); + } + _count += times; + } + + /// Removes one copy of [value]. The caller must remove a value it added. + void remove(num value) { + final at = value.isFinite && value == value.truncate() + ? _find(value.toInt()) + : -1; + if (at < 0) throw StateError('Cannot remove a value that is not present'); + if (--_counts[at] == 0) { + _values.removeAt(at); + _counts.removeAt(at); + } + _count--; + } + + /// Merges [other] into this histogram. + /// Throws [StateError] if the total count would overflow int64. + void merge(IntHistogram other) { + if (_count > _maxInt64 - other._count) { + throw StateError('Histogram count would overflow int64'); + } + final values = List.of(other._values), counts = List.of(other._counts); + for (var i = 0; i < values.length; i++) { + add(values[i], counts[i]); + } + } + + /// The [p]th percentile (0 to 100), interpolated between the two + /// neighbouring order statistics exactly as `percentileSorted` does. Null + /// when empty, never 0. + double? percentile(double p) { + if (_count == 0) return null; + if (!(p >= 0 && p <= 100)) throw ArgumentError.value(p, 'p', '0 to 100'); + if (_count == 1) return _values[0].toDouble(); + final rank = (p / 100) * (_count - 1); + final lo = rank.floor(); + final hi = rank.ceil(); + final low = _orderStatistic(lo); + if (lo == hi) return low; + final frac = rank - lo; + return low + (_orderStatistic(hi) - low) * frac; + } + + /// The value at zero-based sorted position [k]. + double _orderStatistic(int k) { + var seen = 0; + for (var i = 0; i < _values.length; i++) { + seen += _counts[i]; + if (k < seen) return _values[i].toDouble(); + } + throw StateError('Order statistic out of range'); + } + + Map toJson() => { + 'version': 1, + 'type': 'IntHistogram', + 'values': List.of(_values), + 'counts': List.of(_counts), + }; +} + +/// Fixed-grid spectral sums. Raw snapshots establish that each sync is a +/// genuine prefix; callers may revise or remove any earlier sample. +// Press & Rybicki 1989 lombScargle() on the identical fixed grid using retained +// trigonometric sums and centered moments. Algebraically equal; long-run drift +// fixtures check 1e-9, not bit identity. +class IncrementalLombScargle { + final List _frequencies; + List _t = [], _y = []; + List> _sums; + RunningMoments _moments = RunningMoments(); + int _processedPoints = 0; + bool _incremental = true; + double _origin = 0, _yOrigin = 0, _tMin = 0, _tMax = 0; + + IncrementalLombScargle(List frequencies) + : _frequencies = List.of(frequencies), + _sums = [for (final _ in frequencies) List.filled(7, 0.0)]; + + factory IncrementalLombScargle.fromJson(Map json) { + _checkVersion(json, 'IncrementalLombScargle'); + final result = IncrementalLombScargle(_readDoubles(json['frequencies'])); + result._t = _readDoubles(json['t']); + result._y = _readDoubles(json['y']); + result._processedPoints = _readCount(json['processedPoints']); + final incremental = json['incremental']; + if (incremental is! bool || result._t.length != result._y.length) { + throw const FormatException('Invalid spectral input state'); + } + result._incremental = incremental; + result._origin = _readFinite(json['origin']); + result._yOrigin = _readFinite(json['yOrigin']); + result._tMin = _readFinite(json['tMin']); + result._tMax = _readFinite(json['tMax']); + final moments = json['moments']; + if (moments is! Map) { + throw const FormatException('Missing spectral moments'); + } + result._moments = RunningMoments.fromJson(moments); + final sums = json['sums']; + if (sums is! List || sums.length != result._frequencies.length) { + throw const FormatException('Invalid spectral sum dimensions'); + } + result._sums = [ + for (final row in sums) _readDoubles(row, finite: true), + ]; + if (result._sums.any((row) => row.length != 7) || + (incremental && + (!result._allFinite(result._t, result._y) || + result._moments.count != result._t.length || + result._processedPoints < result._t.length)) || + (!incremental && result._moments.count != 0)) { + throw const FormatException('Invalid spectral accumulator'); + } + if (result._t.isEmpty || !incremental) { + if (result._origin != 0 || + result._yOrigin != 0 || + result._tMin != 0 || + result._tMax != 0 || + result._sums.any((row) => row.any((value) => value != 0))) { + throw const FormatException('Invalid empty spectral state'); + } + } + if (incremental && result._t.isNotEmpty) { + if (result._origin != result._t.first || + result._yOrigin != result._y.first || + result._tMin != result._t.reduce(math.min) || + result._tMax != result._t.reduce(math.max)) { + throw const FormatException('Invalid spectral origin or extrema'); + } + for (final row in result._sums) { + if (row[2] < 0 || + row[3] < 0 || + ((row[2] + row[3]) - result._t.length).abs() > + 1e-8 * result._t.length) { + throw const FormatException('Invalid trigonometric sums'); + } + } + } + return result; + } + + int get processedPoints => _processedPoints; + + bool _allFinite(List t, List y) => + _frequencies.every((x) => x.isFinite) && + t.every((x) => x.isFinite) && + y.every((x) => x.isFinite); + + LombScargle? sync(List t, List y, {bool force = false}) { + if (t.length != y.length) { + _reset(); + return lombScargle(t, y, _frequencies); + } + if (!_allFinite(t, y)) { + // The batch function defines the NaN/infinity behavior. Never fold these + // values into summaries that could contaminate later finite prefixes. + final result = lombScargle(t, y, _frequencies); + _reset(); + _incremental = false; + _t = List.of(t); + _y = List.of(y); + if (t.length >= 4 && _frequencies.isNotEmpty) + _processedPoints += t.length; + return result; + } + var prefix = !force && _incremental && t.length >= _t.length; + if (prefix) { + for (var i = 0; i < _t.length; i++) { + if (_t[i] != t[i] || _y[i] != y[i]) { + prefix = false; + break; + } + } + } + if (!prefix) _reset(); + final start = _t.length; + if (start == 0 && t.isNotEmpty) { + _origin = _tMin = _tMax = t.first; + _yOrigin = y.first; + } + for (var i = start; i < t.length; i++) { + _moments.add(y[i]); + _tMin = math.min(_tMin, t[i]); + _tMax = math.max(_tMax, t[i]); + for (var j = 0; j < _frequencies.length; j++) { + final w = 2 * math.pi * _frequencies[j]; + final c = math.cos(w * (t[i] - _origin)); + final s = math.sin(w * (t[i] - _origin)); + final centeredY = y[i] - _yOrigin; + final sums = _sums[j]; + sums[0] += c; + sums[1] += s; + sums[2] += c * c; + sums[3] += s * s; + sums[4] += c * s; + sums[5] += centeredY * c; + sums[6] += centeredY * s; + } + _processedPoints++; + } + _t = List.of(t); + _y = List.of(y); + return _render(); + } + + void _reset() { + _t = []; + _y = []; + _sums = [for (final _ in _frequencies) List.filled(7, 0.0)]; + _moments = RunningMoments(); + _origin = _yOrigin = _tMin = _tMax = 0; + _incremental = true; + } + + LombScargle? _render() { + final n = _moments.count; + final span = _tMax - _tMin; + if (n < 4 || _frequencies.isEmpty || _moments._m2 <= 0 || span <= 0) { + return null; + } + final centeredMean = (_moments._origin - _yOrigin) + _moments._meanOffset; + final spectrum = []; + for (var j = 0; j < _frequencies.length; j++) { + final frequency = _frequencies[j]; + if (frequency == 0) { + spectrum.add(const LsPoint(0, 0)); + continue; + } + final sums = _sums[j]; + final phi = 0.5 * math.atan2(2 * sums[4], sums[2] - sums[3]); + final a = math.cos(phi), b = math.sin(phi); + final uc = sums[5] - centeredMean * sums[0]; + final us = sums[6] - centeredMean * sums[1]; + final cNum = a * uc + b * us, sNum = a * us - b * uc; + final cDen = a * a * sums[2] + b * b * sums[3] + 2 * a * b * sums[4]; + final sDen = a * a * sums[3] + b * b * sums[2] - 2 * a * b * sums[4]; + final term1 = cDen == 0 ? 0.0 : cNum * cNum / cDen; + final term2 = sDen == 0 ? 0.0 : sNum * sNum / sDen; + spectrum.add(LsPoint(frequency, span / (n - 1) * (term1 + term2))); + } + return LombScargle(spectrum); + } + + Map toJson() => { + 'version': 1, + 'type': 'IncrementalLombScargle', + 'frequencies': _writeDoubles(_frequencies), + 't': _writeDoubles(_t), + 'y': _writeDoubles(_y), + 'processedPoints': _processedPoints, + 'incremental': _incremental, + 'origin': _origin, + 'yOrigin': _yOrigin, + 'tMin': _tMin, + 'tMax': _tMax, + 'moments': _moments.toJson(), + 'sums': [for (final row in _sums) List.of(row)], + }; +} + +void _checkVersion(Map json, String type) { + if (json['version'] is! int || json['version'] != 1 || json['type'] != type) { + throw FormatException('Unsupported $type checkpoint'); + } +} + +int _readCount(Object? value) { + if (value is! int || value < 0) throw const FormatException('Invalid count'); + return value; +} + +double _readFinite(Object? value) { + if (value is! num || !value.isFinite) + throw const FormatException('Invalid number'); + return value.toDouble(); +} + +double _readM2(Object? value) { + final result = decodeDouble(value); + if ((!result.isFinite && value is! String) || + result.isNaN || + result.isNegative) { + throw const FormatException('Invalid number'); + } + return result; +} + +List _readDoubles(Object? value, {bool finite = false}) { + if (value is! List) throw const FormatException('Expected a numeric list'); + return [for (final item in value) _readDouble(item, finite: finite)]; +} + +double _readDouble(Object? value, {bool finite = false}) { + if (value is num && (!finite || value.isFinite)) return value.toDouble(); + if (!finite) { + if (value == 'NaN') return double.nan; + if (value == 'Infinity') return double.infinity; + if (value == '-Infinity') return double.negativeInfinity; + } + throw const FormatException('Invalid number'); +} + +List _writeDoubles(List values) => [ + for (final value in values) + if (value.isFinite) + value + else if (value.isNaN) + 'NaN' + else if (value > 0) + 'Infinity' + else + '-Infinity', + ]; + +class _CalculationEntry { + final Object? dependencies; + final Object? value; + const _CalculationEntry(this.dependencies, this.value); +} + +/// Bounded least-recently-used results with owned dependency snapshots. +class CalculationCache { + final int maxEntries; + final Map _entries = {}; + int _computations = 0, _hits = 0; + + CalculationCache({this.maxEntries = 128}) { + if (maxEntries < 1) throw ArgumentError.value(maxEntries, 'maxEntries'); + } + + int get computations => _computations; + int get hits => _hits; + + T evaluate(String key, Object? dependencies, T Function() calculate, + {bool full = false}) { + final old = _entries[key]; + if (!full && old != null && _deepEqual(old.dependencies, dependencies)) { + final result = _copyValue(old.value) as T; + _entries.remove(key); + _entries[key] = old; + _hits++; + return result; + } + // Snapshot before calling user code, which may itself mutate dependencies. + // Prepare both copies before committing, so exceptions preserve the old entry. + final snapshot = _snapshotDependency(dependencies); + final value = _copyValue(calculate()); + final result = _copyValue(value) as T; + _entries.remove(key); + _entries[key] = _CalculationEntry(snapshot, value); + while (_entries.length > maxEntries) { + _entries.remove(_entries.keys.first); + } + _computations++; + return result; + } + + void clear() => _entries.clear(); +} + +Object? _snapshotDependency(Object? value) { + if (value is List) + return [for (final item in value) _snapshotDependency(item)]; + if (value is Map) { + return { + for (final entry in value.entries) + entry.key: _snapshotDependency(entry.value) + }; + } + return value; +} + +bool _deepEqual(Object? a, Object? b) { + if (identical(a, b)) return true; + if (a is num && b is num && a.isNaN && b.isNaN) return true; + if (a is List && b is List) { + if (a.length != b.length) return false; + for (var i = 0; i < a.length; i++) { + if (!_deepEqual(a[i], b[i])) return false; + } + return true; + } + if (a is Map && b is Map) { + if (a.length != b.length) return false; + for (final key in a.keys) { + if (!b.containsKey(key) || !_deepEqual(a[key], b[key])) return false; + } + return true; + } + return a == b; +} + +Object? _copyValue(Object? value) { + if (value is List) { + // toList dispatches on the original List and keeps its reified E. + final copy = value.toList(); + for (var i = 0; i < copy.length; i++) { + copy[i] = _copyValue(value[i]); + } + return copy; + } + if (value is Map) { + // Preserve the usual JSON map types, including typed scalar/list values. + // Unlike List.toList, Map.map creates new key/value types from its callback. + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>>) + return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map>) return _copyMap(value); + if (value is Map) return _copyMap(value); + return _copyMap(value); + } + // Metric and other immutable domain values retain their identity and type. + return value; +} + +Map _copyMap(Map value) => { + for (final entry in value.entries) + entry.key: _copyValue(entry.value) as V, + }; diff --git a/lib/src/onehz/respiration/cvhr_apnea.dart b/lib/src/onehz/respiration/cvhr_apnea.dart index fe1ada73..348c2aff 100644 --- a/lib/src/onehz/respiration/cvhr_apnea.dart +++ b/lib/src/onehz/respiration/cvhr_apnea.dart @@ -127,6 +127,20 @@ Metric cvhrApneaScreen( } final tSec = [for (final t in nnTimesMs) t / 1000.0]; + if (tSec.any((t) => !t.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite beat time', + ); + } + if (hasRegressingBeatTimes(nnTimesMs)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'regressing beat time', + ); + } if (tSec.last - tSec.first <= 0) { return const Metric.absent( tier: Tier.high, diff --git a/lib/src/onehz/respiration/resp_rate.dart b/lib/src/onehz/respiration/resp_rate.dart index 6cd3452a..8e57c1d9 100644 --- a/lib/src/onehz/respiration/resp_rate.dart +++ b/lib/src/onehz/respiration/resp_rate.dart @@ -187,6 +187,20 @@ Metric rsaRespRate( ); } final tSec = [for (final t in nnTimesMs) t / 1000.0]; + if (tSec.any((t) => !t.isFinite)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'non-finite beat time', + ); + } + if (hasRegressingBeatTimes(nnTimesMs)) { + return const Metric.absent( + tier: Tier.high, + inputs_used: inputs, + note: 'regressing beat time', + ); + } final spanSec = tSec.last - tSec.first; if (spanSec <= 0) { return const Metric.absent( diff --git a/lib/src/onehz/sleep/circadian_np.dart b/lib/src/onehz/sleep/circadian_np.dart index c80c4789..d5365969 100644 --- a/lib/src/onehz/sleep/circadian_np.dart +++ b/lib/src/onehz/sleep/circadian_np.dart @@ -74,10 +74,19 @@ Metric circadianNonparametric( final lEp = l5Epochs ?? (epochsPerDay * 5 ~/ 24); final grand = mean(x)!; - var ssTot = 0.0; - for (final v in x) { - final d = v - grand; - ssTot += d * d; + // Welford (1962) centered variance; Witting (1990), van Someren (1999) + // indices retain their formulas. Circular profile sums keep input order. + var ssTot = 0.0, center = 0.0, count = 0; + for (final value in x) { + count++; + if (!value.isFinite) { + ssTot = double.nan; + center = x.take(count).fold(0, (a, b) => a + b) / count; + continue; + } + final delta = value - center; + center += delta / count; + ssTot += delta * (value - center); } if (ssTot == 0) { return const Metric.absent( diff --git a/lib/src/onehz/sleep/sleep.dart b/lib/src/onehz/sleep/sleep.dart index dc21184c..e37b330c 100644 --- a/lib/src/onehz/sleep/sleep.dart +++ b/lib/src/onehz/sleep/sleep.dart @@ -15,6 +15,9 @@ // honesty-bounded; stager.dart holds only the shared post-processing // (StagerResult, Webster rescore, consolidation) it reuses. The // autonomic-HR-only stager that used to live in stager.dart is DELETED. +// - CAUSAL (online) wake/NREM/REM stager (causal_stager.dart) — +// the same features/weights/cutoffs on past-only windows, abstains with a +// reason; for live decisions, NOT a replacement for cardioStager. // - Nonparametric circadian IS/IV/RA/L5/M10 (circadian_np.dart) // // Pure Dart, built on the package foundations (RR correction, Lomb-Scargle, @@ -27,6 +30,7 @@ export 'segment.dart'; export 'hr_fallback.dart'; export 'advanced_stager.dart'; export 'band_offset.dart'; + export 'sri.dart'; export 'accounting.dart'; // stager.dart provides StagerResult + the shared consolidation post-step. Its @@ -40,6 +44,10 @@ export 'cardio_stager.dart' cleanBeatsInWindowForTest, cleanBeatDiffsInWindowForTest, windowRmssdRkForTest; + + + + export 'circadian_np.dart'; export 'cycles.dart'; export 'night_hrv_shape.dart'; diff --git a/lib/src/onehz/util.dart b/lib/src/onehz/util.dart index 634880dd..e9ccd4a8 100644 --- a/lib/src/onehz/util.dart +++ b/lib/src/onehz/util.dart @@ -345,6 +345,7 @@ LombScargle? lombScargle(List t, List y, List freqsHz) { } final span = tMax - tMin; if (span <= 0) return null; + final t0 = t[0]; // One-sided periodogram → density: 2·Δt, with Δt the MEAN sample interval. final psdScale = 2.0 * span / (n - 1); @@ -358,14 +359,14 @@ LombScargle? lombScargle(List t, List y, List freqsHz) { // τ: phase reference for time-shift invariance. var sin2 = 0.0, cos2 = 0.0; for (final ti in t) { - sin2 += math.sin(2 * w * ti); - cos2 += math.cos(2 * w * ti); + sin2 += math.sin(2 * w * (ti - t0)); + cos2 += math.cos(2 * w * (ti - t0)); } final tau = math.atan2(sin2, cos2) / (2 * w); var cNum = 0.0, cDen = 0.0, sNum = 0.0, sDen = 0.0; for (var i = 0; i < n; i++) { - final arg = w * (t[i] - tau); + final arg = w * ((t[i] - t0) - tau); final c = math.cos(arg); final s = math.sin(arg); cNum += yc[i] * c; @@ -499,3 +500,14 @@ List benjaminiHochberg(List ps) { } return out; } + +/// Whether supplied beat times move backward. Equal times remain allowed: +/// band timestamps can be quantized to whole seconds. +bool hasRegressingBeatTimes(List times) { + for (var i = 1; i < times.length; i++) { + // Non-finite timestamps have their own refusal note at the call site. + if (times[i].isFinite && times[i - 1].isFinite && + times[i] < times[i - 1]) return true; + } + return false; +} diff --git a/lib/src/onehz/workout/hr_zones.dart b/lib/src/onehz/workout/hr_zones.dart index 16c66e37..d2efec62 100644 --- a/lib/src/onehz/workout/hr_zones.dart +++ b/lib/src/onehz/workout/hr_zones.dart @@ -182,33 +182,29 @@ class HeartRateZones { HeartRateZoneSet zoneSet, ) { final sorted = [...hr]..sort((a, b) => a.tsMs.compareTo(b.tsMs)); - final zoneSeconds = List.filled(5, 0); - var below = 0.0; - - final tailSeconds = - sampleCadenceSeconds([for (final s in sorted) s.tsMs / 1000.0]); - if (tailSeconds == null) return null; - for (var i = 0; i < sorted.length; i++) { - final sample = sorted[i]; - if (!sample.valid) continue; - final durSeconds = i < sorted.length - 1 - ? _boundedGapSeconds(sorted[i + 1].tsMs - sample.tsMs, tailSeconds) - : tailSeconds; - final zone = zoneSet.zoneNumber(sample.hr); - if (zone >= 1) { - zoneSeconds[zone - 1] += durSeconds; - } else { - below += durSeconds; + // Karvonen (1957) supports reserve zones only. Duration credit is an + // internal ESTIMATE rule. Count equal gaps before summing their credit. + final cadence = sampleCadenceSeconds([for (final s in sorted) s.tsMs / 1000.0]); + if (cadence == null) return null; + final credits = List.generate(6, (_) => {}); + for (var i = 1; i < sorted.length; i++) { + final previous = sorted[i - 1]; + if (!previous.valid) continue; + final gap = (sorted[i].tsMs - previous.tsMs) / 1000.0; + final histogram = credits[zoneSet.zoneNumber(previous.hr)]; + histogram[gap] = (histogram[gap] ?? 0) + 1; + } + final seconds = List.filled(6, 0); + for (var zone = 0; zone < credits.length; zone++) { + for (final entry in credits[zone].entries) { + final gap = entry.key; + seconds[zone] += entry.value * (gap > 0 ? math.min(gap, cadence) : cadence); } } - return TimeInHeartRateZone(seconds: zoneSeconds, belowZone1: below); - } - - static double _boundedGapSeconds(double gapMs, double fallbackSeconds) { - final gapSeconds = gapMs / 1000.0; - return gapSeconds > 0 - ? math.min(gapSeconds, fallbackSeconds) - : fallbackSeconds; + if (sorted.isNotEmpty && sorted.last.valid) { + seconds[zoneSet.zoneNumber(sorted.last.hr)] += cadence; + } + return TimeInHeartRateZone(seconds: seconds.sublist(1), belowZone1: seconds[0]); } } diff --git a/test/aggregate/aggregator_core_test.dart b/test/aggregate/aggregator_core_test.dart new file mode 100644 index 00000000..215f7724 --- /dev/null +++ b/test/aggregate/aggregator_core_test.dart @@ -0,0 +1,242 @@ +// Aggregator core: the Batch implementations, fold (with ranges) / run / +// runList, map and both. + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +/// Sum of the int rows. State is a one-slot list so combine/addSlice can mutate. +class _SumRows extends Aggregator, ListBatch, int> { + const _SumRows(); + @override + List empty() => [0]; + @override + List addSlice(List s, ListBatch b, int start, int end) { + for (var i = start; i < end; i++) { + s[0] += b.rows[i]; + } + return s; + } + + @override + List combine(List left, List right) { + left[0] += right[0]; + return left; + } + + @override + int finish(List s) => s[0]; +} + +/// Rows seen, in order, as a list (state mutated in place). +class _Rows extends Aggregator, ListBatch, List> { + const _Rows(); + @override + List empty() => []; + @override + List addSlice(List s, ListBatch b, int start, int end) => + s..addAll(b.rows.sublist(start, end)); + @override + List combine(List left, List right) => left..addAll(right); + @override + List finish(List s) => List.of(s); +} + +bool _same(T a, T b) => a.toString() == b.toString(); + +void main() { + group('ListBatch', () { + test('length, rows and copyRange', () { + final b = ListBatch(['a', 'b', 'c', 'd']); + expect(b.length, 4); + expect(b.rows, ['a', 'b', 'c', 'd']); + expect(b.copyRange(1, 3).rows, ['b', 'c']); + expect(b.copyRange(0, 4).rows, ['a', 'b', 'c', 'd']); + expect(b.copyRange(2, 2).length, 0); + expect(b.copyRange(4, 4).length, 0); + expect(b.copyRange(1, 3).length, 2); + }); + + test('copyRange is a copy: later changes to the source do not show', () { + final rows = [1, 2, 3, 4]; + final b = ListBatch(rows); + final c = b.copyRange(1, 4); + rows[1] = 99; + rows[3] = 77; + expect(c.rows, [2, 3, 4]); + }); + + test('join puts this before other and leaves both unchanged', () { + final a = ListBatch([1, 2]), b = ListBatch([3, 4, 5]); + final j = a.join(b); + expect(j.rows, [1, 2, 3, 4, 5]); + expect(j.length, 5); + expect(a.rows, [1, 2]); + expect(b.rows, [3, 4, 5]); + expect(a.join(ListBatch([])).rows, [1, 2]); + expect(ListBatch([]).join(b).rows, [3, 4, 5]); + expect(ListBatch([]).join(ListBatch([])).length, 0); + }); + }); + + group('DoubleColumn', () { + test('length, values and copyRange', () { + final c = col([1, 2, 3, 4]); + expect(c.length, 4); + expect(c.copyRange(1, 3).values, [2.0, 3.0]); + expect(c.copyRange(0, 4).values, [1.0, 2.0, 3.0, 4.0]); + expect(c.copyRange(3, 3).length, 0); + }); + + test('copyRange is a copy: later changes to the source do not show', () { + final v = Float64List.fromList([1, 2, 3, 4]); + final c = DoubleColumn(v).copyRange(1, 3); + v[1] = 99; + v[2] = 77; + expect(c.values, [2.0, 3.0]); + }); + + test('join, with NaN kept as a missing value', () { + final a = col([1, double.nan]), b = col([3]); + final j = a.join(b); + expect(j.length, 3); + expect(j.values[0], 1.0); + expect(j.values[1].isNaN, isTrue); + expect(j.values[2], 3.0); + expect(a.length, 2); + expect(b.values, [3.0]); + expect(a.join(col([])).length, 2); + expect(col([]).join(b).values, [3.0]); + }); + }); + + group('fold, run, runList', () { + const sum = _SumRows(); + final batch = ListBatch([for (var i = 1; i <= 5; i++) i]); + + test('run of nothing is the identity result', () { + expect(sum.run(ListBatch([])), 0); + expect(sum.runList([]), 0); + }); + + test('run equals finish(fold) and folds in order', () { + expect(sum.run(batch), 15); + expect(sum.finish(sum.fold(batch)), 15); + expect(const _Rows().run(batch), [1, 2, 3, 4, 5]); + }); + + test('runList is run over a ListBatch', () { + expect(sum.runList([4, 5, 6]), 15); + expect(const _Rows().runList([9, 8, 7]), [9, 8, 7]); + }); + + test('fold takes a half-open range; end defaults to the length', () { + expect(sum.finish(sum.fold(batch, 1, 4)), 2 + 3 + 4); + expect(sum.finish(sum.fold(batch, 0, 5)), 15); + expect(sum.finish(sum.fold(batch, 2)), 3 + 4 + 5); + expect(sum.finish(sum.fold(batch, 5)), 0); + expect(sum.finish(sum.fold(batch, 2, 2)), 0); + expect(sum.finish(sum.fold(batch, 5, 5)), 0); + expect(const _Rows().finish(const _Rows().fold(batch, 1, 3)), [2, 3]); + }); + + test('fold ranges combine back to the whole at every split', () { + final xs = ListBatch([for (var i = 1; i <= 20; i++) i]); + for (var k = 0; k <= 20; k++) { + final s = sum.combine(sum.fold(xs, 0, k), sum.fold(xs, k, 20)); + expect(sum.finish(s), 210, reason: 'split at $k'); + } + }); + + test('addSlice continues a state: several slices equal one', () { + var s = sum.empty(); + s = sum.addSlice(s, batch, 0, 2); + s = sum.addSlice(s, batch, 2, 2); + s = sum.addSlice(s, batch, 2, 5); + expect(sum.finish(s), 15); + }); + + test('empty() returns independent states', () { + final a = sum.empty(), b = sum.empty(); + sum.addSlice(a, batch, 0, 5); + expect(sum.finish(b), 0); + }); + + test('chunked and sliced runs agree with the whole run', () { + const agg = _Rows(); + final r = Rng(99); + for (final b in randomIntBatches(1)) { + for (var t = 0; t < 6; t++) { + final bounds = randomBounds(r, b.length); + expect(runChunked(agg, b, bounds), b.rows, reason: '$bounds'); + expect(runChunked(agg, b, bounds, copy: true), b.rows, + reason: 'copy $bounds'); + expect(runChunkedRight(agg, b, bounds), b.rows, reason: '$bounds'); + expect(runSliced(agg, b, bounds), b.rows, reason: '$bounds'); + } + } + }); + }); + + group('map', () { + test('transforms the result only', () { + final a = const _SumRows().map((n) => 'n=$n'); + expect(a.runList([1, 2, 3]), 'n=6'); + expect(a.runList([]), 'n=0'); + expectLawful(a, randomIntBatches(2)); + }); + + test('composes', () { + final a = Count>().map((n) => n * 2).map((n) => n + 1); + expect(a.runList([7, 7, 7]), 7); + }); + + test('keeps ranges and seams working', () { + final a = const _SumRows().map((n) => n * 10); + final b = ListBatch([1, 2, 3, 4]); + expect(a.finish(a.combine(a.fold(b, 0, 2), a.fold(b, 2, 4))), 100); + expect(a.finish(a.fold(b, 1, 3)), 50); + }); + }); + + group('both', () { + final a = both(const _SumRows(), Count>()); + + test('runs two aggregators over the same rows', () { + expect(a.runList([3, 4, 5]), (12, 3)); + expect(a.runList([]), (0, 0)); + }); + + test('nests and stays lawful', () { + final n = both(a, const _Rows()); + final r = n.runList([3, 9, 5]); // records holding lists compare by identity + expect(r.$1, (17, 3)); + expect(r.$2, [3, 9, 5]); + expectLawful(n, randomIntBatches(3), equals: _same); + }); + + test('each side has its own fresh state', () { + final p = a.addSlice(a.empty(), ListBatch([5]), 0, 1); + final q = a.empty(); + expect(a.finish(q), (0, 0)); + expect(a.finish(p), (5, 1)); + }); + + test('combines across a seam and honours fold ranges', () { + final b = ListBatch([1, 2, 3, 4]); + expect(a.finish(a.combine(a.fold(b, 0, 2), a.fold(b, 2, 4))), (10, 4)); + expect(a.finish(a.fold(b, 1, 3)), (5, 2)); + }); + + test('Pair is a mutable holder', () { + final p = Pair(1, 'a'); + p.first = 2; + p.second = 'b'; + expect((p.first, p.second), (2, 'b')); + }); + }); +} diff --git a/test/aggregate/ewma_prefix_test.dart b/test/aggregate/ewma_prefix_test.dart new file mode 100644 index 00000000..9fac2a73 --- /dev/null +++ b/test/aggregate/ewma_prefix_test.dart @@ -0,0 +1,286 @@ +// Affine, Ewma and Prefix (spec: research/r7-ewma.md). +// +// Decisions the implementation follows: +// * `const Affine(double scale, double offset)`, positional; identity is +// `Affine(1, 0)`; `apply(x) = scale * x + offset`; +// `a.then(b)` = b after a = (b.scale*a.scale, b.scale*a.offset + b.offset). +// * `Ewma(col, lambda)` is an Aggregator<_, B, Affine>, positional args. +// Its Affine maps the running value c to the value after all rows: +// one row x is c -> (1 - lambda) * c + lambda * x. NaN rows are skipped +// (identity). No rows (or only NaN rows) finish as Affine(1, 0). +// * `Prefix(k, head, tail)` (type args inferred) is an +// Aggregator<_, B, (R1, R2)>. Head sees rows [0, k) of the whole input via +// head.run on stored rows (so head.combine is never called), tail sees rows +// [k, n) in order. k = 0 gives head an empty batch. The stored prefix is a +// copy: mutating the source column after folding changes nothing. + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +const nan = double.nan; + +List _aff(Affine a) => [a.scale, a.offset]; +final _affEq = fieldsEqual(_aff, relTol: 1e-12, absTol: 1e-15); + + +Ewma _ewma(double lambda) => + Ewma((b) => b.values, lambda); + +/// Reference: apply x_i one by one to the running value, skipping NaN. +double _loop(double seed, List xs, double lambda) { + var c = seed; + for (final x in xs) { + if (x.isNaN) continue; + c = c + lambda * (x - c); + } + return c; +} + +void _expectAffine(Affine got, Affine want, String tag) => expect( + _affEq(got, want), isTrue, + reason: '$tag got ${_aff(got)} want ${_aff(want)}'); + +typedef _Rows = List; + +/// Records every row it is shown, in order. Head variant never combines. +class _Rec extends Aggregator<_Rows, DoubleColumn, _Rows> { + const _Rec({this.combinable = true}); + final bool combinable; + @override + _Rows empty() => []; + @override + _Rows addSlice(_Rows s, DoubleColumn b, int start, int end) => + s..addAll([for (var i = start; i < end; i++) b.values[i]]); + @override + _Rows combine(_Rows l, _Rows r) { + if (!combinable) throw StateError('head.combine must not be called'); + return l..addAll(r); + } + + @override + _Rows finish(_Rows s) => List.of(s); +} + +DoubleColumn _iota(int n) => col([for (var i = 0; i < n; i++) i * 1.0]); + +void _is(Object? got, Object? want, {String? reason}) => + expect(got.toString(), want.toString(), reason: reason); + +bool _str(Object? a, Object? b) => a.toString() == b.toString(); + +void main() { + group('Affine', () { + test('apply, identity and then', () { + const a = Affine(2, 3); + const b = Affine(5, -1); + expect(a.apply(4), 11); + expect(const Affine(1, 0).apply(7.5), 7.5); + final ab = a.then(b); // b after a + expect(ab.scale, 10); + expect(ab.offset, 5 * 3 + -1); + for (final x in [-3.0, 0.0, 1.5, 100.0]) { + expect(ab.apply(x), b.apply(a.apply(x))); + } + _expectAffine(const Affine(1, 0).then(a), a, 'id.then(a)'); + _expectAffine(a.then(const Affine(1, 0)), a, 'a.then(id)'); + }); + + test('then is associative (seeded)', () { + final r = Rng(0xAF1); + Affine rnd() => Affine((r.nextDouble() - .5) * 4, (r.nextDouble() - .5) * 100); + for (var t = 0; t < 100; t++) { + final a = rnd(), b = rnd(), c = rnd(); + _expectAffine(a.then(b).then(c), a.then(b.then(c)), 't$t'); + } + }); + }); + + group('Ewma', () { + test('one row: (1 - lambda) c + lambda x', () { + final f = _ewma(0.25).run(col([8])); + expect(f.scale, closeTo(0.75, 1e-15)); + expect(f.offset, closeTo(2, 1e-15)); + expect(f.apply(4), closeTo(0.75 * 4 + 2, 1e-15)); + }); + + test('no rows, and only NaN rows, are the identity', () { + final e = _ewma(0.1); + _expectAffine(e.run(col([])), const Affine(1, 0), 'empty'); + _expectAffine(e.finish(e.empty()), const Affine(1, 0), 'empty state'); + _expectAffine(e.run(col([nan, nan, nan])), const Affine(1, 0), 'nans'); + }); + + test('values equal a direct loop (seeded, several lambdas)', () { + final r = Rng(0xE111); + for (final lambda in [0.5, 1 - 0.9, 0.02, 0.1331, 1.0, 0.0]) { + final e = _ewma(lambda); + for (var t = 0; t < 40; t++) { + final n = r.intIn(0, 30); + final xs = [ + for (var i = 0; i < n; i++) + r.nextBool(.2) ? nan : (r.nextDouble() - .3) * 500 + ]; + final seed = (r.nextDouble() - .5) * 300; + final got = e.run(col(xs)).apply(seed); + final want = _loop(seed, xs, lambda); + expect(got, closeTo(want, 1e-12 * (1 + want.abs())), + reason: 'lambda=$lambda t=$t'); + } + } + }); + + test('NaN rows are skipped, not decayed', () { + final e = _ewma(0.3); + final a = e.run(col([10, nan, 20, nan, nan, 5])); + final b = e.run(col([10, 20, 5])); + _expectAffine(a, b, 'nan vs removed'); + }); + + test('combine is left.then(right): chunking does not matter', () { + final r = Rng(0xE112); + final e = _ewma(0.07); + for (var t = 0; t < 60; t++) { + final n = r.intIn(0, 60); + final c = col([ + for (var i = 0; i < n; i++) r.nextBool(.1) ? nan : r.nextDouble() * 400 + ]); + final want = e.run(c); + final b = randomBounds(r, n); + _expectAffine(runChunked(e, c, b), want, 'chunked $b'); + _expectAffine(runChunked(e, c, b, copy: true), want, 'copy $b'); + _expectAffine(runChunkedRight(e, c, b), want, 'right $b'); + _expectAffine(runSliced(e, c, b), want, 'sliced $b'); + _expectAffine(runChunked(e, c, singletonBounds(n)), want, 'single'); + } + }); + + test('is lawful', () { + expectLawful(_ewma(0.05), randomColumns(0xE113, nanP: .2, maxLen: 40), + equals: _affEq); + }); + }); + + group('Prefix', () { + for (final k in [0, 1, 3, 7]) { + group('k = $k', () { + final p = Prefix(k, const _Rec(combinable: false), const _Rec()); + + (List, List) want(int n) => ( + [for (var i = 0; i < n && i < k; i++) i * 1.0], + [for (var i = k; i < n; i++) i * 1.0], + ); + + test('head sees exactly rows [0, k), tail rows [k, n), inputs of ' + 'every length', () { + for (var n = 0; n <= 20; n++) { + _is(p.run(_iota(n)), want(n), reason: 'n=$n'); + } + }); + + test('every single and double cut, all four evaluation paths', () { + for (final n in [0, 1, k, k + 1, 12]) { + final c = _iota(n); + for (var a = 0; a <= n; a++) { + for (var b = a; b <= n; b++) { + final bd = [0, a, b, n]; + _is(runChunked(p, c, bd), want(n), reason: 'chunk $bd'); + _is(runChunked(p, c, bd, copy: true), want(n), + reason: 'copy $bd'); + _is(runChunkedRight(p, c, bd), want(n), reason: 'right $bd'); + _is(runSliced(p, c, bd), want(n), reason: 'sliced $bd'); + } + } + } + }); + + test('random chunkings and slicings, empty chunks included', () { + final r = Rng(0x9F00 + k); + for (var t = 0; t < 80; t++) { + final n = r.intIn(0, 40); + final c = _iota(n); + final b = randomBounds(r, n); + _is(runChunked(p, c, b), want(n), reason: 'chunk $b n=$n'); + _is(runChunked(p, c, b, copy: true), want(n), reason: 'copy'); + _is(runChunkedRight(p, c, b), want(n), reason: 'right $b n=$n'); + _is(runSliced(p, c, b), want(n), reason: 'sliced $b n=$n'); + final s = singletonBounds(n); + _is(runChunked(p, c, s), want(n), reason: 'singletons'); + _is(runChunkedRight(p, c, s), want(n), reason: 'right singles'); + _is(runSliced(p, c, s), want(n), reason: 'sliced singles'); + } + }); + + test('is lawful with a recording head and tail', () { + expectLawful(p, [for (var n = 0; n <= 14; n++) _iota(n), _iota(40)], + equals: _str); + }); + + test('is lawful with Count and Sum', () { + final eq = fieldsEqual<(int, double)>((x) => [x.$1, x.$2], + relTol: 1e-12, absTol: 1e-9); + expectLawful( + Prefix(k, Count(), Sum((b) => b.values)), + randomColumns(0x9F10 + k, nanP: .2), + equals: eq); + final eq2 = fieldsEqual<(double, int)>((x) => [x.$1, x.$2], + relTol: 1e-12, absTol: 1e-9); + expectLawful( + Prefix(k, Sum((b) => b.values), Count()), + randomColumns(0x9F20 + k, nanP: .2), + equals: eq2); + }); + }); + } + + test('k = 3: seams of one-row chunks fill the prefix then the tail', () { + final p = Prefix(3, const _Rec(combinable: false), const _Rec()); + final c = _iota(6); + var s = p.fold(c, 0, 1); + s = p.combine(s, p.fold(c, 1, 2)); + _is(p.finish(s), ([0, 1], [])); + s = p.combine(s, p.fold(c, 2, 3)); + s = p.combine(s, p.fold(c, 3, 4)); + _is(p.finish(s), ([0, 1, 2], [3])); + // A right chunk bigger than the room left in the left prefix. + final r = p.combine(p.fold(c, 0, 2), p.fold(c, 2, 6)); + _is(p.finish(r), ([0, 1, 2], [3, 4, 5])); + // Empty left, empty right. + _is(p.finish(p.combine(p.empty(), p.fold(c, 0, 6))), ([0, 1, 2], [3, 4, 5])); + _is(p.finish(p.combine(p.fold(c, 0, 6), p.empty())), ([0, 1, 2], [3, 4, 5])); + }); + + test('the stored prefix is a copy of the rows', () { + final p = Prefix(3, const _Rec(combinable: false), const _Rec()); + final c = _iota(6); + final s = p.fold(c, 0, 6); + final s2 = p.fold(c, 0, 2); + for (var i = 0; i < 6; i++) { + c.values[i] = -99; + } + _is(p.finish(s), ([0, 1, 2], [3, 4, 5])); + _is(p.finish(s2), ([0, 1], [])); + }); + + test('composes with Ewma as the tail: mean seed, then decay', () { + final p = Prefix( + 4, Sum((b) => b.values), _ewma(0.2)); + final r = Rng(0x9F30); + for (var t = 0; t < 40; t++) { + final n = r.intIn(0, 30); + final xs = [for (var i = 0; i < n; i++) r.nextDouble() * 300]; + final c = col(xs); + final b = randomBounds(r, n); + final (sum, f) = runChunked(p, c, b); + final head = n < 4 ? n : 4; + expect(sum, closeTo(xs.take(head).fold(0.0, (a, x) => a + x), 1e-9)); + final seed = 50.0; + final got = f.apply(seed); + final want = _loop(seed, xs.skip(4).toList(), 0.2); + expect(got, closeTo(want, 1e-9 * (1 + want.abs())), reason: 't$t $b'); + } + }); + }); +} diff --git a/test/aggregate/group_by_test.dart b/test/aggregate/group_by_test.dart new file mode 100644 index 00000000..c6e064a3 --- /dev/null +++ b/test/aggregate/group_by_test.dart @@ -0,0 +1,259 @@ +// GroupBy: consecutive rows with equal key go to the inner aggregator as one +// slice; a key seen again later is combined into its existing state; finish +// maps every key through inner.finish. + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart' show RunningMoments; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +Float64List _v(DoubleColumn c) => c.values; +int _mod3(DoubleColumn c, int i) => c.values[i].toInt() % 3; +int _mod2(DoubleColumn c, int i) => c.values[i].toInt() % 2; +int _tens(DoubleColumn c, int i) => c.values[i] ~/ 10; + +/// Exact deep equality for (nested) maps; Map == is identity in Dart. +bool _mapsEqual(Object? a, Object? b) { + if (a is Map && b is Map) { + if (a.length != b.length) return false; + for (final k in a.keys) { + if (!b.containsKey(k) || !_mapsEqual(a[k], b[k])) return false; + } + return true; + } + return a == b; +} + +List _momentFields(Map m) { + final keys = m.keys.toList()..sort(); + return [ + keys.length, + for (final k in keys) ...[k, m[k]!.count, m[k]!.mean, m[k]!.sampleSd], + ]; +} + +/// Records every slice the inner is handed, as a list of its rows. +class _Slices extends Aggregator>, DoubleColumn, List>> { + const _Slices(); + @override + List> empty() => >[]; + @override + List> addSlice( + List> s, DoubleColumn b, int start, int end) { + if (end > start) s.add(b.values.sublist(start, end)); + return s; + } + + @override + List> combine(List> l, List> r) => + l..addAll(r); + @override + List> finish(List> s) => s; +} + +DoubleColumn _ints(List xs) => col([for (final x in xs) x * 1.0]); + +void main() { + final count3 = GroupBy(_mod3, Count()); + final sum3 = GroupBy(_mod3, Sum(_v)); + + group('values', () { + test('counts per key', () { + expect(count3.run(_ints([1, 2, 3, 4, 5, 6, 7])), {0: 2, 1: 3, 2: 2}); + }); + + test('sums per key', () { + final a = GroupBy(_mod2, Sum(_v)); + expect(a.run(_ints([1, 2, 3, 4, 5])), {1: 9.0, 0: 6.0}); + }); + + test('empty input gives an empty map', () { + expect(count3.run(col([])), isEmpty); + expect(count3.finish(count3.empty()), {}); + }); + + test('a key that never occurs is absent', () { + expect(count3.run(_ints([3, 6])).keys, [0]); + }); + + test('the key function sees the batch and a row index', () { + final rows = []; + final a = GroupBy( + (b, i) { + rows.add(i); + return 0; + }, + Count()); + a.run(_ints([5, 6, 7])); + expect(rows.toSet(), {0, 1, 2}); + }); + + test('works over a ListBatch', () { + final a = GroupBy, dynamic, int>( + (b, i) => b.rows[i].length, Count>()); + expect(a.runList(['a', 'bb', 'c', 'dd', 'e']), {1: 3, 2: 2}); + }); + }); + + group('runs of equal keys', () { + const slices = _Slices(); + final g = GroupBy>>(_tens, slices); + + test('consecutive equal keys reach the inner as one slice', () { + final r = g.run(col([10, 11, 12, 20, 21])); + expect(r, { + 1: [ + [10.0, 11.0, 12.0] + ], + 2: [ + [20.0, 21.0] + ], + }); + }); + + test('a key that reappears later is combined into its existing state, ' + 'in row order', () { + final r = g.run(col([10, 11, 20, 21, 22, 12, 30, 13, 14])); + expect(r.keys.toSet(), {1, 2, 3}); + expect(r[1]!.expand((s) => s).toList(), [10.0, 11.0, 12.0, 13.0, 14.0]); + expect(r[2], [ + [20.0, 21.0, 22.0] + ]); + expect(r[3], [ + [30.0] + ]); + // each run stayed together; the two runs of key 1 stay separate slices + expect(r[1], [ + [10.0, 11.0], + [12.0], + [13.0, 14.0] + ]); + }); + + test('alternating keys give one-row slices', () { + final r = g.run(col([10, 20, 11, 21, 12])); + expect(r[1], [ + [10.0], + [11.0], + [12.0] + ]); + expect(r[2], [ + [20.0], + [21.0] + ]); + }); + + test('a fold range groups only the rows inside it', () { + final c = col([10, 11, 20, 21, 12, 13]); + final r = g.finish(g.fold(c, 1, 5)); + expect(r[1], [ + [11.0], + [12.0] + ]); + expect(r[2], [ + [20.0, 21.0] + ]); + }); + + test('a run cut by a slice boundary arrives as two slices', () { + final c = col([10, 11, 12, 13]); + var s = g.empty(); + s = g.addSlice(s, c, 0, 2); + s = g.addSlice(s, c, 2, 4); + final r = g.finish(s); + expect(r[1]!.expand((x) => x).toList(), [10.0, 11.0, 12.0, 13.0]); + }); + }); + + group('combine', () { + test('disjoint keys are adopted', () { + final s = sum3.combine(sum3.fold(_ints([3, 6])), sum3.fold(_ints([1, 4]))); + expect(sum3.finish(s), {0: 9.0, 1: 5.0}); + }); + + test('overlapping keys merge with the inner combine', () { + final s = sum3.combine( + sum3.fold(_ints([1, 2, 3])), sum3.fold(_ints([4, 5, 6]))); + expect(sum3.finish(s), {0: 9.0, 1: 5.0, 2: 7.0}); + }); + + test('mixed shared and one-sided keys', () { + final s = sum3.combine(sum3.fold(_ints([1, 3])), sum3.fold(_ints([4, 5]))); + expect(sum3.finish(s), {0: 3.0, 1: 5.0, 2: 5.0}); + }); + + test('combining with empty on either side changes nothing', () { + final xs = _ints([1, 2, 3, 4]); + expect(sum3.finish(sum3.combine(sum3.empty(), sum3.fold(xs))), + sum3.run(xs)); + expect(sum3.finish(sum3.combine(sum3.fold(xs), sum3.empty())), + sum3.run(xs)); + }); + + test('order within a key is preserved across a seam', () { + final g = GroupBy>>( + _tens, const _Slices()); + final c = col([10, 20, 11, 12, 21, 13]); + final r = g.finish(g.combine(g.fold(c, 0, 3), g.fold(c, 3, 6))); + expect(r[1]!.expand((s) => s).toList(), [10.0, 11.0, 12.0, 13.0]); + expect(r[2]!.expand((s) => s).toList(), [20.0, 21.0]); + }); + }); + + group('laws', () { + final cols = randomColumns(0x6B01, nanP: 0, value: (r) => r.intIn(-20, 20) * 1.0); + + test('GroupBy(key, Count) is lawful with exact equality', () { + expectLawful(count3, cols, equals: _mapsEqual); + }); + + test('GroupBy(key, Sum) is lawful (integer-valued sums are exact)', () { + expectLawful(sum3, cols, equals: _mapsEqual); + }); + + test('nested GroupBy', () { + final nested = GroupBy>( + _mod2, GroupBy(_mod3, Sum(_v))); + expect(nested.run(_ints([1, 2, 3, 4, 5, 6])), { + 1: {1: 1.0, 0: 3.0, 2: 5.0}, + 0: {2: 2.0, 1: 4.0, 0: 6.0}, + }); + expect(nested.run(col([])), isEmpty); + expectLawful(nested, cols, equals: _mapsEqual); + }); + + test('tolerance-compared inner (Moments per key)', () { + final g = GroupBy( + _mod3, Moments(_v)); + final r = g.run(_ints([1, 2, 3, 4, 5, 10])); + expect(r.keys.toSet(), {0, 1, 2}); + expect(r[1]!.count, 3); + expect(r[1]!.mean, closeTo(5.0, 1e-12)); // (1+4+10)/3 + expect(r[1]!.sampleSd, closeTo(4.58257569495584, 1e-9)); + expectLawful(g, cols, + equals: fieldsEqual>(_momentFields, + relTol: 1e-9, absTol: 1e-9)); + }); + + test('random chunkings and slicings agree with the whole fold', () { + final r = Rng(0x6B02); + for (var t = 0; t < 40; t++) { + final c = _ints([for (var i = r.intIn(0, 60); i > 0; i--) r.intIn(0, 9)]); + final b = randomBounds(r, c.length); + final want = sum3.run(c); + expect(_mapsEqual(runChunked(sum3, c, b), want), isTrue, + reason: 'chunked $b'); + expect(_mapsEqual(runChunked(sum3, c, b, copy: true), want), isTrue, + reason: 'copies $b'); + expect(_mapsEqual(runChunkedRight(sum3, c, b), want), isTrue, + reason: 'right $b'); + expect(_mapsEqual(runSliced(sum3, c, b), want), isTrue, + reason: 'sliced $b'); + } + }); + }); +} diff --git a/test/aggregate/last_rows_test.dart b/test/aggregate/last_rows_test.dart new file mode 100644 index 00000000..e71cfcd2 --- /dev/null +++ b/test/aggregate/last_rows_test.dart @@ -0,0 +1,75 @@ +// LastRows(k): a verbatim copy of the last min(k, n) rows of the whole input, +// null for an empty input. Persistable when batchFromJson is given. + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/checkpoint_support.dart'; + +const nan = double.nan; + +List _vals(DoubleColumn? c) => [c == null, ...?c?.values]; +final _same = fieldsEqual(_vals); // NaN-aware, exact + +LastRows _last(int k) => + LastRows(k, batchFromJson: DoubleColumn.fromJson); + +void main() { + group('values', () { + final x = col([1, 2, 3, 4, 5]); + test('k = 1, 3, 8 on a longer and a shorter input', () { + expect(_last(1).run(x)!.values.toList(), [5]); + expect(_last(3).run(x)!.values.toList(), [3, 4, 5]); + expect(_last(8).run(x)!.values.toList(), [1, 2, 3, 4, 5]); // n < k + expect(_last(3).run(col([7, nan]))!.values.toList(), [7, isNaN]); + }); + test('empty input is null, also through finish(empty())', () { + for (final k in [1, 3, 8]) { + expect(_last(k).run(col([])), isNull); + expect(_last(k).finish(_last(k).empty()), isNull); + } + }); + test('honours the range and returns a copy', () { + final a = _last(3); + expect(a.finish(a.fold(x, 0, 4))!.values.toList(), [2, 3, 4]); + expect(a.finish(a.fold(x, 2, 2)), isNull); + final got = a.run(x)!; + got.values[0] = 99; + expect(x.values.toList(), [1, 2, 3, 4, 5]); + }); + test('combine keeps the last k of left ++ right', () { + final a = _last(3); + List at(int i, int j) => a + .finish(a.combine(a.fold(x, 0, i), a.fold(x, i, j)))! + .values + .toList(); + expect(at(4, 5), [3, 4, 5]); // right has 1 row: 2 from left + expect(at(2, 5), [3, 4, 5]); // right alone has k rows + expect(at(3, 4), [2, 3, 4]); + }); + }); + + group('laws', () { + for (final k in [1, 3, 8]) { + test('k=$k is lawful (slicing, finishPure, round trip)', () { + expectLawful(_last(k), randomColumns(40 + k), equals: _same); + }); + } + }); + + group('persistence', () { + test('persistable only with batchFromJson', () { + expect(LastRows(3).persistable, isFalse); + expect(_last(3).persistable, isTrue); + }); + test('a mid-stream checkpoint resumes to the one-pass result', () { + final c = randomColumns(77, count: 1, maxLen: 30).last; + for (final k in [1, 3, 8]) { + expectCheckpointResumes(_last(k), col([for (var i = 0; i < 20; i++) i * 1.5]), + equals: _same); + expectCheckpointResumes(_last(k), c, equals: _same); + } + }); + }); +} diff --git a/test/aggregate/laws_harness_test.dart b/test/aggregate/laws_harness_test.dart new file mode 100644 index 00000000..9cd219c9 --- /dev/null +++ b/test/aggregate/laws_harness_test.dart @@ -0,0 +1,707 @@ +// The law harness itself: lawful aggregators pass, deliberately broken ones are +// caught under the right law name (including 'slicing'), and fieldsEqual +// compares as documented. + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; + +typedef _B = ListBatch; + +int _sliceSum(_B b, int start, int end) { + var t = 0; + for (var i = start; i < end; i++) { + t += b.rows[i]; + } + return t; +} + +/// A correct sum over int rows. +class _Sum extends Aggregator { + const _Sum(); + @override + int empty() => 0; + @override + int addSlice(int s, _B b, int start, int end) => s + _sliceSum(b, start, end); + @override + int combine(int left, int right) => left + right; + @override + int finish(int s) => s; +} + +/// Sum whose identity is 1: breaks the identity laws. +class _SumFromOne extends _Sum { + const _SumFromOne(); + @override + int empty() => 1; +} + +/// Combine drops the right side. +class _DropRight extends _Sum { + const _DropRight(); + @override + int combine(int left, int right) => left; +} + +/// addSlice ignores the state it is given (restarts from the slice), so a +/// second addSlice onto a state that already holds data loses it. fold (one +/// addSlice onto empty) and combine are both correct, so only 'slicing' sees it. +class _IgnoresState extends _Sum { + const _IgnoresState(); + @override + int addSlice(int s, _B b, int start, int end) => _sliceSum(b, start, end); +} + +/// addSlice always sums the whole batch, whatever range it is asked for. +class _IgnoresRange extends _Sum { + const _IgnoresRange(); + @override + int addSlice(int s, _B b, int start, int end) => + s + _sliceSum(b, 0, b.length); +} + +/// Mean kept as (mean, count); combine averages the two means unweighted. +class _UnweightedMean extends Aggregator, _B, double> { + const _UnweightedMean(); + @override + List empty() => [0, 0]; + @override + List addSlice(List s, _B b, int start, int end) { + for (var i = start; i < end; i++) { + final n = s[1] + 1; + s = [s[0] + (b.rows[i] - s[0]) / n, n]; + } + return s; + } + + @override + List combine(List l, List r) { + if (r[1] == 0) return l; + if (l[1] == 0) return r; + return [(l[0] + r[0]) / 2, l[1] + r[1]]; + } + + @override + double finish(List s) => s[0]; +} + +/// addSlice counts 1 per row, but joining two NON-empty chunks adds a bonus, so +/// the add path and the combine path disagree. Identities still hold. +class _JoinBonus extends Aggregator { + const _JoinBonus(); + @override + int empty() => 0; + @override + int addSlice(int s, _B b, int start, int end) => s + (end - start); + @override + int combine(int left, int right) => + left > 0 && right > 0 ? left + right + 1 : left + right; + @override + int finish(int s) => s; +} + +/// Wrong only when exactly 3 rows meet exactly 2 (a split of a 5-row batch). +class _WrongAtThreeTwo extends Aggregator, _B, int> { + const _WrongAtThreeTwo(); + @override + List empty() => [0, 0]; // sum, count + @override + List addSlice(List s, _B b, int start, int end) => + [s[0] + _sliceSum(b, start, end), s[1] + (end - start)]; + @override + List combine(List l, List r) => + [l[0] + r[0] + (l[1] == 3 && r[1] == 2 ? 1000 : 0), l[1] + r[1]]; + @override + int finish(List s) => s[0]; +} + +/// A correct sum that mutates and consumes both arguments in combine. +class _MutatingSum extends Aggregator, _B, int> { + const _MutatingSum(); + @override + List empty() => [0]; + @override + List addSlice(List s, _B b, int start, int end) { + s[0] += _sliceSum(b, start, end); + return s; + } + + @override + List combine(List left, List right) { + left[0] += right[0]; + right[0] = -999999; // consumed + return left; + } + + @override + int finish(List s) => s[0]; +} + +/// Same, but wipes `left` too and returns a brand new state. +class _ConsumingSum extends _MutatingSum { + const _ConsumingSum(); + @override + List combine(List left, List right) { + final out = [left[0] + right[0]]; + left[0] = -1; + right[0] = -1; + return out; + } +} + +/// A sum whose finish reads the state and then clears it. A single finish per +/// fresh state looks right, so only a law that finishes the SAME state twice +/// (or keeps using it afterwards) can see the damage. +class _DestructiveSum extends Aggregator, _B, int> { + const _DestructiveSum(); + @override + List empty() => [0]; + @override + List addSlice(List s, _B b, int start, int end) { + s[0] += _sliceSum(b, start, end); + return s; + } + + @override + List combine(List left, List right) { + left[0] += right[0]; + return left; + } + + @override + int finish(List s) { + final out = s[0]; + s[0] = 0; // destructive + return out; + } +} + +/// finish returns the right number the first time, but marks the state as +/// closed so later rows are ignored. Two finishes in a row agree; continuing to +/// add afterwards does not. +class _FinishFreezes extends Aggregator, _B, int> { + const _FinishFreezes(); + @override + List empty() => [0, 0]; // sum, frozen + @override + List addSlice(List s, _B b, int start, int end) { + if (s[1] == 0) s[0] += _sliceSum(b, start, end); + return s; + } + + @override + List combine(List left, List right) { + if (left[1] == 0) left[0] += right[0]; + return left; + } + + @override + int finish(List s) { + s[1] = 1; + return s[0]; + } +} + +/// A correct sum whose codec only breaks when the restored state is the RIGHT +/// operand of combine: a state read back from JSON carries a flag, and combine +/// drops the right side's total when that flag is set. Restoring only the left +/// operand (the old roundTrip law) never sees it. +class _RightRestoreDrops extends Aggregator, _B, int> { + const _RightRestoreDrops({this.faithful = false}); + + /// When true the same codec is lawful: combine keeps a restored right side. + final bool faithful; + @override + List empty() => [0, 0]; // total, restored flag + @override + List addSlice(List s, _B b, int start, int end) { + s[0] += _sliceSum(b, start, end); + return s; + } + + @override + List combine(List left, List right) => + [left[0] + (right[1] == 1 && !faithful ? 0 : right[0]), left[1]]; + @override + int finish(List s) => s[0]; + @override + bool get persistable => true; + @override + Object? encodeState(List s) => s[0]; + @override + List decodeState(Object? json) => [json as int, 1]; +} + +Set _laws(List v) => {for (final x in v) x.law}; + +void main() { + final cases = randomIntBatches(11, lo: 1, hi: 50); + + group('lawful aggregators', () { + test('a correct sum has no violations', () { + expect(checkLaws(const _Sum(), cases), isEmpty); + }); + test('a mutating combine is judged lawful when otherwise lawful', () { + expect(checkLaws(const _MutatingSum(), cases), isEmpty); + expect(checkLaws(const _ConsumingSum(), cases), isEmpty); + }); + test('no cases is vacuously lawful', () { + expect(checkLaws(const _Sum(), <_B>[]), isEmpty); + }); + test('empty and one-row cases alone are fine', () { + expect(checkLaws(const _Sum(), [_B([]), _B([5])]), isEmpty); + }); + test('long cases are bounded by maxSplits and still pass', () { + final long = _B([for (var i = 0; i < 300; i++) i]); + expect(checkLaws(const _Sum(), [long], maxSplits: 8), isEmpty); + }); + test('the harness leaves the case batches unchanged', () { + final b = _B([5, 6, 7, 8]); + checkLaws(const _Sum(), [b]); + expect(b.rows, [5, 6, 7, 8]); + }); + test('expectLawful passes for a lawful aggregator and fails for a broken ' + 'one', () { + expectLawful(const _Sum(), cases); + expect(() => expectLawful(const _DropRight(), cases), + throwsA(isA())); + }); + }); + + group('broken aggregators are caught', () { + test('(a) non-identity empty fires the identity laws', () { + final laws = _laws(checkLaws(const _SumFromOne(), cases)); + expect(laws, containsAll(['leftIdentity', 'rightIdentity'])); + }); + + test('(b) combine that drops the right side fires split', () { + final v = checkLaws(const _DropRight(), cases); + expect(_laws(v), contains('split')); + expect(v.every((x) => x.detail.isNotEmpty), isTrue); + }); + + test('(c) unweighted mean combine fires split or associativity', () { + bool close(double a, double b) => (a - b).abs() < 1e-9; + final laws = + _laws(checkLaws(const _UnweightedMean(), cases, equals: close)); + expect(laws.intersection({'split', 'associativity'}), isNotEmpty); + }); + + test('(d) addSlice inconsistent with combine fires singletons (identities ' + 'hold)', () { + final laws = _laws(checkLaws(const _JoinBonus(), cases)); + expect(laws, contains('singletons')); + expect(laws, contains('split')); + expect(laws, isNot(contains('leftIdentity'))); + expect(laws, isNot(contains('rightIdentity'))); + }); + + test('(e) addSlice that ignores the state on a second call fires only ' + '"slicing"', () { + final laws = _laws(checkLaws(const _IgnoresState(), cases)); + expect(laws, {'slicing'}); + }); + + test('(f) addSlice that ignores its range fires split and slicing', () { + final laws = _laws(checkLaws(const _IgnoresRange(), cases)); + expect(laws, containsAll(['split', 'slicing'])); + }); + + test('every split point of a short case is tried', () { + // Only the 3|2 split of a five-row batch is wrong. + final laws = _laws(checkLaws(const _WrongAtThreeTwo(), [ + _B([1, 2, 3, 4, 5]) + ])); + expect(laws, contains('split')); + }); + + test('a lawful case alongside a broken one still reports the broken one', + () { + final v = checkLaws(const _DropRight(), [ + _B([]), + _B([5]), + _B([1, 2, 3]) + ]); + expect(_laws(v), contains('split')); + }); + + test('results are deterministic for a given seed', () { + String render(List v) => + v.map((x) => '${x.law}|${x.detail}').join('\n'); + final a = checkLaws(const _IgnoresState(), cases, seed: 5); + final b = checkLaws(const _IgnoresState(), cases, seed: 5); + expect(render(a), render(b)); + expect(a, isNotEmpty); + }); + + test('a custom equals is honoured', () { + // Everything is "equal", so even a broken aggregator passes. + expect(checkLaws(const _DropRight(), cases, equals: (a, b) => true), + isEmpty); + expect(checkLaws(const _IgnoresState(), cases, equals: (a, b) => true), + isEmpty); + }); + }); + + group('roundTrip restores the right operand too', () { + final small = [ + _B([]), + _B([1, 2, 3]), + _B([4, 5]), + ]; + + test('a codec that breaks only as the right operand is caught as exactly ' + '{roundTrip}', () { + final v = checkLaws(const _RightRestoreDrops(), small); + expect(_laws(v), {'roundTrip'}); + expect(v.every((x) => x.detail.isNotEmpty), isTrue); + }); + + test('the review case [1,2,3] is caught', () { + expect( + _laws(checkLaws(const _RightRestoreDrops(), [ + _B([1, 2, 3]) + ])), + {'roundTrip'}); + }); + + test('checkConversion reports it as law:roundTrip', () { + final f = checkConversion, _B, int>( + (_) => const _RightRestoreDrops(), + small, + (b) => b.rows.fold(0, (s, x) => s + x)); + expect({for (final x in f) x.path}, {'law:roundTrip'}); + }); + + test('expectLawful fails for it', () { + expect(() => expectLawful(const _RightRestoreDrops(), small), + throwsA(isA())); + }); + + test('the same codec with a combine that keeps a restored right side is ' + 'clean', () { + expect(checkLaws(const _RightRestoreDrops(faithful: true), small), + isEmpty); + }); + }); + + group('finishPure', () { + test('a sum whose finish clears the state is caught with exactly ' + '{finishPure}', () { + expect(_laws(checkLaws(const _DestructiveSum(), cases)), {'finishPure'}); + }); + + test('a finish that closes the state to later rows is caught with exactly ' + '{finishPure}', () { + expect(_laws(checkLaws(const _FinishFreezes(), cases)), {'finishPure'}); + }); + + test('the violation names the case', () { + final v = checkLaws(const _DestructiveSum(), [ + _B([1, 2, 3]) + ]); + expect(v, isNotEmpty); + expect(v.every((x) => x.law == 'finishPure' && x.detail.isNotEmpty), + isTrue); + }); + + test('lawful aggregators report no finishPure violation', () { + for (final agg in >[ + const _Sum(), + const _MutatingSum(), + const _ConsumingSum(), + ]) { + expect(checkLaws(agg, cases), isEmpty, reason: '$agg'); + } + }); + + test('expectLawful fails for a destructive finish', () { + expect(() => expectLawful(const _DestructiveSum(), cases), + throwsA(isA())); + }); + }); + + group('fieldsEqual', () { + List same(List x) => x; + + test('two ints compare exactly, with no double conversion', () { + final eq = fieldsEqual>(same, relTol: 1e-9, absTol: 1e-9); + expect(eq([9007199254740992], [9007199254740993]), isFalse); + expect(eq([9007199254740993], [9007199254740993]), isTrue); + final loose = fieldsEqual>(same, relTol: 0.5, absTol: 10); + expect(loose([100], [101]), isFalse); + }); + + test('tolerance is absTol + relTol * max(|a|, |b|)', () { + final eq = fieldsEqual>(same, absTol: 0.1, relTol: 0.05); + // 0.15 <= 0.1 + 0.05 * 1.15; the max(absTol, rel) rule would say no. + expect(eq([1.0], [1.15]), isTrue); + expect(eq([1.15], [1.0]), isTrue); + expect(eq([1.0], [1.2]), isFalse); + }); + + test('a finite value never equals a non-finite one', () { + final eq = fieldsEqual>(same, absTol: 1e300, relTol: 1); + expect(eq([1.0], [double.infinity]), isFalse); + expect(eq([double.negativeInfinity], [1.0]), isFalse); + expect(eq([double.maxFinite], [double.infinity]), isFalse); + expect(eq([1.0], [double.nan]), isFalse); + expect(eq([double.nan], [double.infinity]), isFalse); + }); + + test('NaN equals NaN and each infinity only itself, tolerances or not', () { + final eq = fieldsEqual>(same, absTol: 1, relTol: 1); + expect(eq([double.nan], [double.nan]), isTrue); + expect(eq([double.infinity], [double.infinity]), isTrue); + expect(eq([double.negativeInfinity], [double.negativeInfinity]), isTrue); + expect(eq([double.infinity], [double.negativeInfinity]), isFalse); + }); + + test('an int and a double compare with tolerance', () { + final eq = fieldsEqual>(same, absTol: 0.1); + expect(eq([1], [1.05]), isTrue); + expect(eq([1], [1.5]), isFalse); + }); + + test('identical lists are equal; length mismatch is not', () { + final eq = fieldsEqual>(same); + expect(eq([1, 'a', null], [1, 'a', null]), isTrue); + expect(eq([1, 2], [1, 2, 3]), isFalse); + }); + + test('NaN equals NaN, but not a number', () { + final eq = fieldsEqual>(same); + expect(eq([double.nan], [double.nan]), isTrue); + expect(eq([double.nan], [1.0]), isFalse); + expect(eq([1.0], [double.nan]), isFalse); + }); + + test('null equals only null', () { + final eq = fieldsEqual>(same, relTol: 1, absTol: 1); + expect(eq([null], [null]), isTrue); + expect(eq([null], [0.0]), isFalse); + expect(eq([0.0], [null]), isFalse); + }); + + test('zero tolerance is exact', () { + final eq = fieldsEqual>(same); + expect(eq([1.0], [1.0]), isTrue); + expect(eq([1.0], [1.0000000000000002]), isFalse); + expect(eq([1], [1.0]), isTrue); + }); + + test('relTol scales with magnitude', () { + final eq = fieldsEqual>(same, relTol: 1e-9); + expect(eq([1000.0], [1000.0000001]), isTrue); // 1e-7 abs, 1e-10 rel + expect(eq([1000.0], [1000.00001]), isFalse); // 1e-5 abs, 1e-8 rel + expect(eq([0.0], [1e-12]), isFalse); // relTol alone gives nothing at 0 + }); + + test('absTol covers values near zero', () { + final eq = fieldsEqual>(same, absTol: 1e-6); + expect(eq([0.0], [1e-7]), isTrue); + expect(eq([0.0], [1e-5]), isFalse); + }); + + test('infinities match only themselves', () { + final eq = fieldsEqual>(same, relTol: 1e-9, absTol: 1e-9); + expect(eq([double.infinity], [double.infinity]), isTrue); + expect(eq([double.infinity], [double.negativeInfinity]), isFalse); + }); + + test('non-numeric fields compare with ==', () { + final eq = fieldsEqual>(same, relTol: 1, absTol: 1); + expect(eq(['a'], ['a']), isTrue); + expect(eq(['a'], ['b']), isFalse); + expect(eq([true], [true]), isTrue); + expect(eq([1], ['1']), isFalse); + }); + + test('the projection is applied to both sides', () { + final eq = + fieldsEqual<({int a, double b})>((r) => [r.a, r.b], absTol: 1e-6); + expect(eq((a: 1, b: 2.0), (a: 1, b: 2.0000001)), isTrue); + expect(eq((a: 1, b: 2.0), (a: 2, b: 2.0)), isFalse); + }); + + group('list fields', () { + test('equal-valued distinct List fields are equal', () { + final eq = fieldsEqual>(same); + expect( + eq([ + [1.0, 2.5] + ], [ + [1.0, 2.5] + ]), + isTrue); + }); + + test('Float64List equals a List with the same values', () { + final eq = fieldsEqual>(same); + final typed = Float64List.fromList([1.0, 2.5]); + expect( + eq([typed], [ + [1.0, 2.5] + ]), + isTrue); + expect( + eq([ + [1.0, 2.5] + ], [typed]), + isTrue); + }); + + test('elements use the numeric tolerance', () { + final eq = fieldsEqual>(same, relTol: 1e-3); + expect( + eq([ + [1000.0, 1.0] + ], [ + [1000.5, 1.0] + ]), + isTrue); + expect( + eq([ + [1000.0, 1.0] + ], [ + [1002.0, 1.0] + ]), + isFalse); + }); + + test('different lengths are unequal (guard)', () { + final eq = fieldsEqual>(same, absTol: 1); + expect( + eq([ + [1.0, 2.0] + ], [ + [1.0] + ]), + isFalse); + }); + + test('NaN elements equal each other', () { + final eq = fieldsEqual>(same); + expect( + eq([ + [double.nan, 1.0] + ], [ + [double.nan, 1.0] + ]), + isTrue); + expect( + eq([ + [double.nan] + ], [ + [1.0] + ]), + isFalse); + }); + + test('finite never equals non-finite inside a list (guard)', () { + final eq = fieldsEqual>(same, absTol: 1e300, relTol: 1); + expect( + eq([ + [1.0] + ], [ + [double.infinity] + ]), + isFalse); + }); + + test('int elements compare exactly', () { + final eq = fieldsEqual>(same, relTol: 1e-9, absTol: 1e-9); + expect( + eq([ + [9007199254740992] + ], [ + [9007199254740993] + ]), + isFalse); + expect( + eq([ + [9007199254740993] + ], [ + [9007199254740993] + ]), + isTrue); + }); + + test('null elements equal only null', () { + final eq = fieldsEqual>(same, absTol: 1); + expect( + eq([ + [null, 1.0] + ], [ + [null, 1.0] + ]), + isTrue); + expect( + eq([ + [null] + ], [ + [0.0] + ]), + isFalse); + }); + + test('nested lists recurse', () { + final eq = fieldsEqual>(same, absTol: 0.01); + expect( + eq([ + [ + [1.0, 2.0], + [3.0] + ] + ], [ + [ + [1.001, 2.0], + [3.0] + ] + ]), + isTrue); + expect( + eq([ + [ + [1.0, 2.0], + [3.0] + ] + ], [ + [ + [1.0, 2.0], + [3.5] + ] + ]), + isFalse); + }); + + test('a list never equals a non-list (guard)', () { + final eq = fieldsEqual>(same, absTol: 10, relTol: 1); + expect( + eq([ + [1.0] + ], [1.0]), + isFalse); + expect( + eq([1.0], [ + [1.0] + ]), + isFalse); + expect( + eq([ + [] + ], [null]), + isFalse); + }); + + test('empty lists are equal', () { + final eq = fieldsEqual>(same); + expect(eq([[]], [[]]), isTrue); + expect(eq([Float64List(0)], [[]]), isTrue); + }); + }); + }); +} diff --git a/test/aggregate/lookback_test.dart b/test/aggregate/lookback_test.dart new file mode 100644 index 00000000..4d9c45a9 --- /dev/null +++ b/test/aggregate/lookback_test.dart @@ -0,0 +1,255 @@ +// Lookback(k, inner): the inner aggregator sees every length-k window of the +// whole batch exactly once, in order, however the batch is sliced or chunked. +// +// The probe inner records each window it is shown. Per the contract, inner +// .addSlice(s, b, start, end) must itself handle the windows ending at rows +// start+k-1 .. end-1 (reading rows i-k+1..i of b), so the recorder does that +// and nothing else. Row values are their own index, so a window identifies +// itself. + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/fold_law_support.dart' show foldBounds; +import '../support/property.dart' show Rng; + +typedef _Windows = List>; + +class _Record extends Aggregator<_Windows, DoubleColumn, _Windows> { + const _Record(this.k); + final int k; + @override + _Windows empty() => >[]; + @override + _Windows addSlice(_Windows s, DoubleColumn b, int start, int end) { + for (var i = start + k - 1; i < end; i++) { + s.add([for (var j = i - k + 1; j <= i; j++) b.values[j]]); + } + return s; + } + + @override + _Windows combine(_Windows left, _Windows right) => left..addAll(right); + @override + _Windows finish(_Windows s) => [for (final w in s) List.of(w)]; +} + +Lookback _lookback(int k) => + Lookback(k, _Record(k)); + +/// n rows whose values are 0, 1, 2, ... (distinct, so windows identify). +DoubleColumn _iota(int n) => col([for (var i = 0; i < n; i++) i * 1.0]); + +_Windows _expected(int n, int k) => [ + for (var i = k - 1; i < n; i++) + [for (var j = i - k + 1; j <= i; j++) j * 1.0] + ]; + +bool _same(Object? a, Object? b) => a.toString() == b.toString(); + +void main() { + for (final k in [1, 2, 3, 4]) { + group('k = $k', () { + final lb = _lookback(k); + + test('a single slice sees every window of the batch', () { + for (final n in [0, 1, k - 1, k, k + 1, 2 * k, 17]) { + if (n < 0) continue; + expect(lb.run(_iota(n)), _expected(n, k), reason: 'n=$n'); + } + }); + + test('a fold range sees only windows inside the range', () { + final c = _iota(12); + final got = lb.finish(lb.fold(c, 3, 9)); + expect(got, [ + for (var i = 3 + k - 1; i < 9; i++) + [for (var j = i - k + 1; j <= i; j++) j * 1.0] + ]); + }); + + test('every single split point (combine)', () { + final c = _iota(14); + for (var s = 0; s <= 14; s++) { + final r = lb.finish(lb.combine(lb.fold(c, 0, s), lb.fold(c, s, 14))); + expect(r, _expected(14, k), reason: 'split at $s'); + } + }); + + test('every two-slice cut on the slicing path (addSlice after data)', () { + final c = _iota(14); + for (var s = 0; s <= 14; s++) { + var st = lb.empty(); + st = lb.addSlice(st, c, 0, s); + st = lb.addSlice(st, c, s, 14); + expect(lb.finish(st), _expected(14, k), reason: 'cut at $s'); + } + }); + + test('every three-slice cut on both paths', () { + final c = _iota(10); + for (var a = 0; a <= 10; a++) { + for (var b = a; b <= 10; b++) { + final bounds = [0, a, b, 10]; + expect(runChunked(lb, c, bounds), _expected(10, k), + reason: 'chunked $bounds'); + expect(runChunkedRight(lb, c, bounds), _expected(10, k), + reason: 'right $bounds'); + expect(runSliced(lb, c, bounds), _expected(10, k), + reason: 'sliced $bounds'); + expect(runChunked(lb, c, bounds, copy: true), _expected(10, k), + reason: 'copies $bounds'); + } + } + }); + + test('random slicings and chunkings, empty chunks included', () { + final r = Rng(0x10B0 + k); + for (var t = 0; t < 80; t++) { + final n = r.intIn(0, 40); + final c = _iota(n); + final b = foldBounds(n, r.intIn(0, 9), r.nextInt(1 << 20), + special: [0, 1, 2, 3, 4]); + final want = _expected(n, k); + expect(runChunked(lb, c, b), want, reason: 'chunked $b n=$n'); + expect(runChunked(lb, c, b, copy: true), want, reason: 'copy $b n=$n'); + expect(runChunkedRight(lb, c, b), want, reason: 'right $b n=$n'); + expect(runSliced(lb, c, b), want, reason: 'sliced $b n=$n'); + } + }); + + test('one row per chunk and per slice', () { + for (final n in [0, 1, 2, 3, 9]) { + final c = _iota(n); + final b = singletonBounds(n); + expect(runChunked(lb, c, b), _expected(n, k), reason: 'chunks n=$n'); + expect(runChunked(lb, c, b, copy: true), _expected(n, k), + reason: 'copies n=$n'); + expect(runChunkedRight(lb, c, b), _expected(n, k), + reason: 'right n=$n'); + expect(runSliced(lb, c, b), _expected(n, k), reason: 'slices n=$n'); + } + }); + + test('empty chunks around and between data change nothing', () { + final c = _iota(8); + final want = _expected(8, k); + expect(runChunked(lb, c, [0, 0, 0, 3, 3, 3, 8, 8]), want); + expect(runSliced(lb, c, [0, 0, 3, 3, 8, 8]), want); + expect( + lb.finish(lb.combine(lb.combine(lb.empty(), lb.fold(c)), lb.empty())), + want); + expect( + lb.finish(lb.combine(lb.fold(c, 0, 4), + lb.combine(lb.empty(), lb.fold(c, 4, 8)))), + want); + }); + + test('is lawful', () { + expectLawful(lb, [for (var n = 0; n <= 12; n++) _iota(n), _iota(40)], + equals: _same); + }); + }); + } + + group('chunks shorter than k-1 rows', () { + test('k = 4: all chunks of 1 or 2 rows, mixed with empties', () { + final lb = _lookback(4); + final c = _iota(11); + final want = _expected(11, 4); + for (final b in [ + [0, 1, 2, 4, 5, 6, 8, 9, 11], + [0, 2, 3, 3, 4, 6, 7, 9, 10, 11], + [0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + ]) { + expect(runChunked(lb, c, b), want, reason: '$b'); + expect(runChunked(lb, c, b, copy: true), want, reason: 'copy $b'); + expect(runChunkedRight(lb, c, b), want, reason: 'right $b'); + expect(runSliced(lb, c, b), want, reason: 'sliced $b'); + } + }); + + test('k = 3: a seam of two one-row chunks yields no window by itself', () { + final lb = _lookback(3); + final c = _iota(3); + expect(lb.finish(lb.combine(lb.fold(c, 0, 1), lb.fold(c, 1, 2))), isEmpty); + final s = lb.combine( + lb.combine(lb.fold(c, 0, 1), lb.fold(c, 1, 2)), lb.fold(c, 2, 3)); + expect(lb.finish(s), [ + [0.0, 1.0, 2.0] + ]); + }); + + test('k = 4: no window when the whole batch is shorter than k', () { + final lb = _lookback(4); + expect(lb.run(_iota(3)), isEmpty); + expect(runChunked(lb, _iota(3), [0, 1, 2, 3]), isEmpty); + expect(runSliced(lb, _iota(3), [0, 1, 2, 3]), isEmpty); + }); + }); + + group('seam batches are tiny', () { + test('combining two 50-row chunks feeds the inner one seam of 2*(k-1) rows', + () { + // The inner records the length of every slice it is handed. The two + // folds hand it 50 rows each; the combine hands it exactly one more + // slice: the whole seam batch, last(k-1) + first(k-1) = 4 rows. + final seen = []; + final probe = _Spy(3, seen); + final lb = Lookback(3, probe); + final c = _iota(100); + lb.combine(lb.fold(c, 0, 50), lb.fold(c, 50, 100)); + expect(seen, [50, 50, 4]); + }); + }); + + group('results of the inner pass through', () { + test('Lookback(2, absolute successive differences)', () { + final lb = Lookback, double>( + 2, _AbsDiffSum()); + final c = col([1, 5, 2, 2, 10]); + expect(lb.run(c), 4 + 3 + 0 + 8); + expect(lb.run(col([])), 0.0); + expect(lb.run(col([7])), 0.0); + expectLawful(lb, randomColumns(0x10B9, nanP: 0), + equals: fieldsEqual((x) => [x], relTol: 1e-12, absTol: 1e-9)); + }); + + test('composes with both and map', () { + final a = both( + Lookback(2, _Record(2)), + Count()).map((r) => (r.$1.length, r.$2)); + expect(a.run(_iota(5)), (4, 5)); + expect(a.finish(a.combine(a.fold(_iota(5), 0, 2), a.fold(_iota(5), 2, 5))), + (4, 5)); + }); + }); +} + +class _Spy extends _Record { + _Spy(super.k, this.sizes); + final List sizes; + @override + _Windows addSlice(_Windows s, DoubleColumn b, int start, int end) { + sizes.add(end - start); + return super.addSlice(s, b, start, end); + } +} + +class _AbsDiffSum extends Aggregator, DoubleColumn, double> { + @override + List empty() => [0.0]; + @override + List addSlice(List s, DoubleColumn b, int start, int end) { + for (var i = start + 1; i < end; i++) { + s[0] += (b.values[i] - b.values[i - 1]).abs(); + } + return s; + } + + @override + List combine(List l, List r) => l..[0] += r[0]; + @override + double finish(List s) => s[0]; +} diff --git a/test/aggregate/primitives_test.dart b/test/aggregate/primitives_test.dart new file mode 100644 index 00000000..2cbc6518 --- /dev/null +++ b/test/aggregate/primitives_test.dart @@ -0,0 +1,271 @@ +// Primitives: known values (NaN skipping, empty gives the identity result), +// then the laws on seeded random columns. + +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:openstrap_analytics/onehz.dart' show RunningMoments; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +Float64List _v(DoubleColumn c) => c.values; + +const nan = double.nan; + +List _momentFields(RunningMoments m) => + [m.count, m.mean, m.sampleSd, m.populationSd]; + +// Aggregators over DoubleColumn. +final _count = Count(); +final _finite = CountFinite(_v); +final _positive = CountWhere(_v, (x) => x > 0); +final _sum = Sum(_v); +final _sumSq = SumSq(_v); +final _min = Min(_v); +final _max = Max(_v); +final _moments = Moments(_v); + +/// Every split point of [c] recombines to [want] (compared by [eq]). +void _splitsAgree(Aggregator a, DoubleColumn c, + bool Function(R, R) eq) { + final want = a.run(c); + for (var k = 0; k <= c.length; k++) { + final got = a.finish(a.combine(a.fold(c, 0, k), a.fold(c, k, c.length))); + expect(eq(got, want), isTrue, reason: 'split at $k: $got vs $want'); + } +} + +void main() { + final cols = randomColumns(11); + final pos = randomColumns(12, nanP: .1, value: (r) => 1 + r.nextDouble() * 100); + final ints = randomIntBatches(13); + + group('Count', () { + test('counts rows, NaN included', () { + expect(_count.run(col([1, nan, 3])), 3); + expect(_count.run(col([])), 0); + expect(Count>().runList(['a', 'b']), 2); + }); + test('honours the range', () { + final c = col([1, 2, 3, 4, 5]); + expect(_count.finish(_count.fold(c, 1, 4)), 3); + expect(_count.finish(_count.fold(c, 2, 2)), 0); + }); + }); + + group('CountFinite and CountWhere', () { + test('CountFinite skips NaN', () { + expect(_finite.run(col([1, nan, 3, nan])), 2); + expect(_finite.run(col([nan, nan])), 0); + expect(_finite.run(col([])), 0); + }); + test('CountWhere counts non-NaN rows passing the test', () { + expect(_positive.run(col([-1, 2, nan, 3, 0])), 2); + expect(_positive.run(col([])), 0); + // NaN is skipped before the test, even for a test NaN would pass. + final notPos = CountWhere(_v, (x) => !(x > 0)); + expect(notPos.run(col([-1, nan, 5, 0])), 2); + }); + test('both use the column getter', () { + final second = CountFinite>( + (b) => Float64List.fromList([for (final r in b.rows) r.$2])); + expect(second.runList([(1, nan), (2, 3), (3, 4)]), 2); + }); + }); + + group('Sum', () { + test('plain sums, empty is exactly 0.0', () { + expect(_sum.run(col([1.5, 2.5, -1.0])), 3.0); + final e = _sum.run(col([])); + expect(e, 0.0); + expect(e, isA()); + }); + test('skips NaN; all-NaN is 0.0', () { + expect(_sum.run(col([1, nan, 2, nan])), 3.0); + expect(_sum.run(col([nan, nan])), 0.0); + }); + test('is compensated: ten times 0.1 is exactly 1.0', () { + expect(_sum.run(col(List.filled(10, 0.1))), 1.0); + }); + test('is Neumaier compensated (big, small, minus big)', () { + expect(_sum.run(col([1e16, 1.0, -1e16])), 1.0); + expect(_sum.run(col([1.0, 1e16, -1e16])), 1.0); + }); + test('compensation survives chunking and repeated addSlice', () { + final c = col([1e16, 1.0, -1e16, 1.0]); + for (var k = 0; k <= 4; k++) { + final s = _sum.combine(_sum.fold(c, 0, k), _sum.fold(c, k, 4)); + expect(_sum.finish(s), 2.0, reason: 'split at $k'); + var one = _sum.addSlice(_sum.empty(), c, 0, k); + one = _sum.addSlice(one, c, k, 4); + expect(_sum.finish(one), 2.0, reason: 'sliced at $k'); + } + }); + test('finish does not mutate: finish twice, keep adding', () { + final c = col([1, 2, 3, 4]); + var s = _sum.fold(c, 0, 3); + expect(_sum.finish(s), 6.0); + expect(_sum.finish(s), 6.0); + s = _sum.addSlice(s, c, 3, 4); + expect(_sum.finish(s), 10.0); + }); + }); + + group('SumSq', () { + test('sums squares, skips NaN, empty is 0.0', () { + expect(_sumSq.run(col([1, -2, 3])), 14.0); + expect(_sumSq.run(col([2, nan, 3])), 13.0); + expect(_sumSq.run(col([])), 0.0); + expect(_sumSq.run(col([nan])), 0.0); + }); + }); + + group('Min and Max', () { + test('values', () { + final c = col([3, -1, 2]); + expect(_min.run(c), -1.0); + expect(_max.run(c), 3.0); + expect(_min.run(col([7])), 7.0); + expect(_max.run(col([-4, -9])), -4.0); + }); + test('null when no finite row', () { + expect(_min.run(col([])), isNull); + expect(_max.run(col([])), isNull); + expect(_min.run(col([nan, nan])), isNull); + expect(_max.run(col([nan])), isNull); + }); + test('NaN rows are skipped', () { + final c = col([nan, 5, nan, 2, 9, nan]); + expect(_min.run(c), 2.0); + expect(_max.run(c), 9.0); + }); + test('empty and all-NaN sides of a seam change nothing', () { + final full = col([4, 2]); + final nans = col([nan, nan]); + expect(_max.finish(_max.combine(_max.empty(), _max.fold(full))), 4.0); + expect(_max.finish(_max.combine(_max.fold(full), _max.empty())), 4.0); + expect(_max.finish(_max.combine(_max.fold(nans), _max.fold(full))), 4.0); + expect(_min.finish(_min.combine(_min.fold(full), _min.fold(nans))), 2.0); + expect(_max.finish(_max.combine(_max.empty(), _max.empty())), isNull); + expect(_min.finish(_min.combine(_min.fold(nans), _min.fold(nans))), isNull); + }); + }); + + group('Moments', () { + test('textbook sample', () { + final m = _moments.run(col([2, 4, 4, 4, 5, 5, 7, 9])); + expect(m.count, 8); + expect(m.mean, closeTo(5.0, 1e-12)); + expect(m.populationSd, closeTo(2.0, 1e-12)); + expect(m.sampleSd, closeTo(math.sqrt(32 / 7), 1e-12)); + }); + test('empty, one value, all NaN', () { + final e = _moments.run(col([])); + expect((e.count, e.mean, e.sampleSd), (0, null, null)); + final one = _moments.run(col([4])); + expect((one.count, one.mean, one.sampleSd), (1, 4.0, null)); + final n = _moments.run(col([nan, nan])); + expect((n.count, n.mean), (0, null)); + }); + test('skips NaN rows', () { + final m = _moments.run(col([nan, 2, 4, nan, 6])); + expect(m.count, 3); + expect(m.mean, closeTo(4.0, 1e-12)); + expect(m.sampleSd, closeTo(2.0, 1e-12)); + }); + test('merged across a seam matches the whole', () { + final c = col([for (var i = 0; i < 30; i++) math.sin(i) * 100 + 500]); + final whole = _moments.run(c); + for (final k in [0, 1, 15, 29, 30]) { + final r = _moments + .finish(_moments.combine(_moments.fold(c, 0, k), _moments.fold(c, k, 30))); + expect(r.count, whole.count); + expect(r.mean, closeTo(whole.mean!, 1e-9)); + expect(r.sampleSd, closeTo(whole.sampleSd!, 1e-9)); + } + }); + }); + + group('First and Last', () { + final first = First>(), last = Last>(); + + test('a one-row copy, null when empty', () { + final f = first.runList(['x', 'y', 'z'])!; + final l = last.runList(['x', 'y', 'z'])!; + expect(f.rows, ['x']); + expect(l.rows, ['z']); + expect(first.runList([]), isNull); + expect(last.runList([]), isNull); + }); + test('work on a column and on a range', () { + final c = col([10, 20, 30, 40]); + expect(First().run(c)!.values, [10.0]); + expect(Last().run(c)!.values, [40.0]); + final f = First(), l = Last(); + expect(f.finish(f.fold(c, 1, 3))!.values, [20.0]); + expect(l.finish(l.fold(c, 1, 3))!.values, [30.0]); + expect(f.finish(f.fold(c, 2, 2)), isNull); + }); + test('the copy does not alias the source', () { + final v = Float64List.fromList([1, 2, 3]); + final l = Last().run(DoubleColumn(v))!; + v[2] = 99; + expect(l.values, [3.0]); + }); + test('keep a seam in order, skipping empty chunks', () { + final f = First(), l = Last(); + final c = col([10, 20, 30]); + final fs = f.combine(f.combine(f.fold(c, 0, 0), f.fold(c, 0, 2)), f.fold(c, 2, 3)); + expect(f.finish(fs)!.values, [10.0]); + final ls = l.combine(l.combine(l.fold(c, 0, 2), l.fold(c, 2, 3)), l.fold(c, 3, 3)); + expect(l.finish(ls)!.values, [30.0]); + }); + }); + + group('every primitive is lawful on seeded random cases', () { + final tol = fieldsEqual((x) => [x], relTol: 1e-12, absTol: 1e-6); + List batchFields(DoubleColumn? b) => + b == null ? [null] : [b.length, ...b.values]; + + test('Count', () => expectLawful(_count, cols)); + test('CountFinite', () => expectLawful(_finite, cols)); + test('CountWhere', () => expectLawful(_positive, cols)); + test('Sum', () => expectLawful(_sum, cols, equals: tol)); + test('SumSq', () => expectLawful(_sumSq, cols, equals: tol)); + test('Min', () => expectLawful(_min, cols)); + test('Max', () => expectLawful(_max, cols)); + test('Moments (tolerance)', () { + expectLawful(_moments, pos, + equals: fieldsEqual(_momentFields, + relTol: 1e-9, absTol: 1e-9)); + }); + test('First', () { + expectLawful(First(), cols, + equals: fieldsEqual(batchFields)); + expectLawful(First>(), ints, + equals: (a, b) => a?.rows.toString() == b?.rows.toString()); + }); + test('Last', () { + expectLawful(Last(), cols, + equals: fieldsEqual(batchFields)); + expectLawful(Last>(), ints, + equals: (a, b) => a?.rows.toString() == b?.rows.toString()); + }); + test('both of primitives with mixed result types', () { + expectLawful( + both(_min, _count).map((r) => [r.$1, r.$2]), cols, + equals: (a, b) => a.toString() == b.toString()); + }); + test('splits agree on a column with NaN gaps', () { + final r = Rng(5); + final c = col([ + for (var i = 0; i < 25; i++) r.nextBool(.3) ? nan : r.intIn(-9, 9) * 1.0 + ]); + _splitsAgree(_finite, c, (a, b) => a == b); + _splitsAgree(_max, c, (a, b) => a == b); + }); + }); +} diff --git a/test/aggregate/sum_of_test.dart b/test/aggregate/sum_of_test.dart new file mode 100644 index 00000000..0b9e0b88 --- /dev/null +++ b/test/aggregate/sum_of_test.dart @@ -0,0 +1,149 @@ +// SumOf(col, f): Neumaier-compensated sum of f(x) over the rows where x is +// not NaN and f(x) is not NaN. +// +// Decisions the implementation follows: +// * f is never called with NaN (a NaN input row is skipped before f). An +// infinite input IS passed to f. +// * f returning NaN skips that row (it does not poison the sum). +// * The result is a double; nothing counted (or an empty batch) is exactly 0.0. +// * Compensation is the same as Sum: combine adds the other state's sum and +// correction, so the compensation survives chunking and addSlice. +// * Exported from package:openstrap_analytics/aggregate.dart. + +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import '../support/aggregator_laws.dart'; +import '../support/property.dart' show Rng; + +Float64List _v(DoubleColumn c) => c.values; + +const nan = double.nan; + +final _tol = fieldsEqual((x) => [x], relTol: 1e-12, absTol: 1e-6); + +SumOf _sumOf(double Function(double) f) => + SumOf(_v, f); + +void main() { + final id = _sumOf((x) => x); + final square = _sumOf((x) => x * x); + // Square root of positives; NaN (skip) for everything else. + final rootPos = _sumOf((x) => x > 0 ? math.sqrt(x) : nan); + + group('values', () { + test('plain sums of f(x)', () { + expect(id.run(col([1.5, 2.5, -1.0])), 3.0); + expect(square.run(col([1, 2, 3])), 14.0); + expect(_sumOf((x) => 2 * x + 1).run(col([1, 2, 3])), 15.0); + }); + test('empty is exactly 0.0, as a double', () { + final e = square.run(col([])); + expect(e, 0.0); + expect(e, isA()); + expect(square.finish(square.empty()), 0.0); + }); + test('NaN inputs are skipped, all-NaN is 0.0', () { + expect(square.run(col([1, nan, 2, nan])), 5.0); + expect(square.run(col([nan, nan])), 0.0); + }); + test('f is never called with NaN', () { + final seen = []; + final spy = _sumOf((x) { + seen.add(x); + return x; + }); + expect(spy.run(col([1, nan, 2, nan, nan, 3])), 6.0); + expect(seen, [1.0, 2.0, 3.0]); + }); + test('f returning NaN skips the row', () { + expect(rootPos.run(col([4, -1, 9, 0, nan, 16])), 9.0); + expect(rootPos.run(col([-1, -2, 0])), 0.0); + expect(_sumOf((_) => nan).run(col([1, 2, 3])), 0.0); + }); + test('an infinite input reaches f', () { + final finiteOnly = _sumOf((x) => x.isFinite ? x : nan); + expect(finiteOnly.run(col([1, double.infinity, 2, -double.infinity])), + 3.0); + expect(id.run(col([1, double.infinity])), double.infinity); + }); + test('honours the range', () { + final c = col([1, 2, 3, 4, 5]); + expect(square.finish(square.fold(c, 1, 4)), 4.0 + 9 + 16); + expect(square.finish(square.fold(c, 2, 2)), 0.0); + }); + test('reads the column through the getter, on any batch', () { + final second = SumOf>( + (b) => Float64List.fromList([for (final r in b.rows) r.$2]), + (x) => x * 10); + expect(second.runList([(1, nan), (2, 3), (3, 4)]), 70.0); + }); + }); + + group('compensation', () { + test('ten times 0.1 is exactly 1.0', () { + expect(id.run(col(List.filled(10, 0.1))), 1.0); + }); + test('Neumaier: big, small, minus big', () { + expect(id.run(col([1e16, 1.0, -1e16])), 1.0); + expect(id.run(col([1.0, 1e16, -1e16])), 1.0); + }); + test('compensation applies to f(x), not x', () { + // f(1e8) = 1e16, f(-1e8) = -1e16; the 1.0 would vanish in a naive sum. + final signedSquare = _sumOf((x) => x >= 0 ? x * x : -(x * x)); + expect(signedSquare.run(col([1e8, 1, -1e8])), 1.0); + }); + test('survives every split and repeated addSlice', () { + final c = col([1e16, 1.0, -1e16, 1.0]); + for (var k = 0; k <= 4; k++) { + final s = id.combine(id.fold(c, 0, k), id.fold(c, k, 4)); + expect(id.finish(s), 2.0, reason: 'split at $k'); + var one = id.addSlice(id.empty(), c, 0, k); + one = id.addSlice(one, c, k, 4); + expect(id.finish(one), 2.0, reason: 'sliced at $k'); + } + }); + test('finish does not mutate: finish twice, keep adding', () { + final c = col([1, 2, 3, 4]); + var s = id.fold(c, 0, 3); + expect(id.finish(s), 6.0); + expect(id.finish(s), 6.0); + s = id.addSlice(s, c, 3, 4); + expect(id.finish(s), 10.0); + }); + }); + + group('random parity (seeded)', () { + test('matches Sum of the mapped column under random chunkings', () { + final r = Rng(0x50F); + final sum = Sum(_v); + for (final c in randomColumns(21, count: 25)) { + final mapped = DoubleColumn(Float64List.fromList([ + for (final x in c.values) x.isNaN || x <= 0 ? nan : math.sqrt(x) + ])); + final want = sum.run(mapped); + for (var t = 0; t < 3; t++) { + final b = randomBounds(r, c.length); + for (final got in [ + runChunked(rootPos, c, b), + runChunked(rootPos, c, b, copy: true), + runChunkedRight(rootPos, c, b), + runSliced(rootPos, c, b), + ]) { + expect(_tol(got, want), isTrue, reason: '$got vs $want, $b'); + } + } + } + }); + }); + + group('lawful', () { + final cols = randomColumns(22); + test('identity f', () => expectLawful(id, cols, equals: _tol)); + test('square', () => expectLawful(square, cols, equals: _tol)); + test('f that skips rows', () => expectLawful(rootPos, cols, equals: _tol)); + }); +} diff --git a/test/fixtures/output_batch_parity.json b/test/fixtures/output_batch_parity.json new file mode 100644 index 00000000..9ddfdaff --- /dev/null +++ b/test/fixtures/output_batch_parity.json @@ -0,0 +1 @@ +{"source":"580e16733ea6cd0dec81066417d2e89972bdbb54","provenance":"Public batch outputs recorded on the batch-only output commit, before the framework refactor. Inputs are reproductionCases in tool/reproduce_outputs.dart.","cases":[{"rr":[201,0,1],"spectral":2191.9035532963126,"cosinor":[60.20705778058244,1.994548567920997,0.99021822497395,3.584442855968536e-94],"near_singular":false,"hrv":[3.615265234174367,21.52663924532641,1.0673833338741983,21.55214616158443,0.984955296232754],"nonfinite":false,"trimp":10.252064196950478,"ctl":[93.59303565371741,96.28901109956863,-2.695975445851218],"zones":[0.0,50.00499999999995,0.0,0.0,0.0],"circadian":[1.0,0.02086956521739154,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[199,17,0],"spectral":240.47476890807894,"cosinor":[60.20546901353539,1.9944760220371391,0.9906874461342637,3.6450131402730934e-95],"near_singular":false,"hrv":[14.544681533327767,21.511404623818784,0.2902542164251966,21.539302061762694,0.7620460195717869],"nonfinite":false,"trimp":13.023941743036463,"ctl":[94.97450463635269,101.20837984038745,-6.23387520403476],"zones":[0.0,61.00609999999995,0.0,0.0,0.0],"circadian":[1.0,0.2553191489361734,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[199,31,0],"spectral":4.814151167014313,"cosinor":[60.20670784840657,1.9932291959984847,0.9907102525961057,3.252232588931257e-95],"near_singular":false,"hrv":[39.550764408744726,21.5122083009717,0.40991536302833925,21.55029614999938,-0.7370544026973745],"nonfinite":false,"trimp":15.28358496194033,"ctl":[94.92528070209704,96.00888145442389,-1.083600752326845],"zones":[0.0,72.00719999999993,0.0,0.0,0.0],"circadian":[1.0,0.3312676056338072,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[199,32,0],"spectral":358.69230942776977,"cosinor":[60.206138700452115,1.9956478767563555,0.9905970400705738,5.7121975661113537e-95],"near_singular":false,"hrv":[42.09631660353519,21.499141419377167,0.29941934343651677,21.512944620556805,-0.9682323671330902],"nonfinite":false,"trimp":18.40304984916745,"ctl":[96.58887865497881,103.10914807456258,-6.520269419583769],"zones":[0.0,83.00829999999988,0.0,0.0,0.0],"circadian":[1.0,0.3688421052631628,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[199,33,0],"spectral":19.577975582737867,"cosinor":[60.206354166666685,1.99256198692952,0.9903700870164812,1.7315692772094542e-94],"near_singular":false,"hrv":[3.615265234174367,21.52663924532641,1.0673833338741983,21.55214616158443,0.984955296232754],"nonfinite":false,"trimp":20.354197211570078,"ctl":[95.91769269304092,96.02635036170024,-0.10865766865931903],"zones":[0.0,94.00939999999984,0.0,0.0,0.0],"circadian":[1.0,0.02086956521739154,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[200,33,0],"spectral":11606.502913565982,"cosinor":[60.2065027398695,1.994982877334022,0.9902341170988117,3.323420791543799e-94],"near_singular":false,"hrv":[14.544681533327767,21.511404623818784,0.2902542164251966,21.539302061762694,0.7620460195717869],"nonfinite":false,"trimp":23.84820212239254,"ctl":[97.91559020097185,105.11939669789578,-7.2038064969239315],"zones":[0.0,105.0104999999998,0.0,0.0,0.0],"circadian":[1.0,0.2553191489361734,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[201,0,1],"spectral":179.12281661844955,"cosinor":[60.20613433801807,1.993285411438357,0.9902158539353855,3.625067770914454e-94],"near_singular":false,"hrv":[39.54331943857053,21.508583812826043,0.27113546544007344,21.551316007749993,-0.7371891534838562],"nonfinite":false,"trimp":25.46704673479356,"ctl":[96.68685633705937,96.44160847368228,0.24524786337708804],"zones":[0.0,116.01159999999977,0.0,0.0,0.0],"circadian":[1.0,0.3312676056338072,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[199,17,0],"spectral":527.7181728812988,"cosinor":[60.20664182391473,1.993388211364034,0.990292034354844,2.520360883833827e-94],"near_singular":false,"hrv":[42.11090878105368,21.502568386886722,0.29644031390575065,21.528250446057537,-0.9685070047216077],"nonfinite":false,"trimp":28.750997708819092,"ctl":[98.08372430811947,101.79002204026516,-3.7062977321456856],"zones":[0.0,127.01269999999977,0.0,0.0,0.0],"circadian":[1.0,0.3688421052631628,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[199,31,0],"spectral":21.91751421630071,"cosinor":[60.206102619835434,1.9948632846663268,0.9904511511664523,1.168751936831454e-94],"near_singular":false,"hrv":[3.6114616155271952,21.50300002384371,0.916280807418444,21.477133240134368,0.9853180909453777],"nonfinite":false,"trimp":30.6254910760735,"ctl":[97.31583171818741,97.22062215666931,0.09520956151810367],"zones":[0.0,138.0137999999999,0.0,0.0,0.0],"circadian":[1.0,0.02086956521739154,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[199,32,0],"spectral":94.441233233861,"cosinor":[60.206607219531165,1.9918709847273792,0.9905159686967243,8.514781581412862e-95],"near_singular":false,"hrv":[14.550213162597531,21.50963654798735,0.2980266237242866,21.540300395940907,0.7627458277372915],"nonfinite":false,"trimp":33.68218231936987,"ctl":[98.15600470749584,99.32476832904142,-1.1687636215455797],"zones":[0.0,149.01489999999987,0.0,0.0,0.0],"circadian":[1.0,0.2553191489361734,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[199,33,0],"spectral":12.677409836990822,"cosinor":[60.20622592520504,1.9957534839219526,0.9905696980951562,6.537929637620507e-95],"near_singular":false,"hrv":[39.54331943857053,21.508583812826043,0.27113546544007344,21.551316007749993,-0.7371891534838562],"nonfinite":false,"trimp":35.83311260770479,"ctl":[97.86366868742407,98.31752817052501,-0.45385948310094193],"zones":[0.0,160.01599999999982,0.0,0.0,0.0],"circadian":[1.0,0.3312676056338072,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]},{"rr":[200,33,0],"spectral":35.03266947297398,"cosinor":[60.20646575449843,1.9913371893547758,0.9904828062401032,1.0015309381130758e-94],"near_singular":false,"hrv":[42.11090878105368,21.502568386886722,0.29644031390575065,21.528250446057537,-0.9685070047216077],"nonfinite":false,"trimp":38.64417150766156,"ctl":[98.18187926518608,97.60954398514504,0.5723352800410453],"zones":[0.0,171.01709999999983,0.0,0.0,0.0],"circadian":[1.0,0.3688421052631628,3.505000000000011,1.3600000000000079,14,0],"irregular_gap":[22.360679774997894,83.33333333333333]}]} diff --git a/test/fixtures/output_changes.json b/test/fixtures/output_changes.json new file mode 100644 index 00000000..f3c232f5 --- /dev/null +++ b/test/fixtures/output_changes.json @@ -0,0 +1,134 @@ +{ + "provenance": "Synthetic deterministic arithmetic cases, no personal data. Formula inputs are expanded by tool/reproduce_outputs.dart. Old values measured on up/main 3a7ca34; new values measured on f8e7d3c plus #98.", + "rr": { + "normal": 900.0, + "before": 100, + "run": 33, + "artifact": 250.0, + "after": 100 + }, + "spectral": { + "count": 96, + "origin": 1800000000.0, + "step": 0.8125, + "frequency": 0.137 + }, + "cosinor": { + "count": 96, + "step": 0.7 + }, + "near_singular": { + "count": 20000, + "origin": 3.0, + "step": 1e-09 + }, + "hrv": { + "count": 1800 + }, + "load": { + "count": 90 + }, + "zones": { + "count": 120 + }, + "circadian": { + "days": 3, + "epochs": 24 + }, + "old": { + "rr": [ + 199, + 34, + 0 + ], + "spectral": 32627.260306094682, + "cosinor": [ + 60.20591440999805, + 1.9937577719142656, + 0.9902626829318173, + 2.900200339561918e-94 + ], + "near_singular": true, + "hrv": [ + 3.615265234174367, + 21.52663924532639, + 1.067383333873884, + 21.55214616158443, + 0.9849552962327551 + ], + "nonfinite": "throws", + "trimp": 32.71197112260296, + "ctl": [ + 98.49620732954641, + 103.12182624417179, + -4.625618914625377 + ], + "zones": [ + 0.0, + 120.01199999999982, + 0.0, + 0.0, + 0.0 + ], + "circadian": [ + 1.0, + 0.331267605633807, + 3.505000000000011, + 1.3600000000000079, + 14, + 0 + ], + "irregular_gap": [ + 22.360679774997894, + 83.33333333333333 + ] + }, + "new": { + "rr": [ + 200, + 33, + 0 + ], + "spectral": 32627.260527635714, + "cosinor": [ + 60.205914409998044, + 1.9937577719142632, + 0.9902626829318162, + 2.9002003395774255e-94 + ], + "near_singular": false, + "hrv": [ + 3.615265234174367, + 21.52663924532641, + 1.0673833338741983, + 21.55214616158443, + 0.984955296232754 + ], + "nonfinite": false, + "trimp": 32.711971122602975, + "ctl": [ + 98.49620732954642, + 103.12182624417177, + -4.625618914625349 + ], + "zones": [ + 0.0, + 120.01199999999977, + 0.0, + 0.0, + 0.0 + ], + "circadian": [ + 1.0, + 0.3312676056338072, + 3.505000000000011, + 1.3600000000000079, + 14, + 0 + ], + "irregular_gap": [ + 22.360679774997894, + 83.33333333333333 + ] + } +} diff --git a/test/onehz/incremental/int_histogram_test.dart b/test/onehz/incremental/int_histogram_test.dart new file mode 100644 index 00000000..b9f2f8c3 --- /dev/null +++ b/test/onehz/incremental/int_histogram_test.dart @@ -0,0 +1,210 @@ +import 'dart:io'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; + +/// Exact equality, not a tolerance: the histogram must return the very double +/// `percentileSorted` returns for the expanded sorted list. +void _same(IntHistogram h, List values, {String? reason}) { + final sorted = [...values]..sort(); + expect(h.count, values.length, reason: reason); + for (final p in [0, 1, 5, 25, 30, 33.3, 50, 66.6, 75, 90, 95, 99, 100]) { + final expected = percentileSorted(sorted, p.toDouble()); + final actual = h.percentile(p.toDouble()); + expect(actual, expected, reason: '$reason p=$p'); + } + expect(h.median, percentileSorted(sorted, 50), reason: reason); + expect(h.median, median(values), reason: reason); +} + +List _realNightHr() => [ + for (final line in File('test/onehz/fixtures/real_night_2026_07_onehz.csv') + .readAsLinesSync() + .skip(1)) + if (line.trim().isNotEmpty) double.parse(line.split(',')[1]) + ]; + +void main() { + test('real overnight HR: exact parity at every prefix and few distinct values', + () { + final hr = _realNightHr(); + expect(hr.length, greaterThan(20000)); + final h = IntHistogram(); + for (var i = 0; i < hr.length; i++) { + h.add(hr[i].toInt()); + if (i % 997 == 0 || i == hr.length - 1) { + _same(h, hr.sublist(0, i + 1), reason: 'prefix ${i + 1}'); + } + } + expect(h.distinct, lessThan(80), reason: 'compaction is the point'); + }); + + test('real overnight HR: sliding window add/remove stays exact', () { + final hr = _realNightHr(); + const window = 1800; + final h = IntHistogram(); + for (var i = 0; i < 6000; i++) { + h.add(hr[i].toInt()); + if (i >= window) h.remove(hr[i - window].toInt()); + if (i % 251 == 0) { + _same(h, hr.sublist(math.max(0, i + 1 - window), i + 1), + reason: 'window ending $i'); + } + } + }); + + for (final seed in [1, 7, 42, 914]) { + test('randomized integer data, add/remove sequence seed=$seed', () { + final rnd = math.Random(seed); + final h = IntHistogram(); + final values = []; + for (var step = 0; step < 1500; step++) { + if (values.isNotEmpty && rnd.nextInt(10) < 4) { + final v = values.removeAt(rnd.nextInt(values.length)); + h.remove(v.toInt()); + } else { + // Includes negatives and a wide spread, not just heart rates. + final v = rnd.nextInt(400) - 100; + values.add(v.toDouble()); + h.add(v); + } + if (step % 7 == 0 || values.length < 6) { + _same(h, values, reason: 'step $step n=${values.length}'); + } + } + while (values.isNotEmpty) { + h.remove(values.removeLast().toInt()); + } + _same(h, values); + }); + } + + test('odd and even counts, single value, all-equal values', () { + for (final values in [ + [5.0], + [5.0, 5.0], + [4.0, 9.0], + [3.0, 1.0, 2.0], + [3.0, 1.0, 2.0, 10.0], + List.filled(7, 61.0), + [60.0, 60.0, 61.0], + ]) { + final h = IntHistogram(); + for (final v in values) { + h.add(v.toInt()); + } + _same(h, values, reason: '$values'); + } + }); + + test('p=0 and p=1 hit the extremes; p=100 is not a special case', () { + final h = IntHistogram(); + final values = [72.0, 55.0, 90.0, 61.0, 61.0]; + for (final v in values) { + h.add(v.toInt()); + } + expect(h.percentile(0), 55); + expect(h.percentile(100), 90); + expect(h.percentile(1), percentileSorted([...values]..sort(), 1)); + expect(h.percentile(0.5), percentileSorted([...values]..sort(), 0.5)); + }); + + test('empty histogram abstains with null, as the batch functions do', () { + final h = IntHistogram(); + expect(h.count, 0); + expect(h.distinct, 0); + expect(h.median, isNull); + expect(h.median, median(const [])); + expect(h.percentile(0), isNull); + expect(h.percentile(100), isNull); + expect(h.percentile(50), percentile(const [], 50)); + h.add(70); + h.remove(70); + expect(h.median, isNull, reason: 'emptied again'); + expect(h.distinct, 0, reason: 'no zero-count bins are retained'); + }); + + test('add with a repeat count, and removing one of several', () { + final h = IntHistogram() + ..add(60, 3) + ..add(70, 2); + _same(h, [60, 60, 60, 70, 70]); + h.remove(60); + _same(h, [60, 60, 70, 70]); + expect(h.distinct, 2); + }); + + test('removing a value that is not present throws and changes nothing', () { + final h = IntHistogram()..add(60); + expect(() => h.remove(61), throwsStateError); + expect(() => IntHistogram().remove(1), throwsStateError); + _same(h, [60]); + }); + + test('rejects values that are not whole numbers and bad arguments', () { + final h = IntHistogram(); + expect(() => h.add(double.nan), throwsArgumentError); + expect(() => h.add(double.infinity), throwsArgumentError); + expect(() => h.add(70.5), throwsArgumentError); + expect(() => h.add(70, 0), throwsArgumentError); + expect(() => h.add(70, -1), throwsArgumentError); + h.add(70.0); // a whole-number double is a whole number + expect(h.count, 1); + expect(() => h.percentile(-1), throwsArgumentError); + expect(() => h.percentile(100.5), throwsArgumentError); + expect(() => h.percentile(double.nan), throwsArgumentError); + }); + + for (final seed in [3, 11]) { + test('merge equals one histogram over the concatenation seed=$seed', () { + final rnd = math.Random(seed); + final a = [for (var i = 0; i < 300; i++) rnd.nextInt(60) + 40.0]; + final b = [for (var i = 0; i < 211; i++) rnd.nextInt(90) + 30.0]; + final ha = IntHistogram(), hb = IntHistogram(); + for (final v in a) { + ha.add(v.toInt()); + } + for (final v in b) { + hb.add(v.toInt()); + } + ha.merge(hb); + _same(ha, [...a, ...b]); + _same(hb, b, reason: 'the argument is left unchanged'); + ha.merge(IntHistogram()); + _same(ha, [...a, ...b], reason: 'merging empty is a no-op'); + final empty = IntHistogram()..merge(hb); + _same(empty, b, reason: 'merging into empty'); + empty.merge(empty); + _same(empty, [...b, ...b], reason: 'merging with itself doubles counts'); + }); + } + + test('checkpoint round-trips and continues; malformed input throws', () { + final h = IntHistogram(); + final values = []; + for (final v in [70, 70, 64, 91, 64, 64, 120]) { + h.add(v); + values.add(v.toDouble()); + } + final back = IntHistogram.fromJson(checkpoint(h.toJson())); + _same(back, values); + back.add(55); + values.add(55); + _same(back, values); + for (final bad in >[ + {}, + {'version': 2, 'type': 'IntHistogram', 'values': [], 'counts': []}, + {'version': 1, 'type': 'RunningMoments', 'values': [], 'counts': []}, + {'version': 1, 'type': 'IntHistogram', 'values': [1, 2], 'counts': [1]}, + {'version': 1, 'type': 'IntHistogram', 'values': [2, 1], 'counts': [1, 1]}, + {'version': 1, 'type': 'IntHistogram', 'values': [1, 1], 'counts': [1, 1]}, + {'version': 1, 'type': 'IntHistogram', 'values': [1], 'counts': [0]}, + {'version': 1, 'type': 'IntHistogram', 'values': [1.5], 'counts': [1]}, + ]) { + expect(() => IntHistogram.fromJson(bad), throwsFormatException, + reason: '$bad'); + } + }); +} diff --git a/test/onehz/incremental/lomb_scargle_test.dart b/test/onehz/incremental/lomb_scargle_test.dart new file mode 100644 index 00000000..4e9f6d44 --- /dev/null +++ b/test/onehz/incremental/lomb_scargle_test.dart @@ -0,0 +1,99 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void main() { + final frequencies = [0.0, .0033, .01, .04, .07, .1, .15, .25, .4]; + for (final seed in [1, 42, 914]) { + for (final chunk in [1, 17, 128]) { + test('Lomb prefixes and checkpoint seed=$seed chunk=$chunk', () { + final f = incrementalFixture(seed: seed, beats: 160); + final t = f.times.map((x) => x / 1000).toList(); + var state = IncrementalLombScargle(frequencies); + for (final n in prefixSizes(t.length, chunk)) { + final ts = t.sublist(0, n), y = f.nn.sublist(0, n); + spectrumClose(state.sync(ts, y), lombScargle(ts, y, frequencies)); + final work = state.processedPoints; + spectrumClose(state.sync(ts, y), lombScargle(ts, y, frequencies)); + expect(state.processedPoints, work, + reason: 'identical prefix does no math'); + state = IncrementalLombScargle.fromJson(checkpoint(state.toJson())); + } + expect(state.processedPoints, lessThanOrEqualTo(t.length + 4), + reason: 'append work is linear in newly accepted samples'); + }); + } + test('Lomb replacement, shrink, shifted origin and force seed=$seed', () { + final f = incrementalFixture(seed: seed, beats: 160); + final state = IncrementalLombScargle(frequencies); + var t = f.times.map((x) => x / 1000).toList(), y = [...f.nn]; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + y[31] += 79; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + t[44] += .02; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + t = t.sublist(13, 141); + y = y.sublist(13, 141); + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + t = t.map((x) => x + 12345.125).toList(); + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + final work = state.processedPoints; + spectrumClose( + state.sync(t, y, force: true), lombScargle(t, y, frequencies)); + expect(state.processedPoints - work, t.length); + spectrumClose(state.sync([], []), lombScargle([], [], frequencies)); + }); + } + for (final grid in [ + [], + [0.0], + [.04, .04, .13], + [.1, .07, .4] + ]) { + test('Lomb preserves frequency grid $grid', () { + final f = incrementalFixture(beats: 31); + spectrumClose(IncrementalLombScargle(grid).sync(f.times, f.nn), + lombScargle(f.times, f.nn, grid)); + }); + } + test('Lomb constant values, collapsed time, and malformed lengths abstain', + () { + final state = IncrementalLombScargle(frequencies); + for (final input in <(List, List)>[ + ([0.0, 1, 2, 3], [7.0, 7, 7, 7]), + ([1.0, 1, 1, 1], [7.0, 8, 9, 10]), + ([1.0, 2, 3, 4], [7.0]) + ]) { + spectrumClose(state.sync(input.$1, input.$2), + lombScargle(input.$1, input.$2, frequencies)); + } + }); + for (final bad in [double.nan, double.infinity, double.negativeInfinity]) { + test('Lomb nonfinite fallback $bad', () { + final f = incrementalFixture(beats: 31); + final t = f.times.map((x) => x / 1000).toList(), y = [...f.nn]; + final state = IncrementalLombScargle(frequencies); + state.sync(t, y); + y[13] = bad; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + y[13] = f.nn[13]; + t[9] = bad; + spectrumClose(state.sync(t, y), lombScargle(t, y, frequencies)); + }); + } + test('Lomb malformed checkpoint throws an explicit input error', () { + expect(() => IncrementalLombScargle.fromJson({'frequencies': 'bad'}), + throwsA(anyOf(isA(), isA()))); + }); + test('Lomb real capture irregular cleaned beat times match batch', () { + final f = realIncrementalFixture(); + expect(f.nn.length, greaterThan(300)); + final t = f.times.map((x) => x / 1000).toList(); + final state = IncrementalLombScargle(frequencies); + for (final n in prefixSizes(f.nn.length, 173)) { + final ts = t.sublist(0, n), nn = f.nn.sublist(0, n); + spectrumClose(state.sync(ts, nn), lombScargle(ts, nn, frequencies)); + } + }); +} diff --git a/test/onehz/incremental/running_moments_test.dart b/test/onehz/incremental/running_moments_test.dart new file mode 100644 index 00000000..1c2f7cc3 --- /dev/null +++ b/test/onehz/incremental/running_moments_test.dart @@ -0,0 +1,71 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; +import '../support/incremental_compare.dart'; +import '../support/incremental_fixtures.dart'; + +void _check(RunningMoments state, List values) { + expect(state.count, values.length); + numberClose(state.mean, mean(values)); + numberClose(state.sampleSd, stddev(values)); + numberClose(state.populationSd, stddevPop(values)); +} + +void main() { + for (final seed in [1, 42, 914]) { + test('moments add/remove and JSON continuation seed=$seed', () { + var state = RunningMoments(); + final values = []; + _check(state, values); + final nn = incrementalFixture(seed: seed, beats: 100).nn; + for (final x in nn) { + values.add(x); + state.add(x); + _check(state, values); + if (values.length == 53) { + state = RunningMoments.fromJson(checkpoint(state.toJson())); + _check(state, values); + } + } + while (values.isNotEmpty) { + state.remove(values.removeAt(values.length ~/ 2)); + _check(state, values); + } + state.add(17); + _check(state, [17]); + }); + test('moments merge disjoint partitions seed=$seed', () { + final values = incrementalFixture(seed: seed, beats: 101).nn; + final left = RunningMoments(), right = RunningMoments(); + for (final x in values.take(37)) { + left.add(x); + } + for (final x in values.skip(37)) { + right.add(x); + } + left.merge(right); + _check(left, values); + _check(right, values.sublist(37)); + left.merge(RunningMoments()); + _check(left, values); + final copy = RunningMoments()..merge(left); + _check(copy, values); + }); + } + for (final values in [ + [], + [42.0], + List.filled(80, 42), + List.generate(80, (i) => 1e9 + (i % 7) * .25) + ]) { + test( + 'moments degenerate/stable data n=${values.length} first=${values.firstOrNull}', + () { + final state = RunningMoments(); + for (final x in values) { + state.add(x); + } + _check(state, values); + _check(RunningMoments.fromJson(checkpoint(state.toJson())), values); + }); + } +} diff --git a/test/onehz/lomb_scargle_precision_test.dart b/test/onehz/lomb_scargle_precision_test.dart new file mode 100644 index 00000000..0bc98758 --- /dev/null +++ b/test/onehz/lomb_scargle_precision_test.dart @@ -0,0 +1,119 @@ +// Batch Lomb–Scargle on absolute epoch seconds. Trig arguments `w * t` reach +// ~1e10 rad when `t` is ~1.8e9 s, where a double's spacing is ~1e-6 rad, so the +// periodogram used to carry ~1e-6 relative error that depends on the clock, not +// the data. `lombScargle` now measures time from the first sample. +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +const _epoch = 1791028730.0; + +/// A night of beats. Times are multiples of 1/1024 s, so adding [_epoch] (a +/// whole number of seconds) is EXACT in a double: the epoch series and the +/// rebased series have identical differences, bit for bit. +({List t, List y}) _beats(int n, {int seed = 5}) { + final rnd = math.Random(seed); + final t = [], y = []; + var k = 0; + for (var i = 0; i < n; i++) { + final rr = 900 + + 90 * math.sin(i / 300) + + 45 * math.sin(i * 2 * math.pi * .25 * .9) + + rnd.nextDouble() * 30; + k += (rr / 1000 * 1024).round(); + t.add(k / 1024); + y.add(rr); + } + return (t: t, y: y); +} + +double _worstRel(LombScargle a, LombScargle b) { + expect(a.spectrum.length, b.spectrum.length); + var worst = 0.0; + for (var i = 0; i < a.spectrum.length; i++) { + final e = b.spectrum[i].psd; + final d = (a.spectrum[i].psd - e).abs(); + worst = math.max(worst, e == 0 ? d : d / e.abs()); + } + return worst; +} + +/// The same algorithm with every sum Kahan-compensated and time measured from +/// the first sample. It shares no code with `lombScargle`, so it checks the +/// shift and the summation, not just self-consistency. +LombScargle _kahanReference(List t, List y, List f) { + double ksum(Iterable xs) { + var s = 0.0, c = 0.0; + for (final x in xs) { + final v = x - c; + final u = s + v; + c = (u - s) - v; + s = u; + } + return s; + } + + final n = t.length; + final ts = [for (final v in t) v - t.first]; + final my = ksum(y) / n; + final yc = [for (final v in y) v - my]; + final span = ts.reduce(math.max) - ts.reduce(math.min); + final scale = 2.0 * span / (n - 1); + final out = []; + for (final hz in f) { + final w = 2 * math.pi * hz; + final tau = math.atan2(ksum([for (final v in ts) math.sin(2 * w * v)]), + ksum([for (final v in ts) math.cos(2 * w * v)])) / + (2 * w); + final c = [for (final v in ts) math.cos(w * (v - tau))]; + final s = [for (final v in ts) math.sin(w * (v - tau))]; + final cNum = ksum([for (var i = 0; i < n; i++) yc[i] * c[i]]); + final sNum = ksum([for (var i = 0; i < n; i++) yc[i] * s[i]]); + final cDen = ksum([for (final v in c) v * v]); + final sDen = ksum([for (final v in s) v * v]); + out.add(LsPoint(hz, 0.5 * (cNum * cNum / cDen + sNum * sNum / sDen) * scale)); + } + return LombScargle(out); +} + +void main() { + final grid = freqGrid(0.0033, 0.4, 120); + + test('epoch seconds and rebased seconds give the same spectrum', () { + final b = _beats(1200); + final epoch = [for (final v in b.t) v + _epoch]; + expect(epoch[17] - epoch[3], b.t[17] - b.t[3], reason: 'offset is exact'); + final a = lombScargle(epoch, b.y, grid)!; + final r = lombScargle(b.t, b.y, grid)!; + expect(_worstRel(a, r), lessThan(1e-11)); + }); + + test('epoch-second spectrum matches a compensated, shifted reference', () { + final b = _beats(1200, seed: 9); + final epoch = [for (final v in b.t) v + _epoch]; + final a = lombScargle(epoch, b.y, grid)!; + expect(_worstRel(a, _kahanReference(epoch, b.y, grid)), lessThan(1e-11)); + }); + + test('the shift is the first sample, so any time origin gives the same', () { + final b = _beats(400, seed: 2); + final base = lombScargle(b.t, b.y, grid)!; + for (final offset in [-1000.0, 3600.0, 1e6, 1.5e9]) { + final moved = lombScargle([for (final v in b.t) v + offset], b.y, grid)!; + expect(_worstRel(moved, base), lessThan(1e-11), reason: 'offset $offset'); + } + }); + + test('incremental state agrees with batch on epoch seconds, far inside 1e-8', + () { + final b = _beats(900, seed: 4); + final epoch = [for (final v in b.t) v + _epoch]; + final state = IncrementalLombScargle(grid); + for (final n in [10, 100, 450, 900]) { + final inc = state.sync(epoch.sublist(0, n), b.y.sublist(0, n))!; + final batch = lombScargle(epoch.sublist(0, n), b.y.sublist(0, n), grid)!; + expect(_worstRel(inc, batch), lessThan(1e-9), reason: 'n=$n'); + } + }); +} diff --git a/test/onehz/support/incremental_compare.dart b/test/onehz/support/incremental_compare.dart new file mode 100644 index 00000000..a969d77c --- /dev/null +++ b/test/onehz/support/incremental_compare.dart @@ -0,0 +1,78 @@ +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void numberClose(double? actual, double? expected, {String? reason}) { + if (expected == null) { + expect(actual, isNull, reason: reason); + } else if (expected.isNaN) { + expect(actual?.isNaN, isTrue, reason: reason); + } else if (!expected.isFinite) { + expect(actual, expected, reason: reason); + } else { + expect(actual, isNotNull, reason: reason); + expect(actual, closeTo(expected, math.max(1e-9, expected.abs() * 1e-8)), + reason: reason); + } +} + +void metricEnvelope(Metric actual, Metric expected) { + expect(actual.present, expected.present); + numberClose(actual.confidence, expected.confidence); + expect(actual.tier, expected.tier); + expect(actual.inputs_used, expected.inputs_used); + expect(actual.note, expected.note); + expect(actual.drivers?.map((d) => d.toJson()).toList(), + expected.drivers?.map((d) => d.toJson()).toList()); +} + +void hrvClose(Metric actual, Metric expected) { + metricEnvelope(actual, expected); + if (expected.value == null) return; + final a = actual.value!, e = expected.value!; + expect(a.nBeats, e.nBeats); + numberClose(a.rmssd, e.rmssd); + numberClose(a.sdnn, e.sdnn); + numberClose(a.sdann, e.sdann); + numberClose(a.sdnnIndex, e.sdnnIndex); + numberClose(a.pnn50, e.pnn50); + numberClose(a.diffAcf1, e.diffAcf1); +} + +void spectrumClose(LombScargle? actual, LombScargle? expected) { + if (expected == null) { + expect(actual, isNull); + return; + } + expect(actual, isNotNull); + expect(actual!.spectrum.length, expected.spectrum.length); + for (var i = 0; i < expected.spectrum.length; i++) { + expect(actual.spectrum[i].freqHz, expected.spectrum[i].freqHz); + numberClose(actual.spectrum[i].psd, expected.spectrum[i].psd, + reason: 'frequency ${expected.spectrum[i].freqHz}'); + } + for (final band in [(0.0, .04), (.04, .15), (.15, .4)]) { + numberClose(actual.bandPower(band.$1, band.$2), + expected.bandPower(band.$1, band.$2)); + } +} + +void enmoClose(EnmoResult actual, EnmoResult expected) { + numberClose(actual.gRef, expected.gRef); + numberClose(actual.coverage, expected.coverage); + expect(actual.minutes.length, expected.minutes.length); + for (var i = 0; i < expected.minutes.length; i++) { + final a = actual.minutes[i], e = expected.minutes[i]; + expect(a.tsMinStartMs, e.tsMinStartMs); + expect(a.nSamples, e.nSamples); + numberClose(a.enmo, e.enmo); + numberClose(a.mad, e.mad); + numberClose(a.meanMag, e.meanMag); + numberClose(a.dynAmp, e.dynAmp); + } +} + +Map checkpoint(Map json) => + jsonDecode(jsonEncode(json)) as Map; diff --git a/test/onehz/support/incremental_fixtures.dart b/test/onehz/support/incremental_fixtures.dart new file mode 100644 index 00000000..b9277cca --- /dev/null +++ b/test/onehz/support/incremental_fixtures.dart @@ -0,0 +1,113 @@ +import 'dart:io'; +import 'dart:math' as math; + +import 'package:openstrap_analytics/onehz.dart'; + +/// Reproducible inputs; no wall clock, device, network, or unseeded randomness. +class IncrementalFixture { + final List times, nn; + final List accel; + final List minuteKeys; + final List hr; + final List cadence; + const IncrementalFixture( + this.times, this.nn, this.accel, this.minuteKeys, this.hr, this.cadence); +} + +IncrementalFixture incrementalFixture( + {int seed = 42, + int beats = 850, + int seconds = 241, + int minutes = 121, + double originMs = 0, + double cadenceHz = 1, + bool gaps = true, + bool jitter = false}) { + final random = math.Random(seed); + final nn = [], times = []; + var time = originMs; + for (var i = 0; i < beats; i++) { + final interval = jitter + ? 800 + 160 * (random.nextDouble() - .5) + : 800 + + 75 * math.sin(i * .13 + seed * .003) + + 21 * math.sin(i * .031 + seed * .007); + time += interval; + if (gaps && i % 113 == 112) time += 21000; + nn.add(interval); + times.add(time); + } + final accel = []; + for (var i = 0; i < seconds; i++) { + if (gaps && i >= 70 && i < 139) continue; + accel.add(AccelSample( + originMs + i * 1000 / cadenceHz, + .22 * math.sin(i * .29) + random.nextDouble() * .007, + .11 * math.cos(i * .07), + 1.02 + .065 * math.sin(i * .23), + valid: i % 47 != 46)); + } + const hrBranches = [54.0, 95.0, 106.799999, 106.8, 130.0, 190.0, 0.0]; + const cadBranches = [null, 99.999, 100, 110, 120, 130, 160, 0]; + return IncrementalFixture( + times, + nn, + accel, + List.generate(minutes, (i) => 28000000 + i + i ~/ 17), + List.generate(minutes, (i) => hrBranches[(i + seed) % hrBranches.length]), + List.generate( + minutes, (i) => cadBranches[(i + seed) % cadBranches.length])); +} + +List prefixSizes(int length, int chunk) => { + 0, + 1, + 2, + 3, + 4, + 29, + 30, + 31, + for (var i = chunk; i < length; i += chunk) i, + length, + }.where((n) => n <= length).toList() + ..sort(); + +/// Checked-in anonymised WHOOP-4 overnight capture, July 2026. Provenance and +/// documentation: test/onehz/real_night_cardio_stager_test.dart. Timestamps were +/// rebased to zero and only HR, axes, and RR retained. This subset is +/// read directly from the repository; missing fixtures fail, never skip. +IncrementalFixture realIncrementalFixture({int seconds = 900}) { + final accel = [], hrSeconds = []; + for (final line in File('test/onehz/fixtures/real_night_2026_07_onehz.csv') + .readAsLinesSync() + .skip(1)) { + if (line.trim().isEmpty) continue; + final p = line.split(',').map(double.parse).toList(); + if (p[0] >= seconds) break; + hrSeconds.add(p[1]); + accel.add(AccelSample(p[0] * 1000, p[2], p[3], p[4])); + } + final raw = [], times = []; + for (final line in File('test/onehz/fixtures/real_night_2026_07_rr.csv') + .readAsLinesSync() + .skip(1)) { + if (line.trim().isEmpty) continue; + final p = line.split(',').map(double.parse).toList(); + if (p[0] >= seconds * 1000) break; + times.add(p[0]); + raw.add(p[1]); + } + final clean = correctRr(raw, rrTsMs: times); + final hr = []; + for (var i = 0; i < hrSeconds.length; i += 60) { + hr.add(mean(hrSeconds.sublist(i, math.min(i + 60, hrSeconds.length)))!); + } + return IncrementalFixture( + clean.nnTimesMs, + clean.nn, + accel, + List.generate(hr.length, (i) => i), + hr, + List.filled(hr.length, null)); +} diff --git a/test/output_changes/batch_controls_test.dart b/test/output_changes/batch_controls_test.dart new file mode 100644 index 00000000..ebde99c9 --- /dev/null +++ b/test/output_changes/batch_controls_test.dart @@ -0,0 +1,42 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void main() { + test('RR cap boundary preserves uncapped NN for short classified runs', () { + for (final n in [30, 31, 33]) { + final rr = [...List.filled(100, 900), + ...List.filled(n, 250), ...List.filled(100, 900)]; + final capped = correctRr(rr); + if (n <= 31) { + final old = correctRr(rr, maxArtifactRun: null); + expect(capped.nn, old.nn); + expect(capped.nnTimesMs, old.nnTimesMs); + expect(capped.classes, old.classes); + } else { + expect(capped.nn, List.filled(200, 900)); + expect(capped.droppedCount, 33); + } + } + }); + test('batch circular near-ties retain strict first-winner selection', () { + // Equal 10-epoch windows on an alternating profile tie at every start. + // A 5-epoch trough starts at zero, which includes three low epochs. + final profile = [for (var i = 0; i < 24; i++) i.isEven ? 1.0 : 2.0]; + final equal = circadianNonparametric(profile, 24).value!; + expect(equal.m10StartEpoch, 0); + expect(equal.l5StartEpoch, 0); + final near = [...profile]..[0] += 1e-12; + final result = circadianNonparametric(near, 24).value!; + expect(result.m10StartEpoch, 0); + expect(result.l5StartEpoch, 2); + }); + test('batch irregular screen excludes the gap pair already excluded by main', () { + final result = irregularBeatScreen([800, 900, 1000, 1100, 700, 850, 900, 1000], + nnTimesMs: [800, 1700, 2700, 3800, 14500, 15350, 16250, 17250], + minBeats: 4).value!; + // Six successive differences remain: 100,100,100,150,50,100. + // Five exceed 70 ms. The -400 ms difference across the gap is absent. + expect(result.pnnPct, 100 * 5 / 6); + expect(result.nBeats, 8); + }); +} diff --git a/test/output_changes/batch_parity_test.dart b/test/output_changes/batch_parity_test.dart new file mode 100644 index 00000000..d81a7b1e --- /dev/null +++ b/test/output_changes/batch_parity_test.dart @@ -0,0 +1,21 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + final config = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final golden = jsonDecode(File('test/fixtures/output_batch_parity.json').readAsStringSync()) as Map; + final cases = reproductionCases(config); + test('the full synthetic corpus is available', () { + expect(cases.length, 12); + expect(golden['cases'], hasLength(12)); + }); + for (var i = 0; i < 12; i++) { + test('split-2 public batch arithmetic survives refactoring, case $i', () { + // Frozen before aggregator adoption, not computed by the current path. + // Covers high-frequency jitter, time holes, cap boundaries and epochs. + expect(reproduce(cases[i]), golden['cases'][i], reason: 'case $i'); + }); + } +} diff --git a/test/output_changes/circadian_test.dart b/test/output_changes/circadian_test.dart new file mode 100644 index 00000000..480799e7 --- /dev/null +++ b/test/output_changes/circadian_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + test('Compute circadian variance with centered moments: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['circadian']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/output_changes/cosinor_test.dart b/test/output_changes/cosinor_test.dart new file mode 100644 index 00000000..6d0e6d45 --- /dev/null +++ b/test/output_changes/cosinor_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + test('Refuse unidentifiable cosinor fits: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['cosinor', 'near_singular']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/output_changes/ctl_test.dart b/test/output_changes/ctl_test.dart new file mode 100644 index 00000000..0c1d61ad --- /dev/null +++ b/test/output_changes/ctl_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + test('Evaluate training-load decay with affine arithmetic: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['ctl']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/output_changes/hrv_test.dart b/test/output_changes/hrv_test.dart new file mode 100644 index 00000000..05a5a9d9 --- /dev/null +++ b/test/output_changes/hrv_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + test('Compute HRV moments without subtracting a large mean: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['hrv']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/output_changes/nonfinite_test.dart b/test/output_changes/nonfinite_test.dart new file mode 100644 index 00000000..f26f6a1d --- /dev/null +++ b/test/output_changes/nonfinite_test.dart @@ -0,0 +1,31 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import 'package:openstrap_analytics/onehz.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + for (final bad in [double.nan, double.infinity, double.negativeInfinity]) { + test('all batch timestamp guards abstain on $bad', () { + final nn = [for (var i = 0; i < 1500; i++) 900.0 + 40 * (i % 7)]; + final ts = [for (var i = 0; i < nn.length; i++) i * 1100.0]; + ts[700] = bad; + for (final metric in [hrvTime(nn, nnTimesMs: ts), hrvFreq(nn, ts, artifactFraction: 0), + rsaRespRate(nn, ts, artifactFraction: 0), cvhrApneaScreen(nn, ts, artifactFraction: 0), nocturnalRmssd(nn, ts), + sleepSessionRmssdDetail(nn, ts, startSec: 1, endSec: 7200)]) { + expect(metric.present, isFalse); + expect(metric.note, 'non-finite beat time'); + } + }); + } + + test('Abstain when a beat has no finite time: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['nonfinite']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/output_changes/review_inputs_test.dart b/test/output_changes/review_inputs_test.dart new file mode 100644 index 00000000..4e3a3dae --- /dev/null +++ b/test/output_changes/review_inputs_test.dart @@ -0,0 +1,158 @@ +import 'package:openstrap_analytics/onehz.dart'; +import 'package:test/test.dart'; + +void main() { + for (final bad in [double.nan, double.infinity, double.negativeInfinity]) { + test('HRV entry points abstain on non-finite NN interval $bad', () { + final nn = List.filled(1200, 900); + final ts = [for (var i = 0; i < nn.length; i++) 2000.0 + i * 900]; + nn[400] = bad; + for (final metric in [ + hrvTime(nn, nnTimesMs: ts), + nocturnalRmssd(nn, ts), + sleepSessionRmssdDetail(nn, ts, startSec: 1, endSec: 1200), + sleepSessionWindowedRmssd(nn, ts, startSec: 1, endSec: 1200), + ]) { + expect(metric.present, isFalse); + expect(metric.note, 'non-finite NN interval'); + } + expect(hrvTime(nn).present, isFalse); + }); + test('unknown training load $bad abstains in prime and later history', () { + for (final index in [0, 6, 7, 13]) { + final loads = List.filled(14, 50)..[index] = bad; + final metric = ctlAtlTsb(loads); + expect(metric.present, isFalse, reason: 'day $index'); + expect(metric.note, 'non-finite daily TRIMP'); + } + }); + } + test('a five-minute segment containing only NaNs abstains without throwing', () { + final nn = List.filled(1200, 900); + for (var i = 0; i < 334; i++) { nn[i] = double.nan; } + final ts = [for (var i = 0; i < nn.length; i++) i * 900.0]; + expect(hrvTime(nn, nnTimesMs: ts).present, isFalse); + }); + for (final period in [0.0, -24.0, double.nan, double.infinity, + double.negativeInfinity]) { + test('cosinor refuses invalid period $period', () { + final metric = cosinor( + [for (var i = 0; i < 48; i++) i.toDouble()], + [for (var i = 0; i < 48; i++) 60.0 + i % 3], + periodHours: period, + ); + expect(metric.present, isFalse); + expect(metric.note, 'invalid cosinor period'); + }); + } + test('a capped reset keeps source time but invalidates regressing HRV time', () { + // The RR clock exceeds elapsed sensor time before the cap. Re-anchoring + // the next independent segment must retain its actual sensor timestamp. + final rr = [...List.filled(100, 1500), + ...List.filled(33, 3000), ...List.filled(100, 1500)]; + final ts = [for (var i = 0; i < rr.length; i++) 1.7e12 + i * 1000]; + final result = correctRr(rr, rrTsMs: ts); + expect(result.nnTimesMs[99], 150000); + expect(result.nnTimesMs[100], 134500); + for (final metric in [ + hrvTime(result.nn, nnTimesMs: result.nnTimesMs), + nocturnalRmssd(result.nn, result.nnTimesMs), + sleepSessionRmssdDetail(result.nn, result.nnTimesMs, + startSec: 1, endSec: 400), + sleepSessionRmssdDetail(result.nn, result.nnTimesMs, + startSec: 1, endSec: 100), + ]) { + expect(metric.present, isFalse); + expect(metric.note, 'regressing beat time'); + } + }); + + for (final entry in Function(List, List)>{ + 'frequency HRV': (nn, ts) => hrvFreq(nn, ts, artifactFraction: 0), + 'RSA': (nn, ts) => rsaRespRate(nn, ts, artifactFraction: 0), + 'CVHR': (nn, ts) => cvhrApneaScreen(nn, ts, artifactFraction: 0), + }.entries) { + test('${entry.key} refuses a regressing spectral clock', () { + final nn = List.filled(1500, 900); + final ts = [for (var i = 0; i < nn.length; i++) i * 900.0]; + ts[700] = ts[699] - 500; + final metric = entry.value(nn, ts); + expect(metric.present, isFalse); + expect(metric.note, 'regressing beat time'); + }); + } + + for (final times in >[[], [800], [800, 1700, 2500]]) { + test('time HRV refuses a supplied ${times.length}-beat clock for two NN', () { + final metric = hrvTime([800, 900], nnTimesMs: times); + expect(metric.present, isFalse); + expect(metric.note, 'beat time length mismatch'); + // NN-only remains supported, as does a matching two-beat clock. + // One 100 ms difference has RMSSD exactly 100 ms. + expect(hrvTime([800, 900]).value!.rmssd, 100); + expect(hrvTime([800, 900], nnTimesMs: [800, 1700]).value!.rmssd, 100); + }); + } + + test('cosinor refuses overflow from a subnormal positive period', () { + final metric = cosinor( + List.generate(48, (i) => i.toDouble()), + List.generate(48, (i) => 60.0 + i % 3), + periodHours: 1e-320, + ); + expect(metric.present, isFalse); + expect(metric.note, 'non-finite cosinor design'); + }); + for (final bad in [double.nan, double.infinity, double.negativeInfinity]) { + for (final axis in ['time', 'response']) { + test('cosinor refuses non-finite $axis moment $bad', () { + final ts = List.generate(48, (i) => i.toDouble()); + final values = List.generate(48, (i) => 60.0 + i % 3); + (axis == 'time' ? ts : values)[12] = bad; + final metric = cosinor(ts, values); + expect(metric.present, isFalse); + expect(metric.note, 'non-finite cosinor design'); + }); + } + } + test('cosinor refuses non-finite moments from finite overflow', () { + final metric = cosinor( + List.generate(48, (i) => i.toDouble()), + List.generate(48, (i) => i.isEven ? 1e308 : -1e308), + ); + expect(metric.present, isFalse); + expect(metric.note, 'non-finite cosinor design'); + }); + + test('an entirely unknown load history abstains', () { + expect(ctlAtlTsb(List.filled(42, double.nan)).present, isFalse); + }); + + test('over-cap artifacts have an explicit gap label', () { + final rr = [ + ...List.filled(100, 900), + ...List.filled(33, 100), + ...List.filled(100, 900), + ]; + final result = correctRr(rr); + expect(result.classes.sublist(100, 133).map((c) => c.name), + everyElement('gap')); + expect(result.droppedCount, 33); + expect(result.nn, List.filled(200, 900)); + expect(result.nnSourceIndex, + [...List.generate(100, (i) => i), + ...List.generate(100, (i) => i + 133)]); + }); + + test('observed rest days still decay load', () { + // Seven 50-load days prime at 50. Seven observed rest days then apply + // exp(-1/tau) seven times, giving 50*exp(-7/42) and 50*exp(-7/7). + final metric = ctlAtlTsb([ + ...List.filled(7, 50), ...List.filled(7, 0), + ]); + expect(metric.value!.ctl, closeTo(42.3240862445307, 1e-12)); + expect(metric.value!.atl, closeTo(18.3939720585721, 1e-12)); + expect(ctlAtlTsb(List.filled(14, 50)).value!.ctl, + closeTo(50, 1e-12)); + }); +} diff --git a/test/output_changes/rr_test.dart b/test/output_changes/rr_test.dart new file mode 100644 index 00000000..1421dbd6 --- /dev/null +++ b/test/output_changes/rr_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + test('Leave long RR artifact runs as gaps: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['rr']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/output_changes/spectral_test.dart b/test/output_changes/spectral_test.dart new file mode 100644 index 00000000..01e2fce5 --- /dev/null +++ b/test/output_changes/spectral_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + test('Reduce spectral error on epoch timestamps: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['spectral']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/output_changes/trimp_test.dart b/test/output_changes/trimp_test.dart new file mode 100644 index 00000000..afaf57d5 --- /dev/null +++ b/test/output_changes/trimp_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + test('Retain small TRIMP contributions in the total: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['trimp']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/output_changes/zones_test.dart b/test/output_changes/zones_test.dart new file mode 100644 index 00000000..c9b772dc --- /dev/null +++ b/test/output_changes/zones_test.dart @@ -0,0 +1,16 @@ +import 'dart:convert'; +import 'dart:io'; +import 'package:test/test.dart'; +import '../../tool/reproduce_outputs.dart'; + +void main() { + test('Count repeated zone intervals before summing: unrounded synthetic batch regression', () { + // Old/new values were captured independently on 3a7ca34 and f8e7d3c. + // The fixture gives the input formulas and provenance, never personal data. + final fixture = jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + final actual = reproduce(fixture); + for (final key in ['zones']) { + expect(actual[key], fixture['new'][key], reason: key); + } + }); +} diff --git a/test/support/aggregator_laws.dart b/test/support/aggregator_laws.dart new file mode 100644 index 00000000..20a29c06 --- /dev/null +++ b/test/support/aggregator_laws.dart @@ -0,0 +1,128 @@ +// Shared helpers for the column-slice aggregator tests: the law assertion, +// seeded batch generation, and the three ways to evaluate an aggregator over a +// batch (whole, chunked + combined, one state sliced). + +import 'dart:typed_data'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +import 'fold_law_support.dart' show foldBounds; +import 'property.dart' show Rng; + +/// Asserts that [agg] satisfies every aggregator law on [cases]. The failure +/// message lists each violated law with its detail. +void expectLawful, R>( + Aggregator agg, + List cases, { + ResultEquals? equals, +}) { + final violations = checkLaws(agg, cases, equals: equals); + expect(violations, isEmpty, + reason: violations.isEmpty + ? null + : 'aggregator laws violated:\n' + '${violations.map((v) => ' ${v.law}: ${v.detail}').join('\n')}'); +} + +/// Seeded random double columns. Always starts with the empty column, a +/// singleton and a pair, then [count] columns of length `0..maxLen`. A value +/// is NaN with probability [nanP]; [value] draws the finite ones. +List randomColumns(int seed, + {int count = 30, + int maxLen = 40, + double nanP = .15, + double Function(Rng)? value}) { + final r = Rng(seed); + final draw = value ?? (Rng r) => (r.nextDouble() - .5) * 2000; + DoubleColumn col(int n) => DoubleColumn(Float64List.fromList( + [for (var i = 0; i < n; i++) r.nextBool(nanP) ? double.nan : draw(r)])); + return [ + col(0), + col(1), + col(2), + for (var i = 0; i < count; i++) col(r.intIn(0, maxLen)), + ]; +} + +/// Seeded random int batches, same shape as [randomColumns]. +List> randomIntBatches(int seed, + {int count = 30, int maxLen = 40, int lo = -1000, int hi = 1000}) { + final r = Rng(seed); + ListBatch b(int n) => + ListBatch([for (var i = 0; i < n; i++) r.intIn(lo, hi)]); + return [b(0), b(1), b(2), for (var i = 0; i < count; i++) b(r.intIn(0, maxLen))]; +} + +/// A column of the given values. +DoubleColumn col(List xs) => DoubleColumn(Float64List.fromList(xs)); + +/// Chunk bounds `[0, ...cuts, n]` for a batch of [n] rows. Cuts may repeat or +/// sit on 0 / 1 / 2 / 3 / n-1 / n, so there are empty, singleton and very short +/// chunks. +List randomBounds(Rng r, int n) => + foldBounds(n, r.intIn(0, 6), r.nextInt(1 << 30), special: [0, 1, 2, 3]); + +/// Folds each chunk `[bounds[k], bounds[k+1])` from a fresh state and combines +/// the chunk states left to right, then finishes. With [copy] each chunk is +/// first cut out with `copyRange` and folded as a batch of its own (so the +/// inner aggregator cannot peek at rows before the chunk); otherwise chunks are +/// ranges of the one batch. +R runChunked, R>( + Aggregator agg, B batch, List bounds, + {bool copy = false}) { + S chunk(int a, int b) => copy + ? agg.fold(batch.copyRange(a, b)) + : agg.fold(batch, a, b); + var acc = chunk(bounds[0], bounds[1]); + for (var k = 1; k + 1 < bounds.length; k++) { + acc = agg.combine(acc, chunk(bounds[k], bounds[k + 1])); + } + return agg.finish(acc); +} + +/// Same chunks, combined right to left: `c0 + (c1 + (c2 + ...))`. +R runChunkedRight, R>( + Aggregator agg, B batch, List bounds) { + var acc = agg.fold(batch, bounds[bounds.length - 2], bounds.last); + for (var k = bounds.length - 3; k >= 0; k--) { + acc = agg.combine(agg.fold(batch, bounds[k], bounds[k + 1]), acc); + } + return agg.finish(acc); +} + +/// The 'slicing' path: ONE state, repeated `addSlice` over the consecutive +/// slices `[bounds[k], bounds[k+1])` (empty slices included), then finish. +R runSliced, R>( + Aggregator agg, B batch, List bounds) { + var state = agg.empty(); + for (var k = 0; k + 1 < bounds.length; k++) { + state = agg.addSlice(state, batch, bounds[k], bounds[k + 1]); + } + return agg.finish(state); +} + +/// Plain bounds for the chunk-size-one case: `[0, 1, 2, ..., n]` (`[0, 0]` for +/// an empty batch). +List singletonBounds(int n) => n == 0 ? [0, 0] : [for (var i = 0; i <= n; i++) i]; + +/// Asserts that [build] over every case in [cases] reproduces [reference] on +/// every evaluation path of [checkConversion] (and, with [laws], the aggregator +/// laws). The failure reason lists the first few failures, one per line. +void expectConversion, R>( + Aggregator Function(B input) build, + List cases, + R Function(B input) reference, { + ResultEquals? equals, + int chunkings = 8, + int seed = 0, + bool laws = true, +}) { + final failures = checkConversion(build, cases, reference, + equals: equals, chunkings: chunkings, seed: seed, laws: laws); + expect(failures, isEmpty, + reason: failures.isEmpty + ? null + : 'conversion check failed (${failures.length}):\n' + '${failures.take(5).map((f) => ' ${f.path} (case ${f.caseIndex}): ${f.detail}').join('\n')}'); +} diff --git a/test/support/checkpoint_support.dart b/test/support/checkpoint_support.dart new file mode 100644 index 00000000..97005c3e --- /dev/null +++ b/test/support/checkpoint_support.dart @@ -0,0 +1,60 @@ +// Shared by the module aggregator tests: a mid-stream checkpoint must resume. + +import 'dart:convert'; + +import 'package:openstrap_analytics/aggregate.dart'; +import 'package:test/test.dart'; + +/// Asserts that [agg] is persistable and that a mid-stream checkpoint resumes: +/// fold the first half of [batch], checkpoint it through JSON text, restore, +/// keep folding the second half, and compare with the one-pass result. +void expectCheckpointResumes, R>( + Aggregator agg, + B batch, { + ResultEquals? equals, +}) { + expect(agg.persistable, isTrue); + final half = batch.length ~/ 2; + final env = jsonDecode(jsonEncode(encodeCheckpoint( + agg, agg.fold(batch, 0, half), + kind: 'test', version: 1))); + final restored = decodeCheckpoint(agg, env, kind: 'test', version: 1); + expect(restored, isNotNull); + final got = agg.finish(agg.addSlice(restored as S, batch, half, batch.length)); + final want = agg.run(batch); + final same = equals ?? (R a, R b) => a == b; + expect(same(got, want), isTrue, reason: 'resumed $got vs one-pass $want'); +} + +/// The JSON-text form of [agg]'s state after folding all of [batch], inside a +/// checkpoint envelope (kind 'test', version 1). +Map checkpointOf, R>( + Aggregator agg, B batch) => + (jsonDecode(jsonEncode( + encodeCheckpoint(agg, agg.fold(batch), kind: 'test', version: 1))) + as Map) + .cast(); + +/// Asserts that a checkpoint whose children disagree is refused with null. +/// +/// The undamaged checkpoint of [batch] must restore; then [damage] edits a +/// deep copy of its state JSON (in place, or returning a replacement) and the +/// result must restore as null, because the damage is a relationship between +/// children that each decode on their own. +void expectDamagedCheckpointRefused, R>( + Aggregator agg, + B batch, + Object? Function(Object? state) damage, { + String? reason, +}) { + final good = checkpointOf(agg, batch); + expect(decodeCheckpoint(agg, good, kind: 'test', version: 1), isNotNull, + reason: 'the undamaged checkpoint must restore'); + final state = jsonDecode(jsonEncode(good['state'])); + final damaged = damage(state) ?? state; + expect(jsonEncode(damaged), isNot(jsonEncode(good['state'])), + reason: 'the damage must change the state'); + final env = {...good, 'state': damaged}; + expect(decodeCheckpoint(agg, env, kind: 'test', version: 1), isNull, + reason: reason ?? jsonEncode(env)); +} diff --git a/test/support/fold_law_support.dart b/test/support/fold_law_support.dart new file mode 100644 index 00000000..0443fde9 --- /dev/null +++ b/test/support/fold_law_support.dart @@ -0,0 +1,133 @@ +// Pieces the fold-law files share (design 05, pilot cluster C2A): chunk +// boundaries, the mutation outcome vocabulary, and fixed-length int-vector +// generators (a recipe is a handful of ints, so a failing input shrinks to a +// few readable numbers instead of thousands of doubles). + +import 'dart:convert'; +import 'dart:math' as math; + +import 'package:test/test.dart'; + +import 'property.dart'; + +/// Chunk boundaries `[0, ...cuts, n]`. Cuts may repeat or sit on 0 / n, which +/// makes empty chunks; half of them are placed on [special] positions (the +/// edges of a module's windows), the rest uniformly. +List foldBounds(int n, int splits, int cutSeed, + {List special = const [0, 1, 2, 3]}) { + final r = Rng(cutSeed + 17); + final cuts = []; + for (var k = 0; k < splits; k++) { + final v = r.nextBool(.5) + ? (r.nextBool(.2) ? n - 1 + r.nextInt(2) : special[r.nextInt(special.length)]) + : r.intIn(0, n); + cuts.add(v.clamp(0, n)); + } + cuts.sort(); + return [0, ...cuts, n]; +} + +bool hasEmptyChunk(List b) { + for (var k = 0; k + 1 < b.length; k++) { + if (b[k + 1] == b[k]) return true; + } + return false; +} + +/// The seams a law checks against an expensive reference or batch parity check: first, middle, last. +Set pickSeams(int seams) => + {0, seams ~/ 2, seams - 1}..removeWhere((k) => k < 0); + +/// What a mutated checkpoint is allowed to do. +enum Must { + /// Refused with a FormatException, nothing else. + refuse, + + /// Refused, or accepted holding exactly what was written. + faithful, + + /// Refused, or accepted as the ORIGINAL (the change is ignored). + ignored, +} + +/// (kind, a, b): which mutation, and two parameters. +typedef Mut = (int, int, int); + +Object? deepCopy(Object? o) => jsonDecode(jsonEncode(o)); + +/// Checks one outcome of reading a mutated checkpoint: [outcome] is `refused`, +/// `accepted:` or `WRONG ERROR TYPE...`. +void expectMutationOutcome(String outcome, Must must, String faithfulText, + String originalText, String tag) { + expect(outcome, isNot(startsWith('WRONG')), reason: '$tag: only FormatException'); + switch (must) { + case Must.refuse: + expect(outcome, 'refused', reason: '$tag: refused whole'); + case Must.faithful: + if (outcome != 'refused') { + expect(outcome, 'accepted:$faithfulText', + reason: '$tag: accepted means exactly what was written'); + } + case Must.ignored: + if (outcome != 'refused') { + expect(outcome, 'accepted:$originalText', + reason: '$tag: an unknown key is ignored, nothing else moves'); + } + } +} + +class MutKindGen extends Gen { + MutKindGen(this.kinds); + final int kinds; + @override + int generate(Rng r, int size) => r.nextInt(kinds); + @override + Iterable shrink(int v) => G.intIn(0, kinds - 1).shrink(v); +} + +Gen mutGen(int kinds) => + G.triple(MutKindGen(kinds), G.intIn(0, 1 << 20), G.intIn(0, 1 << 20)); + +/// A size: the interesting small values first, then growing with the case +/// index up to [maxN] (cost: a property has two seconds). +class SizeGen extends Gen { + SizeGen(this.maxN, {this.pool = const [0, 1, 2, 3, 4, 5, 10]}); + final int maxN; + final List pool; + @override + int generate(Rng r, int size) { + final p = r.nextDouble(); + if (p < .04) return 0; + if (p < .29) { + final ok = [for (final x in pool) if (x <= maxN) x]; + return ok[r.nextInt(ok.length)]; + } + if (p < .40) return r.intIn(math.min(500, maxN), maxN); + final cap = math.max(120, maxN * size ~/ 100); + if (p < .62) return r.intIn(0, math.min(cap, 400)); + return r.intIn(0, math.min(cap, maxN)); + } + + @override + Iterable shrink(int v) => G.intIn(0, maxN).shrink(v); +} + +/// A fixed-length vector of ints, each slot with its own generator, each +/// shrunk on its own. +class IntsGen extends Gen> { + IntsGen(this.slots); + final List> slots; + @override + List generate(Rng r, int size) => [for (final g in slots) g.generate(r, size)]; + @override + Iterable> shrink(List v) sync* { + for (var i = 0; i < slots.length; i++) { + for (final c in slots[i].shrink(v[i])) { + yield [...v.sublist(0, i), c, ...v.sublist(i + 1)]; + } + } + } + + @override + String show(List v) => '$v'; +} diff --git a/test/support/law_registry.dart b/test/support/law_registry.dart new file mode 100644 index 00000000..7f78f3a1 --- /dev/null +++ b/test/support/law_registry.dart @@ -0,0 +1,124 @@ +// LAW REGISTRY — every law states which situations its default cases must +// reach, and one test per file measures that (design 05, generator reach). +// +// A property that never meets the situation it claims to cover is a green test +// that checks nothing. So each law is registered with a [Reach]: named +// situations, each with a share of the law's default case count it must reach, +// and an observer that names the situations of one input WITHOUT running the +// code under test (the situations come from the generator's model). The reach +// test re-draws the law's default cases (the forced examples are not counted: +// they are the part that is hand-placed) and fails when a share is not met. +// +// PROPERTY_REACH_REPORT=1 prints what each law reached, to retune the shares +// after a generator change. +// +// final laws = LawSet(); +// laws.law('L1 ...', gen, body, reach: Reach({...}, observe)); +// laws.registerReachTest(); // once, last in main() + +import 'dart:io' show Platform; +import 'dart:math' as math; + +import 'package:test/test.dart'; + +import 'property.dart'; + +/// What a law's default cases must reach. +class Reach { + const Reach(this.shares, this.observe); + + /// Situation name -> share of the default case count it must reach + /// (need = max(1, floor(cases * share))). + final Map shares; + + /// Names the situations of one input by calling `bump('name')`. + final void Function(T arg, void Function(String) bump) observe; +} + +class _Registered { + _Registered(this.name, this.cases, this.shares, this.measure); + final String name; + final int cases; + final Map shares; + final Map Function() measure; +} + +class LawSet { + final List<_Registered> _registry = []; + + /// Number of registered laws. + int get length => _registry.length; + + /// Registers [name] as a property (forced [examples] first, then [cases] + /// generated ones) and records its reach. + void law( + String name, + Gen gen, + void Function(T) body, { + required Reach reach, + List examples = const [], + int cases = 30, + String genVersion = 'g1', + Duration? budget, + Object? skip, + }) { + _registry.add(_Registered(name, cases, reach.shares, () { + final got = {for (final k in reach.shares.keys) k: 0}; + final r = runProperty( + name: name, + gen: gen, + config: PropertyConfig( + cases: cases, + // A generous guard, but PROPERTY_BUDGET_MS (the serial acceptance + // run) is the hard limit here too: drawing a law's cases is cheap. + budget: Duration( + milliseconds: int.tryParse( + Platform.environment['PROPERTY_BUDGET_MS'] ?? '') ?? + 60000)), + body: (v) => reach.observe(v, (k) => got[k] = (got[k] ?? 0) + 1), + genVersion: genVersion, + ); + if (!r.passed) fail(r.failure!.report); + return got; + })); + forAll(name, gen, body, + examples: examples, + cases: cases, + genVersion: genVersion, + // The spec's cap is 2 s per property, measured serially on the dev + // desktop. The in-test guard that fails a runaway has 2.5x headroom on + // top, so a suite running 12 test files at once does not trip it. + // PROPERTY_BUDGET_MS replaces it as the HARD limit (see + // PropertyConfig.fromEnvironment): the acceptance run is + // PROPERTY_BUDGET_MS=2000 dart test -j 1 test/properties/ + // and any property over 2 s there fails the run. + budget: budget ?? const Duration(seconds: 5), + skip: skip); + } + + /// The test that every law's default cases reach what the law claims. + void registerReachTest() { + group('generator reach', () { + test('every law sees every situation in its default cases', () { + expect(_registry, isNotEmpty); + final weak = []; + for (final law in _registry) { + final got = law.measure(); + if (Platform.environment.containsKey('PROPERTY_REACH_REPORT')) { + // ignore: avoid_print + print('REACH ${law.name.split(' ').first}/${law.cases} cases: ' + '${got.entries.map((e) => "${e.key}=${e.value}").join("; ")}'); + } + for (final e in law.shares.entries) { + final need = math.max(1, (law.cases * e.value).floor()); + if (got[e.key]! < need) { + weak.add('"${law.name}": ${e.key} in ${got[e.key]} of ' + '${law.cases} cases, need $need'); + } + } + } + expect(weak, isEmpty, reason: weak.join('\n')); + }); + }); + } +} diff --git a/test/support/property.dart b/test/support/property.dart new file mode 100644 index 00000000..89456a3c --- /dev/null +++ b/test/support/property.dart @@ -0,0 +1,753 @@ +// PROPERTY HARNESS — seeded, shrinking, replayable property tests. +// +// Port of the edge harness (edge test/support/property.dart, design 05 pilot): +// same generators, seeds, replay and budget rules, on package:test only (this +// package is pure Dart). Keep the two in step; the RNG is pinned to reference +// outputs in property_test.dart so a replay command means the same thing in +// both repos. +// +// In-house on purpose (design 05, section 3): no dependency, fully +// deterministic, and every failure prints what is needed to replay it. +// +// forAll('no two icons overlap', G.listOf(item, maxLen: 60), (items) { +// ... expect(...) ... +// }, examples: [...forced scenarios...]); +// +// How a run is built +// • SEED. Each property has its own seed, derived from its NAME (so adding or +// reordering properties never changes another property's inputs). +// PROPERTY_SEED overrides it for every property in the run. +// • CASES. Case `i` draws from its own generator stream `caseSeed(seed, i)` and +// a size that depends on `i` only, so PROPERTY_CASE=i replays exactly that +// input whatever the iteration count. Default 200 cases; PROPERTY_ITERATIONS +// overrides it (and scales the wall budget with it). +// • FORCED SCENARIOS. `examples` always run first, in order, before any +// generated case. They are never shrunk (they are explicit). Replay one with +// PROPERTY_CASE=forced:. +// • FAILURE. The first failing input is shrunk (bounded by `shrinkLimit` body +// runs) and printed with seed, case, generator version and a one-line replay +// command. Shrinkers are domain-preserving: a candidate is always something +// the generator itself could have produced. +// • BUDGET. A property that takes longer than its wall budget (default 2 s) +// fails. Shrinking counts against the same budget: when it runs out the +// best failing input so far is reported, marked "over budget (shrinking)". The budget is the ONLY +// use of a real clock; it is injectable for the harness's own tests. +// +// GENERATOR VERSION. A seed only means something for the generator that drew +// it. When a property's generator changes shape, bump its `genVersion`; the +// report prints it so an old replay command is recognised as stale. +// +// Environment: PROPERTY_SEED, PROPERTY_CASE, PROPERTY_ITERATIONS, +// PROPERTY_BUDGET_MS (fixes the budget, no scaling). + +import 'dart:io' show Platform; +import 'dart:math' as math; + +import 'package:test/test.dart'; + +// ── randomness ────────────────────────────────────────────────────────────── + +/// True on dart2js, where `1` and `1.0` are the same value. +const bool _onJs = identical(1, 1.0); + +/// splitmix64 (Steele, Lea, Flood; the public-domain reference generator): +/// tiny, fast, and identical on every Dart VM version (unlike `dart:math` +/// Random, whose stream is not promised to be stable). The first outputs for +/// seeds 0 and 1234567 are pinned in property_test.dart against values computed +/// independently. +/// +/// ASSUMPTION: a Dart VM `int` is a 64-bit two's-complement integer whose `+` +/// and `*` wrap, and `>>>` is a logical shift. That is what the arithmetic +/// below relies on. This harness is VM-only and must not be compiled for the +/// web (ints are doubles there); the constructor refuses to run on JS. +class Rng { + Rng(int seed) : _s = seed { + if (_onJs) { + throw UnsupportedError('test/support/property.dart needs 64-bit VM ints'); + } + } + int _s; + + static const int _gamma = 0x9E3779B97F4A7C15; + + static int mix(int x) { + var z = x; + z = (z ^ (z >>> 30)) * 0xBF58476D1CE4E5B9; + z = (z ^ (z >>> 27)) * 0x94D049BB133111EB; + return z ^ (z >>> 31); + } + + int _next() { + _s += _gamma; + return mix(_s); + } + + /// The next raw 64-bit output (a signed Dart int; same bits as the + /// reference's unsigned value). + int next64() => _next(); + + /// A uniform int in `[0, bound)`. `bound` must be in `1 .. 2^53`. + int nextInt(int bound) { + assert(bound > 0 && bound <= (1 << 53), 'bound out of range: $bound'); + return (_next() >>> 1) % bound; + } + + /// A uniform int in `[min, max]`, both inclusive. + int intIn(int min, int max) { + assert(min <= max); + return min + nextInt(max - min + 1); + } + + /// A uniform double in `[0, 1)`. + double nextDouble() => (_next() >>> 11) / 9007199254740992.0; + + /// True with probability [p]. + bool nextBool([double p = .5]) => nextDouble() < p; +} + +/// The default seed of the property called [name] (32-bit FNV-1a). +int seedFor(String name) { + var h = 0x811C9DC5; + for (final u in name.codeUnits) { + h = ((h ^ u) * 0x01000193) & 0xFFFFFFFF; + } + return h; +} + +/// The generator stream of case [index] of a property seeded [seed]. Depends on +/// nothing else, so one case can be replayed alone. +int caseSeed(int seed, int index) => + Rng.mix(Rng.mix(seed) + index * 0x9E3779B97F4A7C15) & 0x7FFFFFFFFFFF; + +// ── generators ────────────────────────────────────────────────────────────── + +/// Draws values and shrinks them. [shrink] must only return values this +/// generator could itself have produced (domain-preserving), none equal to the +/// input, simplest first, and finitely many per call. +abstract class Gen { + const Gen(); + + /// A value; [size] grows 10..100 across the cases of a run. + T generate(Rng r, int size); + + /// Simpler candidates than [v], simplest first. May be lazy. + Iterable shrink(T v); + + /// A readable, copy-pasteable rendering for the failure report. + String show(T v) => '$v'; +} + +class _IntGen extends Gen { + _IntGen(this.min, this.max, int? origin) + : origin = (origin ?? 0).clamp(min, max).toInt() { + assert(min <= max && max - min < (1 << 53)); + } + final int min, max, origin; + + @override + int generate(Rng r, int size) { + final p = r.nextDouble(); + if (p < .06) return min; + if (p < .12) return max; + if (p < .16) return origin; + return r.intIn(min, max); + } + + @override + Iterable shrink(int v) sync* { + if (v == origin) return; + yield origin; + final d = v - origin; + var step = d ~/ 2; + final seen = {origin}; + while (step != 0) { + final c = v - step; + if (seen.add(c)) yield c; + step ~/= 2; + } + final one = v - d.sign; + if (seen.add(one) && one != v) yield one; + } +} + +class _DoubleGen extends Gen { + _DoubleGen(this.min, this.max, this.boundaries, this.specials, double? origin, + this.integerBias) + : origin = (origin ?? 0).clamp(min, max).toDouble() { + assert(min.isFinite && max.isFinite && min <= max); + } + final double min, max, origin, integerBias; + final List boundaries, specials; + + List get _pool => [ + min, + max, + if (origin != min && origin != max) origin, + for (final b in boundaries) + if (b >= min && b <= max) b, + ]; + + @override + double generate(Rng r, int size) { + final p = r.nextDouble(); + if (specials.isNotEmpty && p < .08) { + return specials[r.nextInt(specials.length)]; + } + if (p < .08 + .17) { + final pool = _pool; + return pool[r.nextInt(pool.length)]; + } + var v = min + r.nextDouble() * (max - min); + if (r.nextBool(integerBias)) v = v.roundToDouble(); + return v.clamp(min, max).toDouble(); + } + + @override + Iterable shrink(double v) sync* { + if (!v.isFinite) { + yield origin; // a special shrinks to the simplest finite value + return; + } + if (v == origin) return; + final d = v - origin; + if (d.abs() < 1e-6) { + yield origin; + return; + } + final seen = {v}; + Iterable cand() sync* { + yield origin; + for (final b in _pool) { + if ((b - origin).abs() < d.abs()) yield b; + } + yield v.truncateToDouble(); + yield v - d / 2; + yield v - d / 4; + yield v - d / 8; + } + + for (final c in cand()) { + if (c >= min && c <= max && seen.add(c)) yield c; + } + } +} + +class _BoolGen extends Gen { + @override + bool generate(Rng r, int size) => r.nextBool(); + @override + Iterable shrink(bool v) => v ? const [false] : const []; +} + +class _ElementsGen extends Gen { + _ElementsGen(this.values) { + assert(values.isNotEmpty); + } + final List values; + @override + T generate(Rng r, int size) => values[r.nextInt(values.length)]; + @override + Iterable shrink(T v) { + final i = values.indexOf(v); + return i <= 0 ? const [] : values.take(i); + } +} + +class _ListGen extends Gen> { + _ListGen(this.elem, this.minLen, this.maxLen) { + assert(minLen >= 0 && minLen <= maxLen); + } + final Gen elem; + final int minLen, maxLen; + + @override + List generate(Rng r, int size) { + final cap = minLen + ((maxLen - minLen) * (size + 1) / 101).ceil(); + final p = r.nextDouble(); + final n = p < .08 + ? minLen + : p < .16 + ? maxLen + : r.intIn(minLen, math.max(minLen, cap)); + return [for (var i = 0; i < n; i++) elem.generate(r, size)]; + } + + @override + Iterable> shrink(List v) sync* { + // Shorter first: drop chunks, big to small. + if (v.length > minLen) { + for (var chunk = math.max(1, v.length ~/ 2); chunk >= 1; chunk ~/= 2) { + if (v.length - chunk < minLen) continue; + for (var at = 0; at + chunk <= v.length; at += chunk) { + yield [...v.sublist(0, at), ...v.sublist(at + chunk)]; + } + } + } + // Then simpler elements, in place. + for (var i = 0; i < v.length; i++) { + for (final e in elem.shrink(v[i])) { + yield [...v.sublist(0, i), e, ...v.sublist(i + 1)]; + } + } + } + + @override + String show(List v) => '[${v.map(elem.show).join(', ')}]'; +} + +class _NullableGen extends Gen { + _NullableGen(this.inner, this.nullProbability); + final Gen inner; + final double nullProbability; + @override + T? generate(Rng r, int size) => + r.nextBool(nullProbability) ? null : inner.generate(r, size); + @override + Iterable shrink(T? v) sync* { + if (v == null) return; + yield null; + yield* inner.shrink(v); + } + + @override + String show(T? v) => v == null ? 'null' : inner.show(v); +} + +/// Candidate lists of the components of a tuple, shrunk SEPARATELY and then +/// TOGETHER (the k-th simpler candidate of every component at once), so a law +/// that only fails on correlated components (a == b) still shrinks. +Iterable> _shrinkTuple( + List> gens, List v) sync* { + final cands = [ + for (var i = 0; i < gens.length; i++) gens[i].shrink(v[i]).take(12).toList() + ]; + // Joint steps first: they make the biggest strides and are the only ones + // that work when the components are correlated. + final joint = cands.map((c) => c.length).reduce(math.min); + for (var k = 0; k < joint; k++) { + yield [for (final c in cands) c[k]]; + } + for (var i = 0; i < gens.length; i++) { + for (final c in gens[i].shrink(v[i])) { + yield [...v.sublist(0, i), c, ...v.sublist(i + 1)]; + } + } +} + +class _TupleGen extends Gen { + _TupleGen(this.gens, this.build, this.parts); + final List> gens; + final R Function(List) build; + final List Function(R) parts; + + @override + R generate(Rng r, int size) => + build([for (final g in gens) g.generate(r, size)]); + @override + Iterable shrink(R v) => _shrinkTuple(gens, parts(v)).map(build); + @override + String show(R v) { + final p = parts(v); + return '(${[for (var i = 0; i < gens.length; i++) gens[i].show(p[i])].join(', ')})'; + } +} + +/// The generator constructors. +abstract final class G { + /// An int in `[min, max]`; shrinks toward [origin] (default: zero, or the + /// bound nearest zero). + static Gen intIn(int min, int max, {int? origin}) => + _IntGen(min, max, origin); + + /// A double in `[min, max]` (finite). [boundaries] are drawn deliberately; + /// [specials] (NaN, infinities) are drawn occasionally and shrink to + /// [origin]. [integerBias] is the share of draws snapped to whole numbers + /// (ties and exact thresholds are where the interesting laws bend). + static Gen doubleIn(double min, double max, + {List boundaries = const [], + List specials = const [], + double? origin, + double integerBias = .25}) => + _DoubleGen(min, max, boundaries, specials, origin, integerBias); + + static Gen boolean() => _BoolGen(); + + /// One of [values]; shrinks toward the earlier ones. + static Gen elements(List values) => _ElementsGen(values); + + /// A list of `minLen .. maxLen` elements (the cap is a hard bound). + static Gen> listOf(Gen elem, + {int minLen = 0, required int maxLen}) => + _ListGen(elem, minLen, maxLen); + + static Gen nullable(Gen inner, {double nullProbability = .25}) => + _NullableGen(inner, nullProbability); + + static Gen<(A, B)> pair(Gen a, Gen b) => _TupleGen( + [a, b], (p) => (p[0] as A, p[1] as B), (r) => [r.$1, r.$2]); + + static Gen<(A, B, C)> triple(Gen a, Gen b, Gen c) => + _TupleGen([a, b, c], (p) => (p[0] as A, p[1] as B, p[2] as C), + (r) => [r.$1, r.$2, r.$3]); + + static Gen<(A, B, C, D)> quad( + Gen a, Gen b, Gen c, Gen d) => + _TupleGen( + [a, b, c, d], + (p) => (p[0] as A, p[1] as B, p[2] as C, p[3] as D), + (r) => [r.$1, r.$2, r.$3, r.$4]); +} + +// ── configuration and result ──────────────────────────────────────────────── + +class PropertyConfig { + const PropertyConfig({ + this.seed, + this.cases = 200, + this.caseOnly, + this.budget = const Duration(seconds: 2), + this.shrinkLimit = 300, + this.testFile, + }); + + /// Reads PROPERTY_SEED / PROPERTY_CASE / PROPERTY_ITERATIONS / + /// PROPERTY_BUDGET_MS from [env]. A malformed value throws: a typo must not + /// silently run the default. + factory PropertyConfig.fromEnvironment( + Map env, { + String? testFile, + int? seed, + int defaultCases = 200, + Duration defaultBudget = const Duration(seconds: 2), + int shrinkLimit = 300, + }) { + int? number(String key, {int min = 0}) { + final raw = env[key]; + if (raw == null || raw.isEmpty) return null; + final n = int.tryParse(raw); + if (n == null || n < min) { + throw FormatException('$key must be an integer >= $min', raw); + } + return n; + } + + final envSeed = number('PROPERTY_SEED', min: -(1 << 62)); + final iterations = number('PROPERTY_ITERATIONS', min: 1); + final budgetMs = number('PROPERTY_BUDGET_MS', min: 1); + final caseRaw = env['PROPERTY_CASE']; + if (caseRaw != null && + caseRaw.isNotEmpty && + !RegExp(r'^(forced:)?\d+$').hasMatch(caseRaw)) { + throw FormatException('PROPERTY_CASE must be N or forced:N', caseRaw); + } + final cases = iterations ?? defaultCases; + final scale = math.max(1.0, cases / defaultCases); + return PropertyConfig( + seed: envSeed ?? seed, + cases: cases, + caseOnly: caseRaw == null || caseRaw.isEmpty ? null : caseRaw, + budget: budgetMs != null + ? Duration(milliseconds: budgetMs) + : iterations != null + ? Duration( + milliseconds: (defaultBudget.inMilliseconds * scale).round()) + : defaultBudget, + shrinkLimit: shrinkLimit, + testFile: testFile, + ); + } + + /// null: derived from the property name. + final int? seed; + final int cases; + + /// Replay only this case: `N` or `forced:N`. null: the whole run. + final String? caseOnly; + final Duration budget; + + /// Most body runs spent shrinking one failure. + final int shrinkLimit; + + /// Repo-relative path printed in the replay command. + final String? testFile; +} + +enum FailureKind { counterexample, forcedExample, budget, generator } + +class PropertyFailure { + const PropertyFailure({ + required this.kind, + required this.seed, + required this.caseId, + required this.input, + required this.error, + required this.replay, + required this.shrinkSteps, + required this.report, + this.shrinkOverBudget = false, + }); + final FailureKind kind; + final int seed; + + /// `17` for a generated case, `forced:1` for an explicit example. + final String caseId; + + /// The (shrunk) failing input, as the generator shows it. + final String input; + final String error; + final String replay; + + /// Body runs spent shrinking. + final int shrinkSteps; + final String report; + + /// Shrinking stopped because the property's wall budget ran out; [input] is + /// the best (smallest) failing input found so far, not necessarily minimal. + final bool shrinkOverBudget; +} + +class PropertyResult { + const PropertyResult( + {required this.casesRun, required this.forcedRun, this.failure}); + final int casesRun; + final int forcedRun; + final PropertyFailure? failure; + bool get passed => failure == null; +} + +// ── running ───────────────────────────────────────────────────────────────── + +String _errorText(Object e, StackTrace st) { + final msg = e.toString(); + final text = msg.length > 900 ? '${msg.substring(0, 900)}...' : msg; + if (e is TestFailure) return text; + // A non-assertion error (RangeError, StateError, ...) needs its origin. + final frames = st.toString().split('\n').where((l) => l.isNotEmpty).take(6); + return '$text\n ${frames.join('\n ')}'; +} + +/// Runs the property and returns the outcome; never throws for a failing body. +/// [forAll] is the test-registering wrapper. +PropertyResult runProperty({ + required String name, + required Gen gen, + required void Function(T) body, + List examples = const [], + PropertyConfig config = const PropertyConfig(), + String genVersion = 'g1', + Duration Function()? elapsed, +}) { + final seed = config.seed ?? seedFor(name); + late final Duration Function() clock; + if (elapsed != null) { + clock = elapsed; + } else { + final sw = Stopwatch()..start(); + clock = () => sw.elapsed; + } + + String? attempt(T v) { + try { + body(v); + return null; + } catch (e, st) { + return _errorText(e, st); + } + } + + String replayFor(String? caseId) => + 'PROPERTY_SEED=$seed ${caseId == null ? '' : 'PROPERTY_CASE=$caseId '}' + 'dart test ' + "${config.testFile ?? ''} " + "--plain-name '${name.replaceAll("'", r"'\''")}'"; + + PropertyFailure fail(FailureKind kind, String caseId, String input, + String error, + {int shrinkSteps = 0, + int accepted = 0, + String? original, + bool overBudget = false}) { + final replay = replayFor(kind == FailureKind.budget ? null : caseId); + final b = StringBuffer() + ..writeln('Property "$name" FAILED (${kind.name})') + ..writeln(' seed: $seed case: $caseId generator: $genVersion' + ' shrink limit: ${config.shrinkLimit}'); + if (kind == FailureKind.budget) { + b.writeln(' $error'); + } else { + b.writeln(shrinkSteps > 0 + ? ' input (shrunk: $accepted accepted of $shrinkSteps tried): $input' + : ' input: $input'); + if (overBudget) { + b.writeln(' over budget (shrinking): stopped after ${config.budget.inMilliseconds} ' + 'ms; this is the best failing input so far, not necessarily minimal'); + } + if (original != null && original != input) { + b.writeln(' original input: $original'); + } + b.writeln(' error: $error'); + } + b.writeln(' replay: $replay'); + return PropertyFailure( + kind: kind, + seed: seed, + caseId: caseId, + input: input, + error: error, + replay: replay, + shrinkSteps: shrinkSteps, + report: b.toString(), + shrinkOverBudget: overBudget); + } + + PropertyFailure shrunk(String caseId, T original, String error) { + var cur = original; + var err = error; + var tried = 0, accepted = 0; + var overBudget = false; + try { + outer: + while (true) { + var progressed = false; + for (final c in gen.shrink(cur)) { + if (tried >= config.shrinkLimit) break outer; + // The wall budget covers shrinking too (a replay is exempt). + if (config.caseOnly == null && clock() > config.budget) { + overBudget = true; + break outer; + } + tried++; + final e = attempt(c); + if (e != null) { + cur = c; + err = e; + accepted++; + progressed = true; + break; + } + } + if (!progressed) break; + } + } catch (_) { + // A shrinker that throws ends shrinking; the best input so far stands. + } + return fail(FailureKind.counterexample, caseId, gen.show(cur), err, + shrinkSteps: tried, + accepted: accepted, + original: gen.show(original), + overBudget: overBudget); + } + + PropertyResult done(int cases, int forced, [PropertyFailure? f]) => + PropertyResult(casesRun: cases, forcedRun: forced, failure: f); + + // One generated case; returns a failure or null. + PropertyFailure? runCase(int i) { + final T input; + try { + input = gen.generate(Rng(caseSeed(seed, i)), math.min(100, 10 + i)); + } catch (e, st) { + return fail(FailureKind.generator, '$i', '', + _errorText(e, st)); + } + final e = attempt(input); + return e == null ? null : shrunk('$i', input, e); + } + + PropertyFailure? runForced(int i) { + final e = attempt(examples[i]); + return e == null + ? null + : fail(FailureKind.forcedExample, 'forced:$i', gen.show(examples[i]), e); + } + + final only = config.caseOnly; + if (only != null) { + if (only.startsWith('forced:')) { + final i = int.parse(only.substring(7)); + if (i >= examples.length) { + throw StateError('PROPERTY_CASE=$only: only ${examples.length} forced ' + 'examples in "$name"'); + } + return done(0, 1, runForced(i)); + } + return done(1, 0, runCase(int.parse(only))); + } + + PropertyFailure? overBudget(int cases, int forced) => clock() > config.budget + ? fail( + FailureKind.budget, + 'n/a', + '', + 'wall budget ${config.budget.inMilliseconds} ms exceeded after ' + '$cases cases + $forced forced examples ' + '(${clock().inMilliseconds} ms)') + : null; + + var forcedRun = 0, casesRun = 0; + for (var i = 0; i < examples.length; i++) { + final f = runForced(i); + forcedRun++; + if (f != null) return done(casesRun, forcedRun, f); + final b = overBudget(casesRun, forcedRun); + if (b != null) return done(casesRun, forcedRun, b); + } + for (var i = 0; i < config.cases; i++) { + final f = runCase(i); + casesRun++; + if (f != null) return done(casesRun, forcedRun, f); + final b = overBudget(casesRun, forcedRun); + if (b != null) return done(casesRun, forcedRun, b); + } + return done(casesRun, forcedRun); +} + +String? _callerTestFile() { + final m = RegExp(r'test/[\w/.\-]+\.dart'); + for (final line in StackTrace.current.toString().split('\n')) { + final hit = m.firstMatch(line)?.group(0); + if (hit != null && !RegExp(r'support/(property|law_registry)\.dart$').hasMatch(hit)) { + return hit; + } + } + return null; +} + +/// Registers one `test` that checks [body] on [examples] (forced, first) and on +/// generated cases. Replay with the command in the failure report. +/// +/// [cases], [budget] and [seed] are this property's defaults; the PROPERTY_* +/// environment wins. [skip] is for a law that currently fails because of a +/// suspected bug: say which in the reason. +void forAll( + String name, + Gen gen, + void Function(T) body, { + List examples = const [], + int? cases, + int? seed, + Duration? budget, + int? shrinkLimit, + String genVersion = 'g1', + Object? skip, +}) { + final file = _callerTestFile(); + test(name, () { + final config = PropertyConfig.fromEnvironment( + Platform.environment, + testFile: file, + seed: seed, + defaultCases: cases ?? 200, + defaultBudget: budget ?? const Duration(seconds: 2), + shrinkLimit: shrinkLimit ?? 300, + ); + final r = runProperty( + name: name, + gen: gen, + body: body, + examples: examples, + config: config, + genVersion: genVersion); + if (!r.passed) fail(r.failure!.report); + }, skip: skip); +} diff --git a/test/support/property_test.dart b/test/support/property_test.dart new file mode 100644 index 00000000..69938a31 --- /dev/null +++ b/test/support/property_test.dart @@ -0,0 +1,778 @@ +// Self-tests of the property harness (test/support/property.dart). +// +// The harness decides whether the property suites can be trusted, so its own +// guarantees are pinned here: deterministic and independent seeds, replay of a +// single case, forced examples first, domain-preserving bounded shrinking, the +// failure report (seed, case, input, generator version, replay command) and +// the per-property wall budget. + +import 'dart:math' as math; + +import 'package:test/test.dart'; + +import 'property.dart'; + +/// Runs [gen] through the harness with a body that records every input and +/// never fails. Returns the recorded inputs (forced examples first). +List _inputs(Gen gen, + {String name = 'rec', + int cases = 30, + List examples = const [], + int? seed, + String? caseOnly}) { + final seen = []; + final r = runProperty( + name: name, + gen: gen, + body: seen.add, + examples: examples, + config: PropertyConfig(seed: seed, cases: cases, caseOnly: caseOnly), + ); + expect(r.passed, isTrue, reason: r.failure?.report); + return seen; +} + +PropertyFailure _fails(Gen gen, bool Function(T) holds, + {List examples = const [], + int cases = 200, + int shrinkLimit = 300, + String name = 'p', + int seed = 7, + String? caseOnly, + String? testFile}) { + final r = runProperty( + name: name, + gen: gen, + body: (v) { + if (!holds(v)) throw StateError('does not hold: $v'); + }, + examples: examples, + config: PropertyConfig( + seed: seed, + cases: cases, + shrinkLimit: shrinkLimit, + caseOnly: caseOnly, + testFile: testFile), + ); + expect(r.passed, isFalse, reason: 'the property should have failed'); + return r.failure!; +} + +void main() { + group('rng', () { + test('the same seed gives the same sequence, another seed another', () { + List draw(int seed) { + final r = Rng(seed); + return [for (var i = 0; i < 20; i++) r.nextInt(1 << 20)]; + } + + expect(draw(5), draw(5)); + expect(draw(5), isNot(draw(6))); + }); + + test('intIn is inclusive, nextDouble is in [0, 1), nextBool honours p', () { + final r = Rng(42); + final ints = {for (var i = 0; i < 400; i++) r.intIn(-2, 2)}; + expect(ints, {-2, -1, 0, 1, 2}); + for (var i = 0; i < 400; i++) { + final d = r.nextDouble(); + expect(d >= 0 && d < 1, isTrue); + } + expect(r.intIn(3, 3), 3); + expect([for (var i = 0; i < 50; i++) r.nextBool(0)], everyElement(false)); + expect([for (var i = 0; i < 50; i++) r.nextBool(1)], everyElement(true)); + }); + + test('next64 is SplitMix64: the first outputs match the reference values', + () { + // Computed independently with python3, masking every step to 64 bits: + // s = (s + 0x9E3779B97F4A7C15) & M + // z = ((s ^ (s >> 30)) * 0xBF58476D1CE4E5B9) & M + // z = ((z ^ (z >> 27)) * 0x94D049BB133111EB) & M; out = z ^ (z >> 31) + // Seed 0 starts 0xE220A8397B1DCDAF, the published SplitMix64 test value. + // Dart's signed int holds the same 64 bits, so compare as hex. + String hex(int v) => + (v >>> 32).toRadixString(16).padLeft(8, '0') + + (v & 0xFFFFFFFF).toRadixString(16).padLeft(8, '0'); + final z = Rng(0); + expect([for (var i = 0; i < 5; i++) hex(z.next64())], [ + 'e220a8397b1dcdaf', + '6e789e6aa1b965f4', + '06c45d188009454f', + 'f88bb8a8724c81ec', + '1b39896a51a8749b', + ]); + final k = Rng(1234567); + expect([for (var i = 0; i < 5; i++) hex(k.next64())], [ + '599ed017fb08fc85', + '2c73f08458540fa5', + '883ebce5a3f27c77', + '3fbef740e9177b3f', + 'e3b8346708cb5ecd', + ]); + }); + + test('the stream is pinned: a replay command stays valid across SDKs', () { + final r = Rng(1); + expect([for (var i = 0; i < 5; i++) r.nextInt(1000)], + [232, 259, 295, 117, 380]); + expect(Rng(1).nextDouble(), 0.5665615751722809); + expect(seedFor('a law'), 1308619236); + expect(caseSeed(9, 3), 136612318154068); + }); + + test('draws are spread: a long run touches every bucket of 16', () { + final r = Rng(1); + final buckets = {for (var i = 0; i < 2000; i++) r.nextInt(16)}; + expect(buckets.length, 16); + }); + }); + + group('rng ranges', () { + forAll<(int, int)>( + 'rng draws stay in range: nextInt(bound) in [0, bound), nextDouble in [0, 1)', + G.pair( + G.intIn(-(1 << 40), 1 << 40), + G.intIn(0, 9)), + (arg) { + final (seed, pick) = arg; + const bounds = [1, 2, 3, 7, 1000, 1 << 20, (1 << 53) - 1, 1 << 53]; + final r = Rng(seed); + for (var i = 0; i < 40; i++) { + final b = bounds[(pick + i) % bounds.length]; + final n = r.nextInt(b); + expect(n >= 0 && n < b, isTrue, reason: 'nextInt($b) = $n'); + final d = r.nextDouble(); + expect(d >= 0 && d < 1, isTrue, reason: 'nextDouble() = $d'); + final k = r.intIn(0, b - 1); + expect(k >= 0 && k < b, isTrue, reason: 'intIn(0, ${b - 1}) = $k'); + } + expect(Rng(seed).nextInt(1), 0); + }, examples: const [(0, 0), (-1, 7), (1 << 40, 3)]); + }); + + group('seeds', () { + test('seedFor is stable per name and differs between names', () { + expect(seedFor('a law'), seedFor('a law')); + expect(seedFor('a law'), isNot(seedFor('another law'))); + }); + + test('case seeds are independent of each other and of the run length', () { + expect(caseSeed(9, 3), caseSeed(9, 3)); + expect(caseSeed(9, 3), isNot(caseSeed(9, 4))); + expect(caseSeed(9, 3), isNot(caseSeed(10, 3))); + }); + + test('two properties with different names draw different inputs', () { + final gen = G.listOf(G.intIn(0, 1 << 20), maxLen: 8); + expect(_inputs(gen, name: 'one', cases: 5), + isNot(_inputs(gen, name: 'two', cases: 5))); + }); + + test('the same property is reproduced exactly', () { + final gen = G.pair(G.intIn(-50, 50), G.doubleIn(0, 1)); + expect(_inputs(gen, cases: 40), _inputs(gen, cases: 40)); + }); + }); + + group('config from the environment', () { + test('defaults: 200 cases, 2 s, no seed, no single case', () { + final c = PropertyConfig.fromEnvironment(const {}); + expect(c.cases, 200); + expect(c.budget, const Duration(seconds: 2)); + expect(c.seed, isNull); + expect(c.caseOnly, isNull); + }); + + test('PROPERTY_SEED, PROPERTY_CASE and PROPERTY_ITERATIONS override', () { + final c = PropertyConfig.fromEnvironment(const { + 'PROPERTY_SEED': '12345', + 'PROPERTY_CASE': '17', + 'PROPERTY_ITERATIONS': '1000', + }, testFile: 'test/x_test.dart'); + expect(c.seed, 12345); + expect(c.caseOnly, '17'); + expect(c.cases, 1000); + expect(c.testFile, 'test/x_test.dart'); + }); + + test('raising the iterations scales the wall budget with them', () { + final c = + PropertyConfig.fromEnvironment(const {'PROPERTY_ITERATIONS': '1000'}); + expect(c.budget, const Duration(seconds: 10)); + final fixed = PropertyConfig.fromEnvironment(const { + 'PROPERTY_ITERATIONS': '1000', + 'PROPERTY_BUDGET_MS': '3000', + }); + expect(fixed.budget, const Duration(milliseconds: 3000)); + }); + + test('a malformed number is loud, not ignored', () { + expect(() => PropertyConfig.fromEnvironment(const {'PROPERTY_SEED': 'x'}), + throwsFormatException); + expect( + () => PropertyConfig.fromEnvironment( + const {'PROPERTY_ITERATIONS': '0'}), + throwsFormatException); + }); + }); + + group('running', () { + test('runs exactly the configured number of cases', () { + final r = runProperty( + name: 'n', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig(cases: 37), + ); + expect(r.passed, isTrue); + expect(r.casesRun, 37); + expect(r.forcedRun, 0); + }); + + test('forced examples run first, in order, then the generated cases', () { + final seen = _inputs(G.intIn(100, 200), + examples: [5, 6, 7], cases: 10); + expect(seen.take(3), [5, 6, 7]); + expect(seen.length, 13); + final r = runProperty( + name: 'f', + gen: G.intIn(0, 9), + body: (_) {}, + examples: [1, 2], + config: const PropertyConfig(cases: 4)); + expect((r.forcedRun, r.casesRun), (2, 4)); + }); + + test('generated cases stay inside the generator, even the first ones', () { + final seen = _inputs(G.intIn(-3, 11), cases: 200); + expect(seen.every((v) => v >= -3 && v <= 11), isTrue); + }); + + test('a single case replays exactly the input it had in the full run', () { + final gen = G.pair(G.intIn(-1000, 1000), G.listOf(G.boolean(), maxLen: 9)); + final all = _inputs(gen, cases: 60, seed: 99); + for (final k in [0, 1, 17, 59]) { + expect( + _inputs(gen, cases: 60, seed: 99, caseOnly: '$k') + .map(gen.show) + .toList(), + [gen.show(all[k])], + reason: 'case $k'); + } + }); + + test('a case replays the same whatever the iteration count', () { + final gen = G.listOf(G.intIn(0, 1 << 20), maxLen: 30); + final short = _inputs(gen, cases: 25, seed: 3); + final long = _inputs(gen, cases: 80, seed: 3); + expect(long.take(25), short); + }); + + test('a forced example replays by its forced id, generated cases skip it', + () { + final seen = _inputs(G.intIn(100, 200), + examples: [5, 6, 7], cases: 10, caseOnly: 'forced:1'); + expect(seen, [6]); + final one = _inputs(G.intIn(100, 200), + examples: [5, 6, 7], cases: 10, seed: 1, caseOnly: '2'); + expect(one.length, 1); + expect(one.single, inInclusiveRange(100, 200)); + }); + + test('a replay of a case beyond the run length still runs that case', () { + final seen = _inputs(G.intIn(0, 9), cases: 5, caseOnly: '40'); + expect(seen.length, 1); + }); + + test('a body that completes normally is passed, whatever it returns', () { + final r = runProperty( + name: 'ret', gen: G.intIn(0, 3), body: (_) => 5 as dynamic); + expect(r.passed, isTrue); + }); + }); + + group('failure report', () { + test('names the seed, case, shrunk input, generator version, replay', () { + final f = _fails(G.intIn(0, 1000), (v) => v < 50, + name: 'ints stay small', seed: 4242, testFile: 'test/x_test.dart'); + expect(f.kind, FailureKind.counterexample); + expect(f.seed, 4242); + expect(f.input, '50'); + expect(f.caseId, matches(RegExp(r'^\d+$'))); + expect(f.error, contains('does not hold')); + expect( + f.replay, + 'PROPERTY_SEED=4242 PROPERTY_CASE=${f.caseId} ' + "dart test test/x_test.dart --plain-name 'ints stay small'"); + for (final s in [ + 'ints stay small', + 'seed: 4242', + 'case: ${f.caseId}', + 'generator: g1', + 'shrink limit: 300', + 'input', + '50', + f.replay, + ]) { + expect(f.report, contains(s)); + } + }); + + test('the generator version is whatever the property declares', () { + final r = runProperty( + name: 'v', + gen: G.intIn(0, 9), + body: (_) => throw StateError('x'), + genVersion: 'layout-gen/3', + config: const PropertyConfig(seed: 1)); + expect(r.failure!.report, contains('generator: layout-gen/3')); + }); + + test('a quote in the name is escaped in the replay command', () { + final f = _fails(G.intIn(0, 9), (_) => false, name: "it's a law"); + expect(f.replay, contains(r"--plain-name 'it'\''s a law'")); + }); + + test('an unknown test file is a visible placeholder, not omitted', () { + final f = _fails(G.intIn(0, 9), (_) => false); + expect(f.replay, contains('dart test ')); + }); + + test('the replay command reproduces the failure', () { + final gen = G.listOf(G.intIn(0, 99), maxLen: 25); + final f = _fails(gen, (l) => l.length < 20, seed: 11); + final again = + _fails(gen, (l) => l.length < 20, seed: 11, caseOnly: f.caseId); + expect(again.input, f.input); + expect(again.caseId, f.caseId); + }); + + test('is deterministic: the same run produces the same report', () { + final gen = G.pair(G.intIn(0, 500), G.doubleIn(0, 1)); + String report() => _fails(gen, (p) => p.$1 < 400).report; + expect(report(), report()); + }); + + test('a thrown TestFailure keeps its message', () { + final r = runProperty( + name: 'tf', + gen: G.intIn(0, 3), + body: (v) => expect(v, 99, reason: 'my reason'), + config: const PropertyConfig(seed: 1)); + expect(r.failure!.error, contains('my reason')); + }); + + test('a failing forced example is reported as forced, not shrunk', () { + final f = _fails(G.intIn(0, 1000), (v) => v != 777, examples: [3, 777]); + expect(f.kind, FailureKind.forcedExample); + expect(f.caseId, 'forced:1'); + expect(f.input, '777'); + expect(f.shrinkSteps, 0); + expect(f.replay, contains('PROPERTY_CASE=forced:1')); + }); + + test('a generator that throws is reported as a generator failure', () { + final r = runProperty( + name: 'g', + gen: _Boom(), + body: (_) {}, + config: const PropertyConfig(seed: 1, cases: 3)); + expect(r.failure!.kind, FailureKind.generator); + expect(r.failure!.report, contains('boom')); + }); + }); + + group('shrinking ints', () { + test('finds the smallest failing int above zero', () { + expect(_fails(G.intIn(0, 1000), (v) => v < 50).input, '50'); + }); + + test('shrinks negatives toward zero', () { + expect(_fails(G.intIn(-100, 100), (v) => v > -7).input, '-7'); + }); + + test('shrinks toward the bound nearest zero when zero is out of range', + () { + expect(_fails(G.intIn(10, 100), (_) => false).input, '10'); + expect(_fails(G.intIn(-100, -10), (_) => false).input, '-10'); + }); + + test('an explicit origin is the shrink target', () { + expect(_fails(G.intIn(0, 100, origin: 60), (_) => false).input, '60'); + }); + + test('never leaves the generator domain', () { + final tried = []; + runProperty( + name: 'dom', + gen: G.intIn(13, 91), + body: (v) { + tried.add(v); + if (v > 20) throw StateError('x'); + }, + config: const PropertyConfig(seed: 5)); + expect(tried.every((v) => v >= 13 && v <= 91), isTrue); + }); + + test('a boolean shrinks to false', () { + expect(_fails(G.boolean(), (_) => false).input, 'false'); + }); + + test('elements shrink toward the earlier ones', () { + expect( + _fails(G.elements(['a', 'b', 'c', 'd']), (v) => v == 'a').input, 'b'); + }); + }); + + group('shrinking doubles', () { + test('reaches zero or the nearest bound when everything fails', () { + expect(_fails(G.doubleIn(-5, 5), (_) => false).input, '0.0'); + expect(_fails(G.doubleIn(5, 10), (_) => false).input, '5.0'); + }); + + test('lands just above a threshold, not far from it', () { + final f = _fails(G.doubleIn(0, 100), (v) => v < 3.5); + final v = double.parse(f.input); + expect(v, inInclusiveRange(3.5, 5)); + }); + + test('generation reaches the declared boundaries exactly', () { + final seen = _inputs( + G.doubleIn(-10, 10, boundaries: const [-10, 10, 0, 1e-9]), + cases: 200); + expect(seen, containsAll([-10.0, 10.0, 0.0, 1e-9])); + expect(seen.every((v) => v >= -10 && v <= 10), isTrue); + }); + + test('specials (NaN, infinities) appear when declared, never otherwise', + () { + final withSpecials = _inputs( + G.doubleIn(0, 1, specials: const [double.nan, double.infinity]), + cases: 200); + expect(withSpecials.any((v) => v.isNaN), isTrue); + expect(withSpecials.any((v) => v == double.infinity), isTrue); + final plain = _inputs(G.doubleIn(0, 1), cases: 200); + expect(plain.every((v) => v.isFinite && v >= 0 && v <= 1), isTrue); + }); + + test('a failing special shrinks to a simple finite value if that fails too', + () { + expect( + _fails(G.doubleIn(0, 1, specials: const [double.nan]), (_) => false) + .input, + '0.0'); + }); + + test('shrink candidates stay in range, differ from the value, are bounded', + () { + final g = G.doubleIn(-3, 100, boundaries: const [50]); + for (final v in [99.5, -2.25, 0.0, 3.0, 50.0, -3.0, 1e-12]) { + final c = g.shrink(v).toList(); + expect(c.length, lessThanOrEqualTo(40), reason: '$v'); + for (final x in c) { + expect(x >= -3 && x <= 100, isTrue, reason: '$v -> $x'); + expect(x, isNot(v), reason: '$v -> $x'); + } + } + expect(g.shrink(0.0), isEmpty, reason: 'the origin is already simplest'); + }); + + test('following the closest candidate always terminates', () { + final g = G.doubleIn(-3, 100); + var v = 87.123456; + var steps = 0; + while (true) { + final c = g.shrink(v).toList(); + if (c.isEmpty) break; + v = c.last; + expect(++steps, lessThan(200), reason: 'stuck at $v'); + } + expect(v, 0); + }); + + test('int shrink candidates: origin first, in range, never the value', () { + final g = G.intIn(-20, 40); + for (final v in [-20, -1, 0, 1, 7, 40]) { + final c = g.shrink(v).toList(); + if (v == 0) { + expect(c, isEmpty); + continue; + } + expect(c.first, 0); + expect(c.every((x) => x >= -20 && x <= 40 && x != v), isTrue); + expect(c.length, lessThanOrEqualTo(12)); + } + }); + }); + + group('shrinking lists, tuples, nullables', () { + test('a list shrinks to the shortest failing length of zeros', () { + final f = _fails(G.listOf(G.intIn(0, 9), maxLen: 20), (l) => l.length < 3); + expect(f.input, '[0, 0, 0]'); + }); + + test('a list keeps the one element that matters', () { + final f = _fails( + G.listOf(G.intIn(0, 20), maxLen: 20), (l) => !l.any((e) => e >= 5)); + expect(f.input, '[5]'); + }); + + test('a list never goes under its minimum or over its cap', () { + final lens = []; + runProperty>( + name: 'len', + gen: G.listOf(G.intIn(0, 9), minLen: 2, maxLen: 7), + body: (l) { + lens.add(l.length); + throw StateError('always'); + }, + config: const PropertyConfig(seed: 3)); + expect(lens.every((n) => n >= 2 && n <= 7), isTrue); + expect(_fails(G.listOf(G.intIn(0, 9), minLen: 2, maxLen: 7), (_) => false) + .input, '[0, 0]'); + }); + + test('the size cap holds across all generated cases', () { + final seen = _inputs(G.listOf(G.boolean(), maxLen: 60), cases: 200); + expect(seen.map((l) => l.length).reduce(math.max), 60, + reason: 'the cap itself is reached'); + expect(seen.every((l) => l.length <= 60), isTrue); + }); + + test('tuple components shrink TOGETHER when only a joint step still fails', + () { + // Fails only while a and b are within 3 of each other and a >= 5: moving + // ONE component far breaks the closeness, so only a joint step makes + // progress. + final f = _fails(G.pair(G.intIn(0, 1000), G.intIn(0, 1000)), + (p) => !((p.$1 - p.$2).abs() <= 3 && p.$1 >= 5), + cases: 2000, seed: 5); + final nums = RegExp(r'\d+').allMatches(f.input).map((m) => int.parse(m[0]!)); + final (a, b) = (nums.first, nums.last); + expect((a - b).abs(), lessThanOrEqualTo(3)); + expect(a, lessThanOrEqualTo(12), reason: f.input); + }); + + test('a triple and a quad shrink every component', () { + expect( + _fails(G.triple(G.intIn(0, 9), G.intIn(-9, 9), G.boolean()), + (_) => false) + .input, + '(0, 0, false)'); + expect( + _fails( + G.quad(G.intIn(0, 9), G.doubleIn(0, 1), G.boolean(), + G.elements(const ['x', 'y'])), + (_) => false) + .input, + '(0, 0.0, false, x)'); + }); + + test('a nullable shrinks to null first', () { + expect(_fails(G.nullable(G.intIn(5, 9)), (_) => false).input, 'null'); + expect( + _fails(G.nullable(G.intIn(5, 9), nullProbability: 0), + (v) => v == null || v < 7) + .input, + '7'); + }); + + test('a list and an int inside a tuple each shrink to their own minimum', + () { + final f = _fails( + G.pair(G.listOf(G.intIn(0, 9), minLen: 1, maxLen: 12), G.intIn(0, 11)), + (p) => !(p.$1.length >= 6 && p.$2 >= 4)); + expect(f.input, '([0, 0, 0, 0, 0, 0], 4)'); + }); + }); + + group('shrink bound', () { + test('the shrink limit caps the extra body runs', () { + var calls = 0; + final r = runProperty>( + name: 'limit', + gen: G.listOf(G.intIn(0, 1000), minLen: 40, maxLen: 50), + body: (_) { + calls++; + throw StateError('always'); + }, + config: const PropertyConfig(seed: 3, shrinkLimit: 5)); + expect(r.passed, isFalse); + expect(calls, lessThanOrEqualTo(1 + 5)); + expect(r.failure!.shrinkSteps, lessThanOrEqualTo(5)); + expect(r.failure!.report, contains('shrink limit: 5')); + }); + + test('with a limit of zero the original input is reported', () { + final f = _fails(G.intIn(500, 1000), (_) => false, shrinkLimit: 0); + expect(f.shrinkSteps, 0); + expect(int.parse(f.input), inInclusiveRange(500, 1000)); + }); + + test('the report shows the original input when shrinking changed it', () { + final f = _fails(G.intIn(0, 1000), (v) => v < 50, seed: 8); + expect(f.report, contains('original input')); + final z = _fails(G.intIn(0, 1000), (_) => false, examples: [0]); + expect(z.report, isNot(contains('original input'))); + }); + }); + + group('wall budget', () { + test('PROPERTY_BUDGET_MS is the hard limit over a property\'s looser default', + () { + // A law registers a 5 s in-test guard (headroom for a loaded parallel + // suite); the serial acceptance run sets PROPERTY_BUDGET_MS=2000 and + // that must be what fails a property, not the 5 s default. + final c = PropertyConfig.fromEnvironment( + const {'PROPERTY_BUDGET_MS': '2000'}, + defaultBudget: const Duration(seconds: 5), + ); + expect(c.budget, const Duration(milliseconds: 2000)); + var ticks = 0; + final r = runProperty( + name: 'serial acceptance', + gen: G.intIn(0, 9), + body: (_) {}, + config: c, + // 1 s per reading: over 2 s on the third, well inside the 5 s default. + elapsed: () => Duration(seconds: ticks++), + ); + expect(r.failure!.kind, FailureKind.budget); + expect(r.failure!.report, contains('2000 ms')); + }); + + test('a property over its budget fails, naming budget and cases run', () { + var ticks = 0; + final r = runProperty( + name: 'slow', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig(seed: 1, cases: 200), + elapsed: () => Duration(milliseconds: 500 * ticks++), + ); + expect(r.failure!.kind, FailureKind.budget); + expect(r.casesRun, lessThan(200)); + expect(r.failure!.report, contains('2000 ms')); + expect(r.failure!.report, contains('budget')); + }); + + test('shrinking counts against the budget and reports the best so far', () { + // The clock reads past the budget after the 6th reading: the failure at + // case 0 is found, shrinking gets a few attempts, then stops. + var reads = 0; + final r = runProperty>( + name: 'slow shrink', + gen: G.listOf(G.intIn(0, 1000), minLen: 40, maxLen: 50), + body: (_) => throw StateError('always'), + config: const PropertyConfig(seed: 1), + elapsed: () => Duration(milliseconds: 500 * reads++), + ); + final f = r.failure!; + expect(f.kind, FailureKind.counterexample); + expect(f.shrinkOverBudget, isTrue); + expect(f.report, contains('over budget (shrinking)')); + expect(f.shrinkSteps, lessThanOrEqualTo(6), + reason: 'unbudgeted, this shrink takes dozens of runs'); + // The best-so-far input is still a valid, failing input of the domain. + final len = RegExp(', ').allMatches(f.input).length + 1; + expect(len, inInclusiveRange(40, 50)); + }); + + test('shrinking inside the budget is not marked', () { + final r = runProperty( + name: 'quick shrink', + gen: G.intIn(500, 1000), + body: (_) => throw StateError('always'), + config: const PropertyConfig(seed: 1), + elapsed: () => const Duration(milliseconds: 1), + ); + expect(r.failure!.shrinkOverBudget, isFalse); + expect(r.failure!.report, isNot(contains('over budget'))); + expect(r.failure!.input, '500'); + }); + + test('a single-case replay keeps shrinking whatever the clock says', () { + final r = runProperty( + name: 'replay shrink', + gen: G.intIn(500, 1000), + body: (_) => throw StateError('always'), + config: const PropertyConfig(seed: 1, caseOnly: '0'), + elapsed: () => const Duration(hours: 1), + ); + expect(r.failure!.shrinkOverBudget, isFalse); + expect(r.failure!.input, '500'); + }); + + test('a property inside its budget passes', () { + final r = runProperty( + name: 'fast', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig(seed: 1, cases: 20), + elapsed: () => const Duration(milliseconds: 5), + ); + expect(r.passed, isTrue); + }); + + test('a counterexample is reported before the budget', () { + final r = runProperty( + name: 'both', + gen: G.intIn(0, 9), + body: (_) => throw StateError('x'), + config: const PropertyConfig(seed: 1), + elapsed: () => const Duration(hours: 1), + ); + expect(r.failure!.kind, FailureKind.counterexample); + }); + + test('the default clock is a real stopwatch', () { + final r = runProperty( + name: 'spin', + gen: G.intIn(0, 9), + body: (_) { + final sw = Stopwatch()..start(); + while (sw.elapsedMilliseconds < 4) {} + }, + config: const PropertyConfig( + seed: 1, cases: 50, budget: Duration(milliseconds: 1)), + ); + expect(r.failure?.kind, FailureKind.budget); + }); + + test('a single-case replay is not held to the budget', () { + final r = runProperty( + name: 'replay', + gen: G.intIn(0, 9), + body: (_) {}, + config: const PropertyConfig(seed: 1, caseOnly: '3'), + elapsed: () => const Duration(hours: 1), + ); + expect(r.passed, isTrue); + }); + }); + + group('forAll', () { + var ran = 0; + forAll('self: forAll runs examples and generated cases', + G.intIn(0, 100), (v) { + ran++; + expect(v >= 0 && v <= 100, isTrue); + }, examples: const [0, 100], cases: 25); + + test('forAll ran its forced examples plus its cases', () { + expect(ran, anyOf(0, 27), + reason: '0 when this test alone is selected by name'); + }); + + forAll<(int, int)>('self: addition commutes', G.pair(G.intIn(-99, 99), G.intIn(-99, 99)), + (p) => expect(p.$1 + p.$2, p.$2 + p.$1)); + }); +} + +class _Boom extends Gen { + @override + int generate(Rng r, int size) => throw StateError('boom'); + @override + Iterable shrink(int v) => const []; +} diff --git a/tool/reproduce_outputs.dart b/tool/reproduce_outputs.dart new file mode 100644 index 00000000..93fdc2b4 --- /dev/null +++ b/tool/reproduce_outputs.dart @@ -0,0 +1,81 @@ +// Synthetic reproduction cases. Run with --json to capture unrounded values. +// Uses only public batch functions, so the same file runs on up/main and #99. +import 'dart:convert'; +import 'dart:io'; +import 'dart:math' as math; +import 'package:openstrap_analytics/onehz.dart'; +import 'package:openstrap_analytics/src/onehz/util.dart' as u; + +// Deterministic differential cases, shared by the tool and parity tests. +List> reproductionCases(Map base) => [ + for (var i = 0; i < 12; i++) (() { + final f = jsonDecode(jsonEncode(base)) as Map; + f['hrv']['count'] = i < 6 ? 1800 : 5400; + f['hrv']['frequency'] = [.17, .7, 2.4, 2.9][i % 4]; + f['hrv']['gapped'] = i.isOdd; + f['spectral']['origin'] = i.isEven ? 1800000000.0 : 0.0; + f['spectral']['frequency'] = .037 + i * .019; + f['rr']['run'] = [1, 16, 30, 31, 32, 33][i % 6]; + f['cosinor']['step'] = .31 + i * .11; + f['load']['count'] = 30 + i * 7; + f['zones']['count'] = 50 + i * 11; + f['circadian']['days'] = 1 + i % 4; + return f; + })() +]; + +Map reproduce(Map f) { + final out = {}; + final r = f['rr'] as Map; + final rr = [for (var i=0;i<(r['before'] as int);i++) (r['normal'] as num).toDouble(), + for (var i=0;i<(r['run'] as int);i++) (r['artifact'] as num).toDouble(), + for (var i=0;i<(r['after'] as int);i++) (r['normal'] as num).toDouble()]; + final clean = correctRr(rr); + out['rr'] = [clean.nn.length, clean.droppedCount, clean.correctedCount]; + final s=f['spectral'] as Map; + final t=[for(var i=0;i<(s['count'] as int);i++) (s['origin'] as num).toDouble()+i*(s['step'] as num)]; + final y=[for(var i=0;i; + final ct=[for(var i=0;i<(c['count'] as int);i++) i*(c['step'] as num).toDouble()]; + final cy=[for(var i=0;i; + final nt=[for(var i=0;i<(n['count'] as int);i++) (n['origin'] as num).toDouble()+i*(n['step'] as num)]; + out['near_singular']=cosinor(nt,[for(var i=0;i0 && i%400==0 ? 10000 : 0)]; + final hv=hrvTime(nn,nnTimesMs:times).value!; + out['hrv']=[hv.rmssd,hv.sdnn,hv.sdann,hv.sdnnIndex,hv.diffAcf1]; + try { + final bad=[...times]..[times.length-1]=double.infinity; + out['nonfinite']=hrvTime(nn,nnTimesMs:bad).present; + } catch(e) { out['nonfinite']='throws'; } + final load=[for(var i=0;i<(f['load']['count'] as int);i++) 80.1+(i%13)*3.17]; + out['trimp']=banisterTrimp(load,restingHr:60,maxHr:190,sex:Sex.male).value; + final lv=ctlAtlTsb(load).value!; + out['ctl']=[lv.ctl,lv.atl,lv.tsb]; + final z=[for(var i=0;i<(f['zones']['count'] as int);i++) HrSample(i*1000.1,120.0)]; + out['zones']=HeartRateZones.timeInZone(z,HeartRateZones.zonesFromMaxHr(190))!.seconds; + final x=[for(var i=0;i<(f['circadian']['days'] as int)*(f['circadian']['epochs'] as int);i++) 1.1+(i%24)*.13+(i%3)*1e-14]; + final ci=circadianNonparametric(x,f['circadian']['epochs'] as int).value!; + out['circadian']=[ci.interdailyStability,ci.intradailyVariability,ci.m10,ci.l5,ci.m10StartEpoch,ci.l5StartEpoch]; + final ir=irregularBeatScreen([800,900,1000,1100,700,850,900,1000].map((v)=>v.toDouble()).toList(), + nnTimesMs:[800,1700,2700,3800,14500,15350,16250,17250],minBeats:4).value!; + out['irregular_gap']=[ir.sd1,ir.pnnPct]; + return out; +} +void main(List args) { + final f=jsonDecode(File('test/fixtures/output_changes.json').readAsStringSync()) as Map; + if(args.contains('--corpus-json')) { print(jsonEncode([for(final c in reproductionCases(f)) reproduce(c)])); return; } + final values=reproduce(f); + if(args.contains('--json')) { print(jsonEncode(values)); return; } + print('| Change | up/main 3a7ca34 | Current batch |'); + print('|---|---|---|'); + final old=f['old'] as Map; + for(final entry in values.entries) { + print('| ${entry.key} | ${old[entry.key]} | ${entry.value} |'); + } +}