diff --git a/lib/app.dart b/lib/app.dart index c4bfc9b4e..d2c32afbd 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -76,10 +76,16 @@ class _OpenStrapAppState extends State with WidgetsBindingObserver WidgetsBinding.instance.platformDispatcher.platformBrightness); } + @override + void didHaveMemoryPressure() { + context.read().handleMemoryPressure(); + } + @override void didChangeAppLifecycleState(AppLifecycleState state) { final app = context.read(); if (state == AppLifecycleState.resumed) { + app.resumeDerivationRetention(); // The user may have flipped our notification switch either way in OS // Settings while we were backgrounded. Drop the cached authorization // decision so the next present/schedule re-reads reality — a denial used diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 2610e7726..695e546a3 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -70,6 +70,8 @@ import 'hr_max.dart' show estimatedMaxHr, kHrFloorBpm, smoothedMaxHr, smoothedMinHr; import 'movement_floor_policy.dart' as mfp; import 'sleep_profile_policy.dart'; +import 'staged_sleep_cache.dart'; +import 'substrate_retention.dart'; import 'derive_prepare.dart'; import 'onehz_pipeline.dart'; import 'step_cadence.dart'; @@ -3092,6 +3094,182 @@ class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; + // ED-2a RED seams. Empty cache views and no-op controls describe the current + // uncached engine. Counters below observe real worker work, never results. + // E12/E13 back staging and retention; E18 backs finished-session reuse. + final _stagedSleepCache = StagedSleepCache(); + @visibleForTesting + bool debugDisableStagedSleepCache = false; + @visibleForTesting + int get debugStagedSleepCacheMaxDays => _stagedSleepCache.maxDays; + @visibleForTesting + set debugStagedSleepCacheMaxDays(int value) => _stagedSleepCache.maxDays = value; + @visibleForTesting + int get debugStagedSleepCacheSize => _stagedSleepCache.size; + @visibleForTesting + void debugStagedSleepCacheRekey(int version) => _stagedSleepCache.rekey(version); + @visibleForTesting + bool get debugDisableSubstrateRetention => _disableSubstrateRetention; + @visibleForTesting + set debugDisableSubstrateRetention(bool value) { + _disableSubstrateRetention = value; + if (value) _clearRetainedSubstrate(); + } + bool _disableSubstrateRetention = false; + @visibleForTesting + int get debugRetainedSubstrateMaxDays => _retained.maxDays; + @visibleForTesting + set debugRetainedSubstrateMaxDays(int value) => _retained.maxDays = value; + @visibleForTesting + Set get debugRetainedSubstrateDays => _retained.days; + @visibleForTesting + void debugRetainedSubstrateRekey(int version) => _retained.rekey(version); + void trimForBackground() { + _retentionInBackground = true; + _clearRetainedSubstrate(); + } + void resumeFromBackground() => _retentionInBackground = false; + void handleMemoryPressure() => _clearRetainedSubstrate(); + void dispose() { + _retentionDisposed = true; + _clearRetainedSubstrate(); + } + + final _retained = SubstrateRetention(); + int _retentionEpoch = 0; + int? _retentionPassEpoch; + bool _retentionInBackground = false; + bool _retentionDisposed = false; + final _retentionPreparedDays = {}; + bool get _canRetainSubstrate => !debugDisableSubstrateRetention && + !background && !_retentionInBackground && !_retentionDisposed && + (_retentionPassEpoch == null || _retentionPassEpoch == _retentionEpoch); + + void _clearRetainedSubstrate() { + _retentionEpoch++; + _retained.clear(); + } + + void _beginRetentionPass() { + _retentionPassEpoch = _retentionEpoch; + _retentionPreparedDays.clear(); + if (!_canRetainSubstrate) _retained.clear(); + } + + void _endRetentionPass() { + _retained.retainDays(_retentionPreparedDays); + _retentionPreparedDays.clear(); + _retentionPassEpoch = null; + } + + void _dropRetainedDay(String day) { + _retained.remove(day); + _retentionPreparedDays.remove(day); + } + @visibleForTesting + bool debugDisableSessionReuse = false; + @visibleForTesting + int debugCeilingComputed = 0; + @visibleForTesting + void debugSessionCacheRekey(int version) { + for (final e in _ceilingCache.entries.toList()) { + _ceilingCache[e.key] = (version: version, sig: e.value.sig, + result: e.value.result); + } + } + + // One complete envelope and the unrounded bpm used to choose the winner. + // Entries survive passes on this engine; oldest entries are evicted first. + final _ceilingCache = <(String, String), + ({int version, String sig, _SessionCeiling result})>{}; + static const _ceilingCacheMax = 128; + final _ceilingInput = Expando<_CeilingInput>(); + + Future<_CeilingInput?> _ceilingInputRevision(String day) async { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + final lo = _localDayLabelToSec(day); + final hi = _localNextDayLabelToSec(day) - 1; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [lo ~/ 900, hi ~/ 900]); + return (generation: generation, revisions: {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}); + } catch (_) { + return null; // Unavailable provenance always recomputes. + } + } + + Future _sessionCeilingRevision() async { + try { + return await LocalDb.sessionCeilingRevision(); + } catch (_) { + return null; + } + } + + Future> _ceilingSignatures( + PreparedDerivationDay day, List> sessions, + int? sessionRevision) async { + final input = _ceilingInput[day]; + final sub = day.daySub; + if (input == null || sub.isEmpty || sessionRevision == null) return const {}; + final current = await _ceilingInputRevision(day.date); + if (current == null || input.generation != current.generation || + sessionRevision != await _sessionCeilingRevision()) { + return const {}; + } + final out = {}; + for (final row in sessions) { + final id = row['id']?.toString(); + final start = (row['start_ts'] as num?)?.toInt(); + final end = (row['end_ts'] as num?)?.toInt(); + if (id == null || row['status'] != 'done' || start == null || + end == null || end <= start) { + continue; + } + final lo = math.max(start, sub.tsSec.first); + final hi = math.min(end, sub.tsSec.last); + if (hi < lo) continue; + // E4 buckets are conservative at the clipped window's edges. Compare + // before/after the load so a racing decoded write cannot certify a hit. + String windowKey(_CeilingInput fp) => jsonEncode([ + fp.generation, for (final e in fp.revisions.entries) + if (e.key >= lo ~/ 900 && e.key <= hi ~/ 900) [e.key, e.value], + ]); + final revision = windowKey(input); + if (revision != windowKey(current)) continue; + out[id] = jsonEncode([ + revision, sessionRevision, id, row['type'], row['status'], start, end, + lo, hi, sub.deviceFamily, priorityKey(day.priority), + for (final signal in const [InputSignal.hr1Hz, InputSignal.accel1Hz]) + [signal.name, for (final span in day.ownership[signal] ?? const []) + if (span.end > lo && span.start <= hi) + [span.start, span.end, span.deviceId]], + ]); + } + return out; + } + @visibleForTesting + int debugStagingComputations = 0; + @visibleForTesting + void Function(bool mayFold, int segmentedDays, String? observation)? debugOnStagingWalk; + @visibleForTesting + bool debugDisableTargetDaySkip = false; + @visibleForTesting + Map debugRetainedSubstrate(String day) => _retained.extras(day); + @visibleForTesting + Future debugSleepCandidate(String day) => + _sleepCandidateForDay(day); + @visibleForTesting + Future debugPrepareDay(String day) => + _prepareTargetDay(day); + @visibleForTesting + Future debugLoadSubstrate(int from, int to, {required String day, + Map> ownership = const {}}) => + _loadSubstrateRange(from, to, dayId: day, ownership: ownership); + /// True while a band offload is draining. The scheduler never STARTS a /// derive during one, but one can start mid-pass; the raw prune checks this /// before archiving each substrate bucket and stops for the pass, so a drain @@ -3151,9 +3329,11 @@ class DerivationEngine { static Future _withRunLock(T busy, Future Function() body) async { if (_running) return busy; _running = true; + beginWearableDayIndexPass(); try { return await body(); } finally { + endWearableDayIndexPass(); _running = false; } } @@ -3389,6 +3569,8 @@ class DerivationEngine { }) async { if (_running) return 0; _running = true; + _beginRetentionPass(); + beginWearableDayIndexPass(); final startedAt = DateTime.now().millisecondsSinceEpoch; _diag ..['running'] = true @@ -3651,7 +3833,8 @@ class DerivationEngine { } else if (prepared != null) { _diag['prepared_days'] = (_diag['prepared_days'] as int) + 1; final complete = await _derivePreparedDay( - prepared, profile, dataNowSec, history); + prepared, profile, dataNowSec, history, + allowSessionReuse: !heavy && !force); done++; _diag['done_days'] = done; // Stored only when the derive was a fixed point; see @@ -3751,6 +3934,7 @@ class DerivationEngine { } catch (e, st) { _diag['last_error'] = '$e'; lastRunError = '$e'; + _clearRetainedSubstrate(); _log('derive ERROR: $e\n$st'); return 0; } finally { @@ -3763,90 +3947,95 @@ class DerivationEngine { } catch (e) { _log('[derive] outcome callback threw: $e'); } - // Storage housekeeping runs here, after everything, still holding - // `_running`. See _runStorageHousekeeping — this is the only place every - // entry path and every early return actually reaches. - _diag['stage'] = 'housekeeping'; - await _runStorageHousekeeping(long: heavy || force); - // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days - // inside the raw window re-derive from substrate above; everything older - // has none, so its headline is rebuilt from the stored TRIMP + wake - // window instead. No-ops after the first successful pass - // (`compute_freshness`), and never fatal — a failed rescale must not take - // the derive cycle down with it. - // - // AFTER the sweep, not before it. It used to be the first statement of - // run(), so the one launch where it actually fires — the first after the - // version bump, the launch the user is watching — spent a DB round trip - // and a bundle decode+encode PER DAY OF HISTORY before today's data was - // touched at all. Nothing in the sweep reads what it rewrites (it only - // ever touches days older than `rawRetentionDays`, which no pass - // re-derives), so the only thing the old ordering bought was that the - // FIRST cross-day rollup after the bump saw the rescaled history rather - // than the next one. Still holds `_running`, so nothing else can be - // reading day_result while it rewrites. - _diag['stage'] = 'strain_rescale'; try { - final rescaled = await backfillStrainScale( - female: workoutSex(profile.sex) == 'female', - ); - if (rescaled.didWork) { - _log('[derive] strain rescale: ${rescaled.bundleDays} day(s) ' - 'rebuilt, ${rescaled.skipped} skipped (no TRIMP or no wake ' - 'window)'); + // Storage housekeeping runs here, after everything, still holding + // `_running`. See _runStorageHousekeeping — this is the only place every + // entry path and every early return actually reaches. + _diag['stage'] = 'housekeeping'; + _endRetentionPass(); + await _runStorageHousekeeping(long: heavy || force); + // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days + // inside the raw window re-derive from substrate above; everything older + // has none, so its headline is rebuilt from the stored TRIMP + wake + // window instead. No-ops after the first successful pass + // (`compute_freshness`), and never fatal — a failed rescale must not take + // the derive cycle down with it. + // + // AFTER the sweep, not before it. It used to be the first statement of + // run(), so the one launch where it actually fires — the first after the + // version bump, the launch the user is watching — spent a DB round trip + // and a bundle decode+encode PER DAY OF HISTORY before today's data was + // touched at all. Nothing in the sweep reads what it rewrites (it only + // ever touches days older than `rawRetentionDays`, which no pass + // re-derives), so the only thing the old ordering bought was that the + // FIRST cross-day rollup after the bump saw the rescaled history rather + // than the next one. Still holds `_running`, so nothing else can be + // reading day_result while it rewrites. + _diag['stage'] = 'strain_rescale'; + try { + final rescaled = await backfillStrainScale( + female: workoutSex(profile.sex) == 'female', + ); + if (rescaled.didWork) { + _log('[derive] strain rescale: ${rescaled.bundleDays} day(s) ' + 'rebuilt, ${rescaled.skipped} skipped (no TRIMP or no wake ' + 'window)'); + } + } catch (e) { + _log('[derive] strain rescale failed (kept old values): $e'); } - } catch (e) { - _log('[derive] strain rescale failed (kept old values): $e'); - } - // ONE-SHOT, after the sweep for the same reason as the strain rescale: - // the rest of the readiness_z history, then a re-score of every day - // below v112 that has no raw substrate left (substrate days were just - // re-derived, or are retried next pass). A re-score rewrites stored - // days, so the baselines and the cross-day rollup are refreshed here — - // `finally` is reached by every early return, including "nothing to - // derive" and "no decoded data". - _diag['stage'] = 'readiness_calibration'; - try { - final r = await backfillReadinessCalibration( - // Both substrate sources, as `_deriveScope` reads them: a wearable - // day with rows is the derive's, not the re-score's. - rawDays: await substrateDays(), - zHistoryLoader: () async { - final h = await _BaselineHistoryCache.load(); - return (day, fam) => h.valuesBefore('readiness_z', day, - family: _baselineFamily('readiness_z', fam)); - }, - ); - if (r.seeded > 0 || r.rescored > 0) { - _log('[derive] readiness calibration: ${r.seeded} z seeded, ' - '${r.rescored} day(s) re-scored' - '${r.forced > 0 ? ' (${r.forced} with substrate whose derive ' - 'never landed, from stored inputs)' : ''}'); - await _refreshBaselines(); + // ONE-SHOT, after the sweep for the same reason as the strain rescale: + // the rest of the readiness_z history, then a re-score of every day + // below v112 that has no raw substrate left (substrate days were just + // re-derived, or are retried next pass). A re-score rewrites stored + // days, so the baselines and the cross-day rollup are refreshed here — + // `finally` is reached by every early return, including "nothing to + // derive" and "no decoded data". + _diag['stage'] = 'readiness_calibration'; + try { + final r = await backfillReadinessCalibration( + // Both substrate sources, as `_deriveScope` reads them: a wearable + // day with rows is the derive's, not the re-score's. + rawDays: await substrateDays(), + zHistoryLoader: () async { + final h = await _BaselineHistoryCache.load(); + return (day, fam) => h.valuesBefore('readiness_z', day, + family: _baselineFamily('readiness_z', fam)); + }, + ); + if (r.seeded > 0 || r.rescored > 0) { + _log('[derive] readiness calibration: ${r.seeded} z seeded, ' + '${r.rescored} day(s) re-scored' + '${r.forced > 0 ? ' (${r.forced} with substrate whose derive ' + 'never landed, from stored inputs)' : ''}'); + await _refreshBaselines(); + } + if (r.rescored > 0) await _runCrossDay(profile); + } catch (e) { + _log('[derive] readiness calibration failed (will retry): $e'); } - if (r.rescored > 0) await _runCrossDay(profile); - } catch (e) { - _log('[derive] readiness calibration failed (will retry): $e'); - } - // Restamp Today's freshness here, not in each caller: the iOS - // background pass has no caller to do it, and getToday reads the - // night from `overnight_day`, so a night settled in the background - // kept serving the previous one. - try { - await LocalDb.refreshComputeFreshness(); - } catch (e) { - _log('[derive] freshness refresh failed: $e'); - } - _running = false; - final finishedAt = DateTime.now().millisecondsSinceEpoch; - _diag - ..['running'] = false - ..['stage'] = 'idle' - ..['active_days'] = const [] - ..['finished_at'] = finishedAt - ..['duration_ms'] = finishedAt - startedAt; + // Restamp Today's freshness here, not in each caller: the iOS + // background pass has no caller to do it, and getToday reads the + // night from `overnight_day`, so a night settled in the background + // kept serving the previous one. + try { + await LocalDb.refreshComputeFreshness(); + } catch (e) { + _log('[derive] freshness refresh failed: $e'); + } + } finally { + endWearableDayIndexPass(); + _running = false; + final finishedAt = DateTime.now().millisecondsSinceEpoch; + _diag + ..['running'] = false + ..['stage'] = 'idle' + ..['active_days'] = const [] + ..['finished_at'] = finishedAt + ..['duration_ms'] = finishedAt - startedAt; - try { await runTrace?.stop(); } catch (_) {} + try { await runTrace?.stop(); } catch (_) {} + } } } @@ -3859,6 +4048,8 @@ class DerivationEngine { if (days.isEmpty) return 0; if (_running) return 0; _running = true; + _beginRetentionPass(); + beginWearableDayIndexPass(); final startedAt = DateTime.now().millisecondsSinceEpoch; _diag ..['running'] = true @@ -3969,17 +4160,23 @@ class DerivationEngine { } return done; } catch (e, st) { + _clearRetainedSubstrate(); _log('derive selected ERROR: $e\n$st'); return 0; } finally { - await _runStorageHousekeeping(); - final finishedAt = DateTime.now().millisecondsSinceEpoch; - _diag - ..['running'] = false - ..['stage'] = 'idle' - ..['finished_at'] = finishedAt - ..['duration_ms'] = finishedAt - startedAt; - _running = false; + try { + _endRetentionPass(); + await _runStorageHousekeeping(); + final finishedAt = DateTime.now().millisecondsSinceEpoch; + _diag + ..['running'] = false + ..['stage'] = 'idle' + ..['finished_at'] = finishedAt + ..['duration_ms'] = finishedAt - startedAt; + } finally { + endWearableDayIndexPass(); + _running = false; + } } } @@ -3988,6 +4185,21 @@ class DerivationEngine { static const int _maxDayRawPages = 300; Future _prepareTargetDay(String dayId) async { + try { + final prepared = await _prepareTargetDayPayload(dayId); + if (prepared == null || prepared.daySub.isEmpty) { + _dropRetainedDay(dayId); + } else if (_retentionPassEpoch != null) { + _retentionPreparedDays.add(dayId); + } + return prepared; + } catch (_) { + _dropRetainedDay(dayId); + rethrow; + } + } + + Future _prepareTargetDayPayload(String dayId) async { // Per-day page/row totals used to live in the shared `_diag` map (reset // then accumulated across this day's 2-3 substrate loads). Under // concurrent per-day processing (see `run()`), multiple days resetting/ @@ -4021,6 +4233,7 @@ class DerivationEngine { ].reduce(math.max); final (ownership, priority) = await _resolveOwnership(unionFrom, unionTo); + final ceilingInput = await _ceilingInputRevision(dayId); // Load the day PLUS the nap boundary buffer in ONE pass (each // _loadSubstrateRange spawns its own isolate, so a second load would @@ -4034,6 +4247,7 @@ class DerivationEngine { dayId: dayId, stats: stats, ownership: ownership, + load: 'day', ); // A day the primary band never saw is the ACTIVE WEARABLE's, when one is // chosen and flagged on (inputs/canonical.dart). Never mixed: a day with @@ -4062,6 +4276,7 @@ class DerivationEngine { dayId: dayId, stats: stats, ownership: ownership, + load: 'sleep', ); } // Single safe merge into the shared max-tracking diagnostics — one @@ -4088,7 +4303,7 @@ class DerivationEngine { dayEndSec: dayEnd, sleepOffsetSec: candidate.present ? candidate.sleepOffsetSec : 0, ); - return candidate.toPreparedDay( + final prepared = candidate.toPreparedDay( daySub: daySub, napSub: napSub, sleepSub: sleepSub, @@ -4102,6 +4317,12 @@ class DerivationEngine { ownership: ownership, priority: priority, ); + // Wearable construction and supplied imports have other inputs; only the + // canonical decoded load above can carry this revision provenance. + if (wearable == null && ceilingInput != null) { + _ceilingInput[prepared] = ceilingInput; + } + return prepared; } /// [next] (the following day's main sleep) when it is TONIGHT's from this @@ -4241,6 +4462,7 @@ class DerivationEngine { // A user sleep override is the source of truth — never serve the cached auto // candidate, and don't cache the override result (so a later edit / clear is // not shadowed by a stale artifact). The auto path keeps its finalized cache. + // The RAM worker cache below keys the override itself, including edits. final overrideRow = await LocalDb.getSleepOverride(dayId); final override = overrideRow == null ? null @@ -4282,13 +4504,31 @@ class DerivationEngine { // exists — and it has to be: staging the night off both devices' rows // while `napSub`/`sleepSub` score it off one device's is a window found in // a night that was never scored. - final (searchOwnership, _) = await _resolveOwnership(range.$1, range.$2); + final (searchOwnership, searchPriority) = + await _resolveOwnership(range.$1, range.$2); + List>? revisions; + if (!debugDisableStagedSleepCache) { + try { + revisions = []; + final first = DateTime.fromMillisecondsSinceEpoch(range.$1 * 1000); + for (var d = DateTime(first.year, first.month, first.day); + d.millisecondsSinceEpoch ~/ 1000 <= range.$2; + d = DateTime(d.year, d.month, d.day + 1)) { + final label = dayLabelOf(d); + final fp = await LocalDb.inputDayFingerprint(label); + revisions.add([label, fp.revisionSum, fp.revisionMax, fp.generation]); + } + } catch (_) { + revisions = null; // Unavailable provenance costs a staging, never a hit. + } + } final searchSub = await _loadSubstrateRange( range.$1, range.$2, dayId: dayId, stats: stats, ownership: searchOwnership, + load: 'search', ); // PERSONALIZED STAGER (v42): stage on a WORKER isolate, NOT the main/UI // thread. cardioStager reads analytics "ambient" globals — the rolling sleep @@ -4354,12 +4594,36 @@ class DerivationEngine { ? await wearableNight(range.$1, range.$2) : null; if (ownNight != null) vendorNights = [...vendorNights, ownNight]; + String? stagingKey; + if (revisions != null) { + try { + stagingKey = stagingInputKey( + fromSec: range.$1, + toSec: range.$2, + revisions: revisions, + sub: searchSub, + ownership: searchOwnership, + priority: searchPriority, + override: override, + profileJson: profileJson, + priorSleep: priorSleep, + vendorNights: vendorNights, + activeWearable: override == null ? await activeWearable() : null, + ); + } catch (_) { + // A context that cannot be keyed still takes the ordinary worker path. + } + } // Cancellable + TIMED OUT. This site previously used a bare `Isolate.run` // with no timeout at all, so a hung staging pass never completed its future // — `_running` stayed true and `DeriveScheduler._drain` never returned, i.e. // all derivation was dead until app restart. - final (candidateJson, observationJson) = - await _runIsolateCancellable(() { + final skipUnusedDays = !mayFold && !debugDisableTargetDaySkip; + // Read here: a `_perDayTimeout` inside `stage` would capture `this` into + // the context the worker closure ships, and the engine's callbacks (the + // app's own fields) are not sendable. + final stageTimeout = _perDayTimeout; + Future<(String, String?, int)> stage() => _runIsolateCancellable(() { try { final p = profileJson == null ? null @@ -4380,9 +4644,11 @@ class DerivationEngine { } ana.cardioRecordObservations = true; ana.resetCardioObservations(); + debugSegmentedDayCount = 0; final candidate = prepareSleepSessionCandidate( searchSub, targetDay: dayId, + skipUnusedDays: skipUnusedDays, override: override, priorSleep: priorSleep, vendorNights: vendorNights, @@ -4427,8 +4693,32 @@ class DerivationEngine { } } } - return (jsonEncode(candidate.toJson()), observationJson); - }, _perDayTimeout, label: 'sleep-staging $dayId'); + return (jsonEncode(candidate.toJson()), observationJson, debugSegmentedDayCount); + }, stageTimeout, label: 'sleep-staging $dayId'); + final hit = stagingKey == null ? null : _stagedSleepCache.lookup( + dayId, kAlgoVersion, stagingKey, mayFold: mayFold); + final String candidateJson; + final String? observationJson; + // Reuse worker output only; banked-night and fold checks below still run. + if (hit != null) { + candidateJson = hit.candidateJson; + observationJson = mayFold ? hit.observationJson : null; + } else { + debugStagingComputations++; + final result = await stage(); + candidateJson = result.$1; + observationJson = result.$2; + debugOnStagingWalk?.call(mayFold, result.$3, observationJson); + if (stagingKey != null) { + _stagedSleepCache.store(dayId, StagedSleepEntry( + algoVersion: kAlgoVersion, + key: stagingKey, + mayFold: mayFold, + candidateJson: candidateJson, + observationJson: observationJson, + )); + } + } final candidate = SleepSessionCandidate.fromJson( (jsonDecode(candidateJson) as Map).cast()); for (final f in candidate.flags) { @@ -4657,6 +4947,62 @@ class DerivationEngine { } Future _loadSubstrateRange( + int fromRecTs, + int toRecTs, { + required String dayId, + _PrepareStats? stats, + Map> ownership = const {}, + String load = 'direct', + }) async { + if (toRecTs < fromRecTs) { + _dropRetainedDay(dayId); + return Substrate.empty; + } + final epoch = _retentionEpoch; + String? context; + Map? revisions; + if (_canRetainSubstrate) { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [fromRecTs ~/ 900, toRecTs ~/ 900]); + revisions = {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}; + context = substrateLoadContext(fromRecTs, toRecTs, generation, ownership); + } catch (_) { + // Missing revision provenance always takes the ordinary cold path. + _dropRetainedDay(dayId); + } + } else { + _dropRetainedDay(dayId); + } + final old = context == null ? null : _retained.lookup(dayId, load); + final readFrom = old?.readFrom(kAlgoVersion, context!, revisions!, toRecTs); + if (readFrom != null && readFrom > toRecTs) return old!.sub; + try { + final (sub, extras) = await _readSubstrateRange( + readFrom ?? fromRecTs, toRecTs, dayId: dayId, stats: stats, + ownership: ownership, seed: readFrom == null ? null : old, + retain: context != null); + if (_canRetainSubstrate && epoch == _retentionEpoch && + context != null && extras.isNotEmpty) { + _retained.store(dayId, load, RetainedSubstrate( + version: kAlgoVersion, context: context, revisions: revisions!, + sub: sub, extras: extras, + cut: math.max(fromRecTs, ((sub.lastTs ?? fromRecTs) ~/ 900) * 900))); + } else { + _dropRetainedDay(dayId); + } + return sub; + } catch (_) { + _dropRetainedDay(dayId); + rethrow; + } + } + + Future<(Substrate, Map)> _readSubstrateRange( int fromRecTs, int toRecTs, { required String dayId, @@ -4667,8 +5013,10 @@ class DerivationEngine { // single-device install resolves to anyway and what the direct-load tests // pass. Map> ownership = const {}, + RetainedSubstrate? seed, + bool retain = false, }) async { - if (toRecTs < fromRecTs) return Substrate.empty; + if (toRecTs < fromRecTs) return (Substrate.empty, const {}); final port = ReceivePort(); // onError/onExit are LOAD-BEARING. Without them, an uncaught throw inside // the worker (a malformed SQLite row reaching one of the numeric reads in @@ -4686,6 +5034,7 @@ class DerivationEngine { ); final ready = Completer(); final result = Completer(); + Map extras = const {}; // A failure completes BOTH completers, but we may bail out via `ready` and // never await `result` — register a listener so that error is never an // unobserved async error. (The real error still propagates via `ready`.) @@ -4706,7 +5055,8 @@ class DerivationEngine { if (kind == 'substrate') { final payload = ((message['payload'] as Map?) ?? const {}) .cast(); - if (!result.isCompleted) result.complete(Substrate.fromJson(payload)); + extras = ((message['retain'] as Map?) ?? const {}).cast(); + if (!result.isCompleted) result.complete(Substrate.fromTransfer(payload)); } return; } @@ -4737,11 +5087,17 @@ class DerivationEngine { } try { final worker = await ready.future; - worker.send(const {'type': 'config', 'mode': 'substrate'}); + worker.send({'type': 'config', 'mode': 'substrate', 'retain': retain}); + if (seed != null) worker.send(seed.seed(fromRecTs)); int? afterRecTs; int? afterCursor; - var rangePages = 0; - var rangeRows = 0; + final rawRowsByBucket = { + if (seed != null) + for (final e in (seed.extras['raw_rows_by_bucket'] as Map).entries) + if (e.key < fromRecTs ~/ 900) e.key: e.value, + }; + var rangeRows = rawRowsByBucket.values.fold(0, (int n, v) => n + v); + var rangePages = rangeRows ~/ _rawDecodeBatchSize; // Low water mark for the BEAT read, which is NOT the same window as the // frame read. Paging is driven by `decoded_onehz`, but a beat can exist // for a second with no 1 Hz row at all (gen4 R10-lite is hr-only and @@ -4773,6 +5129,25 @@ class DerivationEngine { ); if (decodedRows.isNotEmpty) { _trackPrepareBatch(decodedRows.length); + if (retain) { + // Count sorted bucket runs, including rows ownership will drop. + // This keeps the existing full-load row budget exact on a tail read. + var i = 0; + while (i < decodedRows.length) { + final bucket = (decodedRows[i]['rec_ts'] as num).toInt() ~/ 900; + var lo = i + 1, hi = decodedRows.length; + while (lo < hi) { + final mid = (lo + hi) >> 1; + if ((decodedRows[mid]['rec_ts'] as num).toInt() < (bucket + 1) * 900) { + lo = mid + 1; + } else { + hi = mid; + } + } + rawRowsByBucket[bucket] = (rawRowsByBucket[bucket] ?? 0) + lo - i; + i = lo; + } + } rangePages += 1; rangeRows += decodedRows.length; if (stats != null) { @@ -4871,12 +5246,15 @@ class DerivationEngine { // BOUNDED. `result.future` had no timeout at all, so any path that left // the worker unable to answer hung this call — and with it the whole // engine — permanently. - return await result.future.timeout( + final loaded = await result.future.timeout( _perDayTimeout, onTimeout: () => throw TimeoutException( 'substrate prepare for $dayId timed out after $_perDayTimeout', ), ); + return (loaded, { + ...extras, if (extras.isNotEmpty) 'raw_rows_by_bucket': rawRowsByBucket, + }); } finally { // ALWAYS tear down: on success, on error, and on timeout. The isolate is // killed rather than abandoned so a wedged worker can never outlive the @@ -5311,6 +5689,8 @@ class DerivationEngine { }) async { if (_running) return 0; _running = true; + _beginRetentionPass(); + beginWearableDayIndexPass(); try { await refreshSessionSensorSources(); // as [run] does // Baseline gate: compute the CURRENT signature and compare to the stored @@ -5435,11 +5815,17 @@ class DerivationEngine { await LocalDb.refreshComputeFreshness(); return done; } catch (e, st) { + _clearRetainedSubstrate(); _log('rescan ERROR: $e\n$st'); return 0; } finally { - await _runStorageHousekeeping(); - _running = false; + try { + _endRetentionPass(); + await _runStorageHousekeeping(); + } finally { + endWearableDayIndexPass(); + _running = false; + } } } @@ -5753,6 +6139,7 @@ class DerivationEngine { Future refreshActivityReviews(Profile profile) async { if (_running) return false; _running = true; + beginWearableDayIndexPass(); try { final revisions = await LocalDb.applyPendingActivityReviews(); if (revisions.isEmpty) return true; @@ -5763,6 +6150,7 @@ class DerivationEngine { await LocalDb.finishActivityReviews(revisions); return (await (await LocalDb.instance).query('activity_review_days', limit: 1)).isEmpty; } finally { + endWearableDayIndexPass(); _running = false; } } @@ -5851,6 +6239,26 @@ class DerivationEngine { _BaselineHistoryCache history, { bool forceFinalize = false, bool suppliedSubstrate = false, + bool allowSessionReuse = false, + }) async { + try { + return await _derivePreparedDayPayload(day, profile, dataNowSec, history, + forceFinalize: forceFinalize, suppliedSubstrate: suppliedSubstrate, + allowSessionReuse: allowSessionReuse); + } catch (_) { + _dropRetainedDay(day.date); + rethrow; + } + } + + Future _derivePreparedDayPayload( + PreparedDerivationDay day, + Profile profile, + int dataNowSec, + _BaselineHistoryCache history, { + bool forceFinalize = false, + bool suppliedSubstrate = false, + bool allowSessionReuse = false, }) async { final daySub = day.daySub; final sleepSub = day.sleepSub; @@ -6043,6 +6451,7 @@ class DerivationEngine { if (_isRealDayResult(existing)) { _log('derive ${day.date}: no substrate (raw pruned) — kept the ' 'existing result rather than blanking it'); + _dropRetainedDay(day.date); return false; } } @@ -6118,6 +6527,7 @@ class DerivationEngine { "under a day that already had real night scalars — kept the " 'existing result rather than overwriting the readiness baseline ' '(edge#305)'); + _dropRetainedDay(day.date); return false; } } @@ -6159,10 +6569,24 @@ class DerivationEngine { // sample, so a completed session the band's PPG lost contact for // entirely (zero HR samples at all, e.g. a wrist-gripping lift) can // fall outside them and never reach the zero-coverage credit below. + final sessionRevision = await _sessionCeilingRevision(); final savedSessions = await LocalDb.sessionsInRange( _localDayLabelToSec(day.date), localNextMidnightSecForDayLabel(day.date), ); + final ceilingSigs = debugDisableSessionReuse ? const {} + : await _ceilingSignatures(day, savedSessions, sessionRevision); + final ceilingReuse = {}; + if (allowSessionReuse && !debugDisableSessionReuse) { + for (final e in ceilingSigs.entries) { + final hit = _ceilingCache[(day.date, e.key)]; + if (hit != null && hit.version == kAlgoVersion && hit.sig == e.value) { + ceilingReuse[e.key] = hit.result; + } + } + } + // Removed, live or changed sessions cannot resurrect a former result. + _ceilingCache.removeWhere((key, _) => key.$1 == day.date); // Off-wrist / charging spans over the NAP window (which runs past this // day's end), read here because the isolate has no DB handle. These are @@ -6261,6 +6685,7 @@ class DerivationEngine { } final napEdits = await ActivityStore.napEdits(reviewDb, day.date); final blocksInput = _DayBlocksInput( + ceilingReuse: ceilingReuse, daySub: daySub, napSub: day.napSub, napEdits: napEdits, @@ -6324,6 +6749,19 @@ class DerivationEngine { final blocks = await _runDayBlocksCancellable(blocksInput, _perDayTimeout); + debugCeilingComputed += blocks.ceilingComputations; + if (!debugDisableSessionReuse) { + for (final e in {...ceilingReuse, ...blocks.ceilingComputed}.entries) { + final sig = ceilingSigs[e.key]; + if (sig == null) continue; + _ceilingCache[(day.date, e.key)] = + (version: kAlgoVersion, sig: sig, result: e.value); + while (_ceilingCache.length > _ceilingCacheMax) { + _ceilingCache.remove(_ceilingCache.keys.first); + } + } + } + // Merge the computed blocks back into the isolate-1 bundle. scMap is the // CastMap view over bundle['scalars'], so addAll writes through — nap_min / // hrr_bpm reach the persisted series map below. @@ -6458,6 +6896,7 @@ class DerivationEngine { await _persistWakeDayFeatures(dayId: day.date, wake: blocks.wake); } catch (e, st) { secondHalfOk = false; + _dropRetainedDay(day.date); _log('day-blocks (offloaded second half) failed for ${day.date} — ' 'persisting headline day (partial): $e'); TelemetryService.instance.recordNonFatal(e, st, reason: 'day_blocks_failed'); @@ -6724,6 +7163,7 @@ class DerivationEngine { 'hr_ceiling_bpm': sc('hr_ceiling_bpm'), }, ); + if (effectiveFinalized || effectivePartial) _dropRetainedDay(day.date); // AFTER the row: retired only once the row that proves it is durable. if (quietSettled) await LocalDb.deleteComputeFreshness(quietKey); // NOTE: the sweep's `history` snapshot is deliberately NOT updated here. @@ -10572,7 +11012,11 @@ class DerivationEngine { bundlePatch['workout_suggestions'] = wc.boutJson; if (wc.hrrBpm != null) scMap['hrr_bpm'] = wc.hrrBpm; if (wc.hrrTauS != null) scMap['hrr_tau_s'] = wc.hrrTauS; - final ceiling = _dayHrCeiling(daySub, inp.savedSessions); + var ceilingComputations = 0; + final ceilingComputed = {}; + final ceiling = _dayHrCeiling(daySub, inp.savedSessions, + reuse: inp.ceilingReuse, computed: ceilingComputed, + onCompute: () => ceilingComputations++); bundlePatch['hr_ceiling'] = ceiling; // TS-03 — the day's observed ceiling as a SCALAR too, so the app-wide // "highest we've seen" is a max over `metric_series` (one small query the @@ -10629,6 +11073,8 @@ class DerivationEngine { // than no diagnostic. return _DayBlocksOutput( + ceilingComputations: ceilingComputations, + ceilingComputed: ceilingComputed, bundlePatch: bundlePatch, seriesPatch: seriesPatch, scalarPatch: scMap, @@ -10819,50 +11265,66 @@ class DerivationEngine { ) => _dayHrCeiling(s, saved); + // Reuse carries the full Metric JSON, including absence, and its unrounded + // bpm. Comparing serialized/rounded values could choose a different session. static Map _dayHrCeiling( Substrate s, - List> saved, - ) { - ana.Metric? best; + List> saved, { + Map reuse = const {}, + Map? computed, + void Function()? onCompute, + }) { + Map? bestJson; + double? bestBpm; String? bestId; String? bestType; - ana.Metric? lastAbsent; + Map? lastAbsentJson; for (final row in saved) { final start = (row['start_ts'] as num?)?.toInt(); final end = (row['end_ts'] as num?)?.toInt(); if (start == null || end == null || end <= start) continue; - final hr = []; - final accel = []; - for (var i = 0; i < s.length; i++) { - final t = s.tsSec[i]; - if (t < start) continue; - if (t > end) break; - final tsMs = t * 1000.0; - hr.add(ana.HrSample(tsMs, s.hr[i].toDouble())); - accel.add(ana.AccelSample(tsMs, s.ax[i], s.ay[i], s.az[i], - valid: s.accelPresentAt(i))); - } - if (hr.isEmpty) continue; - final m = ana.sessionHrCeiling(hr, accel, deviceFamily: s.deviceFamily); - if (!m.present) { - lastAbsent = m; + final id = row['id']?.toString(); + var result = row['status'] == 'done' ? reuse[id] : null; + if (result == null) { + final hr = []; + final accel = []; + for (var i = 0; i < s.length; i++) { + final t = s.tsSec[i]; + if (t < start) continue; + if (t > end) break; + final tsMs = t * 1000.0; + hr.add(ana.HrSample(tsMs, s.hr[i].toDouble())); + accel.add(ana.AccelSample(tsMs, s.ax[i], s.ay[i], s.az[i], + valid: s.accelPresentAt(i))); + } + if (hr.isEmpty) continue; + onCompute?.call(); + final m = ana.sessionHrCeiling(hr, accel, deviceFamily: s.deviceFamily); + result = (json: m.toJson((v) => v.toJson()), + bpm: m.present ? m.value!.bpm : null); + if (id != null && row['status'] == 'done') computed?[id] = result; + } + final bpm = result.bpm; + if (bpm == null) { + lastAbsentJson = result.json; continue; } - if (best == null || m.value!.bpm > best.value!.bpm) { - best = m; - bestId = row['id']?.toString(); + if (bestBpm == null || bpm > bestBpm) { + bestJson = result.json; + bestBpm = bpm; + bestId = id; bestType = row['type']?.toString(); } } - final m = best ?? - lastAbsent ?? + final json = bestJson ?? + lastAbsentJson ?? const ana.Metric.absent( tier: ana.Tier.high, inputs_used: ['hr_1hz', 'accel_1hz', 'device_family'], note: 'no saved session on this day to observe a ceiling in', - ); + ).toJson((v) => v.toJson()); return { - ...m.toJson((v) => v.toJson()), + ...json, 'session_id': ?bestId, 'session_type': ?bestType, }; @@ -11214,7 +11676,7 @@ class _IsolateValue { /// Test seam for [DerivationEngine._runIsolateCancellable] — the isolate /// lifecycle guarantees (value / error / killed-on-timeout) are what the engine /// depends on to never hang, so they're pinned directly. -@visibleForTesting +/// Also used by wearable and session-window preparation in production. Future runCancellableIsolate( FutureOr Function() compute, Duration timeout, { @@ -11222,11 +11684,15 @@ Future runCancellableIsolate( }) => DerivationEngine._runIsolateCancellable(compute, timeout, label: label); +typedef _SessionCeiling = ({Map json, double? bpm}); +typedef _CeilingInput = ({int generation, Map revisions}); + /// Sendable input for [DerivationEngine._computeDayBlocks] — crosses the /// `Isolate.run` boundary, so every field is plain data (Substrate is int/double /// lists; Profile is a primitive data class). DB reads that the /// pure compute needs are performed by the caller and passed in here. class _DayBlocksInput { + final Map ceilingReuse; final Substrate daySub; final Substrate napSub; final Substrate sleepSub; @@ -11321,6 +11787,7 @@ class _DayBlocksInput { /// `[startSec, endSec]`. Nap suggestions overlapping one are dropped. final List> rejectedSleepSpans; const _DayBlocksInput({ + this.ceilingReuse = const {}, required this.daySub, required this.napSub, required this.sleepSub, @@ -11360,6 +11827,8 @@ class _DayBlocksInput { /// isolate; [wake] is persisted; candidates are reconciled and session HRR /// writes are applied by the database-owning caller. class _DayBlocksOutput { + final Map ceilingComputed; + final int ceilingComputations; final List> napSuggestions; final Map bundlePatch; final Map seriesPatch; @@ -11368,6 +11837,8 @@ class _DayBlocksOutput { final List> suggestionsToPersist; final List<(String, double)> sessionHrrWrites; const _DayBlocksOutput({ + this.ceilingComputed = const {}, + this.ceilingComputations = 0, required this.bundlePatch, this.napSuggestions = const [], required this.seriesPatch, diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index 316077e2b..b008c9473 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -347,6 +347,11 @@ void derivationPrepareWorker(SendPort mainSendPort) { targetDay = message['target_day']?.toString(); final cfgMode = message['mode']?.toString(); if (cfgMode != null && cfgMode.isNotEmpty) mode = cfgMode; + state.retain = message['retain'] == true; + return; + } + if (type == 'seed') { + state.seed(message); return; } if (type == 'finish') { @@ -357,6 +362,7 @@ void derivationPrepareWorker(SendPort mainSendPort) { 'type': 'result', 'kind': 'substrate', 'payload': substrate.toJson(), + if (state.retainable) 'retain': state.retainExtras(), }); } else { final payload = prepareDerivationPayload( @@ -408,6 +414,7 @@ PreparedDerivationPayload prepareDerivationPayload( SleepWindowOverride? override, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + bool skipUnusedDays = false, }) { if (sub.isEmpty || sub.lastTs == null) { return const PreparedDerivationPayload(dataNowSec: 0, days: []); @@ -416,6 +423,8 @@ PreparedDerivationPayload prepareDerivationPayload( for (final day in calendarDays( sub, override: override, + targetDay: targetDay, + skipUnusedDays: skipUnusedDays, priorSleep: priorSleep, vendorNights: vendorNights, )) { @@ -470,6 +479,7 @@ SleepSessionCandidate prepareSleepSessionCandidate( SleepWindowOverride? override, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + bool skipUnusedDays = false, }) { // An UNCLAIMED device night (our rows saw under half of it) ending today IS // the night: above our detection, below the user's word. It never reaches @@ -515,6 +525,7 @@ SleepSessionCandidate prepareSleepSessionCandidate( final payload = prepareDerivationPayload(sub, targetDay: targetDay, override: override, + skipUnusedDays: skipUnusedDays, priorSleep: priorSleep, vendorNights: claimed); if (payload.days.isEmpty) return SleepSessionCandidate.absent(targetDay); @@ -635,10 +646,68 @@ class _PrepareAccumulator { /// worker used to hang the whole engine (see [derivationPrepareWorker]). static num? _num(Object? v) => v is num ? v : null; + // Retention is recorded only for decoded loads requested by the engine. + bool retain = false; + bool _retainBroken = false; + bool get retainable => retain && !_retainBroken; + final List _rrStart = []; + final Map _buckets = {}; + Float64List? _rrTsRaw; + + void seed(Map message) { + List ints(String k) => message[k] as List; + List dbls(String k) => message[k] as List; + final ts = ints('ts_sec'); + final cut = message['cut'] as int; + var n = 0; + while (n < ts.length && ts[n] < cut) { n++; } + final starts = ints('rr_start'); + final beats = n < ts.length ? starts[n] : dbls('rr_ms').length; + void take(List into, String k) => into.addAll(ints(k).getRange(0, n)); + take(tsSec, 'ts_sec'); + take(hr, 'hr'); + take(spo2Red, 'spo2_red'); + take(spo2Ir, 'spo2_ir'); + take(skinTemp, 'skin_temp'); + take(skinContact, 'skin_contact'); + take(stepCount, 'step_count'); + take(hrValid, 'hr_valid'); + take(bandSleepState, 'band_sleep_state'); + ax.addAll(dbls('ax').getRange(0, n)); + ay.addAll(dbls('ay').getRange(0, n)); + az.addAll(dbls('az').getRange(0, n)); + // Start from the raw axis: buildSubstrate applies the running max once + // across the entire rebuilt load, including the prefix/tail seam. + rrTsMs.addAll(dbls('rr_ts_raw').getRange(0, beats)); + rrMs.addAll(dbls('rr_ms').getRange(0, beats)); + _rrStart.addAll(starts.getRange(0, n)); + for (final wire in message['meta'] as List) { + final m = wire as List; + final bucket = m[0] as int; + if (bucket >= cut ~/ 900) continue; + final acc = _buckets[bucket] = _BucketAcc() + ..families.addAll((m[1] as List).cast()) + ..deviceIds.addAll((m[2] as List).cast()) + ..skinUnit = m[3] as String?; + _families.addAll(acc.families); + _deviceIds.addAll(acc.deviceIds); + _skinTempUnit ??= acc.skinUnit; + } + } + + Map retainExtras() => { + 'rr_start': Int32List.fromList(_rrStart), + 'rr_ts_raw': _rrTsRaw!, + 'meta': [for (final e in _buckets.entries) + [e.key, e.value.families.toList(), e.value.deviceIds.toList(), + e.value.skinUnit]], + }; + void addRawPage(List hexes) { if (hexes.isEmpty) return; final sub = decodeSubstrate(hexes); if (sub.isEmpty) return; + _retainBroken = true; tsSec.addAll(sub.tsSec); hr.addAll(sub.hr); rrTsMs.addAll(sub.rrTsMs); @@ -707,6 +776,10 @@ class _PrepareAccumulator { final row = frameByRecTs[recTs]; _noteFamily(row?['device_family']); _noteDeviceId(row?['device_id']); + if (retain) { + _rrStart.add(rrMs.length); + _buckets.putIfAbsent(recTs ~/ 900, _BucketAcc.new).note(row); + } tsSec.add(recTs); // NULL (absent), a beat-only second (no row at all) and an impossible // byte all land on the same 0 — the array's one "no usable HR this @@ -794,6 +867,7 @@ class _PrepareAccumulator { // `addRawPage` each place their own beats correctly and then APPEND, so an // overlap between the last record of one page and the first of the next is // only visible once every page is in. See [monotonizeBeatAxis]. + if (retainable) _rrTsRaw = Float64List.fromList(rrTsMs); monotonizeBeatAxis(rrTsMs); // `_skinTempUnitDrops` (see [_skinTempFor]) is structurally 0 today — // `derivableSourceSql()` renders `source IS NULL` (db.dart:116), so a @@ -807,23 +881,41 @@ class _PrepareAccumulator { // which M0 must not take; M5 — which is what makes two-device admission // reachable for real — turns this into a refusal note on // `wellness.skin_temp` instead. + List packed(List values) => + retainable ? Int64List.fromList(values) : values; return Substrate( - tsSec: tsSec, - hr: hr, + tsSec: packed(tsSec), + hr: packed(hr), rrTsMs: rrTsMs, rrMs: rrMs, ax: ax, ay: ay, az: az, - spo2Red: spo2Red, - spo2Ir: spo2Ir, - skinTemp: skinTemp, - skinContact: skinContact, - stepCount: stepCount, - hrValid: hrValid, - bandSleepState: bandSleepState, + spo2Red: packed(spo2Red), + spo2Ir: packed(spo2Ir), + skinTemp: packed(skinTemp), + skinContact: packed(skinContact), + stepCount: packed(stepCount), + hrValid: packed(hrValid), + bandSleepState: packed(bandSleepState), deviceFamily: deviceFamily, deviceIds: _deviceIds, ); } } + +class _BucketAcc { + final Set families = {}; + final Set deviceIds = {}; + String? skinUnit; + + void note(Map? row) { + if (row == null) return; + final family = row['device_family']; + if (family is String && family.isNotEmpty) families.add(family); + final id = row['device_id']; + if (id is String) deviceIds.add(id); + skinUnit ??= row['skin_temp_raw'] is num + ? 'raw' : (row['skin_temp_c'] is num ? 'c' : null); + } +} diff --git a/lib/compute/inputs/canonical.dart b/lib/compute/inputs/canonical.dart index ff4b2ca8b..17950e9fc 100644 --- a/lib/compute/inputs/canonical.dart +++ b/lib/compute/inputs/canonical.dart @@ -18,7 +18,7 @@ import '../../ble/adapters/_registry.dart' import '../../data/day_label.dart'; import '../../data/db.dart'; import '../crossday_pipeline.dart' show buildCrossDayBundle; -import '../derivation_engine.dart' show DerivationEngine, kAlgoVersion; +import '../derivation_engine.dart' show DerivationEngine, kAlgoVersion, runCancellableIsolate; import '../substrate.dart'; import '../vendor_sleep.dart'; import 'colmi_inputs.dart'; @@ -390,6 +390,36 @@ Future refreshSessionSensorSources() async { LocalDb.sessionWearableSource = (await activeWearable())?.$2; } +bool debugDisableWearableWorker = false; +void Function(String)? debugOnWearableConstruction; +// Called beside the actual row loops. A worker extraction must return its +// execution marker to this observer, not label a dispatch on the caller. +void debugRecordWearableConstruction([String? isolateName]) => + debugOnWearableConstruction?.call( + isolateName ?? Isolate.current.debugName ?? 'unnamed'); + +// Only plain rows, columns and bounds are captured by these builders. Return +// the execution marker with the value; observers stay on the calling isolate. +Future prepareWearable(R Function() build) async { + (R, String) construct() => (build(), Isolate.current.debugName ?? 'unnamed'); + final result = debugDisableWearableWorker + ? construct() + : await runCancellableIsolate(construct, const Duration(seconds: 30), + label: 'wearable construction'); + debugRecordWearableConstruction(result.$2); + return result.$1; +} + +Future constructWearable(Substrate Function() build, + {Substrate? unchanged}) async { + final columns = await prepareWearable(() { + final value = build(); + return identical(value, unchanged) ? null : value.toJson(); + }); + // A no-op overlay preserves even a direct caller's empty optional columns. + return columns == null ? unchanged! : Substrate.fromTransfer(columns); +} + /// The active wearable's rows over `[from, to]` as a substrate, for a day the /// primary band left empty, with a strap's session windows laid over them /// ([withStrapSessions]). Empty for an adapter with no inputs mapping. A day @@ -460,6 +490,12 @@ Future withStrapSessions(Substrate sub, int from, int to) async { 'AND d.rec_ts >= ? AND d.rec_ts <= ? ORDER BY d.rec_ts', [from, to], ); + return constructWearable(() => _overlayStrapSessions(sub, beats, all), + unchanged: sub); +} + +Substrate _overlayStrapSessions(Substrate sub, + List> beats, List> all) { final owner = ownerBy([for (final r in all) (r['sid'], r['src'])]); final rows = [ for (final r in all) @@ -616,10 +652,32 @@ Future> flagOnOrPrimary(List ids, return out; } +// Pass-lifetime index. The hook runs after an actual scan, so a test can +// count scans and inject a post-scan failure. +Map<(String, String), Future>>? _passDayIndex; +bool _disableWearableDayIndex = false; +bool get debugDisableWearableDayIndex => _disableWearableDayIndex; +set debugDisableWearableDayIndex(bool value) { + _disableWearableDayIndex = value; + if (value) _passDayIndex?.clear(); +} +void beginWearableDayIndexPass() => _passDayIndex = {}; +void endWearableDayIndexPass() => _passDayIndex = null; +int get debugWearableDayIndexSize => _passDayIndex?.length ?? 0; +void debugWearableDayIndexRekey(int version) => _passDayIndex?.clear(); +void Function((String, String), Map)? debugOnWearableDayIndexScan; + /// `{local day -> last second}` the active wearable has rows for, and the /// days a flagged-on strap recorded a session in while the wearable was off /// or charging ([wearableSubstrate] lays the session over the empty day). Future> wearableRecTsMaxByDay( + (String, String) wearable) { + final cache = debugDisableWearableDayIndex ? null : _passDayIndex; + if (cache == null) return _scanWearableRecTsMaxByDay(wearable); + return cache.putIfAbsent(wearable, () => _scanWearableRecTsMaxByDay(wearable)); +} + +Future> _scanWearableRecTsMaxByDay( (String, String) wearable) async { final out = { ...await switch (wearable.$2) { @@ -637,6 +695,7 @@ Future> wearableRecTsMaxByDay( in (await _strapSessionRecTsMaxByDay()).entries) { out[day] = math.max(out[day] ?? t, t); } + debugOnWearableDayIndexScan?.call(wearable, out); return out; } diff --git a/lib/compute/inputs/colmi_inputs.dart b/lib/compute/inputs/colmi_inputs.dart index b9823df78..cc7ebb2a7 100644 --- a/lib/compute/inputs/colmi_inputs.dart +++ b/lib/compute/inputs/colmi_inputs.dart @@ -51,10 +51,9 @@ Future> colmiSkinTemps( int to, ) async { final db = await LocalDb.instance; - final out = {}; // A reply is banked on or after the day it describes and at most the // ring's history (kColmiHistoryDays) later: scan only those captures. - for (final r in await db.rawQuery( + final rows = await db.rawQuery( 'SELECT hex, COALESCE(rec_ts * 1000, captured_at) AS anchor_ms ' 'FROM ${LocalDb.rawArchiveView} WHERE device_id = ? ' "AND reason = 'colmi_big_0x${kColmiBigTemperature.toRadixString(16)}' " @@ -64,7 +63,14 @@ Future> colmiSkinTemps( (from - 86400) * 1000, (to + (kColmiHistoryDays + 1) * 86400) * 1000, ], - )) { + ); + return prepareWearable(() => _colmiSkinTemps(rows, from, to)); +} + +List<(int, int, int)> _colmiSkinTemps( + List> rows, int from, int to) { + final out = {}; + for (final r in rows) { final hex = r['hex'] as String; final bytes = [ for (var i = 0; i + 1 < hex.length; i += 2) @@ -99,6 +105,10 @@ Future> colmiSkinTemps( Future colmiHr(String deviceId, int from, int to) async { final sub = await sparseHrSubstrate(kColmiFamily, deviceId, from, to); if (sub.length < 2) return sub; + return constructWearable(() => _colmiHr(sub)); +} + +Substrate _colmiHr(Substrate sub) { final gaps = [ for (var i = 1; i < sub.length; i++) sub.tsSec[i] - sub.tsSec[i - 1], ]..sort(); @@ -162,6 +172,10 @@ Future colmiSubstrate(String deviceId, int from, int to) async { final sub = await colmiHr(deviceId, from, to); if (sub.isEmpty) return sub; final temps = await colmiSkinTemps(deviceId, from, to); + return constructWearable(() => _colmiSubstrate(sub, temps)); +} + +Substrate _colmiSubstrate(Substrate sub, List<(int, int, int)> temps) { final skin = []; var j = 0; for (final t in sub.tsSec) { diff --git a/lib/compute/inputs/garmin_inputs.dart b/lib/compute/inputs/garmin_inputs.dart index 47c771e2b..ebe695dce 100644 --- a/lib/compute/inputs/garmin_inputs.dart +++ b/lib/compute/inputs/garmin_inputs.dart @@ -107,6 +107,11 @@ Future sparseHrSubstrate( [deviceId, family, from, to], ); if (rows.isEmpty) return Substrate.empty; + return constructWearable(() => _sparseHrSubstrate(family, deviceId, rows)); +} + +Substrate _sparseHrSubstrate(String family, String deviceId, + List> rows) { final n = rows.length; final zeros = List.filled(n, 0); final none = List.filled(n, -1); diff --git a/lib/compute/inputs/oura_inputs.dart b/lib/compute/inputs/oura_inputs.dart index 2eed4d12f..aaa578aaf 100644 --- a/lib/compute/inputs/oura_inputs.dart +++ b/lib/compute/inputs/oura_inputs.dart @@ -68,6 +68,18 @@ Future ouraSubstrate(String deviceId, int from, int to) async { if (sub.isEmpty) return _ouraTempSubstrate(deviceId, from, to); final c = kHrCadenceSec[kOuraFamily]!; final temps = await _ouraTempRows(deviceId, from - c, to + c); + final db = await LocalDb.instance; + final beats = await db.rawQuery( + 'SELECT COALESCE(beat_ts_ms, rr_ts_ms) AS t, rr_ms AS rr FROM decoded_rr ' + 'WHERE device_id = ? AND device_family = ? AND rec_ts >= ? AND rec_ts <= ? ' + 'ORDER BY t, beat_index', + [deviceId, kOuraFamily, from, to], + ); + return constructWearable(() => _ouraSubstrate(sub, temps, beats, c)); +} + +Substrate _ouraSubstrate(Substrate sub, List> temps, + List> beats, int c) { final skin = []; var j = 0; for (final t in sub.tsSec) { @@ -80,13 +92,6 @@ Future ouraSubstrate(String deviceId, int from, int to) async { ((temps[j]['rec_ts'] as num) - t).abs() <= c; skin.add(near ? ((temps[j]['skin_temp_c'] as num) * 100).round() : 0); } - final db = await LocalDb.instance; - final beats = await db.rawQuery( - 'SELECT COALESCE(beat_ts_ms, rr_ts_ms) AS t, rr_ms AS rr FROM decoded_rr ' - 'WHERE device_id = ? AND device_family = ? AND rec_ts >= ? AND rec_ts <= ? ' - 'ORDER BY t, beat_index', - [deviceId, kOuraFamily, from, to], - ); final rr = [ for (final b in beats) if (plausibleRrOrNull(b['rr'] as num) case final ms?) @@ -104,6 +109,10 @@ Future ouraSubstrate(String deviceId, int from, int to) async { Future _ouraTempSubstrate(String deviceId, int from, int to) async { final rows = await _ouraTempRows(deviceId, from, to); if (rows.isEmpty) return Substrate.empty; + return constructWearable(() => _ouraTempColumns(deviceId, rows)); +} + +Substrate _ouraTempColumns(String deviceId, List> rows) { final n = rows.length; final zeros = List.filled(n, 0); final none = List.filled(n, -1); diff --git a/lib/compute/inputs/ultrahuman_inputs.dart b/lib/compute/inputs/ultrahuman_inputs.dart index dd467784d..300c022e8 100644 --- a/lib/compute/inputs/ultrahuman_inputs.dart +++ b/lib/compute/inputs/ultrahuman_inputs.dart @@ -47,13 +47,18 @@ Future> ultrahumanRecords( int to, ) async { final db = await LocalDb.instance; - final out = []; - for (final r in await db.rawQuery( + final rows = await db.rawQuery( "SELECT hex FROM ${LocalDb.rawArchiveView} WHERE device_id = ? " "AND reason = 'ultrahuman_record' AND rec_ts >= ? AND rec_ts <= ? " 'ORDER BY rec_ts', [deviceId, from, to], - )) { + ); + return prepareWearable(() => _ultrahumanRecords(rows)); +} + +List _ultrahumanRecords(List> rows) { + final out = []; + for (final r in rows) { final hex = r['hex'] as String; final rec = parseUltrahumanRecord([ for (var i = 0; i + 1 < hex.length; i += 2) @@ -77,8 +82,13 @@ Future ultrahumanSubstrate(String deviceId, int from, int to) async { final sub = await sparseHrSubstrate(kUltrahumanFamily, deviceId, from, to); if (sub.isEmpty) return sub; + final records = await ultrahumanRecords(deviceId, from, to); + return constructWearable(() => _ultrahumanSubstrate(sub, records)); +} + +Substrate _ultrahumanSubstrate(Substrate sub, List records) { final temp = { - for (final r in await ultrahumanRecords(deviceId, from, to)) + for (final r in records) if (_skinC(r) case final c?) r.tsA: (c * 100).round(), }; return Substrate( diff --git a/lib/compute/staged_sleep_cache.dart b/lib/compute/staged_sleep_cache.dart new file mode 100644 index 000000000..30fe3d638 --- /dev/null +++ b/lib/compute/staged_sleep_cache.dart @@ -0,0 +1,132 @@ +import 'dart:convert'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'sleep_profile_policy.dart'; +import 'substrate.dart'; +import 'vendor_sleep.dart'; + +class StagedSleepEntry { + StagedSleepEntry({ + required this.algoVersion, + required this.key, + required this.mayFold, + required this.candidateJson, + required this.observationJson, + }); + + int algoVersion; + final String key; + final bool mayFold; + final String candidateJson; + final String? observationJson; +} + +/// Per-engine RAM only. One worker result per day, least recently used first. +class StagedSleepCache { + final _byDay = {}; + int _maxDays = 14; + + int get size => _byDay.length; + int get maxDays => _maxDays; + set maxDays(int value) { + _maxDays = value < 0 ? 0 : value; + while (_byDay.length > _maxDays) { + _byDay.remove(_byDay.keys.first); + } + } + + StagedSleepEntry? lookup(String day, int algoVersion, String key, + {required bool mayFold}) { + final entry = _byDay[day]; + if (entry == null || entry.algoVersion != algoVersion || entry.key != key || + (mayFold && !entry.mayFold)) { + return null; + } + _byDay.remove(day); + _byDay[day] = entry; + return entry; + } + + void store(String day, StagedSleepEntry entry) { + _byDay.remove(day); + if (_maxDays == 0) return; + _byDay[day] = entry; + maxDays = _maxDays; + } + + void rekey(int version) { + for (final entry in _byDay.values) { + entry.algoVersion = version; + } + } +} + +/// Revisions are captured before loading rows. A concurrent write then makes +/// the next request miss. No sample columns are walked on the UI isolate. +String stagingInputKey({ + required int fromSec, + required int toSec, + required List> revisions, + required Substrate sub, + required Map> ownership, + required Map> priority, + required SleepWindowOverride? override, + required String? profileJson, + required List<({int startSec, int endSec, String dayKey})> priorSleep, + required List vendorNights, + required (String, String)? activeWearable, +}) { + // Count local midnights, including DST days, rather than dividing by 86400. + var walked = 0; + if (sub.tsSec.isNotEmpty) { + final first = DateTime.fromMillisecondsSinceEpoch(sub.tsSec.first * 1000); + for (var d = DateTime(first.year, first.month, first.day); + d.millisecondsSinceEpoch ~/ 1000 <= sub.tsSec.last; + d = DateTime(d.year, d.month, d.day + 1)) { + walked++; + } + } + final effectiveHistory = + {for (final h in priorSleep) h.dayKey}.length + walked < 14 + ? const <({int startSec, int endSec, String dayKey})>[] + : priorSleep; + + // Fold bookkeeping is re-read by the caller. Only a warmed-up profile can + // reach staging, but all of its fields (including updated_at_ms) are keyed. + Map? blendedProfile; + if (profileJson != null) { + final p = (jsonDecode(profileJson) as Map).cast(); + if (SleepProfilePolicy.shouldBlend((p['nights'] as num?)?.toInt())) { + blendedProfile = {...p}..remove(SleepProfilePolicy.foldedDaysKey); + } + } + + // Local day boundaries and historical midsleeps read offsets at their own + // instants. Key those too: equal current zone names/offsets are insufficient. + final zoneOffsets = >[]; + for (var t = fromSec - 86400; t <= toSec + 86400; t += 900) { + zoneOffsets.add([t, tzOffsetSecondsAt(t)]); + } + for (final h in effectiveHistory) { + final mid = (h.startSec + h.endSec) ~/ 2; + zoneOffsets.add([mid, tzOffsetSecondsAt(mid)]); + } + final now = DateTime.now(); + return jsonEncode([ + fromSec, toSec, revisions, + [for (final sig in InputSignal.values) [sig.name, + [for (final o in ownership[sig] ?? const []) + [o.start, o.end, o.deviceId]], priority[sig]]], + sub.deviceFamily, (sub.deviceIds.toList()..sort()), + sub.length, sub.firstTs, sub.lastTs, sub.rrMs.length, + now.timeZoneName, now.timeZoneOffset.inSeconds, zoneOffsets, + override == null ? null : + [override.dayId, override.onsetSec, override.offsetSec, override.source], + blendedProfile, + [for (final h in effectiveHistory) [h.startSec, h.endSec, h.dayKey]], + activeWearable == null ? null : [activeWearable.$1, activeWearable.$2], + [for (final n in vendorNights) [n.deviceId, n.source, n.decodedAtSec, + [for (final e in n.epochs) [e.startSec, e.endSec, e.stage]]]], + ]); +} diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 08d8664f3..08b534fe9 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -621,15 +621,27 @@ class Substrate { 'device_ids': deviceIds.toList(), }; - static Substrate fromJson(Map m) { - List ints(Map m, String k) => - ((m[k] as List?) ?? const []).map((e) => (e as num).toInt()).toList(); - // Straight into a Float64List — see [_packed]. This runs on the RECEIVING - // (main) isolate for every substrate handed back from a worker, so the - // growable intermediate was ~400 000 boxes allocated and immediately - // thrown away, on the UI thread. + /// Adopt the columns received from a worker. Packing/copies happen in the + /// worker or isolate transport; correctly typed arrays need no element walk + /// here. Loose numeric lists retain the same normalization as [fromJson]. + static Substrate fromTransfer(Map m) => + _fromColumns(m, adopt: true); + + static Substrate fromJson(Map m) => + _fromColumns(m, adopt: false); + + static Substrate _fromColumns(Map m, {required bool adopt}) { + List ints(Map m, String k) { + final src = m[k]; + if (adopt && src is List) return src; + return ((src as List?) ?? const []) + .map((e) => (e as num).toInt()).toList(); + } + // Convert loose lists straight into packed doubles, without a boxed + // intermediate. The worker already packs these through the constructor. Float64List dbls(String k) { final src = (m[k] as List?) ?? const []; + if (adopt && src is Float64List) return src; final out = Float64List(src.length); for (var i = 0; i < src.length; i++) { out[i] = (src[i] as num).toDouble(); @@ -1208,6 +1220,11 @@ class SleepWindowOverride { String localDateLabel(int epochSec) => dayLabelOf(DateTime.fromMillisecondsSinceEpoch(epochSec * 1000)); +// RED controls: the walk still segments every day. The counter observes its +// actual body; GREEN must preserve the full walk for folding callers. +int debugSegmentedDayCount = 0; +bool debugStageEveryWalkedDay = false; + /// Split the substrate into CALENDAR days (local midnight → next local midnight). /// /// Each day owns its 24 h of data. A day's SLEEP is the main sleep that ENDED @@ -1237,12 +1254,19 @@ String localDateLabel(int epochSec) => /// always fell back to the fixed 03:30 cold-start anchor, and a night-shift /// sleeper's 4 h main block lost the alignment bonus to a shorter block nearer /// 03:30. Days found in THIS call still win (a full restage sees them all). +/// +/// With [skipUnusedDays], only [targetDay] is needed. Earlier days can be +/// skipped when even one new history key per day cannot reach the prior's +/// 14-night threshold. Callers needing profile observations must keep the +/// full walk: its longest observation can belong to an earlier day. List calendarDays( Substrate sub, { SleepWindowOverride? override, int Function(int epochSec)? tzOffsetAt, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + String? targetDay, + bool skipUnusedDays = false, }) { final tzOffset = tzOffsetAt ?? tzOffsetSecondsAt; if (sub.isEmpty) return const []; @@ -1256,6 +1280,20 @@ List calendarDays( ...priorSleep, ]; var dayStart = _localMidnight(dataStart); + final prunable = targetDay != null && + skipUnusedDays && + !debugStageEveryWalkedDay && + (override == null || override.dayId == targetDay); + var skipBefore = false; + if (prunable) { + var earlier = 0; + for (var d = dayStart, g = 0; + d < dataEnd && g++ < 400 && localDateLabel(d).compareTo(targetDay) < 0; + d = _nextLocalMidnight(d)) { + earlier++; + } + skipBefore = {for (final h in sleepHistory) h.dayKey}.length + earlier < 14; + } var guard = 0; while (dayStart < dataEnd && guard++ < 400) { final dayEnd = _nextLocalMidnight(dayStart); @@ -1266,6 +1304,16 @@ List calendarDays( continue; } + final dayLabel = localDateLabel(dayStart); + if (prunable) { + final order = dayLabel.compareTo(targetDay); + if (order > 0) break; + if (order < 0 && skipBefore) { + dayStart = dayEnd; + continue; + } + } + debugSegmentedDayCount++; // The main sleep that ENDS in this calendar day: search from the previous // local noon through this local midnight, then let the sleep selector pick // the overnight main block from any naps / split fragments it sees. The old @@ -1281,7 +1329,6 @@ List calendarDays( var sleepLo = 0, sleepHi = 0; var sleepSource = 'none'; final vendorIgnored = []; - final dayLabel = localDateLabel(dayStart); // Does the user have an override (manual / confirmed) for THIS day? final ov = (override != null && override.dayId == dayLabel) ? override : null; diff --git a/lib/compute/substrate_retention.dart b/lib/compute/substrate_retention.dart new file mode 100644 index 000000000..af77a2b6a --- /dev/null +++ b/lib/compute/substrate_retention.dart @@ -0,0 +1,102 @@ +import 'dart:convert'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'substrate.dart'; + +/// One exact load, plus the worker metadata needed to rebuild its open bucket. +class RetainedSubstrate { + RetainedSubstrate({required this.version, required this.context, + required this.revisions, required this.sub, required this.extras, + required this.cut}); + + int version; + final String context; + final Map revisions; + final Substrate sub; + final Map extras; + final int cut; + + /// A changed closed bucket requires a cold load. Otherwise rebuild the tail + /// from [cut], or return the whole load when every bucket is unchanged. + int? readFrom(int version, String context, Map revisions, int to) { + if (version != this.version || context != this.context) return null; + var changed = false; + for (final b in {...this.revisions.keys, ...revisions.keys}) { + if ((revisions[b] ?? 0) == (this.revisions[b] ?? 0)) continue; + if (b < cut ~/ 900) return null; + changed = true; + } + return changed ? cut : to + 1; + } + + Map seed(int cut) => { + ...sub.toJson(), ...extras, 'type': 'seed', 'cut': cut, + }; +} + +/// At most three loads per day: search, buffered day and selected sleep. +/// Both days and loads are ordered least recently used first. +class SubstrateRetention { + final _days = >{}; + int _maxDays = 2; + + int get maxDays => _maxDays; + set maxDays(int value) { + _maxDays = value < 0 ? 0 : value; + while (_days.length > _maxDays) { + _days.remove(_days.keys.first); + } + } + Set get days => _days.keys.toSet(); + + RetainedSubstrate? lookup(String day, String load) { + final loads = _days.remove(day); + if (loads == null) return null; + _days[day] = loads; + final entry = loads.remove(load); + if (entry != null) loads[load] = entry; + return entry; + } + + void store(String day, String load, RetainedSubstrate entry) { + if (_maxDays == 0) return; + final loads = _days.remove(day) ?? {}; + loads.remove(load); + loads[load] = entry; + while (loads.length > 3) { + loads.remove(loads.keys.first); + } + _days[day] = loads; + maxDays = _maxDays; + } + + Map extras(String day) => + _days[day]?.values.last.extras ?? const {}; + void remove(String day) => _days.remove(day); + void clear() => _days.clear(); + void retainDays(Set days) => + _days.removeWhere((day, _) => !days.contains(day)); + void rekey(int version) { + for (final loads in _days.values) { + for (final entry in loads.values) { + entry.version = version; + } + } + } +} + +/// Only inputs consumed by the decoded loader belong here. Resolved spans +/// already carry priority, coverage and device eligibility decisions. +String substrateLoadContext(int from, int to, int generation, + Map> ownership) { + final now = DateTime.now(); + return jsonEncode([ + from, to, generation, now.timeZoneName, now.timeZoneOffset.inSeconds, + [for (var t = from - 86400; t <= to + 86400; t += 900) + [t, tzOffsetSecondsAt(t)]], + [for (final sig in InputSignal.values) [sig.name, + [for (final s in ownership[sig] ?? const []) + [s.start, s.end, s.deviceId]]]], + ]); +} diff --git a/lib/data/db.dart b/lib/data/db.dart index 578df4b9c..b940f179b 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -33,6 +33,7 @@ import '../coach/coach_db.dart' show CoachDb; import '../compute/derivation_engine.dart' show kAlgoVersion, + runCancellableIsolate, kOvernightGiveUpSec, kPinWakeToleranceSec, pinnableNight, @@ -64,6 +65,9 @@ import 'observation.dart'; import 'series_codec.dart'; import 'substrate_archive.dart'; +part 'db_input_revisions.dart'; +part 'db_session_ceiling_revisions.dart'; + /// The outcome of a database rebuild: why the old file would not open, where it /// was parked, and how many rows came back per table. typedef DbRebuild = ({ @@ -72,6 +76,11 @@ typedef DbRebuild = ({ Map salvaged, }); +typedef SessionHrReads = ({ + Map> stats, + Map> samples, +}); + // ── SUBSTRATE ADMISSION — the one predicate that decides what may become a // number ─────────────────────────────────────────────────────────────────── // @@ -166,7 +175,13 @@ String derivableSourceSql([String col = 'source']) => kDerivableSources.isEmpty /// ponytail: on that gap the workout screen and the day disagree; give the /// band day the strap's window too (or drop the override there) once the /// owner decides whether a strap may fill a band gap. -List sessionWindowRows>(List rows) { +List sessionWindowRows>(List rows) => + _sessionWindowRowsFor(rows, sensors: LocalDb.sessionSensorSources, + wearable: LocalDb.sessionWearableSource); + +// Workers receive the flags explicitly; isolate-local statics start empty. +List _sessionWindowRowsFor>( + List rows, {required Set sensors, required String? wearable}) { bool derivable(Object? s) => s == null || kDerivableSources.contains(s); List plain() => [ for (final r in rows) @@ -177,7 +192,7 @@ List sessionWindowRows>(List rows) { for (final r in rows) { final s = r['source']; if (s == null) return plain(); - if (LocalDb.sessionSensorSources.contains(s) && worn(r)) { + if (sensors.contains(s) && worn(r)) { n[s] = (n[s] ?? 0) + 1; } } @@ -195,7 +210,6 @@ List sessionWindowRows>(List rows) { if (lo == null || t < lo) lo = t; if (hi == null || t > hi) hi = t; } - final wearable = LocalDb.sessionWearableSource; return [ for (final r in rows) if (r['source'] == owner || @@ -479,7 +493,46 @@ class LocalDb { /// pass it: sqflite throws `ArgumentError('onCreate must be null if no /// version is specified')` BEFORE opening anything when `onCreate` is given /// without `version` (sqflite_common database_mixin.dart). - static const int schemaVersion = 58; + static const int schemaVersion = 59; + + /// In-memory cache identity, advanced after open, wipe or committed import. + /// It is local to this isolate; no cache survives a process restart. + static int _storeGeneration = 0; + static int get storeGeneration => _storeGeneration; + + static void _markStoreReplaced() => _storeGeneration++; + + /// Revisions over every absolute 15-minute bucket intersecting the local + /// calendar day. Legacy/missing buckets read zero without a raw-row backfill. + /// RR-only days participate, and all devices share these conservative buckets. + static Future<({int revisionSum, int revisionMax, int generation})> + inputDayFingerprint(String dayId) async { + final (lo, hi) = _localDayWindow(dayId); + final db = await instance; + final generation = storeGeneration; + final rows = await db.rawQuery( + 'SELECT COALESCE(SUM(rev), 0) AS revision_sum, ' + 'COALESCE(MAX(rev), 0) AS revision_max FROM input_rev ' + 'WHERE bucket >= ? AND bucket <= ?', + [lo ~/ 900, (hi - 1) ~/ 900], + ); + return ( + revisionSum: (rows.single['revision_sum'] as num).toInt(), + revisionMax: (rows.single['revision_max'] as num).toInt(), + generation: generation, + ); + } + + /// Session identity/window writes invalidate ceiling reuse, including an + /// identical REPLACE or UPDATE. Derived score writes do not affect it. + static Future sessionCeilingRevision() async { + final rows = await (await instance).query('session_ceiling_rev'); + return rows.isEmpty ? 0 : (rows.single['rev'] as num).toInt(); + } + + @visibleForTesting + static Future repairOpenSchemaForTest(Database db) => + _repairOpenSchema(db); /// SQLite caps host parameters per statement (`SQLITE_MAX_VARIABLE_NUMBER` — /// only 999 on the builds shipped with older Android/iOS). Any `IN (?, ?, …)` @@ -540,7 +593,7 @@ class LocalDb { static Future _open() async { final dir = await getDatabasesPath(); final path = p.join(dir, dbName); - return openDatabase( + final opened = await openDatabase( path, onConfigure: (db) async { // WRITE PERFORMANCE. The default rollback journal (journal_mode=delete) @@ -1249,6 +1302,12 @@ class LocalDb { // A band's own hypnogram (`vendor_staged`). New table only. await _createVendorSleepEpoch(db); } + if (oldV < 59) { + // Additive revision counters only: no decoded scan or backfill. + // Attach after any earlier rung that rebuilds decoded tables. + await _createInputRev(db); + await _createSessionCeilingRev(db); + } }, onOpen: (db) async { await _repairOpenSchema(db); @@ -1264,6 +1323,8 @@ class LocalDb { }, version: schemaVersion, ); + _markStoreReplaced(); + return opened; } static Future _repairOpenSchema(Database db) async { @@ -1366,12 +1427,20 @@ class LocalDb { await _addColumnIfMissing(db, 'device', 'battery_ts', 'INTEGER'); await _createStepCalibration(db); await _ensureLiveStepCalibrationCounterVersion(db); + // A decoded-table rebuild above drops its triggers; attach them last. + await _createInputRev(db); + await _createSessionCeilingRev(db); // Views LAST — they depend on metric_series / day_result / baselines / sessions // / notifications all existing. DROP+CREATE so a shape change takes effect. await _ensureCoachViews(db); await _dropRawStore(db); } + /// One revision counter per absolute 15-minute input bucket, shared by the + /// canonical 1 Hz rows and sparse beats. Triggers cover every write path and + /// roll back with the input transaction. No raw rows are read at creation. + static Future _createInputRev(Database db) => _ldbCreateInputRev(db); + /// Older WHOOP CSV imports filed the export's absolute skin temp (°C) under /// `scalars.skin_temp_z`, which every reader treats as SDs off baseline, so /// 33 °C read as +33 SD in the cross-day temp inputs and the skin temp card. @@ -5567,6 +5636,10 @@ class LocalDb { '''); } + // Schema 59 and every-open repair; no session scan or backfill. + static Future _createSessionCeilingRev(Database db) => + _ldbCreateSessionCeilingRev(db); + static Future _createSyncState(DatabaseExecutor db) async { await db.execute(''' CREATE TABLE IF NOT EXISTS sync_ledger ( @@ -11199,7 +11272,9 @@ class LocalDb { final db = await instance; final rows = await db.rawQuery( "SELECT name FROM sqlite_master WHERE type = 'table' " - "AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata'", + "AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata' " + // Clear revisions after the decoded/session DELETE triggers have finished. + "ORDER BY name IN ('input_rev', 'session_ceiling_rev')", ); var deleted = 0; await db.transaction((txn) async { @@ -11208,6 +11283,7 @@ class LocalDb { if (t is String) deleted += await txn.delete(t); } }); + _markStoreReplaced(); return deleted; } @@ -11588,7 +11664,9 @@ class LocalDb { /// file saved mid-move). Every row is rebuilt through [rawArchiveRow], so a /// frame already here, or in the backup twice, lands once. Paged on rowid, /// like every other table the merge reads. Returns rows offered. - static Future _mergeRawArchive(Database src, Database db) async { + static Future _mergeRawArchive(Database src, Database db, { + required void Function() onPageCommitted, + }) async { var copied = 0; Future pageFrom(String table, String hexSql) async { if (await _objectType(src, table) != 'table') return; @@ -11627,6 +11705,7 @@ class LocalDb { copied++; } await batch.commit(noResult: true); + onPageCommitted(); } } @@ -11661,6 +11740,27 @@ class LocalDb { String path, { List? only, bool tolerant = false, + }) async { + var pageCommitted = false; + try { + return await _mergeFromDbFileBody(path, + only: only, + tolerant: tolerant, + onPageCommitted: () => pageCommitted = true, + ); + } finally { + // A later page, source close or follow-up write can fail after a durable + // replacement. Invalidate on every such exit, but never on a rolled-back + // first page. Keep the flag local so concurrent merges cannot share it. + if (pageCommitted) _markStoreReplaced(); + } + } + + static Future> _mergeFromDbFileBody( + String path, { + List? only, + bool tolerant = false, + required void Function() onPageCommitted, }) async { final db = await instance; final src = await openDatabase(path, readOnly: true); @@ -11979,6 +12079,7 @@ class LocalDb { await txn.execute('DROP TABLE temp._restore_rr_carried'); } }); + onPageCommitted(); lower = upper; } failed = false; @@ -12051,7 +12152,8 @@ class LocalDb { for (final t in (only ?? tables)) { if (t == 'raw_archive') { try { - counts[t] = await _mergeRawArchive(src, db); + counts[t] = await _mergeRawArchive(src, db, + onPageCommitted: onPageCommitted); } catch (_) { if (!tolerant) rethrow; counts[t] = 0; @@ -12562,6 +12664,7 @@ class LocalDb { } await flush(); }); + onPageCommitted(); // Advance past the last row this page actually delivered. Read the // cursor BEFORE dropping the page, and stop on a short page rather // than issuing one more query to discover the end. @@ -14901,13 +15004,38 @@ class LocalDb { ]; } + @visibleForTesting + static bool debugDisableSessionWindowsWorker = false; + @visibleForTesting + static void Function(String)? debugOnSessionWindowsConstruction; + + // Test forwarding seam for the production combined read. + @visibleForTesting + static Future sessionHrReadsForTest(int from, int to, + {int maxHrCeiling = 0, int minHrFloor = 0}) => + sessionHrReads(from, to, + maxHrCeiling: maxHrCeiling, minHrFloor: minHrFloor); + + /// Both HR reads for the workout list. With a sensor enabled, fetch each + /// window once and construct its stats and samples together in a worker. + static Future sessionHrReads(int from, int to, + {int maxHrCeiling = 0, int minHrFloor = 0}) async { + if (sessionSensorSources.isEmpty) { + return (stats: await _sessionHrStatsSql(from, to, maxHrCeiling, minHrFloor), + samples: await _sessionHrSamplesSql(from, to)); + } + return _sessionWindows(from, to, maxHrCeiling, minHrFloor); + } + /// Every session in [fromTs, toTs] (epoch SECONDS) with the worn rows of /// its window a session read keeps ([sessionWindowRows]), ascending. Only /// with a workout sensor flagged on; a session with none is absent, as from /// the band-only joins. - static Future>>> _sessionWindows( + static Future _sessionWindows( int fromTs, int toTs, + int maxHrCeiling, + int minHrFloor, ) async { final db = await instance; final rows = await db.rawQuery( @@ -14919,20 +15047,53 @@ class LocalDb { 'ORDER BY s.id, d.rec_ts ASC', [fromTs, toTs], ); + final sensors = {...sessionSensorSources}; + final wearable = sessionWearableSource; + (SessionHrReads, String) construct() => _prepareSessionWindows( + rows, sensors, wearable, maxHrCeiling, minHrFloor); + final result = debugDisableSessionWindowsWorker + ? construct() + : await runCancellableIsolate(construct, const Duration(seconds: 30), + label: 'session windows'); + debugOnSessionWindowsConstruction?.call(result.$2); + return result.$1; + } + + static (SessionHrReads, String) _prepareSessionWindows( + List> rows, Set sensors, String? wearable, + int maxHrCeiling, int minHrFloor) { final by = >>{}; for (final r in rows) { final id = r['id'] as String?; if (id != null) (by[id] ??= []).add(r); } - return { - for (final MapEntry(key: id, value: rs) in by.entries) - if ([ - for (final r in sessionWindowRows(rs)) - if (((r['hr'] as num?) ?? 0) > 0) r, - ] - case final kept when kept.isNotEmpty) - id: kept, - }; + final stats = >{}; + final samples = >{}; + for (final MapEntry(key: id, value: rs) in by.entries) { + final hr = [ + for (final r in _sessionWindowRowsFor(rs, + sensors: sensors, wearable: wearable)) + if (((r['hr'] as num?) ?? 0) > 0) (r['hr'] as num).toInt(), + ]; + if (hr.isEmpty) continue; + samples[id] = hr; + final low = [ + for (final h in hr) + if (minHrFloor <= 0 || h >= minHrFloor) h, + ]; + final high = [ + for (final h in hr) + if (maxHrCeiling <= 0 || h <= maxHrCeiling) h, + ]; + stats[id] = { + 'n': hr.length, + 'avg_hr': hr.fold(0, (a, b) => a + b) / hr.length, + 'min_hr': low.isEmpty ? 0 : low.reduce((a, b) => a < b ? a : b), + 'max_hr': high.isEmpty ? 0 : high.reduce((a, b) => a > b ? a : b), + }; + } + return ((stats: stats, samples: samples), + Isolate.current.debugName ?? 'unnamed'); } /// Per-session HR aggregates over the 1 Hz substrate for every session in @@ -14952,28 +15113,14 @@ class LocalDb { int minHrFloor = 0, }) async { if (sessionSensorSources.isNotEmpty) { - return { - for (final MapEntry(key: id, value: rs) - in (await _sessionWindows(fromTs, toTs)).entries) - id: () { - final hr = [for (final r in rs) (r['hr'] as num).toInt()]; - final low = [ - for (final h in hr) - if (minHrFloor <= 0 || h >= minHrFloor) h, - ]; - final high = [ - for (final h in hr) - if (maxHrCeiling <= 0 || h <= maxHrCeiling) h, - ]; - return { - 'n': hr.length, - 'avg_hr': hr.fold(0, (a, b) => a + b) / hr.length, - 'min_hr': low.isEmpty ? 0 : low.reduce((a, b) => a < b ? a : b), - 'max_hr': high.isEmpty ? 0 : high.reduce((a, b) => a > b ? a : b), - }; - }(), - }; + return (await sessionHrReads(fromTs, toTs, + maxHrCeiling: maxHrCeiling, minHrFloor: minHrFloor)).stats; } + return _sessionHrStatsSql(fromTs, toTs, maxHrCeiling, minHrFloor); + } + + static Future>> _sessionHrStatsSql( + int fromTs, int toTs, int maxHrCeiling, int minHrFloor) async { final db = await instance; final ceilClause = maxHrCeiling > 0 ? 'AND d.hr <= $maxHrCeiling ' : ''; final floorClause = minHrFloor > 0 ? 'AND d.hr >= $minHrFloor ' : ''; @@ -15015,12 +15162,13 @@ class LocalDb { int toTs, ) async { if (sessionSensorSources.isNotEmpty) { - return { - for (final MapEntry(key: id, value: rs) - in (await _sessionWindows(fromTs, toTs)).entries) - id: [for (final r in rs) (r['hr'] as num).toInt()], - }; + return (await sessionHrReads(fromTs, toTs)).samples; } + return _sessionHrSamplesSql(fromTs, toTs); + } + + static Future>> _sessionHrSamplesSql( + int fromTs, int toTs) async { final db = await instance; final rows = await db.rawQuery( 'SELECT s.id AS id, d.hr AS hr ' diff --git a/lib/data/db_input_revisions.dart b/lib/data/db_input_revisions.dart new file mode 100644 index 000000000..251cebcb3 --- /dev/null +++ b/lib/data/db_input_revisions.dart @@ -0,0 +1,73 @@ +// Revision DDL executes here so schema creation stays additive and cheap. +part of 'db.dart'; + +Future _ldbCreateInputRev(Database db) async { + await db.execute( + 'CREATE TABLE IF NOT EXISTS input_rev (' + 'bucket INTEGER PRIMARY KEY, rev INTEGER NOT NULL)', + ); + if ((await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'decoded_onehz'", + )).isNotEmpty) { + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_ai + AFTER INSERT ON decoded_onehz WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_au + AFTER UPDATE ON decoded_onehz WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_au_old + AFTER UPDATE ON decoded_onehz WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 AND (NEW.rec_ts IS NULL OR NEW.rec_ts <= 0 OR NEW.rec_ts / 900 != OLD.rec_ts / 900) + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_ad + AFTER DELETE ON decoded_onehz WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + } + if ((await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'decoded_rr'", + )).isNotEmpty) { + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_ai + AFTER INSERT ON decoded_rr WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_au + AFTER UPDATE ON decoded_rr WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_au_old + AFTER UPDATE ON decoded_rr WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 AND (NEW.rec_ts IS NULL OR NEW.rec_ts <= 0 OR NEW.rec_ts / 900 != OLD.rec_ts / 900) + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_ad + AFTER DELETE ON decoded_rr WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + } +} diff --git a/lib/data/db_session_ceiling_revisions.dart b/lib/data/db_session_ceiling_revisions.dart new file mode 100644 index 000000000..cb3d07d8c --- /dev/null +++ b/lib/data/db_session_ceiling_revisions.dart @@ -0,0 +1,29 @@ +part of 'db.dart'; + +Future _ldbCreateSessionCeilingRev(Database db) async { + await db.execute('CREATE TABLE IF NOT EXISTS session_ceiling_rev (' + 'id INTEGER PRIMARY KEY CHECK (id = 1), rev INTEGER NOT NULL)'); + // Sparse legacy schemas get their sessions table during onOpen repair. + final present = await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'sessions'", + ); + if (present.isEmpty) return; + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_ai + AFTER INSERT ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_au + AFTER UPDATE OF id, type, status, start_ts, end_ts ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_ad + AFTER DELETE ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); +} diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index ceb77017d..caeb96ee1 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -2405,7 +2405,7 @@ class LocalRepositoryImpl extends LocalRepository { // "no data" (avg_hr == 0) even when the window is full of worn HR. try { final age = _profileAge(); - final stats = await LocalDb.sessionHrStats( + final (:stats, samples: rawBySession) = await LocalDb.sessionHrReads( fromTs, nowSec, maxHrCeiling: hrCeilingForAge(age), @@ -2414,10 +2414,6 @@ class LocalRepositoryImpl extends LocalRepository { // Spike-suppressed max/min per session (issue #127): smooth the raw 1 Hz // over one batched join so the list agrees with getWorkout's on-read // recompute. - final rawBySession = await LocalDb.sessionHrSamplesBySession( - fromTs, - nowSec, - ); for (final w in workouts) { final s = stats[w['id']]; final raw = rawBySession[w['id']]; diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index a97a59523..99721b721 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -82,6 +82,8 @@ const Map kSliceTables = { 'baselines': (SliceKeep.all, null, 'personal baselines'), 'compute_freshness': (SliceKeep.all, null, 'compute bookkeeping'), 'derive_fingerprint': (SliceKeep.all, null, 'which days changed since they were last scored'), + 'input_rev': (SliceKeep.all, null, 'per-bucket input revisions'), + 'session_ceiling_rev': (SliceKeep.all, null, 'session ceiling revisions'), 'sleep_nap': (SliceKeep.all, null, 'naps'), 'sleep_override': (SliceKeep.all, null, 'sleep times the user edited'), // Workouts and activity: one row each. diff --git a/lib/data/restore_classification.dart b/lib/data/restore_classification.dart new file mode 100644 index 000000000..8551fb0f1 --- /dev/null +++ b/lib/data/restore_classification.dart @@ -0,0 +1,79 @@ +// Restore policy for each schema table. Revision counters are local, +// rebuildable bookkeeping: imported inputs fire the destination triggers. +enum RestoreClass { merge, localOnly, rebuildable } + +const kRestoreClassification = { + 'decoded_onehz': RestoreClass.merge, + 'decoded_rr': RestoreClass.merge, + 'samples': RestoreClass.merge, + 'raw_archive': RestoreClass.merge, + 'raw_archive_frames': RestoreClass.localOnly, + 'raw_archive_legacy': RestoreClass.localOnly, + 'events': RestoreClass.merge, + 'band_events': RestoreClass.merge, + 'band_battery': RestoreClass.merge, + 'band_backlog': RestoreClass.localOnly, + 'sync_ledger': RestoreClass.localOnly, + 'sync_quarantine': RestoreClass.localOnly, + 'external_hr': RestoreClass.merge, + 'imported_measurement': RestoreClass.merge, + 'observation': RestoreClass.merge, + 'vendor_sleep_epoch': RestoreClass.merge, + 'live_coverage': RestoreClass.merge, + 'device_coverage': RestoreClass.merge, + 'workout_route': RestoreClass.merge, + 'live_workout_tally': RestoreClass.localOnly, + 'compute_jobs': RestoreClass.localOnly, + 'ecg_reading': RestoreClass.merge, + 'ecg_reading_packet': RestoreClass.merge, + 'ecg_raw_packet': RestoreClass.merge, + 'sleep_session_candidates': RestoreClass.localOnly, + 'wake_day_features': RestoreClass.localOnly, + 'substrate_archive': RestoreClass.merge, + 'day_result': RestoreClass.merge, + 'derived_day': RestoreClass.localOnly, + 'metric_series': RestoreClass.merge, + 'metric_series_version': RestoreClass.merge, + 'metric_method': RestoreClass.merge, + 'baselines': RestoreClass.merge, + 'compute_freshness': RestoreClass.localOnly, + 'derive_fingerprint': RestoreClass.rebuildable, + 'input_rev': RestoreClass.rebuildable, + 'session_ceiling_rev': RestoreClass.rebuildable, + 'sleep_nap': RestoreClass.merge, + 'sleep_override': RestoreClass.merge, + 'sessions': RestoreClass.merge, + 'session_sensor': RestoreClass.merge, + 'session_score_mask': RestoreClass.localOnly, + 'workout_split': RestoreClass.merge, + 'strength_set': RestoreClass.merge, + 'workout_suggestions': RestoreClass.merge, + 'imported_workout': RestoreClass.merge, + 'activity_suggestions': RestoreClass.merge, + 'activity_review_days': RestoreClass.merge, + 'activity_review_meta': RestoreClass.merge, + 'breathing_session': RestoreClass.merge, + 'exercise_def': RestoreClass.merge, + 'journal': RestoreClass.merge, + 'journal_metric': RestoreClass.merge, + 'journal_field_def': RestoreClass.merge, + 'cycle_log': RestoreClass.merge, + 'cycle_symptom': RestoreClass.merge, + 'lab_result': RestoreClass.merge, + 'lab_marker_def': RestoreClass.merge, + 'med_def': RestoreClass.merge, + 'med_dose': RestoreClass.merge, + 'food_def': RestoreClass.merge, + 'food_entry': RestoreClass.merge, + 'device': RestoreClass.merge, + 'signal_priority': RestoreClass.merge, + 'step_calibration': RestoreClass.merge, + 'step_calibration_day': RestoreClass.merge, + 'live_step_calibration': RestoreClass.merge, + 'live_step_calibration_run': RestoreClass.merge, + 'sync_cursor': RestoreClass.merge, + 'alarm_schedule': RestoreClass.localOnly, + 'notifications': RestoreClass.merge, + 'notif_fired': RestoreClass.localOnly, + 'notif_slots': RestoreClass.localOnly, +}; diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index b581d5318..83c947fef 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -2030,6 +2030,7 @@ class AppState extends ChangeNotifier { } _sync.cancelQuietTimer(); _disposed = true; + _derive.dispose(); _ecg?.dispose(); _ecgTransport?.dispose(); // EVERY timer this object owns, not just three of them. @@ -3486,7 +3487,13 @@ class AppState extends ChangeNotifier { /// drops (band out of range / app jettisoned) — see [_onEngineState] / [_armRecovery]. /// /// On Android the Edge Tracking foreground service keeps the process + connection alive. - Future pauseForBackground() => _sync.pauseForBackground(); + Future pauseForBackground() { + _derive.trimForBackground(); + return _sync.pauseForBackground(); + } + + void resumeDerivationRetention() => _derive.resumeFromBackground(); + void handleMemoryPressure() => _derive.handleMemoryPressure(); /// A feature session (workout, breathing, ECG capture) is running — the /// "nothing else in flight" bar the one-off VACUUM waits for. ECG matters diff --git a/pubspec.lock b/pubspec.lock index eee7705f5..45d6b23b5 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -18,7 +18,7 @@ packages: source: hosted version: "1.3.77" analyzer: - dependency: transitive + dependency: "direct dev" description: name: analyzer sha256: de7148ed2fcec579b19f122c1800933dfa028f6d9fd38a152b04b1516cec120b diff --git a/pubspec.yaml b/pubspec.yaml index c20440bfd..209434e25 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -670,6 +670,8 @@ dev_dependencies: flutter_test: sdk: flutter flutter_lints: ^6.0.0 + # DB part guards inspect declarations and SQL arguments through the Dart AST. + analyzer: ^10.0.1 test: ^1.25.8 # no_runtime_font_fetch_test parses pubspec.yaml to assert every family the # type scale uses is bundled rather than fetched at runtime. diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart index 89194293e..9e49ccf66 100644 --- a/test/derive_unchanged_skip_test.dart +++ b/test/derive_unchanged_skip_test.dart @@ -49,6 +49,15 @@ Future _freshDb(String name) async { } } +// The fixture starts at 18:00 on the previous date. That short first day +// finalizes on its first derive, while its daytime summaries read the NEXT +// day's sleep window. Bank that real window before either parity run so the +// foreground worker order cannot decide whether the first day's evening is +// excluded as tonight's sleep. Every stored output is still compared below. +Future _primeFirstNight() async { + expect(await DerivationEngine().runDays(_profile, {'2026-07-13'}), 1); +} + Future<({int derived, int unchanged})> _pass({Profile profile = _profile}) async { final e = DerivationEngine(); final n = await e.run(profile, heavy: true); @@ -160,6 +169,7 @@ void main() { () async { const passes = 5; await _freshDb('unchanged_skip_on.db'); + await _primeFirstNight(); final log = <({int derived, int unchanged})>[]; for (var i = 0; i < passes; i++) { log.add(await _pass()); @@ -167,6 +177,7 @@ void main() { final withSkip = await _snapshot(); await _freshDb('unchanged_skip_off.db'); + await _primeFirstNight(); DerivationEngine.debugDeriveEveryDay = true; for (var i = 0; i < passes; i++) { final r = await _pass(); diff --git a/test/ed2a_e12_staging_cache_test.dart b/test/ed2a_e12_staging_cache_test.dart new file mode 100644 index 000000000..692f34b8c --- /dev/null +++ b/test/ed2a_e12_staging_cache_test.dart @@ -0,0 +1,275 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/sleep_profile_policy.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +Future _profile({String field = 'hr_floor_p5', int delta = 1}) async { + final p = {...const ana.SleepUserProfile(nights: 14, hrFloorP5: 48, + hrFloorP25: 51, hrSleepMedian: 55, hrArousal: 70, rmssdMed: 40, + rmssdMad: 5, enmoStillCut: 0.02, enmoMoveCut: 0.1, + lfhfMed: 1, rkMed: 2).toJson(), 'folded_days': [edDay]}; + p[field] = (p[field] as num) + delta; + await LocalDb.putBaseline('sleep_user_profile', jsonEncode(p)); +} + +Future _history({int shift = 0}) async { + final db = await LocalDb.instance; + for (var i = 1; i <= 14; i++) { + final d = DateTime(2020, 1, 2 - i); + final start = DateTime(d.year, d.month, d.day, 1).millisecondsSinceEpoch; + await db.insert('day_result', {'day_id': dayLabelOf(d), + 'algo_version': kAlgoVersion, 'payload_json': '{}', 'computed_at': 1, + 'window_json': jsonEncode({'onset_ms': start + shift * 1000, + 'offset_ms': start + (6 * 3600 + shift) * 1000})}, + conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +Future _vendor({String stage = 'light', String id = ''}) async { + final db = await LocalDb.instance; + for (final (offset, duration, label) in [(0, 7200, stage), + (7200, 3600, 'deep'), (10800, 3600, 'rem')]) { + await db.insert('vendor_sleep_epoch', { + 'device_id': id, 'night_onset_ts': edStart, 'start_ts': edStart + offset, + 'end_ts': edStart + offset + duration, 'stage': label, + 'source': 'gen4', 'decoded_at': 1, + }, conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +void main() { + final originalZone = Platform.environment['TZ'] ?? 'UTC'; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e12.db'); + await edRows(edStart, edStart + 1200, hr: 60, az: 1); + }); + tearDown(() async { edSetZone(originalZone); await edDrop(); }); + + test('unchanged staging input is computed once, including absent sleep', () async { + final e = DerivationEngine(); + final first = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, 1); + final again = await e.debugSleepCandidate(edDay); + expect(jsonEncode(again.toJson()), jsonEncode(first.toJson())); + expect(e.debugStagingComputations, 1, + reason: 'the second call must reuse the worker result, even absence'); + }); + + test('the staging worker closure does not drag the engine into the isolate', () async { + // AppState hands its engine a callback over its own fields (BLE phase + // futures among them). If the worker closure captured the engine, that + // callback would ride into Isolate.run and the send would be refused. + final unsendable = Completer(); + final e = DerivationEngine(offloadActive: () => unsendable.isCompleted); + final c = await e.debugSleepCandidate(edDay); + expect(c.toJson(), isNotNull); + expect(e.debugStagingComputations, 1); + }); + + final changes = Function(DerivationEngine)>{ + 'identical input replacement': (_) => edRows(edStart, edStart + 1, hr: 60, az: 1), + 'late RR revision': (_) async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': edStart * 1000, 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, 'beat_index': 0, 'rr_ms': 900}); + }, + 'search lookback revision on previous local day': (_) => + edRows(DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000, + DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000 + 1), + 'store generation with unchanged revisions': (_) async { + final before = await LocalDb.inputDayFingerprint(edDay); + await LocalDb.close(); await LocalDb.instance; + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.generation, greaterThan(before.generation)); + }, + 'algorithm': (e) async { e.debugStagedSleepCacheRekey(kAlgoVersion - 1); }, + 'HR ownership': (_) => edOwnership(signals: ['hr1Hz']), + 'accel ownership': (_) => edOwnership(signals: ['accel1Hz']), + 'PPG ownership': (_) => edOwnership(signals: ['ppgRedIr']), + 'skin-temperature ownership': (_) => edOwnership(signals: ['skinTempRaw']), + 'RR ownership': (_) async { + await (await LocalDb.instance).insert('device_coverage', { + 'device_id': 'rr-owner', 'signal': 'rrIntervals', 'start_ts': edStart, + 'end_ts': edStart + 600}); + await (await LocalDb.instance).insert('signal_priority', { + 'device_id': 'rr-owner', 'signal': 'rrIntervals', 'rank': 0, 'user_set': 1}); + }, + 'priority': (_) async { + await (await LocalDb.instance).insert('signal_priority', {'signal': 'hr1Hz', + 'device_id': '', 'rank': 4, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'family': (_) async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}); }, + 'history at 14 distinct nights': (_) => _history(shift: 60), + 'manual override': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'manual'), + 'confirmed wake override': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'confirmed'), + 'override onset edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart + 30, offsetTs: edStart + 900, source: 'manual'), + 'override offset edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 960, source: 'manual'), + 'override source edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'rejected'), + 'override removal': (_) => LocalDb.deleteSleepOverride(edDay), + 'vendor source': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'source': 'miband234'}); }, + 'vendor decode revision': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'decoded_at': 2}); }, + 'vendor window': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'end_ts': edStart + 14460}, + where: 'start_ts = ?', whereArgs: [edStart + 10800]); }, + 'vendor stages': (_) => _vendor(stage: 'deep'), + 'vendor device': (_) => _vendor(id: 'vendor-other'), + 'vendor flag': (_) => LocalDb.setCursor('wearable_enabled:garmin', '1'), + 'active wearable': (_) => LocalDb.setCursor('active_wearable', 'garmin-y'), + 'zone': (_) async { edSetZone('Pacific/Auckland'); }, + // Denver and Phoenix agree in winter, but disagree in the summer search + // window. The separate historical-zone case below pins that distinction. + for (final field in ['nights', 'hr_floor_p5', 'hr_floor_p25', + 'hr_sleep_median', 'hr_arousal', 'rmssd_med', 'rmssd_mad', + 'enmo_still_cut', 'enmo_move_cut', 'lfhf_med', 'rk_med', 'updated_at_ms']) + 'profile $field': (_) => _profile(field: field), + }; + for (final entry in changes.entries) { + test('${entry.key}: miss, exact output, then hit on the new key', () async { + if (entry.key == 'override removal' || entry.key.startsWith('override ') && entry.key.endsWith(' edit')) { + await LocalDb.putSleepOverride(dayId: edDay, onsetTs: edStart, + offsetTs: edStart + 900, source: 'manual'); + } + if (entry.key == 'priority') await edPriorityOrder(['', 'other']); + if (entry.key.startsWith('profile ')) await _profile(delta: 0); + if (entry.key.startsWith('history ')) await _history(); + if (entry.key.startsWith('vendor ') || entry.key == 'active wearable') { + await _vendor(); + await LocalDb.upsertDevice(id: 'garmin-x', adapterId: 'garmin', label: 'watch'); + await LocalDb.setCursor('active_wearable', 'garmin-x'); + await _vendor(id: 'garmin-x'); + } + if (entry.key == 'vendor flag' || entry.key == 'active wearable') { + await (await LocalDb.instance).delete('decoded_onehz'); + await (await LocalDb.instance).delete('vendor_sleep_epoch', + where: 'device_id = ?', whereArgs: ['']); + } + if (entry.key == 'vendor device') { + await LocalDb.upsertDevice(id: 'vendor-other', adapterId: 'garmin', label: 'new vendor'); + await LocalDb.setCursor('active_wearable', 'vendor-other'); + await LocalDb.setCursor('wearable_enabled:garmin', '1'); + await (await LocalDb.instance).delete('decoded_onehz'); + } + if (entry.key == 'active wearable') { + await LocalDb.upsertDevice(id: 'garmin-y', adapterId: 'garmin', label: 'watch2'); + await _vendor(id: 'garmin-y'); + await LocalDb.setCursor('wearable_enabled:garmin', '1'); + } + final e = DerivationEngine(); + await e.debugSleepCandidate(edDay); + final count = e.debugStagingComputations; + await entry.value(e); + final changed = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count + 1, + reason: 'changed ${entry.key} must invalidate the staged result'); + final cold = DerivationEngine()..debugDisableStagedSleepCache = true; + expect(jsonEncode(changed.toJson()), + jsonEncode((await cold.debugSleepCandidate(edDay)).toJson())); + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count + 1, + reason: 'the replacement entry must now hit'); + }); + } + + test('same present-day zone name/offset cannot hide historical night offsets', () async { + edSetZone('America/Chihuahua'); + final nowOffset = DateTime.now().timeZoneOffset; + final nowName = DateTime.now().timeZoneName; + final historical = DateTime.fromMillisecondsSinceEpoch(edStart * 1000).timeZoneOffset; + final e = DerivationEngine(); + await e.debugSleepCandidate(edDay); + final before = e.debugStagingComputations; + edSetZone('America/Guatemala'); + expect(DateTime.now().timeZoneOffset, nowOffset); + expect(DateTime.now().timeZoneName, nowName); + expect(DateTime.fromMillisecondsSinceEpoch(edStart * 1000).timeZoneOffset, + isNot(historical), reason: 'fixture zones agree now but differed in January 2020'); + final changed = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, before + 1); + final cold = await DerivationEngine().debugSleepCandidate(edDay); + expect(jsonEncode(changed.toJson()), jsonEncode(cold.toJson())); + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, before + 1, reason: 'new historical-offset key must hit'); + }); + + test('selected derive followed by baseline rescan reuses actual staging work', () async { + final e = DerivationEngine(); + expect(await e.runDays(edProfile, {edDay}), 1); + final before = e.debugStagingComputations; + expect(before, greaterThan(0)); + await LocalDb.setCursor('baseline_sig', 'force-ed2a-rescan'); + expect(await e.rescanRecent(edProfile), greaterThan(0)); + expect(e.debugStagingComputations, before, + reason: 'baseline recompute must not rerun the unchanged staging worker'); + }); + + test('a nonfolding entry cannot satisfy a later mayFold request', () async { + final e = DerivationEngine(); + final modes = []; + e.debugOnStagingWalk = (fold, _, _) => modes.add(fold); + await e.debugSleepCandidate(edDay); + expect(modes, [false]); + final edge = DateTime(2020, 1, 3, 12).millisecondsSinceEpoch ~/ 1000; + await edRows(edge, edge + 1); + final changed = await e.debugSleepCandidate(edDay); + expect(modes, [false, true], reason: 'a hit without an observation cannot suppress a fold'); + final cold = await DerivationEngine().debugSleepCandidate(edDay); + expect(jsonEncode(changed.toJson()), jsonEncode(cold.toJson())); + final count = e.debugStagingComputations; + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count, reason: 'fold-capable result is reusable even when observation is absent'); + }); + + test('bounded staging cache retains only the most recent two days', () async { + final e = DerivationEngine()..debugStagedSleepCacheMaxDays = 2; + for (final day in ['2020-01-01', edDay, '2020-01-03']) { + await e.debugSleepCandidate(day); + expect(e.debugStagedSleepCacheSize, lessThanOrEqualTo(2)); + } + expect(e.debugStagedSleepCacheSize, 2, + reason: 'three distinct staging keys must populate the bounded cache'); + final count = e.debugStagingComputations; + await e.debugSleepCandidate('2020-01-03'); + expect(e.debugStagingComputations, count); + await e.debugSleepCandidate('2020-01-01'); + expect(e.debugStagingComputations, count + 1, reason: 'oldest day was evicted'); + }); + + test('existing fixture: cached and uncached derive JSON stays exact after invalidation', () async { + await edExistingFixture(); + final cached = DerivationEngine(); + final uncached = DerivationEngine()..debugDisableStagedSleepCache = true; + await edPreparedOutput(cached); + expect(await edPreparedOutput(cached), await edPreparedOutput(uncached)); + await edRows(edStart, edStart + 1, hr: 87); + expect(await edPreparedOutput(cached), await edPreparedOutput(uncached)); + }); + + test('profile fold bookkeeping remains idempotent (passing guard)', () { + final p = SleepProfilePolicy.withFoldedDays( + const ana.SleepUserProfile(nights: 3).toJson(), {}, edDay); + expect(SleepProfilePolicy.shouldFold(alreadyFolded: {edDay}, + dayId: edDay, hasOverride: false), isFalse); + expect(SleepProfilePolicy.foldedDays(jsonEncode(p)), {edDay}); + }); +} diff --git a/test/ed2a_e13_substrate_retention_test.dart b/test/ed2a_e13_substrate_retention_test.dart new file mode 100644 index 000000000..8a72ee80a --- /dev/null +++ b/test/ed2a_e13_substrate_retention_test.dart @@ -0,0 +1,247 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/adapters/signals.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/coverage_resolver.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +int _reads(DerivationEngine e) => e.snapshot()['raw_rows'] as int; +final _lo = (edStart ~/ 900) * 900; +final _hi = _lo + 5 * 900 - 1; +Future _load(DerivationEngine e, {int? lo, int? hi, + Map> ownership = const {}}) => + e.debugLoadSubstrate(lo ?? _lo, hi ?? _hi, day: edDay, ownership: ownership); + +void main() { + final originalZone = Platform.environment["TZ"] ?? "UTC"; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e13.db'); + await edRows(_lo, _lo + 4 * 900 + 200); + }); + tearDown(() async { edSetZone(originalZone); await edDrop(); }); + + test('unchanged load reuses retained arrays without rereading rows', () async { + final e = DerivationEngine(); + final first = await _load(e); + final reads = _reads(e); + expect(reads, greaterThan(0)); + final next = await _load(e); + expect(jsonEncode(next.toJson()), jsonEncode(first.toJson())); + expect(_reads(e), reads, reason: 'unchanged load must not decode rows twice'); + }); + + test('append rereads at most the open revision bucket and new tail', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + await edRows(_lo + 4 * 900 + 200, _hi + 1); + final warm = await _load(e); + final cold = await _load(DerivationEngine()..debugDisableSubstrateRetention = true); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e) - reads, lessThanOrEqualTo(900), + reason: 'four closed buckets must survive the tail append'); + }); + + final changes = Function()>{ + 'identical old-bucket revision': () => edRows(_lo + 30, _lo + 31), + 'old-bucket replacement': () => edRows(_lo + 30, _lo + 31, hr: 83), + 'delete seconds and beats': () async { + final db = await LocalDb.instance; + for (final table in ['decoded_onehz', 'decoded_rr']) { + await db.delete(table, where: 'rec_ts >= ? AND rec_ts < ?', + whereArgs: [_lo + 10, _lo + 20]); + } + }, + 'late beat in retained bucket': () async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': (_lo + 30) * 1000, 'rec_ts': _lo + 30, + 'rr_ts_ms': (_lo + 30) * 1000, 'beat_index': 0, 'rr_ms': 850}); + }, + 'zone': () async { edSetZone('Pacific/Auckland'); }, + 'generation': () async { await LocalDb.close(); await LocalDb.instance; }, + 'family': () async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}, + where: 'rec_ts = ?', whereArgs: [_lo + 30]); }, + 'device provenance': () => edRows(_lo + 30, _lo + 31, id: 'second'), + 'skin unit bucket': () async { await (await LocalDb.instance).update( + 'decoded_onehz', {'skin_temp_c': 33.1}, + where: 'rec_ts = ?', whereArgs: [_lo + 30]); }, + }; + for (final change in changes.entries) { + test('${change.key}: invalidate, preserve every byte, then retain again', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + await change.value(); + final warm = await _load(e); + expect(_reads(e), greaterThan(reads), reason: 'closed prefix must be reloaded'); + final cold = await _load(DerivationEngine()..debugDisableSubstrateRetention = true); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + final afterReload = _reads(e); + await _load(e); + expect(_reads(e), afterReload, reason: 'newly retained key must hit'); + }); + } + + test('algorithm rekey rejects retained work and next load hits', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + e.debugRetainedSubstrateRekey(kAlgoVersion - 1); + final warm = await _load(e); + expect(_reads(e), greaterThan(reads)); + expect(jsonEncode(warm.toJson()), jsonEncode((await _load(DerivationEngine())).toJson())); + final after = _reads(e); + await _load(e); + expect(_reads(e), after, reason: 'current-version retention should now hit'); + }); + + for (final signal in [InputSignal.hr1Hz, InputSignal.rrIntervals]) { + test('$signal ownership miss even if revisions are unchanged', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + final ownership = >{signal: [ + (start: _lo, end: _hi, deviceId: 'other')]}; + final warm = await _load(e, ownership: ownership); + final cold = await _load(DerivationEngine(), ownership: ownership); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e), greaterThan(reads)); + final again = _reads(e); + await _load(e, ownership: ownership); + expect(_reads(e), again, reason: 'ownership belongs to the retained key'); + }); + } + + for (final window in ['from', 'to']) { + test('$window window change reloads then hits', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + final lo = window == 'from' ? _lo + 1 : _lo; + final hi = window == 'to' ? _hi - 1 : _hi; + final warm = await _load(e, lo: lo, hi: hi); + final cold = await _load(DerivationEngine(), lo: lo, hi: hi); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e), greaterThan(reads)); + final after = _reads(e); + await _load(e, lo: lo, hi: hi); + expect(_reads(e), after); + }); + } + + test('raw beat times, per-second beat offsets and bucket metadata survive retention', () async { + final db = await LocalDb.instance; + final rawTimes = []; + final intervals = []; + final starts = []; + final b = db.batch(); + b.delete('decoded_onehz', where: 'rec_ts = ?', whereArgs: [_lo + 1]); + // Include beat-only seconds, missing beats, multiple beats and a backward + // raw timestamp at the cut. Monotonizing a retained prefix twice loses it. + for (var t = _lo; t < _lo + 4 * 900 + 200; t++) { + starts.add(rawTimes.length); + for (var i = 0; i < (t % 3 == 0 ? 0 : 1 + t % 2); i++) { + final raw = t * 1000 - (i == 0 && t % 11 == 0 ? 1500 : i * 400); + rawTimes.add(raw.toDouble()); + intervals.add((700 + t % 90).toDouble()); + b.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'beat_index': i, 'rr_ts_ms': t * 1000, + 'beat_ts_ms': raw, 'rr_ms': 700 + t % 90}); + } + } + await b.commit(noResult: true); + final e = DerivationEngine(); + final before = await _load(e); + var runningMax = double.negativeInfinity; + final axis = []; + for (final raw in rawTimes) { + if (raw > runningMax) runningMax = raw; + axis.add(runningMax); + } + expect(before.rrTsMs, axis, reason: 'hold backward placements; never sort or shift beats'); + expect(before.rrMs, intervals, reason: 'interval order is the recorded rhythm'); + final extras = e.debugRetainedSubstrate(edDay); + expect(extras['rr_ts_raw'], rawTimes, + reason: 'the raw axis must be retained separately from its running max'); + expect(extras['rr_start'], starts); + expect((extras['meta'] as List).map((m) => (m as List).first), + [for (var i = 0; i < 5; i++) _lo ~/ 900 + i]); + expect(before.rrTsMs, isNot(rawTimes)); + await edRows(_lo + 4 * 900 + 200, _hi + 1); + final warm = await _load(e); + final cold = await _load(DerivationEngine()); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + }); + + test('retention respects its day bound and evicts the oldest day', () async { + final e = DerivationEngine()..debugRetainedSubstrateMaxDays = 2; + for (var i = 0; i < 3; i++) { + final lo = DateTime(2020, 1, 2 + i, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(lo, lo + 1200); + await e.debugLoadSubstrate(lo, lo + 1200, day: edLabel(lo)); + expect(e.debugRetainedSubstrateDays.length, lessThanOrEqualTo(2)); + } + expect(e.debugRetainedSubstrateDays, {'2020-01-03', '2020-01-04'}, + reason: 'a zero-sized stub must not satisfy the bound'); + }); + + for (final boundary in ['background', 'memory pressure', 'dispose']) { + test('$boundary clears populated retention and next load stays exact', () async { + final e = DerivationEngine(); + final before = await _load(e); + expect(e.debugRetainedSubstrateDays, contains(edDay), reason: 'populated before boundary'); + switch (boundary) { + case 'background': e.trimForBackground(); + case 'memory pressure': e.handleMemoryPressure(); + case 'dispose': e.dispose(); + } + expect(e.debugRetainedSubstrateDays, isEmpty); + final receiver = boundary == 'dispose' ? DerivationEngine() : e; + expect(jsonEncode((await _load(receiver)).toJson()), jsonEncode(before.toJson())); + }); + } + + test('real day finalization drops retained day and beat data', () async { + final e = DerivationEngine(); + await e.runDays(edProfile, {edDay}); + expect(e.debugRetainedSubstrateDays, contains(edDay)); + final later = DateTime(2020, 1, 6, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1200); + await e.runDays(edProfile, {edDay}); + expect((await LocalDb.dayResult(edDay))!['finalized'], 1); + expect(e.debugRetainedSubstrateDays, isNot(contains(edDay))); + expect(e.debugRetainedSubstrate(edDay), isEmpty); + }); + + test('a pass with no preparable day releases previously retained rows', () async { + final e = DerivationEngine(); + await _load(e); + expect(e.debugRetainedSubstrateDays, contains(edDay)); + final db = await LocalDb.instance; + await db.delete('decoded_onehz'); + await db.delete('decoded_rr'); + expect(await e.runDays(edProfile, {edDay}), 0); + expect(e.debugRetainedSubstrateDays, isEmpty); + expect(e.debugRetainedSubstrate(edDay), isEmpty); + }); + + test('existing fixture: retained and cold derive output exact before and after revision', () async { + await edExistingFixture(); + final warm = DerivationEngine(); + final cold = DerivationEngine()..debugDisableSubstrateRetention = true; + await edPreparedOutput(warm); + expect(await edPreparedOutput(warm), await edPreparedOutput(cold)); + await edRows(edStart, edStart + 1, hr: 89); + expect(await edPreparedOutput(warm), await edPreparedOutput(cold)); + }); +} diff --git a/test/ed2a_e15_restricted_target_walk_test.dart b/test/ed2a_e15_restricted_target_walk_test.dart new file mode 100644 index 000000000..331d4a43a --- /dev/null +++ b/test/ed2a_e15_restricted_target_walk_test.dart @@ -0,0 +1,166 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +List<({int startSec, int endSec, String dayKey})> _history(int distinct, + {bool duplicates = false}) => [ + for (var i = 0; i < (duplicates ? 30 : distinct); i++) + (startSec: DateTime(2019, 12, 1 + i % distinct, 1).millisecondsSinceEpoch ~/ 1000, + endSec: DateTime(2019, 12, 1 + i % distinct, 7).millisecondsSinceEpoch ~/ 1000, + dayKey: dayLabelOf(DateTime(2019, 12, 1 + i % distinct))), +]; + +PreparedDerivationPayload _walk(Substrate sub, {required bool skip, + List<({int startSec, int endSec, String dayKey})> history = const [], + SleepWindowOverride? override}) { + debugSegmentedDayCount = 0; + return prepareDerivationPayload(sub, targetDay: edDay, + priorSleep: history, override: override, skipUnusedDays: skip); +} + +void main() { + setUpAll(edInit); + setUp(() => edFresh("ed2a_e15.db")); + tearDown(() async { + debugStageEveryWalkedDay = false; + debugSegmentedDayCount = 0; + ana.cardioRecordObservations = false; + ana.cardioUserProfile = null; + ana.resetCardioObservations(); + await edDrop(); + }); + + // Two local days, with enough seconds for the segmentation body on each. + final sub = edSmallSub(start: DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000, + count: 10 * 3600); + for (final h in [0, 1, 12]) { + test('$h stored nights cannot reach 14: skip earlier day, equal target JSON', () { + final history = _history(h); + final full = _walk(sub, skip: false, history: history); + expect(debugSegmentedDayCount, 2); + final fast = _walk(sub, skip: true, history: history); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), + jsonEncode(edBundle(full.days.single))); + expect(debugSegmentedDayCount, 1, + reason: '$h distinct stored nights + one earlier day is below 14'); + }); + } + + test('30 rows with only 2 distinct night keys still permit the skip', () { + final history = _history(2, duplicates: true); + final full = _walk(sub, skip: false, history: history); + final fast = _walk(sub, skip: true, history: history); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(debugSegmentedDayCount, 1, reason: 'count distinct nights, not rows'); + }); + + for (final h in [13, 14, 15]) { + test('$h stored nights may reach 14: preserve full earlier walk (guard)', () { + final full = _walk(sub, skip: false, history: _history(h)); + final fast = _walk(sub, skip: true, history: _history(h)); + expect(debugSegmentedDayCount, 2, + reason: '13 prior nights plus one walked night can reach the prior threshold'); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), jsonEncode(edBundle(full.days.single))); + }); + } + + test('override on an earlier day preserves its history contribution (guard)', () { + final override = SleepWindowOverride(dayId: '2020-01-01', + onsetSec: sub.tsSec.first, offsetSec: sub.tsSec.first + 3600, source: 'manual'); + final full = _walk(sub, skip: false, override: override); + final fast = _walk(sub, skip: true, override: override); + expect(debugSegmentedDayCount, 2); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + }); + + test('full-walk control preserves upstream observations (guard)', () { + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final full = _walk(sub, skip: false); + final fullEpochs = [for (final o in ana.takeCardioObservations()) o.epochs]; + ana.resetCardioObservations(); + debugStageEveryWalkedDay = true; + final forced = _walk(sub, skip: true); + expect(debugSegmentedDayCount, 2); + expect(jsonEncode(forced.toJson()), jsonEncode(full.toJson())); + expect([for (final o in ana.takeCardioObservations()) o.epochs], fullEpochs); + }); + + test('production nonfolding worker actually requests the restricted skip', () async { + await edFresh('ed2a_e15_nonfold.db'); + await edRows(sub.tsSec.first, sub.tsSec.last + 1, hr: 55, az: 1); + final walked = <(bool, int)>[]; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, count, _) => walked.add((fold, count)); + final got = await e.debugSleepCandidate(edDay); + final full = DerivationEngine()..debugDisableTargetDaySkip = true; + expect(jsonEncode(got.toJson()), + jsonEncode((await full.debugSleepCandidate(edDay)).toJson())); + expect(walked, [(false, 1)], + reason: 'nonfolding production staging must skip the irrelevant earlier day'); + }); + + test('mayFold always walks earlier days and folds once (guard; D36 excluded)', () async { + await edFresh('ed2a_e15_fold.db'); + await edRows(sub.tsSec.first, sub.tsSec.last + 1, hr: 55, az: 1); + final edge = DateTime(2020, 1, 3, 12).millisecondsSinceEpoch ~/ 1000; + await edRows(edge, edge + 1); + final search = await DerivationEngine().debugLoadSubstrate( + sub.tsSec.first, sub.tsSec.last, day: edDay); + ana.cardioUserProfile = null; + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final full = prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: false); + final observations = ana.takeCardioObservations() + ..sort((a, b) => b.epochs.compareTo(a.epochs)); + expect(observations, isNotEmpty); + final o = observations.first; + final fullObservation = jsonEncode({'epochs': o.epochs, + 'hr_floor_p5': o.hrFloorP5, 'hr_floor_p25': o.hrFloorP25, + 'hr_sleep_median': o.hrSleepMedian, 'hr_arousal': o.hrArousal, + 'rmssd_med': o.rmssdMed, 'rmssd_mad': o.rmssdMad, + 'enmo_still_cut': o.enmoStillCut, 'enmo_move_cut': o.enmoMoveCut, + 'lfhf_med': o.lfhfMed, 'rk_med': o.rkMed}); + final walked = <(bool, int, String?)>[]; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, count, obs) => walked.add((fold, count, obs)); + final candidate = await e.debugSleepCandidate(edDay); + expect(jsonEncode(candidate.toJson()), jsonEncode(full.toJson())); + expect(walked.single.$3, fullObservation, reason: "preserve the full walk's actual fold input"); + expect(walked.single.$1, isTrue, reason: 'fixture reaches a folding pass'); + expect(walked.single.$2, 2, + reason: 'the longest observation can still belong to the earlier day upstream'); + expect(walked.single.$3, isNotNull, reason: 'fixture must produce a real observation'); + final profile = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + final p = jsonDecode(profile as String) as Map; + expect(p['nights'], 1); + expect(p['folded_days'], contains(edDay)); + await e.debugSleepCandidate(edDay); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], profile, + reason: 'repeated staging must preserve every profile byte'); + }); + + test('existing fixture has exact target output with skip on and off (guard)', () async { + await edFresh('ed2a_e15_fixture.db'); + await edExistingFixture(); + final e = DerivationEngine(); + final sub = await e.debugLoadSubstrate(1577880000, 1578038400, day: edDay); + final full = _walk(sub, skip: false); + final fast = _walk(sub, skip: true); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), jsonEncode(edBundle(full.days.single))); + }); +} diff --git a/test/ed2a_e18_session_reuse_test.dart b/test/ed2a_e18_session_reuse_test.dart new file mode 100644 index 000000000..cc57c6b1e --- /dev/null +++ b/test/ed2a_e18_session_reuse_test.dart @@ -0,0 +1,262 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:openstrap_edge/data/restore_classification.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +Future _light(DerivationEngine e) async { + expect(await e.run(edProfile), greaterThan(0), reason: 'light pass must reach the real derive'); + expect(e.snapshot()['last_error'], isNull); +} + +Future _sessionEdit(Map patch) async { + await (await LocalDb.instance).update('sessions', patch, + where: 'id = ?', whereArgs: ['s1']); +} + +void main() { + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e18.db'); + await edRows(edStart, edStart + 1200); + await edSession(); + // These worker-level tests must reach the worker even after main's + // day fingerprint settles. The integration test below keeps both enabled. + DerivationEngine.debugDeriveEveryDay = true; + }); + tearDown(() async { + DerivationEngine.debugDeriveEveryDay = false; + await edDrop(); + }); + + test('session revisions upgrade from 58 and repair without backfilling sessions', () async { + var db = await LocalDb.instance; + final sessions = await db.query('sessions'); + for (final event in ['ai', 'au', 'ad']) { + await db.execute('DROP TRIGGER sessions_ceiling_$event'); + } + await db.execute('DROP TABLE session_ceiling_rev'); + await db.setVersion(58); + await LocalDb.close(); + db = await LocalDb.instance; + expect(await db.getVersion(), 59); + expect(await db.query('sessions'), sessions); + expect(await db.query('session_ceiling_rev'), isEmpty, + reason: 'upgrade never scans or backfills existing sessions'); + await _sessionEdit({'type': 'run'}); + expect(await LocalDb.sessionCeilingRevision(), 1); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + expect(await LocalDb.sessionCeilingRevision(), 1); + await LocalDb.setSessionHrr('s1', 20); + expect(await LocalDb.sessionCeilingRevision(), 1, + reason: 'derived-only writes do not invalidate input identity'); + await expectLater(db.transaction((txn) async { + await txn.update('sessions', {'type': 'bike'}); + throw StateError('rollback'); + }), throwsStateError); + expect(await LocalDb.sessionCeilingRevision(), 1); + expect((await db.query('sessions')).single['type'], 'run'); + }); + + test('session revisions travel in slices, rebuild on restore, and empty on wipe', () async { + final db = await LocalDb.instance; + expect(kRestoreClassification['session_ceiling_rev'], RestoreClass.rebuildable); + expect(kSliceTables['session_ceiling_rev']?.$1, SliceKeep.all); + expect(LocalDb.restoreTablesForTest, isNot(contains('session_ceiling_rev'))); + expect(await db.query('session_ceiling_rev'), hasLength(1)); + expect(await db.query('input_rev'), isNotEmpty); + final generation = LocalDb.storeGeneration; + await LocalDb.wipeAll(); + for (final table in await LocalDb.tableNames()) { + expect(await db.query(table, limit: 1), isEmpty, reason: table); + } + expect(LocalDb.storeGeneration, greaterThan(generation)); + await edSession(); + expect(await LocalDb.sessionCeilingRevision(), 1, + reason: 'wipe preserves working triggers'); + }); + + test('main skips settled days and changed samples still replace reused ceilings', () async { + DerivationEngine.debugDeriveEveryDay = false; + final e = DerivationEngine(); + Future settle() async { + for (var i = 0; i < 6; i++) { + final derived = await e.run(edProfile); + expect(e.snapshot()['last_error'], isNull); + if (derived == 0 && e.lastRunUnchangedDays == 1) return; + } + fail('main fingerprint never settled'); + } + await settle(); + final count = e.debugCeilingComputed; + expect(count, 1); + final first = await edStored(edDay); + expect(await e.run(edProfile), 0); + expect(e.lastRunUnchangedDays, 1); + expect(e.debugCeilingComputed, count); + expect(await edStored(edDay), first); + await edRows(edStart, edStart + 1200, hr: 160); + expect(await e.run(edProfile), 1, + reason: 'changed input must defeat main skip and the old ceiling key'); + expect(e.debugCeilingComputed, count + 1); + final changed = await edStored(edDay); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['scalars']['hr_ceiling_bpm'], 160); + await settle(); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + expect(await cold.runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), changed); + }); + + test('finished session ceiling is scanned once across unchanged light passes', () async { + final e = DerivationEngine(); + await _light(e); + expect(e.debugCeilingComputed, 1); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['scalars']['hr_ceiling_bpm'], 170, + reason: 'motion fixture must contain a real held ceiling'); + final first = await edStored(edDay); + await _light(e); + expect(await edStored(edDay), first); + expect(e.debugCeilingComputed, 1, reason: 'a hit must bypass sessionHrCeiling'); + }); + + final edits = Function(DerivationEngine)>{ + 'decoded sample revision (identical REPLACE)': (_) => edRows(edStart + 30, edStart + 31), + 'decoded sample revision (changed HR)': (_) => edRows(edStart + 30, edStart + 300, hr: 160), + 'late RR input revision': (_) async { + final t = edStart + 30; + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': t * 1000, 'rec_ts': t, 'rr_ts_ms': t * 1000, 'beat_index': 0, 'rr_ms': 900}); + }, + 'session revision (identical REPLACE)': (_) => edSession(), + 'session revision (identical UPDATE)': (_) => _sessionEdit({'type': 'run'}), + 'session identity': (_) => _sessionEdit({'id': 's2'}), + 'session type': (_) => _sessionEdit({'type': 'bike'}), + 'session start': (_) => _sessionEdit({'start_ts': edStart + 1}), + 'session end': (_) => _sessionEdit({'end_ts': edStart + 1000}), + 'clipped sample window': (_) async { + await (await LocalDb.instance).delete('decoded_onehz', + where: 'rec_ts < ?', whereArgs: [edStart + 10]); + }, + 'family': (_) async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}); }, + 'HR ownership': (_) async { + // The primary needs coverage too: as the only covering candidate, + // `other` would own the whole window and leave the day without HR. + await (await LocalDb.instance).insert('device_coverage', {'device_id': '', + 'signal': 'hr1Hz', 'start_ts': edStart, 'end_ts': edStart + 1200}); + await edOwnership(signals: ['hr1Hz']); + }, + 'accel ownership': (_) async { + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': 'other', 'signal': 'accel1Hz', + 'start_ts': edStart, 'end_ts': edStart + 600}); + await db.insert('signal_priority', {'device_id': 'other', 'signal': 'accel1Hz', + 'rank': 0, 'user_set': 1}); + }, + 'priority without changed input revisions': (_) async { + await (await LocalDb.instance).insert('signal_priority', {'signal': 'hr1Hz', + 'device_id': '', 'rank': 2, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'store generation': (_) async { await LocalDb.close(); await LocalDb.instance; }, + 'algorithm': (e) async { e.debugSessionCacheRekey(kAlgoVersion - 1); }, + }; + for (final edit in edits.entries) { + test('${edit.key}: miss, exact result, then reuse new key', () async { + if (edit.key.startsWith('priority ')) await edPriorityOrder(['', 'other']); + final e = DerivationEngine(); + await _light(e); + final count = e.debugCeilingComputed; + expect(count, 1); + await edit.value(e); + await _light(e); + expect(e.debugCeilingComputed, count + 1, reason: 'changed ${edit.key} must rescan'); + final warm = await edStored(edDay); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + expect(await cold.runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), warm, + reason: 'includes the full metric envelope and unrounded winning session'); + final after = e.debugCeilingComputed; + await _light(e); + expect(e.debugCeilingComputed, after, reason: 'new key should now hit'); + }); + } + + test('an absent ceiling is reusable without inventing a number', () async { + await edRows(edStart, edStart + 1200, az: 1); + final e = DerivationEngine(); + await _light(e); + final first = await edStored(edDay); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['hr_ceiling']['value'], '—'); + expect(payload['scalars']['hr_ceiling_bpm'], isNull); + await _light(e); + expect(await edStored(edDay), first); + expect(e.debugCeilingComputed, 1, reason: 'absence is a cached result too'); + }); + + test('live sessions always recompute, including done to live transition (guard)', () async { + final e = DerivationEngine(); + await _light(e); + await _sessionEdit({'status': 'live'}); + final before = e.debugCeilingComputed; + await _light(e); + expect(e.debugCeilingComputed, before + 1); + await _light(e); + expect(e.debugCeilingComputed, before + 2); + await _sessionEdit({'status': 'done'}); + await _light(e); + expect(e.debugCeilingComputed, before + 3, reason: 'return to done must not resurrect stale entry'); + }); + + for (final mode in ['heavy', 'force', 'selected']) { + test('$mode pass always recomputes (guard)', () async { + final e = DerivationEngine(); + await _light(e); + final count = e.debugCeilingComputed; + final done = switch (mode) { + 'heavy' => await e.run(edProfile, heavy: true), + 'force' => await e.run(edProfile, force: true), + _ => await e.runDays(edProfile, {edDay}), + }; + expect(done, greaterThan(0)); + expect(e.debugCeilingComputed, greaterThan(count)); + }); + } + + test('session removal does not keep its former ceiling (guard)', () async { + final e = DerivationEngine(); + await _light(e); + await (await LocalDb.instance).delete('sessions', where: 'id = ?', whereArgs: ['s1']); + await _light(e); + final got = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(got['hr_ceiling']['value'], '—'); + expect(got['scalars']['hr_ceiling_bpm'], isNull); + }); + + test('existing fixture: reuse and forced output byte equality after invalidation (guard)', () async { + await edExistingFixture(); + final e = DerivationEngine(); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + await _light(e); + await _light(e); + final before = await edStored(edDay); + await cold.runDays(edProfile, {edDay}); + expect(await edStored(edDay), before); + await edRows(edStart + 30, edStart + 300, hr: 160); + await _light(e); + final changed = await edStored(edDay); + await cold.runDays(edProfile, {edDay}); + expect(await edStored(edDay), changed); + }); +} diff --git a/test/ed2a_e19_wearable_day_index_test.dart b/test/ed2a_e19_wearable_day_index_test.dart new file mode 100644 index 000000000..e96b0dff7 --- /dev/null +++ b/test/ed2a_e19_wearable_day_index_test.dart @@ -0,0 +1,201 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/inputs/canonical.dart'; +import 'package:openstrap_edge/compute/readiness_calibration_backfill.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; +import 'readiness_calibration_backfill_test.dart' show bundle; + +const _watch = ('garmin-a', 'garmin'); +Future _wearable() async { + await LocalDb.upsertDevice(id: _watch.$1, adapterId: _watch.$2, label: 'Watch A'); + await LocalDb.setCursor(kActiveWearableCursor, _watch.$1); + await LocalDb.setCursor(wearableEnabledCursor(_watch.$2), '1'); + await edRows(edStart, edStart + 1200, id: _watch.$1, family: _watch.$2, + source: _watch.$2, hr: 70, az: 0); +} + +void main() { + setUpAll(edInit); + setUp(() async { await edFresh('ed2a_e19.db'); await _wearable(); }); + tearDown(() async { + endWearableDayIndexPass(); + debugOnWearableDayIndexScan = null; + debugDisableWearableDayIndex = false; + await edDrop(); + }); + + test('one real wearable scan per pass; new pass sees newly appended days', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + beginWearableDayIndexPass(); + try { + final first = await wearableRecTsMaxByDay(_watch); + expect(first, {edDay: edStart + 1199}); + expect(await wearableRecTsMaxByDay(_watch), first); + expect(scans, 1, reason: 'second request in the same pass must hit'); + } finally { endWearableDayIndexPass(); } + expect(debugWearableDayIndexSize, 0); + final later = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1, id: _watch.$1, family: _watch.$2, source: _watch.$2); + beginWearableDayIndexPass(); + try { + expect((await wearableRecTsMaxByDay(_watch)).keys, contains('2020-01-03')); + expect(scans, 2, reason: 'an index belongs to one pass, not to the engine lifetime'); + } finally { endWearableDayIndexPass(); } + }); + + for (final change in ['input revision', 'generation', 'algorithm']) { + test('$change cannot lend a stale index to the next request', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + beginWearableDayIndexPass(); + try { await wearableRecTsMaxByDay(_watch); } + finally { endWearableDayIndexPass(); } + switch (change) { + case 'input revision': await edRows(edStart, edStart + 1, + id: _watch.$1, family: _watch.$2, source: _watch.$2, hr: 80); + case 'generation': await LocalDb.close(); await LocalDb.instance; + case 'algorithm': debugWearableDayIndexRekey(kAlgoVersion - 1); + } + beginWearableDayIndexPass(); + try { + final fresh = await wearableRecTsMaxByDay(_watch); + expect(fresh, {edDay: edStart + 1199}); + expect(scans, 2, reason: 'the changed context belongs to a fresh pass'); + expect(await wearableRecTsMaxByDay(_watch), fresh); + expect(scans, 2, reason: 'unchanged second request in that pass must hit'); + } finally { endWearableDayIndexPass(); } + }); + } + + for (final key in ['device', 'family']) { + test('different wearable $key gets its own pass key', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + final other = key == 'device' ? ('garmin-b', 'garmin') : ('garmin-a', 'oura'); + beginWearableDayIndexPass(); + try { + final original = await wearableRecTsMaxByDay(_watch); + final changed = await wearableRecTsMaxByDay(other); + expect(changed, isEmpty); + expect(original, isNotEmpty); + expect(scans, 2, reason: 'a different $key must miss'); + expect(await wearableRecTsMaxByDay(_watch), original); + expect(await wearableRecTsMaxByDay(other), changed); + expect(scans, 2, reason: 'both keys, including empty index, should now hit'); + } finally { endWearableDayIndexPass(); } + }); + } + + test('index is discarded when body throws after a populated lookup', () async { + var scans = 0; + var populated = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + Future failedPass() async { + beginWearableDayIndexPass(); + try { + expect(await wearableRecTsMaxByDay(_watch), isNotEmpty); + populated = debugWearableDayIndexSize; + throw StateError('after index fill'); + } finally { endWearableDayIndexPass(); } + } + await expectLater(failedPass(), throwsStateError); + expect(debugWearableDayIndexSize, 0); + beginWearableDayIndexPass(); + try { await wearableRecTsMaxByDay(_watch); } + finally { endWearableDayIndexPass(); } + expect(scans, 2, reason: 'the failed pass cannot lend its index to a retry'); + expect(populated, 1, reason: 'exception cleanup must be tested with an occupied index'); + }); + + test('real run clears the index and running latch on an injected scan failure', () async { + var scans = 0; + var occupiedDuringFailure = 0; + debugOnWearableDayIndexScan = (_, _) { + scans++; + if (scans == 1) { + occupiedDuringFailure = debugWearableDayIndexSize; + throw StateError('injected after wearable scan'); + } + }; + final e = DerivationEngine(); + await e.run(edProfile); + expect(e.running, isFalse, reason: 'failure must release the process-wide latch'); + expect(e.snapshot()['last_error'], contains('injected after wearable scan')); + expect(debugWearableDayIndexSize, 0, reason: 'outer finally must release the pass index'); + final beforeRetry = scans; + expect(await e.run(edProfile), greaterThan(0)); + expect(scans, greaterThan(beforeRetry), reason: 'retry must build a fresh index'); + expect(occupiedDuringFailure, 1, reason: 'the fault must occur after population'); + }); + + test('actual light run scans the wearable index once through readiness backfill', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + final e = DerivationEngine(); + expect(await e.run(edProfile), greaterThan(0)); + expect(scans, 1, reason: 'scope, edge and readiness backfill share the pass index'); + expect(debugWearableDayIndexSize, 0); + }); + + test('lookups outside a pass stay fresh (passing guard)', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + await wearableRecTsMaxByDay(_watch); + final t = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(t, t + 1, id: _watch.$1, family: _watch.$2, source: _watch.$2); + expect((await wearableRecTsMaxByDay(_watch)).keys, contains('2020-01-03')); + expect(scans, 2); + expect(debugWearableDayIndexSize, 0); + }); + + test('readiness backfill defers wearable substrate and still rescores pruned days (guard)', () async { + for (final day in [edDay, '2020-01-01']) { + final payload = bundle(0.5, 0.5, 0); + await LocalDb.putDayResult(dayId: day, algoVersion: 111, + payloadJson: jsonEncode(payload), windowJson: '{}', finalized: true, + readiness: (payload['scalars'] as Map)['readiness'] as double); + } + final before = await edStored(edDay); + beginWearableDayIndexPass(); + late final ({int seeded, int rescored, int forced}) result; + try { + final raw = await substrateDays(); + expect(raw, contains(edDay)); + result = await backfillReadinessCalibration(rawDays: raw, + zHistoryLoader: () async => (day, family) => []); + } finally { endWearableDayIndexPass(); } + expect(result.rescored, 1, reason: 'pruned history still needs calibration'); + expect(result.forced, 0); + expect((await LocalDb.dayResult('2020-01-01'))!['algo_version'], kAlgoVersion); + expect((await LocalDb.dayResult(edDay))!['algo_version'], 111, + reason: 'wearable substrate must wait for its own derive'); + // readiness_z seeding is intentional; the day bundle must remain untouched. + final row = await LocalDb.dayResult(edDay); + expect(jsonDecode(before)['day_result'][0]['payload_json'], row!['payload_json']); + }); + + test('existing fixture: pass index enabled/disabled derive JSON remains exact (guard)', () async { + await edExistingFixture(); + final e = DerivationEngine(); + expect(await e.runDays(edProfile, {edDay}), 1); + final indexed = await edStored(edDay); + debugDisableWearableDayIndex = true; + expect(await DerivationEngine().runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), indexed); + debugDisableWearableDayIndex = false; + await edRows(edStart, edStart + 1, hr: 89); + expect(await e.runDays(edProfile, {edDay}), 1); + final changed = await edStored(edDay); + debugDisableWearableDayIndex = true; + expect(await DerivationEngine().runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), changed); + }); +} diff --git a/test/ed2a_e20_worker_preparation_test.dart b/test/ed2a_e20_worker_preparation_test.dart new file mode 100644 index 000000000..76b8a2641 --- /dev/null +++ b/test/ed2a_e20_worker_preparation_test.dart @@ -0,0 +1,204 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:isolate'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/inputs/canonical.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/local_repository_impl.dart'; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +String get _mainIsolate => Isolate.current.debugName ?? 'unnamed'; +Future> _sessionReads() async { + final r = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200); + return {'stats': r.stats, 'samples': r.samples}; +} + +void main() { + final sql = []; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e20.db'); + // Log the real join, rather than counting calls to a mock query function. + await LocalDb.close(); + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions(type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? v) { + if (v is Map) { + if (v['sql'] case final String s) sql.add(s); + for (final x in v.values) { collect(x); } + } else if (v is List) { + for (final x in v) { collect(x); } + } + } + collect(event.arguments); + })); + await LocalDb.instance; + sql.clear(); + }); + tearDown(() async { + debugDisableWearableWorker = false; + debugOnWearableConstruction = null; + LocalDb.debugDisableSessionWindowsWorker = false; + LocalDb.debugOnSessionWindowsConstruction = null; + await edDrop(); + databaseFactory = databaseFactoryFfi; + }); + + for (final family in ['garmin', 'colmi', 'oura', 'ultrahuman', 'pebble', 'miband234']) { + test('$family construction runs off the caller isolate with exact output', () async { + final id = '$family-a'; + await edRows(edStart, edStart + 1200, id: id, family: family, source: family, hr: 71, az: 0); + debugDisableWearableWorker = true; + final inline = await wearableSubstrate((id, family), edStart, edStart + 1199); + expect(inline.isEmpty, isFalse, reason: 'fixture must enter a real builder'); + final locations = []; + debugOnWearableConstruction = locations.add; + debugDisableWearableWorker = false; + final worker = await wearableSubstrate((id, family), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + final a = prepareDerivationPayload(inline, targetDay: edDay).days.single; + final b = prepareDerivationPayload(worker, targetDay: edDay).days.single; + expect(jsonEncode(edBundle(b)), jsonEncode(edBundle(a))); + expect(locations, isNotEmpty, reason: 'actual construction must report execution location'); + expect(locations, everyElement(isNot(_mainIsolate)), + reason: 'row construction still executes on the UI/caller isolate'); + }); + } + + test('two wearable devices at one second remain separate (passing guard)', () async { + await edRows(edStart, edStart + 1200, id: 'a', family: 'garmin', source: 'garmin', hr: 65, az: 0); + await edRows(edStart, edStart + 1200, id: 'b', family: 'garmin', source: 'garmin', hr: 95, az: 0); + for (final id in ['a', 'b']) { + debugDisableWearableWorker = true; + final inline = await wearableSubstrate((id, 'garmin'), edStart, edStart + 1199); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate((id, 'garmin'), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.hr.toSet(), {id == 'a' ? 65 : 95}); + expect(worker.deviceIds, {id}); + } + }); + + test('two straps keep owner, beat timing and session overlay exact (guard)', () async { + LocalDb.sessionSensorSources = {'ble_hrs', 'polar'}; + LocalDb.sessionWearableSource = 'garmin'; + await edRows(edStart, edStart + 1200, id: 'watch', family: 'garmin', source: 'garmin', hr: 70, az: 0); + await edRows(edStart + 100, edStart + 700, id: 'strap-a', family: 'ble_hrs', source: 'ble_hrs', hr: 150, az: 0); + await edRows(edStart + 100, edStart + 300, id: 'strap-b', family: 'polar', source: 'polar', hr: 180, az: 0); + await edSession(start: edStart, end: edStart + 800); + final db = await LocalDb.instance; + for (final (id, source, rr) in [('strap-a', 'ble_hrs', 850), ('strap-b', 'polar', 900)]) { + for (var i = 0; i < (id == 'strap-a' ? 3 : 1); i++) { + final t = edStart + 150 + i; + await db.insert('decoded_rr', {'device_id': id, 'source': source, + 'ts_ms': t * 1000, 'rec_ts': t, 'rr_ts_ms': t * 1000, + 'beat_ts_ms': t * 1000 - 123, 'rr_ms': rr, 'beat_index': 0}); + } + } + debugDisableWearableWorker = true; + final inline = await wearableSubstrate(('watch', 'garmin'), edStart, edStart + 1199); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate(('watch', 'garmin'), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.hr, isNot(contains(180)), reason: 'minority source must not mix into a session'); + expect(worker.rrMs, [850, 850, 850]); + expect(worker.rrTsMs.first, (edStart + 150) * 1000 - 123); + }); + + test('existing two-device fixture: on/off worker derive JSON is byte-identical (guard)', () async { + await edExistingFixture(); + debugDisableWearableWorker = true; + final inline = await wearableSubstrate(('oura-a1b2c3d4', 'oura'), 1577923200, 1577952000); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate(('oura-a1b2c3d4', 'oura'), 1577923200, 1577952000); + expect(inline.isEmpty, isFalse); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.skinTemp.where((v) => v > 0), isNotEmpty); + expect(jsonEncode(edBundle(prepareDerivationPayload(worker, targetDay: edLabel(worker.tsSec.last)).days.single)), + jsonEncode(edBundle(prepareDerivationPayload(inline, targetDay: edLabel(inline.tsSec.last)).days.single))); + }); + + Future sessionFixture() async { + LocalDb.sessionSensorSources = {'ble_hrs', 'polar'}; + LocalDb.sessionWearableSource = 'garmin'; + await edSession(); + await edRows(edStart, edStart + 1200, id: 'watch', family: 'garmin', source: 'garmin', hr: 70); + await edRows(edStart + 100, edStart + 700, id: 'strap-a', family: 'ble_hrs', source: 'ble_hrs', hr: 150); + await edRows(edStart + 100, edStart + 300, id: 'strap-b', family: 'polar', source: 'polar', hr: 180); + } + + test('combined session stats and samples use one SELECT', () async { + await sessionFixture(); + sql.clear(); + final r = await _sessionReads(); + expect((r['samples'] as Map)['s1'], isNotEmpty); + final joins = sql.where((s) => s.contains('JOIN decoded_onehz') && s.contains('FROM sessions')); + expect(joins, hasLength(1), reason: 'stats and samples currently reread the same window twice'); + }); + + test('session grouping executes off the caller isolate with exact JSON', () async { + await sessionFixture(); + LocalDb.debugDisableSessionWindowsWorker = true; + final inline = await _sessionReads(); + final locations = []; + LocalDb.debugOnSessionWindowsConstruction = locations.add; + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await _sessionReads(); + expect(jsonEncode(worker), jsonEncode(inline)); + expect(locations, isNotEmpty); + expect(locations, everyElement(isNot(_mainIsolate)), + reason: 'grouping and owner selection still loop over rows on the caller'); + }); + + test('workout repository uses the combined read in its actual list flow', () async { + await sessionFixture(); + final repo = LocalRepositoryImpl(getProfileMap: () => edProfile.toMap()); + sql.clear(); + final got = await repo.getWorkouts(range: 'all'); + expect(got['workouts'], isNotEmpty); + final joins = sql.where((s) => s.contains('JOIN decoded_onehz') && s.contains('FROM sessions')); + expect(joins, hasLength(1), reason: 'GREEN must wire the real getWorkouts consumer too'); + }); + + test('bounded session aggregates and empty windows stay exact (guard)', () async { + await sessionFixture(); + LocalDb.debugDisableSessionWindowsWorker = true; + final inline = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200, + maxHrCeiling: 140, minHrFloor: 80); + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200, + maxHrCeiling: 140, minHrFloor: 80); + expect(jsonEncode({'stats': worker.stats, 'samples': worker.samples}), + jsonEncode({'stats': inline.stats, 'samples': inline.samples})); + expect(worker.stats['s1']!['min_hr'], 150); + expect(worker.stats['s1']!['max_hr'], 70); + expect(worker.samples['s1'], containsAll([70, 150])); + final empty = await LocalDb.sessionHrReadsForTest(edStart - 100, edStart - 1); + expect(empty.stats, isEmpty); + expect(empty.samples, isEmpty); + }); + + test('session windows keep two-device ownership and last-second samples (guard)', () async { + await sessionFixture(); + final inline = await _sessionReads(); + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await _sessionReads(); + expect(jsonEncode(worker), jsonEncode(inline)); + final samples = (worker['samples'] as Map)['s1'] as List; + expect(samples, isNot(contains(180))); + expect(samples.first, 70); + expect(samples.last, 70, reason: 'inclusive session end must survive preparation'); + final stats = (worker['stats'] as Map)['s1'] as Map; + expect(stats['n'], 1101); + }); +} diff --git a/test/ed2a_input_revision_test.dart b/test/ed2a_input_revision_test.dart new file mode 100644 index 000000000..442c3b03c --- /dev/null +++ b/test/ed2a_input_revision_test.dart @@ -0,0 +1,556 @@ +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:openstrap_edge/data/restore_classification.dart'; +// ED-2a E4 RED contract. Run with flutter test --concurrency=1. +// Only the input portion of fork 561f2838 and generation portions of +// 74c0b37b/30cd944e apply. No store_rev, row_rev, scheduler or checkpoints. +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/models.dart'; + +const _inputs = ['decoded_onehz', 'decoded_rr']; +final _midnight = DateTime(2026, 10, 20); +final _lo = _midnight.millisecondsSinceEpoch ~/ 1000; +final _day = dayLabelOf(_midnight); + +Map _row(String table, int ts, {String device = ''}) => { + 'device_id': device, 'ts_ms': ts * 1000, 'rec_ts': ts, + if (table == 'decoded_onehz') ...{'counter': 1, 'hr': 61}, + if (table == 'decoded_rr') ...{ + 'beat_index': 0, 'rr_ts_ms': ts * 1000, 'rr_ms': 850, + }, +}; + +Future _hasRevisions(Database db) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='input_rev'", +)).isNotEmpty; + +// Missing infrastructure is revision zero, so mutation tests fail an assertion +// about an unchanged revision, never a spurious 'no such table' exception. +Future _rev(Database db, int ts) async { + if (!await _hasRevisions(db)) return 0; + final rows = await db.query('input_rev', columns: ['rev'], + where: 'bucket = ?', whereArgs: [ts ~/ 900]); + return rows.isEmpty ? 0 : (rows.single['rev'] as num).toInt(); +} + +Future>> _revisionRows(Database db) async => + await _hasRevisions(db) ? db.query('input_rev', orderBy: 'bucket') : []; + +Future>> _legacyFingerprint(Database db) => + db.rawQuery('SELECT MAX(rec_ts) AS mx, COUNT(*) AS n FROM decoded_onehz ' + 'WHERE rec_ts >= ? AND rec_ts < ?', [_lo, _lo + 7200]); + +Future> _schema(Database db) async { + final out = {}; + final entries = await db.rawQuery( + "SELECT type, name, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' " + "AND name != 'android_metadata' ORDER BY type, name", + ); + for (final entry in entries) { + final name = entry['name'] as String; + // Compare column definitions rather than CREATE text: ALTER and CREATE + // paths have different whitespace and IF NOT EXISTS spelling. + out[name] = entry['type'] == 'table' + ? await db.rawQuery('PRAGMA table_info("$name")') + : (entry['sql'] as String?)?.replaceAll(RegExp(r'\s+'), ' ').trim(); + } + return out; +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late Directory tmp; + late String livePath; + late String sourcePath; + late DatabaseFactory? previousFactory; + late String previousName; + final sql = []; + final legacyRows = {}; + final legacyColumns = {}; + + setUp(() async { + sqfliteFfiInit(); + previousFactory = databaseFactoryOrNull; + previousName = LocalDb.dbName; + await LocalDb.close(); + SharedPreferences.setMockInitialValues({}); + tmp = await Directory.systemTemp.createTemp('ed2a_rev_'); + livePath = p.join(tmp.path, 'live.db'); + sourcePath = p.join(tmp.path, 'source.db'); + // Public sqflite logger observes actual SQL, including nested batches. + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions( + type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? value) { + if (value is Map) { + if (value['sql'] case final String statement) sql.add(statement); + for (final v in value.values) { collect(v); } + } else if (value is List) { + for (final v in value) { collect(v); } + } + } + collect(event.arguments); + }, + )); + LocalDb.dbName = livePath; + LocalDb.lastRebuild = null; + sql.clear(); + }); + + tearDown(() async { + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + await databaseFactory.deleteDatabase(sourcePath); + databaseFactoryOrNull = previousFactory; + LocalDb.dbName = previousName; + await tmp.delete(recursive: true); + }); + + Future open() async { + final db = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull, + reason: 'a broken rung must not pass by quarantining the original DB'); + return db; + } + + Future seed58() async { + final db = await databaseFactory.openDatabase(livePath, + options: OpenDatabaseOptions(version: 58)); + // Frozen v58 decoded DDL is below. Other tables use the existing repair + // seam; remove E4 infrastructure before setting the old version and seeding. + for (final ddl in _v58DecodedDdl) { await db.execute(ddl); } + await LocalDb.repairOpenSchemaForTest(db); + final triggers = await db.rawQuery( + "SELECT name FROM sqlite_master WHERE type='trigger' " + "AND tbl_name IN ('decoded_onehz','decoded_rr')", + ); + for (final t in triggers) { await db.execute('DROP TRIGGER "${t['name']}"'); } + await db.execute('DROP TABLE IF EXISTS input_rev'); + await db.setVersion(58); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 3)); + await db.insert('decoded_rr', _row('decoded_rr', _lo + 3)); + for (final table in _inputs) { + legacyRows[table] = await db.query(table); + legacyColumns[table] = await db.rawQuery('PRAGMA table_info($table)'); + } + await db.close(); + } + + test('every table is classified for restore and diagnostic slices', () async { + await open(); + final tables = await LocalDb.tableNames(); + expect(tables.toSet().difference(kRestoreClassification.keys.toSet()), isEmpty); + expect(tables.toSet().difference(kSliceTables.keys.toSet()), isEmpty); + expect(kRestoreClassification['input_rev'], RestoreClass.rebuildable); + expect(LocalDb.restoreTablesForTest, isNot(contains('input_rev'))); + }); + + test('rung 59 is reserved for additive input revisions', () async { + expect(LocalDb.schemaVersion, 59, reason: 'upstream is still schema 58'); + final db = await open(); + expect(await db.getVersion(), 59); + expect(await _hasRevisions(db), isTrue); + final tables = await LocalDb.tableNames(); + expect(tables, isNot(contains('store_rev'))); + expect(tables, isNot(contains('row_rev'))); + }); + + test('58 to 59 preserves raw rows and creates all mutation triggers', () async { + await seed58(); + final db = await open(); + expect((await db.query('decoded_onehz')).single['hr'], 61); + expect((await db.query('decoded_rr')).single['rr_ms'], 850); + for (final table in _inputs) { + expect(await db.query(table), legacyRows[table], reason: '$table data unchanged'); + expect(await db.rawQuery('PRAGMA table_info($table)'), legacyColumns[table], + reason: 'rung 59 must not change raw columns/defaults'); + } + expect(await db.getVersion(), 59); + expect(await _hasRevisions(db), isTrue); + final triggers = await db.rawQuery( + "SELECT tbl_name, sql FROM sqlite_master WHERE type='trigger'", + ); + for (final table in _inputs) { + final body = triggers.where((t) => t['tbl_name'] == table) + .map((t) => (t['sql'] as String).toUpperCase()).join('\n'); + for (final op in ['INSERT', 'UPDATE', 'DELETE']) { + expect(body, contains('AFTER $op'), reason: '$table $op revision trigger'); + } + } + expect(await _revisionRows(db), isEmpty, + reason: 'legacy rows must not be backfilled into input_rev'); + expect(await LocalDb.inputDayFingerprint(_day), + (revisionSum: 0, revisionMax: 0, generation: LocalDb.storeGeneration)); + }); + + test('upgrade and every-open repair do not scan or rewrite raw input', () async { + await seed58(); + sql.clear(); + final db = await open(); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + await open(); + final openingSql = List.of(sql); + expect(openingSql, isNotEmpty, reason: 'SQL observation must be active'); + final dataAccess = RegExp( + r'\b(?:FROM|JOIN|UPDATE|INTO|DELETE\s+FROM)\s+["`\[]?' + r'(?:decoded_onehz|decoded_rr|samples|raw_archive)\b', + caseSensitive: false, + ); + expect(openingSql.where((s) { + final ddl = s.trimLeft().toUpperCase(); + // Declaring a trigger/view does not execute its SELECT/UPDATE body. + if (ddl.startsWith('CREATE TRIGGER') || ddl.startsWith('CREATE VIEW')) return false; + // Main probes whether legacy decoded input exists before metadata repair. + // This bounded probe is unrelated to revision creation or backfill. + if (s == 'SELECT 1 FROM decoded_onehz LIMIT 1') return false; + return dataAccess.hasMatch(s); + }), + isEmpty, reason: 'opening may inspect schema, never scan/backfill raw rows'); + expect(await _revisionRows(await LocalDb.instance), isEmpty); + }); + + test('fresh and upgraded schema agree; repair and reopen are idempotent', () async { + await seed58(); + var db = await open(); + expect(await _hasRevisions(db), isTrue, reason: 'input_rev must exist first'); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 100)); + final schema = await _schema(db); + final revs = await _revisionRows(db); + final rows = await db.query('decoded_onehz', orderBy: 'rec_ts'); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + db = await open(); + expect(await _schema(db), schema); + expect(await _revisionRows(db), revs); + expect(await db.query('decoded_onehz', orderBy: 'rec_ts'), rows); + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + db = await open(); + expect(await _schema(db), schema, reason: 'onCreate and onUpgrade must agree'); + }); + + test('absent day and legacy bucket revisions read zero', () async { + await seed58(); + await open(); + for (final day in [_day, '1999-01-01']) { + expect(await LocalDb.inputDayFingerprint(day), + (revisionSum: 0, revisionMax: 0, generation: LocalDb.storeGeneration)); + } + }); + + for (final table in _inputs) { + for (final op in ['INSERT', 'REPLACE', 'UPDATE', 'DELETE']) { + test('$table $op advances only its 15-minute bucket', () async { + final db = await open(); + final ts = _lo + 930; + if (op != 'INSERT') { await db.insert(table, _row(table, ts)); } + final before = await _rev(db, ts); + final adjacent = await _rev(db, ts - 900); + switch (op) { + case 'INSERT': await db.insert(table, _row(table, ts)); + case 'REPLACE': + // Identical values still invalidate; repeat to catch revision reset + // under the outer statement's OR REPLACE conflict policy. + for (var i = 0; i < 3; i++) { + final last = await _rev(db, ts); + await db.insert(table, _row(table, ts), + conflictAlgorithm: ConflictAlgorithm.replace); + expect(await _rev(db, ts), greaterThan(last), + reason: '$table identical REPLACE must never reset the bucket'); + } + case 'UPDATE': + await db.update(table, table == 'decoded_onehz' ? {'hr': 62} : {'rr_ms': 900}); + case 'DELETE': await db.delete(table); + } + expect(await _rev(db, ts), greaterThan(before), reason: '$table $op was invisible'); + expect(await _rev(db, ts - 900), adjacent, reason: 'unrelated bucket changed'); + }); + } + + test('$table timestamp UPDATE invalidates both old and new buckets', () async { + final db = await open(); + final ts = _lo + 899; + await db.insert(table, _row(table, ts)); + final old = await _rev(db, ts), next = await _rev(db, ts + 1); + await db.update(table, {'rec_ts': ts + 1, 'ts_ms': (ts + 1) * 1000}); + expect(await _rev(db, ts), greaterThan(old), reason: 'old window lost a row'); + expect(await _rev(db, ts + 1), greaterThan(next), reason: 'new window gained a row'); + }); + + test('$table invalid timestamps do not create revision buckets', () async { + final db = await open(); + for (final ts in [0, -1]) { await db.insert(table, _row(table, ts)); } + expect(await _revisionRows(db), isEmpty); + }); + } + + test('identical newest-wins replacement defeats MAX/count but moves revision', () async { + final db = await open(); + final row = _row('decoded_onehz', _lo + 10); + await db.insert('decoded_onehz', row); + final old = await _legacyFingerprint(db); + final before = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_onehz', row, conflictAlgorithm: ConflictAlgorithm.replace); + expect(await _legacyFingerprint(db), old, reason: 'MAX/count blind spot reproduced'); + expect((await db.query('decoded_onehz')).length, 1); + expect((await LocalDb.inputDayFingerprint(_day)).revisionSum, + greaterThan(before.revisionSum), reason: 'identical REPLACE must invalidate cached input'); + }); + + test('late RR without any 1 Hz row changes the day fingerprint', () async { + final db = await open(); + final before = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_rr', _row('decoded_rr', _lo + 20)); + expect(await db.query('decoded_onehz'), isEmpty); + expect((await LocalDb.inputDayFingerprint(_day)).revisionSum, + greaterThan(before.revisionSum), reason: 'RR-only inputs must not require MAX(rec_ts)'); + }); + + for (final date in [DateTime(2026, 10, 20), DateTime(2026, 3, 8), DateTime(2026, 11, 1)]) { + test('fingerprint sums/maxes local day buckets for ${dayLabelOf(date)}', () async { + final db = await open(); + final lo = date.millisecondsSinceEpoch ~/ 1000; + final hi = DateTime(date.year, date.month, date.day + 1).millisecondsSinceEpoch ~/ 1000; + // Unequal bucket revisions pin SUM and MAX independently. Outside rows + // must not contribute, including precisely the following midnight. + for (final ts in [lo - 1, lo, lo + 900, hi - 1, hi]) { + await db.insert('decoded_onehz', _row('decoded_onehz', ts)); + } + await db.update('decoded_onehz', {'hr': 63}, where: 'rec_ts = ?', whereArgs: [lo]); + final fp = await LocalDb.inputDayFingerprint(dayLabelOf(date)); + expect(fp.revisionSum, 4, reason: 'two writes in first bucket, one in each of two others'); + expect(fp.revisionMax, 2); + expect(fp.generation, LocalDb.storeGeneration); + expect(await LocalDb.inputDayFingerprint(dayLabelOf(date)), fp, + reason: 'reading the fingerprint must be idempotent'); + }); + } + + test('rolled-back raw writes roll back revisions as well', () async { + final db = await open(); + final before = await LocalDb.inputDayFingerprint(_day); + await expectLater(db.transaction((txn) async { + await txn.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + await txn.insert('decoded_rr', _row('decoded_rr', _lo + 10)); + throw StateError('abort before commit'); + }), throwsStateError); + expect(await db.query('decoded_onehz'), isEmpty); + expect(await _revisionRows(db), isEmpty); + expect(await LocalDb.inputDayFingerprint(_day), before); + }); + + test('reopen advances generation without changing input revisions', () async { + var db = await open(); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + final before = await LocalDb.inputDayFingerprint(_day); + await LocalDb.close(); + db = await open(); + final after = await LocalDb.inputDayFingerprint(_day); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation)); + expect(await db.query('decoded_onehz'), hasLength(1)); + }); + + test('wipe increments generation even when the revision key returns to zero', () async { + final db = await open(); + final before = LocalDb.storeGeneration; + final emptyKey = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + await LocalDb.wipeAll(); + expect(await db.query('decoded_onehz'), isEmpty); + expect(await db.query('input_rev'), isEmpty); + expect(LocalDb.storeGeneration, greaterThan(before)); + expect(await LocalDb.inputDayFingerprint(_day), isNot(emptyKey), + reason: '(day, revision, generation) must reject a pre-wipe cache entry'); + }); + + Future source({bool failAfterCommit = false}) async { + final src = await databaseFactory.openDatabase(sourcePath); + // A valid replacement in an EARLY table, followed by a destination failure + // in a later table. This drives the public importer, not a test-only merge. + await src.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await src.insert('journal', {'date': _day, 'note': 'replacement', 'updated_at': 2}); + if (failAfterCommit) { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + } + + for (final partial in [false, true]) { + test('${partial ? 'partial failing' : 'successful replacement'} import advances generation', () async { + final db = await open(); + await db.insert('journal', {'date': _day, 'note': 'old', 'updated_at': 1}); + await source(failAfterCommit: partial); + final before = LocalDb.storeGeneration; + final key = await LocalDb.inputDayFingerprint(_day); + if (partial) { + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(sourcePath); + } + expect((await db.query('journal')).single['note'], 'replacement', + reason: 'prove a page committed before checking invalidation'); + expect(LocalDb.storeGeneration, greaterThan(before), + reason: 'every exit after durable replacement must invalidate caches'); + expect(await LocalDb.inputDayFingerprint(_day), isNot(key)); + }); + } + + for (final mode in ['success', 'later failure', 'first failure']) { + test('bulk restore $mode invalidates only after a durable page', () async { + final db = await open(); + final src = await databaseFactory.openDatabase(sourcePath); + await src.execute(_v58DecodedDdl.first); + await src.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + if (mode == 'later failure') { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + if (mode == 'first failure') { + await db.execute("CREATE TRIGGER refuse_decoded BEFORE INSERT ON decoded_onehz " + "BEGIN SELECT RAISE(ABORT, 'test bulk first-page failure'); END"); + } + final generation = LocalDb.storeGeneration; + if (mode == 'success') { + await LocalDb.importFromDbFile(sourcePath); + } else { + await expectLater(LocalDb.importFromDbFile(sourcePath), + throwsA(isA())); + } + expect(await db.query('decoded_onehz'), + hasLength(mode == 'first failure' ? 0 : 1)); + expect(LocalDb.storeGeneration, + mode == 'first failure' ? generation : greaterThan(generation)); + }); + } + + for (final mode in ['success', 'later table failure', + 'later archive-page failure', 'first failure']) { + test('archive-only restore $mode tracks durable pages', () async { + final db = await open(); + final src = await databaseFactory.openDatabase(sourcePath); + await src.execute('CREATE TABLE raw_archive (hex TEXT PRIMARY KEY, ' + 'counter INTEGER, packet_type INTEGER, rec_ts INTEGER, ' + 'captured_at INTEGER, reason TEXT)'); + final sourceRows = mode == 'later archive-page failure' ? 501 : 1; + final batch = src.batch(); + for (var i = 1; i <= sourceRows; i++) { + batch.insert('raw_archive', {'hex': i.toRadixString(16).padLeft(4, '0'), + 'counter': i, 'packet_type': 0, 'rec_ts': _lo, + 'captured_at': _lo * 1000, 'reason': 'test_archive'}); + } + await batch.commit(noResult: true); + if (mode == 'later table failure') { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + if (mode == 'first failure' || mode == 'later archive-page failure') { + final rejected = mode == 'first failure' ? 1 : 501; + await db.execute('CREATE TRIGGER refuse_archive BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} WHEN NEW.counter = $rejected ' + "BEGIN SELECT RAISE(ABORT, 'test archive page failure'); END"); + } + final generation = LocalDb.storeGeneration; + if (mode == 'success') { + await LocalDb.importFromDbFile(sourcePath); + } else { + await expectLater(LocalDb.importFromDbFile(sourcePath), + throwsA(isA())); + } + final copied = mode == 'first failure' ? 0 : + mode == 'later archive-page failure' ? 500 : 1; + expect(await db.query(LocalDb.rawArchiveTable), hasLength(copied)); + expect(LocalDb.storeGeneration, + mode == 'first failure' ? generation : greaterThan(generation)); + }); + } + + test('failed first import page leaves generation unchanged', () async { + final db = await open(); + await source(); + await db.execute("CREATE TRIGGER refuse_journal BEFORE INSERT ON journal " + "BEGIN SELECT RAISE(ABORT, 'test first-page failure'); END"); + final before = LocalDb.storeGeneration; + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + expect(await db.query('journal'), isEmpty); + expect(LocalDb.storeGeneration, before, reason: 'nothing committed'); + }); + + test('failed first import page can retry and invalidate the next generation', () async { + final db = await open(); + await source(); + await db.execute("CREATE TRIGGER refuse_journal BEFORE INSERT ON journal " + "BEGIN SELECT RAISE(ABORT, 'test first-page failure'); END"); + final before = LocalDb.storeGeneration; + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + expect(await db.query('journal'), isEmpty); + expect(LocalDb.storeGeneration, before, reason: 'nothing committed'); + await db.execute('DROP TRIGGER refuse_journal'); + await LocalDb.importFromDbFile(sourcePath); + expect((await db.query('journal')).single['note'], 'replacement'); + expect(LocalDb.storeGeneration, greaterThan(before), reason: 'retry must not leave a sticky latch'); + }); + + test('GUARD sync REPLACE shrinks/clears beats only for its device and second', () async { + final db = await open(); + Future commit(String device, int counter, List rr, {int? ts}) { + final at = ts ?? _lo; + return LocalDb.commitSyncBatch( + [RawRecord(counter: counter, packetType: 47, hex: 'feed$counter', + capturedAt: at * 1000, recTs: at)], + [Sample(tsEpoch: at, counter: counter, hr: 60 + counter, + rrIntervalsMs: rr, ax: 0, ay: 0, az: 1, + spo2RedRaw: 0, spo2IrRaw: 0, skinTempRaw: 30000)], + deviceId: device, deviceFamily: 'gen4', + ); + } + await commit('', 10, [800, 810, 820]); + await commit('secondary', 11, [700, 710]); + await commit('', 12, [780], ts: _lo + 1); + await commit('', 1, [900]); + expect((await db.query('decoded_onehz', where: "device_id = '' AND rec_ts = $_lo")).single['counter'], 1); + expect((await db.query('decoded_onehz', where: "device_id = '' AND rec_ts = $_lo")).single['hr'], 61); + expect((await db.query('decoded_rr', where: "device_id = '' AND rec_ts = $_lo")).map((r) => r['rr_ms']), [900]); + await commit('', 2, []); + expect(await db.query('decoded_rr', where: "device_id = '' AND rec_ts = $_lo"), isEmpty); + expect((await db.query('decoded_rr', where: 'rec_ts = ?', whereArgs: [_lo + 1])) + .single['rr_ms'], 780, reason: 'the neighboring second must keep its beats'); + expect((await db.query('decoded_rr', where: "device_id = 'secondary'", orderBy: 'beat_index')) + .map((r) => r['rr_ms']), [700, 710]); + }); +} + +// Frozen c8e63556 v58 base table definitions; optional columns added by the +// existing repair path are captured before upgrade. Never seed input_rev here. +const _v58DecodedDdl = [ + '''CREATE TABLE decoded_onehz ( + device_id TEXT NOT NULL DEFAULT '', ts_ms INTEGER NOT NULL DEFAULT 0, + rec_ts INTEGER, counter INTEGER NOT NULL, hr INTEGER, ax REAL, ay REAL, az REAL, + spo2_red_raw INTEGER, spo2_ir_raw INTEGER, skin_temp_raw INTEGER, + step_count INTEGER, step_cadence INTEGER, activity_class INTEGER, skin_temp_c REAL, + on_wrist INTEGER, hr_valid INTEGER, hr_alt INTEGER, ambient_raw INTEGER, + device_family TEXT, source TEXT, temp_ch2_c REAL, temp_ch3_c REAL, + signal_quality_logvar REAL, dyn_accel_g REAL, PRIMARY KEY(device_id, ts_ms))''', + '''CREATE TABLE decoded_rr ( + device_id TEXT NOT NULL DEFAULT '', ts_ms INTEGER NOT NULL DEFAULT 0, + rec_ts INTEGER NOT NULL, beat_index INTEGER NOT NULL, rr_ts_ms INTEGER NOT NULL, + rr_ms INTEGER NOT NULL, device_family TEXT, source TEXT, + PRIMARY KEY(device_id, ts_ms, beat_index))''', +]; diff --git a/test/ed2a_substrate_transfer_test.dart b/test/ed2a_substrate_transfer_test.dart new file mode 100644 index 000000000..e55df69c4 --- /dev/null +++ b/test/ed2a_substrate_transfer_test.dart @@ -0,0 +1,258 @@ +// ED-2a E5: upstream's existing worker, no E3 annotations/registry/AST guard. +// fromTransfer delegates to fromJson in RED: parity passes, adoption fails. +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart' show kAlgoVersion; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; + +import 'support/dart_source.dart'; + +const _intKeys = ['ts_sec', 'hr', 'spo2_red', 'spo2_ir', 'skin_temp', + 'skin_contact', 'step_count', 'hr_valid', 'band_sleep_state']; +const _doubleKeys = ['rr_ts_ms', 'rr_ms', 'ax', 'ay', 'az']; +const _start = 1577923200; // existing two_device_day fixture, UTC + +Map _columns() => { + 'ts_sec': [_start, _start + 1, _start + 2], + 'hr': [60, 0, 61], + 'rr_ts_ms': Float64List.fromList([1577923199999.5, 1577923201000.25]), + 'rr_ms': Float64List.fromList([850.125, 900.5]), + 'ax': Float64List.fromList([-0.0, double.minPositive, double.nan]), + 'ay': Float64List.fromList([double.infinity, double.negativeInfinity, 0.1 + 0.2]), + 'az': Float64List.fromList([1, -1, 0.5]), + 'spo2_red': [0, 1, 65535], 'spo2_ir': [123, 0, 65534], + 'skin_temp': [30001, 0, 30002], 'skin_contact': [1, 0, -1], + 'step_count': [65535, 0, -1], 'hr_valid': [1, 0, -1], + 'band_sleep_state': [1, 0, -1], + 'device_family': 'gen5', 'device_ids': ['secondary', '', 'secondary'], +}; + +void _same(Substrate got, Substrate oracle) { + final a = got.toJson(), b = oracle.toJson(); + expect(a.keys, b.keys, reason: 'every serialized field must survive'); + for (final key in _intKeys) { expect(a[key], b[key], reason: key); } + for (final key in _doubleKeys) { + expect(a[key], isA(), reason: '$key must remain packed'); + final g = a[key] as Float64List, w = b[key] as Float64List; + expect(g.buffer.asUint64List(g.offsetInBytes, g.length), + w.buffer.asUint64List(w.offsetInBytes, w.length), reason: '$key IEEE-754 bits'); + } + expect(got.deviceFamily, oracle.deviceFamily); + expect(got.deviceIds, oracle.deviceIds); +} + +// Exercise the actual upstream prepare worker. Own every port/subscription and +// kill on every exit, including a failed handshake or malformed-page error. +Future> _worker(List> frames, + List> rr, {Map? page}) async { + final port = ReceivePort(); + Isolate? isolate; + StreamSubscription? subscription; + final ready = Completer(); + final done = Completer>(); + // Either future can fail before the caller reaches it. + unawaited(ready.future.then((_) {}, onError: (Object _, StackTrace _) {})); + unawaited(done.future.then((_) {}, onError: (Object _, StackTrace _) {})); + void fail(Object error) { + if (!ready.isCompleted) ready.completeError(error); + if (!done.isCompleted) done.completeError(error); + } + try { + subscription = port.listen((message) { + if (message is SendPort) { + if (!ready.isCompleted) ready.complete(message); + } else if (message is Map && message['type'] == 'result') { + if (!done.isCompleted) done.complete((message['payload'] as Map).cast()); + } else if (message is Map && message['type'] == 'error') { + fail(StateError('${message['error']}')); + } else if (message is List) { + fail(StateError('uncaught worker error: $message')); + } else if (message == null && !done.isCompleted) { + fail(StateError('worker exited before returning a payload')); + } + }); + isolate = await Isolate.spawn(derivationPrepareWorker, port.sendPort, + onError: port.sendPort, onExit: port.sendPort); + final worker = await ready.future.timeout(const Duration(seconds: 10)); + worker.send({'type': 'config', 'mode': 'substrate'}); + worker.send(page ?? {'type': 'page', 'frames': frames, 'rr': rr}); + worker.send({'type': 'finish'}); + return await done.future.timeout(const Duration(seconds: 10)); + } finally { + await subscription?.cancel(); + port.close(); + isolate?.kill(priority: Isolate.immediate); + } +} + +Map _bundle(Substrate sub) { + final day = prepareDerivationPayload(sub).days.first; + final d = day.daySub, s = day.sleepSub; + return deriveDayBundle(DayBundleInput( + date: day.date, dayTsSec: d.tsSec, dayHr: d.hr, + dayRrTsMs: d.rrTsMs, dayRrMs: d.rrMs, + sleepTsSec: s.tsSec, sleepHr: s.hr, sleepRrTsMs: s.rrTsMs, + sleepRrMs: s.rrMs, sleepSkinTemp: s.skinTemp, + sleepJson: day.sleepJson, hypnoStages: day.hypnoStages, + sleepOnsetSec: day.sleepOnsetSec, sleepOffsetSec: day.sleepOffsetSec, + profile: const Profile().toMap(), dayConfidence: day.confidence, + dayFlags: day.flags, deviceFamily: d.deviceFamily, + sleepSource: day.sleepSource, + ).toJson()); +} + +// Same generator as case 1 in two_device_fixture_test.dart, driven by the +// committed region spec. Only the documented gen4 regions feed that golden. +({List> frames, List> rr}) _fixture() { + final spec = jsonDecode(File('test/fixtures/two_device_day.json').readAsStringSync()) as Map; + int offset(String hhmmss) { + final p = hhmmss.split(':').map(int.parse).toList(); + return p[0] * 3600 + p[1] * 60 + p[2]; + } + final frames = >[], rr = >[]; + for (final region in spec['regions'] as List) { + final r = region as Map; + if (!(r['devices'] as List).contains('gen4')) continue; + final holes = {for (final h in (r['holes'] as List?) ?? []) offset(h as String)}; + for (var i = offset(r['start'] as String); i <= offset(r['end'] as String); i++) { + if (holes.contains(i)) continue; + final ts = _start + i; + frames.add({'rec_ts': ts, 'device_family': 'gen4', 'hr': 60 + ts % 7, + 'ax': 0.0, 'ay': 0.0, 'az': 1.0, 'skin_temp_raw': 30000 + ts % 11}); + rr.add({'rec_ts': ts, 'rr_ms': 1000 - ts % 5 * 10}); + } + } + return (frames: frames, rr: rr); +} + +void main() { + test('engine receives prepare-worker substrate through the transfer seam', () { + final code = stripCommentsAndStrings(File('lib/compute/derivation_engine.dart').readAsStringSync()); + expect(code, matches(RegExp(r'\bSubstrate\.fromTransfer\(\s*payload\s*\)')), + reason: 'the actual receiver still calls fromJson, which copies every column'); + }); + + for (final key in [..._intKeys, ..._doubleKeys]) { + test('transfer adopts correctly typed $key without an element copy', () { + final message = _columns(); + final got = Substrate.fromTransfer(message).toJson(); + expect(identical(got[key], message[key]), isTrue, + reason: '$key was copied on the receiving/UI isolate'); + }); + } + + test('transfer preserves every field and double bit, including band sleep state', () { + final message = _columns(); + final got = Substrate.fromTransfer(message); + _same(got, Substrate.fromJson(message)); + expect(got.bandSleepState, [1, 0, -1], reason: '0 is wake, -1 is absent'); + expect(got.deviceIds, {'secondary', ''}); + }); + + final variants = Function()>{ + 'empty': () => {}, + 'timestamps only': () => {'ts_sec': [_start, _start + 1]}, + 'missing timestamps': () => {..._columns()}..remove('ts_sec'), + 'empty optional columns': () => {..._columns(), 'hr': [], 'ax': Float64List(0), + 'step_count': [], 'hr_valid': [], 'band_sleep_state': []}, + 'short columns and sparse unpaired RR': () => {..._columns(), 'hr': [62], + 'az': Float64List.fromList([0.25]), 'step_count': [1], + 'hr_valid': [1], 'band_sleep_state': [0], + 'rr_ms': Float64List.fromList([850])}, + 'null provenance': () => {..._columns(), 'device_family': null, 'device_ids': null}, + 'loose numeric columns': () => { + for (final e in _columns().entries) + e.key: e.value is List ? List.from(e.value as List) : e.value, + }, + 'mixed integer and double numbers': () => {'ts_sec': [_start.toDouble(), _start + 1], + 'hr': [60.0, 61], 'rr_ms': [850, 900.5], + 'rr_ts_ms': [1577923200123, 1577923201000.5], + 'band_sleep_state': [0.0, 1]}, + 'wide integers': () => {..._columns(), 'ts_sec': [1 << 40, (1 << 40) + 1, (1 << 40) + 2], + 'step_count': [1 << 40, -(1 << 40), 0]}, + }; + for (final variant in variants.entries) { + test('transfer defaults equal fromJson: ${variant.key}', () { + final m = variant.value(); + _same(Substrate.fromTransfer(m), Substrate.fromJson(m)); + }); + } + + test('GUARD constructor optional arrays remain empty; decoder defaults remain absent', () { + final direct = Substrate(tsSec: [_start], hr: [60], rrTsMs: [], rrMs: [], + ax: [0], ay: [0], az: [1], spo2Red: [0], spo2Ir: [0], skinTemp: [0], skinContact: [0]); + expect(direct.stepCount, isEmpty); + expect(direct.hrValid, isEmpty); + expect(direct.bandSleepState, isEmpty); + final decoded = Substrate.fromTransfer({'ts_sec': [_start]}); + expect(decoded.stepCount, [-1]); + expect(decoded.hrValid, [-1]); + expect(decoded.bandSleepState, [-1]); + expect(decoded.hr, [0]); + expect(decoded.ax, [0.0]); + expect(decoded.rrMs, isEmpty); + }); + + test('actual prepare worker emits columnar typed doubles and preserves band sleep', () async { + final frames = [for (var i = 0; i < 3; i++) { + 'rec_ts': _start + i, 'hr': 60 + i, 'device_family': 'gen5', + 'ax': 0.25, 'ay': -0.0, 'az': 1.0, 'step_count': 20 + i, + 'hr_valid': i % 2, 'band_sleep_state': [1, 0, -1][i], + 'device_id': '', 'skin_temp_raw': 30000 + i, + }]; + final rr = [{'rec_ts': _start, 'rr_ms': 850, + 'rr_ts_ms': _start * 1000, 'beat_ts_ms': _start * 1000 + 123}]; + final message = await _worker(frames, rr); + for (final key in _doubleKeys) { expect(message[key], isA(), reason: key); } + for (final key in _intKeys) { expect(message[key], isA>(), reason: key); } + expect(message.values.any((v) => v is List && v.any((e) => e is Map)), isFalse, + reason: 'worker substrate contains columns, never per-row maps'); + final got = Substrate.fromTransfer(message); + _same(got, substrateFromDecodedPage(frames, rr)); + expect(got.bandSleepState, [1, 0, -1]); + expect(got.rrTsMs, [_start * 1000.0 + 123]); + }); + + test('actual worker result adopts its received arrays', () async { + final message = await _worker([{'rec_ts': _start, 'hr': 60, 'az': 1.0}], []); + final adopted = Substrate.fromTransfer(message).toJson(); + for (final key in [..._intKeys, ..._doubleKeys]) { + expect(identical(adopted[key], message[key]), isTrue, reason: '$key copied after worker receive'); + } + }); + + test('a worker error does not prevent the next transfer', () async { + await expectLater(_worker([], [], page: {'type': 'page', 'frames': 'malformed transport'}), throwsStateError); + final good = await _worker([{'rec_ts': _start, 'hr': 60}], []); + expect(Substrate.fromTransfer(good).hr, [60]); + }); + + test('GUARD existing golden is byte-equal with transfer off, on, and across the worker', () async { + // Same timezone condition as the existing golden test; no regeneration. + if (DateTime.fromMillisecondsSinceEpoch(_start * 1000).timeZoneOffset != Duration.zero) { + markTestSkipped('committed golden is UTC; run this test with TZ=UTC'); + return; + } + expect(kAlgoVersion, 119, reason: 'ED-2a changes no output'); + final fixture = _fixture(); + final baseline = substrateFromDecodedPage(fixture.frames, fixture.rr); + final transferred = Substrate.fromTransfer(baseline.toJson()); + final workerMessage = await _worker(fixture.frames, fixture.rr); + final acrossWorker = Substrate.fromTransfer(workerMessage); + _same(transferred, baseline); + _same(acrossWorker, baseline); + final golden = jsonDecode(File('test/fixtures/two_device_day_expected.json').readAsStringSync()) as Map; + final expected = jsonEncode(golden['single_device']); + expect(jsonEncode(_bundle(baseline)), expected, reason: 'transfer off'); + expect(jsonEncode(_bundle(transferred)), expected, reason: 'transfer on'); + expect(jsonEncode(_bundle(acrossWorker)), expected, reason: 'actual worker round trip'); + }, timeout: const Timeout(Duration(minutes: 2))); +} diff --git a/test/guards/db_parts_test.dart b/test/guards/db_parts_test.dart new file mode 100644 index 000000000..86e1f6e45 --- /dev/null +++ b/test/guards/db_parts_test.dart @@ -0,0 +1,62 @@ +import 'dart:io'; + +import 'package:analyzer/dart/analysis/utilities.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:flutter_test/flutter_test.dart'; + +class _ExecuteCalls extends RecursiveAstVisitor { + final arguments = []; + + @override + void visitMethodInvocation(MethodInvocation node) { + if (node.methodName.name == 'execute') { + arguments.add(node.argumentList.arguments.first); + } + super.visitMethodInvocation(node); + } +} + +void main() { + final parts = Directory('lib/data').listSync().whereType() + .where((f) => f.path.endsWith('.dart') && + f.readAsStringSync().contains("part of 'db.dart';")).toList(); + + test('DB parts contain only _ldb functions', () { + expect(parts.map((f) => f.path), contains('lib/data/db_input_revisions.dart')); + for (final file in parts) { + final unit = parseString(content: file.readAsStringSync()).unit; + expect(unit.declarations, isNotEmpty, reason: file.path); + for (final declaration in unit.declarations) { + expect(declaration, isA(), reason: file.path); + expect((declaration as FunctionDeclaration).name.lexeme, + startsWith('_ldb'), reason: file.path); + } + } + }); + + test('DB part execute calls carry literal SQL', () { + var calls = 0; + for (final file in parts) { + final visitor = _ExecuteCalls(); + parseString(content: file.readAsStringSync()).unit.accept(visitor); + calls += visitor.arguments.length; + for (final argument in visitor.arguments) { + expect(argument, isA(), reason: '${file.path}: $argument'); + } + } + expect(calls, greaterThanOrEqualTo(9)); + // Positive control: a variable SQL argument would fail the predicate. + final probe = _ExecuteCalls(); + parseString(content: 'void probe(dynamic db, String sql) { db.execute(sql); }') + .unit.accept(probe); + expect(probe.arguments.single, isNot(isA())); + }); + + test('DB parts stay below 6000 source lines', () { + for (final file in parts) { + expect(file.readAsLinesSync().length, lessThanOrEqualTo(6000), + reason: file.path); + } + }); +} diff --git a/test/support/ed2a_recalc_fixture.dart b/test/support/ed2a_recalc_fixture.dart new file mode 100644 index 000000000..0f0dfb125 --- /dev/null +++ b/test/support/ed2a_recalc_fixture.dart @@ -0,0 +1,184 @@ +// Shared ED-2a inputs. No cache or invalidation policy lives in this harness. +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:ffi/ffi.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +const edProfile = Profile(ageYears: 35, weightKg: 75, heightCm: 178, + sex: 'male', restingHrManual: 54); +const edDay = '2020-01-02'; +final edStart = DateTime(2020, 1, 2, 10).millisecondsSinceEpoch ~/ 1000; + +void edInit() { + TestWidgetsFlutterBinding.ensureInitialized(); + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; +} + +Future edDrop() async { + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + LocalDb.sessionSensorSources = {}; + LocalDb.sessionWearableSource = null; + DerivationEngine.debugSecondHalfHook = null; + DerivationEngine.debugRunning = false; +} + +Future edFresh(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + await edDrop(); + SharedPreferences.setMockInitialValues({}); + await LocalDb.instance; +} + +Future edRows(int from, int to, {int hr = 170, String id = '', + String family = 'gen4', String? source, double az = 1.3}) async { + final db = await LocalDb.instance; + final batch = db.batch(); + for (var ts = from; ts < to; ts++) { + batch.insert('decoded_onehz', {'device_id': id, 'ts_ms': ts * 1000, + 'rec_ts': ts, 'counter': ts, 'hr': hr, 'ax': 0.0, 'ay': 0.0, + 'az': az, 'device_family': family, 'source': source}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await batch.commit(noResult: true); +} + +Future edSession({String id = 's1', String status = 'done', + int? start, int? end}) => LocalDb.putSession({'id': id, 'type': 'run', + 'status': status, 'start_ts': start ?? edStart, + 'end_ts': end ?? edStart + 1100, 'source': 'manual', + 'created_at': edStart * 1000}); + +// The committed two_device_day.json region table and formulas, also used by +// two_device_fixture_test.dart. Keep the fixture intact for exactness laws. +Future edExistingFixture() async { + final spec = jsonDecode(File('test/fixtures/two_device_day.json').readAsStringSync()) as Map; + const start = 1577923200; + int offset(String time) { + final c = time.split(':').map(int.parse).toList(); + return c[0] * 3600 + c[1] * 60 + c[2]; + } + final db = await LocalDb.instance; + final b = db.batch(); + for (final region in spec['regions'] as List) { + final r = region as Map; + final holes = {for (final h in (r['holes'] as List?) ?? []) offset(h as String)}; + for (var i = offset(r['start'] as String); i <= offset(r['end'] as String); i++) { + if (holes.contains(i)) continue; + final ts = start + i; + for (final family in r['devices'] as List) { + final primary = family == 'gen4'; + final id = primary ? '' : 'oura-a1b2c3d4'; + b.insert('decoded_onehz', {'device_id': id, 'ts_ms': ts * 1000, + 'rec_ts': ts, 'counter': ts, 'device_family': family, + 'hr': primary ? 60 + ts % 7 : null, + 'ax': primary ? 0.0 : null, 'ay': primary ? 0.0 : null, + 'az': primary ? 1.0 : null, + 'skin_temp_raw': primary ? 30000 + ts % 11 : null, + 'skin_temp_c': primary ? null : 33.0 + ts % 9 / 10}, + conflictAlgorithm: ConflictAlgorithm.replace); + if (primary) { + b.insert('decoded_rr', {'device_id': '', 'ts_ms': ts * 1000, + 'rec_ts': ts, 'rr_ts_ms': ts * 1000, 'beat_index': 0, 'rr_ms': 1000 - ts % 5 * 10}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + } + } + } + await b.commit(noResult: true); +} + +Map edBundle(PreparedDerivationDay day) { + final d = day.daySub, s = day.sleepSub; + return deriveDayBundle(DayBundleInput(date: day.date, + dayTsSec: d.tsSec, dayHr: d.hr, dayRrTsMs: d.rrTsMs, dayRrMs: d.rrMs, + sleepTsSec: s.tsSec, sleepHr: s.hr, sleepRrTsMs: s.rrTsMs, + sleepRrMs: s.rrMs, sleepSkinTemp: s.skinTemp, sleepJson: day.sleepJson, + hypnoStages: day.hypnoStages, sleepOnsetSec: day.sleepOnsetSec, + sleepOffsetSec: day.sleepOffsetSec, profile: edProfile.toMap(), + dayConfidence: day.confidence, dayFlags: day.flags, + deviceFamily: d.deviceFamily, sleepSource: day.sleepSource).toJson()); +} + +Future edPreparedOutput(DerivationEngine engine) async { + final prepared = await engine.debugPrepareDay(edDay); + expect(prepared, isNotNull); + expect(prepared!.daySub.isEmpty, isFalse, reason: 'fixture must reach compute'); + return jsonEncode({'prepared': prepared.toJson(), 'bundle': edBundle(prepared), + 'profile': (await LocalDb.baseline('sleep_user_profile'))?['payload_json']}); +} + +// Strip only operational timestamps, never metric values, absence, notes, +// confidence, profile state or version. JSON property order stays observable. +Future edStored(String day) async { + final db = await LocalDb.instance; + final rows = await db.query('day_result', where: 'day_id = ?', + whereArgs: [day], orderBy: 'algo_version'); + expect(rows, isNotEmpty, reason: 'the real derive must have persisted a day'); + return jsonEncode({'day_result': [for (final row in rows) { + for (final e in row.entries) + if (e.key == 'payload_json') e.key: jsonEncode( + (jsonDecode(e.value as String) as Map)..remove('computed_at')) + else if (e.key != 'computed_at') e.key: e.value, + }], 'metric_series': await db.query('metric_series', where: 'date = ?', + whereArgs: [day], orderBy: 'key'), + 'profile': (await LocalDb.baseline('sleep_user_profile'))?['payload_json']}); +} + +Future edOwnership({String id = 'other', + List signals = const ['hr1Hz', 'rrIntervals']}) async { + final db = await LocalDb.instance; + for (final signal in signals) { + await db.insert('device_coverage', {'device_id': id, 'signal': signal, + 'start_ts': edStart, 'end_ts': edStart + 600}, + conflictAlgorithm: ConflictAlgorithm.replace); + await db.insert('signal_priority', {'signal': signal, 'device_id': id, + 'rank': 0, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + await db.insert('signal_priority', {'signal': signal, 'device_id': '', + 'rank': 1, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +String edLabel(int second) => dayLabelOf(DateTime.fromMillisecondsSinceEpoch(second * 1000)); +Substrate edSmallSub({int? start, int count = 1200}) => Substrate.fromJson({ + 'ts_sec': [for (var i = 0; i < count; i++) (start ?? edStart) + i], + 'hr': List.filled(count, 60), + 'ax': List.filled(count, 0), 'ay': List.filled(count, 0), + 'az': List.filled(count, 1), 'device_family': 'gen4', +}); + +// Same libc seam used by the fork's zone regressions. No process spawning or +// Dart clock substitution: production DateTime resolves each instant's zone. +typedef _Setenv = Int32 Function(Pointer, Pointer, Int32); +typedef _SetenvDart = int Function(Pointer, Pointer, int); +void edSetZone(String zone) { + final libc = DynamicLibrary.process(); + final key = 'TZ'.toNativeUtf8(), value = zone.toNativeUtf8(); + try { + expect(libc.lookupFunction<_Setenv, _SetenvDart>('setenv')(key, value, 1), 0); + libc.lookupFunction('tzset')(); + } finally { calloc.free(key); calloc.free(value); } +} + +Future edPriorityOrder(List ids) async { + final db = await LocalDb.instance; + for (var rank = 0; rank < ids.length; rank++) { + await db.insert('signal_priority', {'signal': 'hr1Hz', 'device_id': ids[rank], + 'rank': rank, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + } +}