From 748ff8bf229c79e72169bd8f36dd282236a8fe42 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 05:36:49 -0600 Subject: [PATCH 1/5] storage: input revisions (rung 59) and typed substrate transfer - Schema 59 adds input_rev(bucket, rev) with insert, update and delete triggers on decoded_onehz and decoded_rr. A bucket is rec_ts / 900; every write to a bucket advances its revision, including an identical INSERT OR REPLACE. No backfill: existing input stays at revision 0. - inputDayFingerprint(dayId) returns SUM and MAX of the day's bucket revisions plus a store generation that advances on open, after a committed wipe, and after an import that committed a page. - Substrate.fromTransfer adopts the worker's typed columns instead of copying them; the derivation engine receives its prepared substrate through it. - input_rev is copied whole in the diagnostic slice. kAlgoVersion is unchanged (113): no derived number moves. --- lib/compute/derivation_engine.dart | 2 +- lib/compute/substrate.dart | 26 +- lib/data/db.dart | 88 +++- lib/data/db_input_revisions.dart | 73 ++++ lib/data/recent_slice.dart | 1 + lib/data/restore_classification.dart | 78 ++++ pubspec.lock | 2 +- pubspec.yaml | 2 + test/ed2a_input_revision_test.dart | 556 +++++++++++++++++++++++++ test/ed2a_substrate_transfer_test.dart | 258 ++++++++++++ test/guards/db_parts_test.dart | 62 +++ 11 files changed, 1134 insertions(+), 14 deletions(-) create mode 100644 lib/data/db_input_revisions.dart create mode 100644 lib/data/restore_classification.dart create mode 100644 test/ed2a_input_revision_test.dart create mode 100644 test/ed2a_substrate_transfer_test.dart create mode 100644 test/guards/db_parts_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 2610e7726..e3032aaec 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -4706,7 +4706,7 @@ class DerivationEngine { if (kind == 'substrate') { final payload = ((message['payload'] as Map?) ?? const {}) .cast(); - if (!result.isCompleted) result.complete(Substrate.fromJson(payload)); + if (!result.isCompleted) result.complete(Substrate.fromTransfer(payload)); } return; } diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 08d8664f3..4b153d870 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -621,15 +621,27 @@ class Substrate { 'device_ids': deviceIds.toList(), }; - static Substrate fromJson(Map m) { - List ints(Map m, String k) => - ((m[k] as List?) ?? const []).map((e) => (e as num).toInt()).toList(); - // Straight into a Float64List — see [_packed]. This runs on the RECEIVING - // (main) isolate for every substrate handed back from a worker, so the - // growable intermediate was ~400 000 boxes allocated and immediately - // thrown away, on the UI thread. + /// Adopt the columns received from a worker. Packing/copies happen in the + /// worker or isolate transport; correctly typed arrays need no element walk + /// here. Loose numeric lists retain the same normalization as [fromJson]. + static Substrate fromTransfer(Map m) => + _fromColumns(m, adopt: true); + + static Substrate fromJson(Map m) => + _fromColumns(m, adopt: false); + + static Substrate _fromColumns(Map m, {required bool adopt}) { + List ints(Map m, String k) { + final src = m[k]; + if (adopt && src is List) return src; + return ((src as List?) ?? const []) + .map((e) => (e as num).toInt()).toList(); + } + // Convert loose lists straight into packed doubles, without a boxed + // intermediate. The worker already packs these through the constructor. Float64List dbls(String k) { final src = (m[k] as List?) ?? const []; + if (adopt && src is Float64List) return src; final out = Float64List(src.length); for (var i = 0; i < src.length; i++) { out[i] = (src[i] as num).toDouble(); diff --git a/lib/data/db.dart b/lib/data/db.dart index 578df4b9c..4715651a0 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -64,6 +64,8 @@ import 'observation.dart'; import 'series_codec.dart'; import 'substrate_archive.dart'; +part 'db_input_revisions.dart'; + /// The outcome of a database rebuild: why the old file would not open, where it /// was parked, and how many rows came back per table. typedef DbRebuild = ({ @@ -479,7 +481,39 @@ class LocalDb { /// pass it: sqflite throws `ArgumentError('onCreate must be null if no /// version is specified')` BEFORE opening anything when `onCreate` is given /// without `version` (sqflite_common database_mixin.dart). - static const int schemaVersion = 58; + static const int schemaVersion = 59; + + /// In-memory cache identity, advanced after open, wipe or committed import. + /// It is local to this isolate; no cache survives a process restart. + static int _storeGeneration = 0; + static int get storeGeneration => _storeGeneration; + + static void _markStoreReplaced() => _storeGeneration++; + + /// Revisions over every absolute 15-minute bucket intersecting the local + /// calendar day. Legacy/missing buckets read zero without a raw-row backfill. + /// RR-only days participate, and all devices share these conservative buckets. + static Future<({int revisionSum, int revisionMax, int generation})> + inputDayFingerprint(String dayId) async { + final (lo, hi) = _localDayWindow(dayId); + final db = await instance; + final generation = storeGeneration; + final rows = await db.rawQuery( + 'SELECT COALESCE(SUM(rev), 0) AS revision_sum, ' + 'COALESCE(MAX(rev), 0) AS revision_max FROM input_rev ' + 'WHERE bucket >= ? AND bucket <= ?', + [lo ~/ 900, (hi - 1) ~/ 900], + ); + return ( + revisionSum: (rows.single['revision_sum'] as num).toInt(), + revisionMax: (rows.single['revision_max'] as num).toInt(), + generation: generation, + ); + } + + @visibleForTesting + static Future repairOpenSchemaForTest(Database db) => + _repairOpenSchema(db); /// SQLite caps host parameters per statement (`SQLITE_MAX_VARIABLE_NUMBER` — /// only 999 on the builds shipped with older Android/iOS). Any `IN (?, ?, …)` @@ -540,7 +574,7 @@ class LocalDb { static Future _open() async { final dir = await getDatabasesPath(); final path = p.join(dir, dbName); - return openDatabase( + final opened = await openDatabase( path, onConfigure: (db) async { // WRITE PERFORMANCE. The default rollback journal (journal_mode=delete) @@ -1249,6 +1283,11 @@ class LocalDb { // A band's own hypnogram (`vendor_staged`). New table only. await _createVendorSleepEpoch(db); } + if (oldV < 59) { + // Additive revision counters only: no decoded scan or backfill. + // Attach after any earlier rung that rebuilds decoded tables. + await _createInputRev(db); + } }, onOpen: (db) async { await _repairOpenSchema(db); @@ -1264,6 +1303,8 @@ class LocalDb { }, version: schemaVersion, ); + _markStoreReplaced(); + return opened; } static Future _repairOpenSchema(Database db) async { @@ -1366,12 +1407,19 @@ class LocalDb { await _addColumnIfMissing(db, 'device', 'battery_ts', 'INTEGER'); await _createStepCalibration(db); await _ensureLiveStepCalibrationCounterVersion(db); + // A decoded-table rebuild above drops its triggers; attach them last. + await _createInputRev(db); // Views LAST — they depend on metric_series / day_result / baselines / sessions // / notifications all existing. DROP+CREATE so a shape change takes effect. await _ensureCoachViews(db); await _dropRawStore(db); } + /// One revision counter per absolute 15-minute input bucket, shared by the + /// canonical 1 Hz rows and sparse beats. Triggers cover every write path and + /// roll back with the input transaction. No raw rows are read at creation. + static Future _createInputRev(Database db) => _ldbCreateInputRev(db); + /// Older WHOOP CSV imports filed the export's absolute skin temp (°C) under /// `scalars.skin_temp_z`, which every reader treats as SDs off baseline, so /// 33 °C read as +33 SD in the cross-day temp inputs and the skin temp card. @@ -11199,7 +11247,9 @@ class LocalDb { final db = await instance; final rows = await db.rawQuery( "SELECT name FROM sqlite_master WHERE type = 'table' " - "AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata'", + "AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata' " + // Clear revisions after the decoded DELETE triggers have finished. + "ORDER BY name = 'input_rev'", ); var deleted = 0; await db.transaction((txn) async { @@ -11208,6 +11258,7 @@ class LocalDb { if (t is String) deleted += await txn.delete(t); } }); + _markStoreReplaced(); return deleted; } @@ -11588,7 +11639,9 @@ class LocalDb { /// file saved mid-move). Every row is rebuilt through [rawArchiveRow], so a /// frame already here, or in the backup twice, lands once. Paged on rowid, /// like every other table the merge reads. Returns rows offered. - static Future _mergeRawArchive(Database src, Database db) async { + static Future _mergeRawArchive(Database src, Database db, { + required void Function() onPageCommitted, + }) async { var copied = 0; Future pageFrom(String table, String hexSql) async { if (await _objectType(src, table) != 'table') return; @@ -11627,6 +11680,7 @@ class LocalDb { copied++; } await batch.commit(noResult: true); + onPageCommitted(); } } @@ -11661,6 +11715,27 @@ class LocalDb { String path, { List? only, bool tolerant = false, + }) async { + var pageCommitted = false; + try { + return await _mergeFromDbFileBody(path, + only: only, + tolerant: tolerant, + onPageCommitted: () => pageCommitted = true, + ); + } finally { + // A later page, source close or follow-up write can fail after a durable + // replacement. Invalidate on every such exit, but never on a rolled-back + // first page. Keep the flag local so concurrent merges cannot share it. + if (pageCommitted) _markStoreReplaced(); + } + } + + static Future> _mergeFromDbFileBody( + String path, { + List? only, + bool tolerant = false, + required void Function() onPageCommitted, }) async { final db = await instance; final src = await openDatabase(path, readOnly: true); @@ -11979,6 +12054,7 @@ class LocalDb { await txn.execute('DROP TABLE temp._restore_rr_carried'); } }); + onPageCommitted(); lower = upper; } failed = false; @@ -12051,7 +12127,8 @@ class LocalDb { for (final t in (only ?? tables)) { if (t == 'raw_archive') { try { - counts[t] = await _mergeRawArchive(src, db); + counts[t] = await _mergeRawArchive(src, db, + onPageCommitted: onPageCommitted); } catch (_) { if (!tolerant) rethrow; counts[t] = 0; @@ -12562,6 +12639,7 @@ class LocalDb { } await flush(); }); + onPageCommitted(); // Advance past the last row this page actually delivered. Read the // cursor BEFORE dropping the page, and stop on a short page rather // than issuing one more query to discover the end. diff --git a/lib/data/db_input_revisions.dart b/lib/data/db_input_revisions.dart new file mode 100644 index 000000000..251cebcb3 --- /dev/null +++ b/lib/data/db_input_revisions.dart @@ -0,0 +1,73 @@ +// Revision DDL executes here so schema creation stays additive and cheap. +part of 'db.dart'; + +Future _ldbCreateInputRev(Database db) async { + await db.execute( + 'CREATE TABLE IF NOT EXISTS input_rev (' + 'bucket INTEGER PRIMARY KEY, rev INTEGER NOT NULL)', + ); + if ((await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'decoded_onehz'", + )).isNotEmpty) { + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_ai + AFTER INSERT ON decoded_onehz WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_au + AFTER UPDATE ON decoded_onehz WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_au_old + AFTER UPDATE ON decoded_onehz WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 AND (NEW.rec_ts IS NULL OR NEW.rec_ts <= 0 OR NEW.rec_ts / 900 != OLD.rec_ts / 900) + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_onehz_rev_ad + AFTER DELETE ON decoded_onehz WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + } + if ((await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'decoded_rr'", + )).isNotEmpty) { + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_ai + AFTER INSERT ON decoded_rr WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_au + AFTER UPDATE ON decoded_rr WHEN NEW.rec_ts IS NOT NULL AND NEW.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT NEW.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = NEW.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = NEW.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_au_old + AFTER UPDATE ON decoded_rr WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 AND (NEW.rec_ts IS NULL OR NEW.rec_ts <= 0 OR NEW.rec_ts / 900 != OLD.rec_ts / 900) + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS decoded_rr_rev_ad + AFTER DELETE ON decoded_rr WHEN OLD.rec_ts IS NOT NULL AND OLD.rec_ts > 0 + BEGIN + INSERT INTO input_rev(bucket, rev) SELECT OLD.rec_ts / 900, 0 + WHERE NOT EXISTS (SELECT 1 FROM input_rev WHERE bucket = OLD.rec_ts / 900); + UPDATE input_rev SET rev = rev + 1 WHERE bucket = OLD.rec_ts / 900; + END'''); + } +} diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index a97a59523..4ac058cae 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -82,6 +82,7 @@ const Map kSliceTables = { 'baselines': (SliceKeep.all, null, 'personal baselines'), 'compute_freshness': (SliceKeep.all, null, 'compute bookkeeping'), 'derive_fingerprint': (SliceKeep.all, null, 'which days changed since they were last scored'), + 'input_rev': (SliceKeep.all, null, 'per-bucket input revisions'), 'sleep_nap': (SliceKeep.all, null, 'naps'), 'sleep_override': (SliceKeep.all, null, 'sleep times the user edited'), // Workouts and activity: one row each. diff --git a/lib/data/restore_classification.dart b/lib/data/restore_classification.dart new file mode 100644 index 000000000..1e6167745 --- /dev/null +++ b/lib/data/restore_classification.dart @@ -0,0 +1,78 @@ +// Restore policy for each schema table. Revision counters are local, +// rebuildable bookkeeping: imported inputs fire the destination triggers. +enum RestoreClass { merge, localOnly, rebuildable } + +const kRestoreClassification = { + 'decoded_onehz': RestoreClass.merge, + 'decoded_rr': RestoreClass.merge, + 'samples': RestoreClass.merge, + 'raw_archive': RestoreClass.merge, + 'raw_archive_frames': RestoreClass.localOnly, + 'raw_archive_legacy': RestoreClass.localOnly, + 'events': RestoreClass.merge, + 'band_events': RestoreClass.merge, + 'band_battery': RestoreClass.merge, + 'band_backlog': RestoreClass.localOnly, + 'sync_ledger': RestoreClass.localOnly, + 'sync_quarantine': RestoreClass.localOnly, + 'external_hr': RestoreClass.merge, + 'imported_measurement': RestoreClass.merge, + 'observation': RestoreClass.merge, + 'vendor_sleep_epoch': RestoreClass.merge, + 'live_coverage': RestoreClass.merge, + 'device_coverage': RestoreClass.merge, + 'workout_route': RestoreClass.merge, + 'live_workout_tally': RestoreClass.localOnly, + 'compute_jobs': RestoreClass.localOnly, + 'ecg_reading': RestoreClass.merge, + 'ecg_reading_packet': RestoreClass.merge, + 'ecg_raw_packet': RestoreClass.merge, + 'sleep_session_candidates': RestoreClass.localOnly, + 'wake_day_features': RestoreClass.localOnly, + 'substrate_archive': RestoreClass.merge, + 'day_result': RestoreClass.merge, + 'derived_day': RestoreClass.localOnly, + 'metric_series': RestoreClass.merge, + 'metric_series_version': RestoreClass.merge, + 'metric_method': RestoreClass.merge, + 'baselines': RestoreClass.merge, + 'compute_freshness': RestoreClass.localOnly, + 'derive_fingerprint': RestoreClass.rebuildable, + 'input_rev': RestoreClass.rebuildable, + 'sleep_nap': RestoreClass.merge, + 'sleep_override': RestoreClass.merge, + 'sessions': RestoreClass.merge, + 'session_sensor': RestoreClass.merge, + 'session_score_mask': RestoreClass.localOnly, + 'workout_split': RestoreClass.merge, + 'strength_set': RestoreClass.merge, + 'workout_suggestions': RestoreClass.merge, + 'imported_workout': RestoreClass.merge, + 'activity_suggestions': RestoreClass.merge, + 'activity_review_days': RestoreClass.merge, + 'activity_review_meta': RestoreClass.merge, + 'breathing_session': RestoreClass.merge, + 'exercise_def': RestoreClass.merge, + 'journal': RestoreClass.merge, + 'journal_metric': RestoreClass.merge, + 'journal_field_def': RestoreClass.merge, + 'cycle_log': RestoreClass.merge, + 'cycle_symptom': RestoreClass.merge, + 'lab_result': RestoreClass.merge, + 'lab_marker_def': RestoreClass.merge, + 'med_def': RestoreClass.merge, + 'med_dose': RestoreClass.merge, + 'food_def': RestoreClass.merge, + 'food_entry': RestoreClass.merge, + 'device': RestoreClass.merge, + 'signal_priority': RestoreClass.merge, + 'step_calibration': RestoreClass.merge, + 'step_calibration_day': RestoreClass.merge, + 'live_step_calibration': RestoreClass.merge, + 'live_step_calibration_run': RestoreClass.merge, + 'sync_cursor': RestoreClass.merge, + 'alarm_schedule': RestoreClass.localOnly, + 'notifications': RestoreClass.merge, + 'notif_fired': RestoreClass.localOnly, + 'notif_slots': RestoreClass.localOnly, +}; diff --git a/pubspec.lock b/pubspec.lock index eee7705f5..45d6b23b5 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -18,7 +18,7 @@ packages: source: hosted version: "1.3.77" analyzer: - dependency: transitive + dependency: "direct dev" description: name: analyzer sha256: de7148ed2fcec579b19f122c1800933dfa028f6d9fd38a152b04b1516cec120b diff --git a/pubspec.yaml b/pubspec.yaml index c20440bfd..209434e25 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -670,6 +670,8 @@ dev_dependencies: flutter_test: sdk: flutter flutter_lints: ^6.0.0 + # DB part guards inspect declarations and SQL arguments through the Dart AST. + analyzer: ^10.0.1 test: ^1.25.8 # no_runtime_font_fetch_test parses pubspec.yaml to assert every family the # type scale uses is bundled rather than fetched at runtime. diff --git a/test/ed2a_input_revision_test.dart b/test/ed2a_input_revision_test.dart new file mode 100644 index 000000000..442c3b03c --- /dev/null +++ b/test/ed2a_input_revision_test.dart @@ -0,0 +1,556 @@ +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:openstrap_edge/data/restore_classification.dart'; +// ED-2a E4 RED contract. Run with flutter test --concurrency=1. +// Only the input portion of fork 561f2838 and generation portions of +// 74c0b37b/30cd944e apply. No store_rev, row_rev, scheduler or checkpoints. +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/models.dart'; + +const _inputs = ['decoded_onehz', 'decoded_rr']; +final _midnight = DateTime(2026, 10, 20); +final _lo = _midnight.millisecondsSinceEpoch ~/ 1000; +final _day = dayLabelOf(_midnight); + +Map _row(String table, int ts, {String device = ''}) => { + 'device_id': device, 'ts_ms': ts * 1000, 'rec_ts': ts, + if (table == 'decoded_onehz') ...{'counter': 1, 'hr': 61}, + if (table == 'decoded_rr') ...{ + 'beat_index': 0, 'rr_ts_ms': ts * 1000, 'rr_ms': 850, + }, +}; + +Future _hasRevisions(Database db) async => (await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='input_rev'", +)).isNotEmpty; + +// Missing infrastructure is revision zero, so mutation tests fail an assertion +// about an unchanged revision, never a spurious 'no such table' exception. +Future _rev(Database db, int ts) async { + if (!await _hasRevisions(db)) return 0; + final rows = await db.query('input_rev', columns: ['rev'], + where: 'bucket = ?', whereArgs: [ts ~/ 900]); + return rows.isEmpty ? 0 : (rows.single['rev'] as num).toInt(); +} + +Future>> _revisionRows(Database db) async => + await _hasRevisions(db) ? db.query('input_rev', orderBy: 'bucket') : []; + +Future>> _legacyFingerprint(Database db) => + db.rawQuery('SELECT MAX(rec_ts) AS mx, COUNT(*) AS n FROM decoded_onehz ' + 'WHERE rec_ts >= ? AND rec_ts < ?', [_lo, _lo + 7200]); + +Future> _schema(Database db) async { + final out = {}; + final entries = await db.rawQuery( + "SELECT type, name, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' " + "AND name != 'android_metadata' ORDER BY type, name", + ); + for (final entry in entries) { + final name = entry['name'] as String; + // Compare column definitions rather than CREATE text: ALTER and CREATE + // paths have different whitespace and IF NOT EXISTS spelling. + out[name] = entry['type'] == 'table' + ? await db.rawQuery('PRAGMA table_info("$name")') + : (entry['sql'] as String?)?.replaceAll(RegExp(r'\s+'), ' ').trim(); + } + return out; +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + late Directory tmp; + late String livePath; + late String sourcePath; + late DatabaseFactory? previousFactory; + late String previousName; + final sql = []; + final legacyRows = {}; + final legacyColumns = {}; + + setUp(() async { + sqfliteFfiInit(); + previousFactory = databaseFactoryOrNull; + previousName = LocalDb.dbName; + await LocalDb.close(); + SharedPreferences.setMockInitialValues({}); + tmp = await Directory.systemTemp.createTemp('ed2a_rev_'); + livePath = p.join(tmp.path, 'live.db'); + sourcePath = p.join(tmp.path, 'source.db'); + // Public sqflite logger observes actual SQL, including nested batches. + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions( + type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? value) { + if (value is Map) { + if (value['sql'] case final String statement) sql.add(statement); + for (final v in value.values) { collect(v); } + } else if (value is List) { + for (final v in value) { collect(v); } + } + } + collect(event.arguments); + }, + )); + LocalDb.dbName = livePath; + LocalDb.lastRebuild = null; + sql.clear(); + }); + + tearDown(() async { + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + await databaseFactory.deleteDatabase(sourcePath); + databaseFactoryOrNull = previousFactory; + LocalDb.dbName = previousName; + await tmp.delete(recursive: true); + }); + + Future open() async { + final db = await LocalDb.instance; + expect(LocalDb.lastRebuild, isNull, + reason: 'a broken rung must not pass by quarantining the original DB'); + return db; + } + + Future seed58() async { + final db = await databaseFactory.openDatabase(livePath, + options: OpenDatabaseOptions(version: 58)); + // Frozen v58 decoded DDL is below. Other tables use the existing repair + // seam; remove E4 infrastructure before setting the old version and seeding. + for (final ddl in _v58DecodedDdl) { await db.execute(ddl); } + await LocalDb.repairOpenSchemaForTest(db); + final triggers = await db.rawQuery( + "SELECT name FROM sqlite_master WHERE type='trigger' " + "AND tbl_name IN ('decoded_onehz','decoded_rr')", + ); + for (final t in triggers) { await db.execute('DROP TRIGGER "${t['name']}"'); } + await db.execute('DROP TABLE IF EXISTS input_rev'); + await db.setVersion(58); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 3)); + await db.insert('decoded_rr', _row('decoded_rr', _lo + 3)); + for (final table in _inputs) { + legacyRows[table] = await db.query(table); + legacyColumns[table] = await db.rawQuery('PRAGMA table_info($table)'); + } + await db.close(); + } + + test('every table is classified for restore and diagnostic slices', () async { + await open(); + final tables = await LocalDb.tableNames(); + expect(tables.toSet().difference(kRestoreClassification.keys.toSet()), isEmpty); + expect(tables.toSet().difference(kSliceTables.keys.toSet()), isEmpty); + expect(kRestoreClassification['input_rev'], RestoreClass.rebuildable); + expect(LocalDb.restoreTablesForTest, isNot(contains('input_rev'))); + }); + + test('rung 59 is reserved for additive input revisions', () async { + expect(LocalDb.schemaVersion, 59, reason: 'upstream is still schema 58'); + final db = await open(); + expect(await db.getVersion(), 59); + expect(await _hasRevisions(db), isTrue); + final tables = await LocalDb.tableNames(); + expect(tables, isNot(contains('store_rev'))); + expect(tables, isNot(contains('row_rev'))); + }); + + test('58 to 59 preserves raw rows and creates all mutation triggers', () async { + await seed58(); + final db = await open(); + expect((await db.query('decoded_onehz')).single['hr'], 61); + expect((await db.query('decoded_rr')).single['rr_ms'], 850); + for (final table in _inputs) { + expect(await db.query(table), legacyRows[table], reason: '$table data unchanged'); + expect(await db.rawQuery('PRAGMA table_info($table)'), legacyColumns[table], + reason: 'rung 59 must not change raw columns/defaults'); + } + expect(await db.getVersion(), 59); + expect(await _hasRevisions(db), isTrue); + final triggers = await db.rawQuery( + "SELECT tbl_name, sql FROM sqlite_master WHERE type='trigger'", + ); + for (final table in _inputs) { + final body = triggers.where((t) => t['tbl_name'] == table) + .map((t) => (t['sql'] as String).toUpperCase()).join('\n'); + for (final op in ['INSERT', 'UPDATE', 'DELETE']) { + expect(body, contains('AFTER $op'), reason: '$table $op revision trigger'); + } + } + expect(await _revisionRows(db), isEmpty, + reason: 'legacy rows must not be backfilled into input_rev'); + expect(await LocalDb.inputDayFingerprint(_day), + (revisionSum: 0, revisionMax: 0, generation: LocalDb.storeGeneration)); + }); + + test('upgrade and every-open repair do not scan or rewrite raw input', () async { + await seed58(); + sql.clear(); + final db = await open(); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + await open(); + final openingSql = List.of(sql); + expect(openingSql, isNotEmpty, reason: 'SQL observation must be active'); + final dataAccess = RegExp( + r'\b(?:FROM|JOIN|UPDATE|INTO|DELETE\s+FROM)\s+["`\[]?' + r'(?:decoded_onehz|decoded_rr|samples|raw_archive)\b', + caseSensitive: false, + ); + expect(openingSql.where((s) { + final ddl = s.trimLeft().toUpperCase(); + // Declaring a trigger/view does not execute its SELECT/UPDATE body. + if (ddl.startsWith('CREATE TRIGGER') || ddl.startsWith('CREATE VIEW')) return false; + // Main probes whether legacy decoded input exists before metadata repair. + // This bounded probe is unrelated to revision creation or backfill. + if (s == 'SELECT 1 FROM decoded_onehz LIMIT 1') return false; + return dataAccess.hasMatch(s); + }), + isEmpty, reason: 'opening may inspect schema, never scan/backfill raw rows'); + expect(await _revisionRows(await LocalDb.instance), isEmpty); + }); + + test('fresh and upgraded schema agree; repair and reopen are idempotent', () async { + await seed58(); + var db = await open(); + expect(await _hasRevisions(db), isTrue, reason: 'input_rev must exist first'); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 100)); + final schema = await _schema(db); + final revs = await _revisionRows(db); + final rows = await db.query('decoded_onehz', orderBy: 'rec_ts'); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.close(); + db = await open(); + expect(await _schema(db), schema); + expect(await _revisionRows(db), revs); + expect(await db.query('decoded_onehz', orderBy: 'rec_ts'), rows); + await LocalDb.close(); + await databaseFactory.deleteDatabase(livePath); + db = await open(); + expect(await _schema(db), schema, reason: 'onCreate and onUpgrade must agree'); + }); + + test('absent day and legacy bucket revisions read zero', () async { + await seed58(); + await open(); + for (final day in [_day, '1999-01-01']) { + expect(await LocalDb.inputDayFingerprint(day), + (revisionSum: 0, revisionMax: 0, generation: LocalDb.storeGeneration)); + } + }); + + for (final table in _inputs) { + for (final op in ['INSERT', 'REPLACE', 'UPDATE', 'DELETE']) { + test('$table $op advances only its 15-minute bucket', () async { + final db = await open(); + final ts = _lo + 930; + if (op != 'INSERT') { await db.insert(table, _row(table, ts)); } + final before = await _rev(db, ts); + final adjacent = await _rev(db, ts - 900); + switch (op) { + case 'INSERT': await db.insert(table, _row(table, ts)); + case 'REPLACE': + // Identical values still invalidate; repeat to catch revision reset + // under the outer statement's OR REPLACE conflict policy. + for (var i = 0; i < 3; i++) { + final last = await _rev(db, ts); + await db.insert(table, _row(table, ts), + conflictAlgorithm: ConflictAlgorithm.replace); + expect(await _rev(db, ts), greaterThan(last), + reason: '$table identical REPLACE must never reset the bucket'); + } + case 'UPDATE': + await db.update(table, table == 'decoded_onehz' ? {'hr': 62} : {'rr_ms': 900}); + case 'DELETE': await db.delete(table); + } + expect(await _rev(db, ts), greaterThan(before), reason: '$table $op was invisible'); + expect(await _rev(db, ts - 900), adjacent, reason: 'unrelated bucket changed'); + }); + } + + test('$table timestamp UPDATE invalidates both old and new buckets', () async { + final db = await open(); + final ts = _lo + 899; + await db.insert(table, _row(table, ts)); + final old = await _rev(db, ts), next = await _rev(db, ts + 1); + await db.update(table, {'rec_ts': ts + 1, 'ts_ms': (ts + 1) * 1000}); + expect(await _rev(db, ts), greaterThan(old), reason: 'old window lost a row'); + expect(await _rev(db, ts + 1), greaterThan(next), reason: 'new window gained a row'); + }); + + test('$table invalid timestamps do not create revision buckets', () async { + final db = await open(); + for (final ts in [0, -1]) { await db.insert(table, _row(table, ts)); } + expect(await _revisionRows(db), isEmpty); + }); + } + + test('identical newest-wins replacement defeats MAX/count but moves revision', () async { + final db = await open(); + final row = _row('decoded_onehz', _lo + 10); + await db.insert('decoded_onehz', row); + final old = await _legacyFingerprint(db); + final before = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_onehz', row, conflictAlgorithm: ConflictAlgorithm.replace); + expect(await _legacyFingerprint(db), old, reason: 'MAX/count blind spot reproduced'); + expect((await db.query('decoded_onehz')).length, 1); + expect((await LocalDb.inputDayFingerprint(_day)).revisionSum, + greaterThan(before.revisionSum), reason: 'identical REPLACE must invalidate cached input'); + }); + + test('late RR without any 1 Hz row changes the day fingerprint', () async { + final db = await open(); + final before = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_rr', _row('decoded_rr', _lo + 20)); + expect(await db.query('decoded_onehz'), isEmpty); + expect((await LocalDb.inputDayFingerprint(_day)).revisionSum, + greaterThan(before.revisionSum), reason: 'RR-only inputs must not require MAX(rec_ts)'); + }); + + for (final date in [DateTime(2026, 10, 20), DateTime(2026, 3, 8), DateTime(2026, 11, 1)]) { + test('fingerprint sums/maxes local day buckets for ${dayLabelOf(date)}', () async { + final db = await open(); + final lo = date.millisecondsSinceEpoch ~/ 1000; + final hi = DateTime(date.year, date.month, date.day + 1).millisecondsSinceEpoch ~/ 1000; + // Unequal bucket revisions pin SUM and MAX independently. Outside rows + // must not contribute, including precisely the following midnight. + for (final ts in [lo - 1, lo, lo + 900, hi - 1, hi]) { + await db.insert('decoded_onehz', _row('decoded_onehz', ts)); + } + await db.update('decoded_onehz', {'hr': 63}, where: 'rec_ts = ?', whereArgs: [lo]); + final fp = await LocalDb.inputDayFingerprint(dayLabelOf(date)); + expect(fp.revisionSum, 4, reason: 'two writes in first bucket, one in each of two others'); + expect(fp.revisionMax, 2); + expect(fp.generation, LocalDb.storeGeneration); + expect(await LocalDb.inputDayFingerprint(dayLabelOf(date)), fp, + reason: 'reading the fingerprint must be idempotent'); + }); + } + + test('rolled-back raw writes roll back revisions as well', () async { + final db = await open(); + final before = await LocalDb.inputDayFingerprint(_day); + await expectLater(db.transaction((txn) async { + await txn.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + await txn.insert('decoded_rr', _row('decoded_rr', _lo + 10)); + throw StateError('abort before commit'); + }), throwsStateError); + expect(await db.query('decoded_onehz'), isEmpty); + expect(await _revisionRows(db), isEmpty); + expect(await LocalDb.inputDayFingerprint(_day), before); + }); + + test('reopen advances generation without changing input revisions', () async { + var db = await open(); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + final before = await LocalDb.inputDayFingerprint(_day); + await LocalDb.close(); + db = await open(); + final after = await LocalDb.inputDayFingerprint(_day); + expect(after.revisionSum, before.revisionSum); + expect(after.revisionMax, before.revisionMax); + expect(after.generation, greaterThan(before.generation)); + expect(await db.query('decoded_onehz'), hasLength(1)); + }); + + test('wipe increments generation even when the revision key returns to zero', () async { + final db = await open(); + final before = LocalDb.storeGeneration; + final emptyKey = await LocalDb.inputDayFingerprint(_day); + await db.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + await LocalDb.wipeAll(); + expect(await db.query('decoded_onehz'), isEmpty); + expect(await db.query('input_rev'), isEmpty); + expect(LocalDb.storeGeneration, greaterThan(before)); + expect(await LocalDb.inputDayFingerprint(_day), isNot(emptyKey), + reason: '(day, revision, generation) must reject a pre-wipe cache entry'); + }); + + Future source({bool failAfterCommit = false}) async { + final src = await databaseFactory.openDatabase(sourcePath); + // A valid replacement in an EARLY table, followed by a destination failure + // in a later table. This drives the public importer, not a test-only merge. + await src.execute('CREATE TABLE journal (date TEXT PRIMARY KEY, ' + "note TEXT NOT NULL DEFAULT '', updated_at INTEGER NOT NULL)"); + await src.insert('journal', {'date': _day, 'note': 'replacement', 'updated_at': 2}); + if (failAfterCommit) { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + } + + for (final partial in [false, true]) { + test('${partial ? 'partial failing' : 'successful replacement'} import advances generation', () async { + final db = await open(); + await db.insert('journal', {'date': _day, 'note': 'old', 'updated_at': 1}); + await source(failAfterCommit: partial); + final before = LocalDb.storeGeneration; + final key = await LocalDb.inputDayFingerprint(_day); + if (partial) { + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + } else { + await LocalDb.importFromDbFile(sourcePath); + } + expect((await db.query('journal')).single['note'], 'replacement', + reason: 'prove a page committed before checking invalidation'); + expect(LocalDb.storeGeneration, greaterThan(before), + reason: 'every exit after durable replacement must invalidate caches'); + expect(await LocalDb.inputDayFingerprint(_day), isNot(key)); + }); + } + + for (final mode in ['success', 'later failure', 'first failure']) { + test('bulk restore $mode invalidates only after a durable page', () async { + final db = await open(); + final src = await databaseFactory.openDatabase(sourcePath); + await src.execute(_v58DecodedDdl.first); + await src.insert('decoded_onehz', _row('decoded_onehz', _lo + 10)); + if (mode == 'later failure') { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + if (mode == 'first failure') { + await db.execute("CREATE TRIGGER refuse_decoded BEFORE INSERT ON decoded_onehz " + "BEGIN SELECT RAISE(ABORT, 'test bulk first-page failure'); END"); + } + final generation = LocalDb.storeGeneration; + if (mode == 'success') { + await LocalDb.importFromDbFile(sourcePath); + } else { + await expectLater(LocalDb.importFromDbFile(sourcePath), + throwsA(isA())); + } + expect(await db.query('decoded_onehz'), + hasLength(mode == 'first failure' ? 0 : 1)); + expect(LocalDb.storeGeneration, + mode == 'first failure' ? generation : greaterThan(generation)); + }); + } + + for (final mode in ['success', 'later table failure', + 'later archive-page failure', 'first failure']) { + test('archive-only restore $mode tracks durable pages', () async { + final db = await open(); + final src = await databaseFactory.openDatabase(sourcePath); + await src.execute('CREATE TABLE raw_archive (hex TEXT PRIMARY KEY, ' + 'counter INTEGER, packet_type INTEGER, rec_ts INTEGER, ' + 'captured_at INTEGER, reason TEXT)'); + final sourceRows = mode == 'later archive-page failure' ? 501 : 1; + final batch = src.batch(); + for (var i = 1; i <= sourceRows; i++) { + batch.insert('raw_archive', {'hex': i.toRadixString(16).padLeft(4, '0'), + 'counter': i, 'packet_type': 0, 'rec_ts': _lo, + 'captured_at': _lo * 1000, 'reason': 'test_archive'}); + } + await batch.commit(noResult: true); + if (mode == 'later table failure') { + await src.execute('CREATE TABLE baselines (key TEXT PRIMARY KEY)'); + await src.insert('baselines', {'key': 'missing-required-payload'}); + } + await src.close(); + if (mode == 'first failure' || mode == 'later archive-page failure') { + final rejected = mode == 'first failure' ? 1 : 501; + await db.execute('CREATE TRIGGER refuse_archive BEFORE INSERT ON ' + '${LocalDb.rawArchiveTable} WHEN NEW.counter = $rejected ' + "BEGIN SELECT RAISE(ABORT, 'test archive page failure'); END"); + } + final generation = LocalDb.storeGeneration; + if (mode == 'success') { + await LocalDb.importFromDbFile(sourcePath); + } else { + await expectLater(LocalDb.importFromDbFile(sourcePath), + throwsA(isA())); + } + final copied = mode == 'first failure' ? 0 : + mode == 'later archive-page failure' ? 500 : 1; + expect(await db.query(LocalDb.rawArchiveTable), hasLength(copied)); + expect(LocalDb.storeGeneration, + mode == 'first failure' ? generation : greaterThan(generation)); + }); + } + + test('failed first import page leaves generation unchanged', () async { + final db = await open(); + await source(); + await db.execute("CREATE TRIGGER refuse_journal BEFORE INSERT ON journal " + "BEGIN SELECT RAISE(ABORT, 'test first-page failure'); END"); + final before = LocalDb.storeGeneration; + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + expect(await db.query('journal'), isEmpty); + expect(LocalDb.storeGeneration, before, reason: 'nothing committed'); + }); + + test('failed first import page can retry and invalidate the next generation', () async { + final db = await open(); + await source(); + await db.execute("CREATE TRIGGER refuse_journal BEFORE INSERT ON journal " + "BEGIN SELECT RAISE(ABORT, 'test first-page failure'); END"); + final before = LocalDb.storeGeneration; + await expectLater(LocalDb.importFromDbFile(sourcePath), throwsA(isA())); + expect(await db.query('journal'), isEmpty); + expect(LocalDb.storeGeneration, before, reason: 'nothing committed'); + await db.execute('DROP TRIGGER refuse_journal'); + await LocalDb.importFromDbFile(sourcePath); + expect((await db.query('journal')).single['note'], 'replacement'); + expect(LocalDb.storeGeneration, greaterThan(before), reason: 'retry must not leave a sticky latch'); + }); + + test('GUARD sync REPLACE shrinks/clears beats only for its device and second', () async { + final db = await open(); + Future commit(String device, int counter, List rr, {int? ts}) { + final at = ts ?? _lo; + return LocalDb.commitSyncBatch( + [RawRecord(counter: counter, packetType: 47, hex: 'feed$counter', + capturedAt: at * 1000, recTs: at)], + [Sample(tsEpoch: at, counter: counter, hr: 60 + counter, + rrIntervalsMs: rr, ax: 0, ay: 0, az: 1, + spo2RedRaw: 0, spo2IrRaw: 0, skinTempRaw: 30000)], + deviceId: device, deviceFamily: 'gen4', + ); + } + await commit('', 10, [800, 810, 820]); + await commit('secondary', 11, [700, 710]); + await commit('', 12, [780], ts: _lo + 1); + await commit('', 1, [900]); + expect((await db.query('decoded_onehz', where: "device_id = '' AND rec_ts = $_lo")).single['counter'], 1); + expect((await db.query('decoded_onehz', where: "device_id = '' AND rec_ts = $_lo")).single['hr'], 61); + expect((await db.query('decoded_rr', where: "device_id = '' AND rec_ts = $_lo")).map((r) => r['rr_ms']), [900]); + await commit('', 2, []); + expect(await db.query('decoded_rr', where: "device_id = '' AND rec_ts = $_lo"), isEmpty); + expect((await db.query('decoded_rr', where: 'rec_ts = ?', whereArgs: [_lo + 1])) + .single['rr_ms'], 780, reason: 'the neighboring second must keep its beats'); + expect((await db.query('decoded_rr', where: "device_id = 'secondary'", orderBy: 'beat_index')) + .map((r) => r['rr_ms']), [700, 710]); + }); +} + +// Frozen c8e63556 v58 base table definitions; optional columns added by the +// existing repair path are captured before upgrade. Never seed input_rev here. +const _v58DecodedDdl = [ + '''CREATE TABLE decoded_onehz ( + device_id TEXT NOT NULL DEFAULT '', ts_ms INTEGER NOT NULL DEFAULT 0, + rec_ts INTEGER, counter INTEGER NOT NULL, hr INTEGER, ax REAL, ay REAL, az REAL, + spo2_red_raw INTEGER, spo2_ir_raw INTEGER, skin_temp_raw INTEGER, + step_count INTEGER, step_cadence INTEGER, activity_class INTEGER, skin_temp_c REAL, + on_wrist INTEGER, hr_valid INTEGER, hr_alt INTEGER, ambient_raw INTEGER, + device_family TEXT, source TEXT, temp_ch2_c REAL, temp_ch3_c REAL, + signal_quality_logvar REAL, dyn_accel_g REAL, PRIMARY KEY(device_id, ts_ms))''', + '''CREATE TABLE decoded_rr ( + device_id TEXT NOT NULL DEFAULT '', ts_ms INTEGER NOT NULL DEFAULT 0, + rec_ts INTEGER NOT NULL, beat_index INTEGER NOT NULL, rr_ts_ms INTEGER NOT NULL, + rr_ms INTEGER NOT NULL, device_family TEXT, source TEXT, + PRIMARY KEY(device_id, ts_ms, beat_index))''', +]; diff --git a/test/ed2a_substrate_transfer_test.dart b/test/ed2a_substrate_transfer_test.dart new file mode 100644 index 000000000..e55df69c4 --- /dev/null +++ b/test/ed2a_substrate_transfer_test.dart @@ -0,0 +1,258 @@ +// ED-2a E5: upstream's existing worker, no E3 annotations/registry/AST guard. +// fromTransfer delegates to fromJson in RED: parity passes, adoption fails. +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart' show kAlgoVersion; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; + +import 'support/dart_source.dart'; + +const _intKeys = ['ts_sec', 'hr', 'spo2_red', 'spo2_ir', 'skin_temp', + 'skin_contact', 'step_count', 'hr_valid', 'band_sleep_state']; +const _doubleKeys = ['rr_ts_ms', 'rr_ms', 'ax', 'ay', 'az']; +const _start = 1577923200; // existing two_device_day fixture, UTC + +Map _columns() => { + 'ts_sec': [_start, _start + 1, _start + 2], + 'hr': [60, 0, 61], + 'rr_ts_ms': Float64List.fromList([1577923199999.5, 1577923201000.25]), + 'rr_ms': Float64List.fromList([850.125, 900.5]), + 'ax': Float64List.fromList([-0.0, double.minPositive, double.nan]), + 'ay': Float64List.fromList([double.infinity, double.negativeInfinity, 0.1 + 0.2]), + 'az': Float64List.fromList([1, -1, 0.5]), + 'spo2_red': [0, 1, 65535], 'spo2_ir': [123, 0, 65534], + 'skin_temp': [30001, 0, 30002], 'skin_contact': [1, 0, -1], + 'step_count': [65535, 0, -1], 'hr_valid': [1, 0, -1], + 'band_sleep_state': [1, 0, -1], + 'device_family': 'gen5', 'device_ids': ['secondary', '', 'secondary'], +}; + +void _same(Substrate got, Substrate oracle) { + final a = got.toJson(), b = oracle.toJson(); + expect(a.keys, b.keys, reason: 'every serialized field must survive'); + for (final key in _intKeys) { expect(a[key], b[key], reason: key); } + for (final key in _doubleKeys) { + expect(a[key], isA(), reason: '$key must remain packed'); + final g = a[key] as Float64List, w = b[key] as Float64List; + expect(g.buffer.asUint64List(g.offsetInBytes, g.length), + w.buffer.asUint64List(w.offsetInBytes, w.length), reason: '$key IEEE-754 bits'); + } + expect(got.deviceFamily, oracle.deviceFamily); + expect(got.deviceIds, oracle.deviceIds); +} + +// Exercise the actual upstream prepare worker. Own every port/subscription and +// kill on every exit, including a failed handshake or malformed-page error. +Future> _worker(List> frames, + List> rr, {Map? page}) async { + final port = ReceivePort(); + Isolate? isolate; + StreamSubscription? subscription; + final ready = Completer(); + final done = Completer>(); + // Either future can fail before the caller reaches it. + unawaited(ready.future.then((_) {}, onError: (Object _, StackTrace _) {})); + unawaited(done.future.then((_) {}, onError: (Object _, StackTrace _) {})); + void fail(Object error) { + if (!ready.isCompleted) ready.completeError(error); + if (!done.isCompleted) done.completeError(error); + } + try { + subscription = port.listen((message) { + if (message is SendPort) { + if (!ready.isCompleted) ready.complete(message); + } else if (message is Map && message['type'] == 'result') { + if (!done.isCompleted) done.complete((message['payload'] as Map).cast()); + } else if (message is Map && message['type'] == 'error') { + fail(StateError('${message['error']}')); + } else if (message is List) { + fail(StateError('uncaught worker error: $message')); + } else if (message == null && !done.isCompleted) { + fail(StateError('worker exited before returning a payload')); + } + }); + isolate = await Isolate.spawn(derivationPrepareWorker, port.sendPort, + onError: port.sendPort, onExit: port.sendPort); + final worker = await ready.future.timeout(const Duration(seconds: 10)); + worker.send({'type': 'config', 'mode': 'substrate'}); + worker.send(page ?? {'type': 'page', 'frames': frames, 'rr': rr}); + worker.send({'type': 'finish'}); + return await done.future.timeout(const Duration(seconds: 10)); + } finally { + await subscription?.cancel(); + port.close(); + isolate?.kill(priority: Isolate.immediate); + } +} + +Map _bundle(Substrate sub) { + final day = prepareDerivationPayload(sub).days.first; + final d = day.daySub, s = day.sleepSub; + return deriveDayBundle(DayBundleInput( + date: day.date, dayTsSec: d.tsSec, dayHr: d.hr, + dayRrTsMs: d.rrTsMs, dayRrMs: d.rrMs, + sleepTsSec: s.tsSec, sleepHr: s.hr, sleepRrTsMs: s.rrTsMs, + sleepRrMs: s.rrMs, sleepSkinTemp: s.skinTemp, + sleepJson: day.sleepJson, hypnoStages: day.hypnoStages, + sleepOnsetSec: day.sleepOnsetSec, sleepOffsetSec: day.sleepOffsetSec, + profile: const Profile().toMap(), dayConfidence: day.confidence, + dayFlags: day.flags, deviceFamily: d.deviceFamily, + sleepSource: day.sleepSource, + ).toJson()); +} + +// Same generator as case 1 in two_device_fixture_test.dart, driven by the +// committed region spec. Only the documented gen4 regions feed that golden. +({List> frames, List> rr}) _fixture() { + final spec = jsonDecode(File('test/fixtures/two_device_day.json').readAsStringSync()) as Map; + int offset(String hhmmss) { + final p = hhmmss.split(':').map(int.parse).toList(); + return p[0] * 3600 + p[1] * 60 + p[2]; + } + final frames = >[], rr = >[]; + for (final region in spec['regions'] as List) { + final r = region as Map; + if (!(r['devices'] as List).contains('gen4')) continue; + final holes = {for (final h in (r['holes'] as List?) ?? []) offset(h as String)}; + for (var i = offset(r['start'] as String); i <= offset(r['end'] as String); i++) { + if (holes.contains(i)) continue; + final ts = _start + i; + frames.add({'rec_ts': ts, 'device_family': 'gen4', 'hr': 60 + ts % 7, + 'ax': 0.0, 'ay': 0.0, 'az': 1.0, 'skin_temp_raw': 30000 + ts % 11}); + rr.add({'rec_ts': ts, 'rr_ms': 1000 - ts % 5 * 10}); + } + } + return (frames: frames, rr: rr); +} + +void main() { + test('engine receives prepare-worker substrate through the transfer seam', () { + final code = stripCommentsAndStrings(File('lib/compute/derivation_engine.dart').readAsStringSync()); + expect(code, matches(RegExp(r'\bSubstrate\.fromTransfer\(\s*payload\s*\)')), + reason: 'the actual receiver still calls fromJson, which copies every column'); + }); + + for (final key in [..._intKeys, ..._doubleKeys]) { + test('transfer adopts correctly typed $key without an element copy', () { + final message = _columns(); + final got = Substrate.fromTransfer(message).toJson(); + expect(identical(got[key], message[key]), isTrue, + reason: '$key was copied on the receiving/UI isolate'); + }); + } + + test('transfer preserves every field and double bit, including band sleep state', () { + final message = _columns(); + final got = Substrate.fromTransfer(message); + _same(got, Substrate.fromJson(message)); + expect(got.bandSleepState, [1, 0, -1], reason: '0 is wake, -1 is absent'); + expect(got.deviceIds, {'secondary', ''}); + }); + + final variants = Function()>{ + 'empty': () => {}, + 'timestamps only': () => {'ts_sec': [_start, _start + 1]}, + 'missing timestamps': () => {..._columns()}..remove('ts_sec'), + 'empty optional columns': () => {..._columns(), 'hr': [], 'ax': Float64List(0), + 'step_count': [], 'hr_valid': [], 'band_sleep_state': []}, + 'short columns and sparse unpaired RR': () => {..._columns(), 'hr': [62], + 'az': Float64List.fromList([0.25]), 'step_count': [1], + 'hr_valid': [1], 'band_sleep_state': [0], + 'rr_ms': Float64List.fromList([850])}, + 'null provenance': () => {..._columns(), 'device_family': null, 'device_ids': null}, + 'loose numeric columns': () => { + for (final e in _columns().entries) + e.key: e.value is List ? List.from(e.value as List) : e.value, + }, + 'mixed integer and double numbers': () => {'ts_sec': [_start.toDouble(), _start + 1], + 'hr': [60.0, 61], 'rr_ms': [850, 900.5], + 'rr_ts_ms': [1577923200123, 1577923201000.5], + 'band_sleep_state': [0.0, 1]}, + 'wide integers': () => {..._columns(), 'ts_sec': [1 << 40, (1 << 40) + 1, (1 << 40) + 2], + 'step_count': [1 << 40, -(1 << 40), 0]}, + }; + for (final variant in variants.entries) { + test('transfer defaults equal fromJson: ${variant.key}', () { + final m = variant.value(); + _same(Substrate.fromTransfer(m), Substrate.fromJson(m)); + }); + } + + test('GUARD constructor optional arrays remain empty; decoder defaults remain absent', () { + final direct = Substrate(tsSec: [_start], hr: [60], rrTsMs: [], rrMs: [], + ax: [0], ay: [0], az: [1], spo2Red: [0], spo2Ir: [0], skinTemp: [0], skinContact: [0]); + expect(direct.stepCount, isEmpty); + expect(direct.hrValid, isEmpty); + expect(direct.bandSleepState, isEmpty); + final decoded = Substrate.fromTransfer({'ts_sec': [_start]}); + expect(decoded.stepCount, [-1]); + expect(decoded.hrValid, [-1]); + expect(decoded.bandSleepState, [-1]); + expect(decoded.hr, [0]); + expect(decoded.ax, [0.0]); + expect(decoded.rrMs, isEmpty); + }); + + test('actual prepare worker emits columnar typed doubles and preserves band sleep', () async { + final frames = [for (var i = 0; i < 3; i++) { + 'rec_ts': _start + i, 'hr': 60 + i, 'device_family': 'gen5', + 'ax': 0.25, 'ay': -0.0, 'az': 1.0, 'step_count': 20 + i, + 'hr_valid': i % 2, 'band_sleep_state': [1, 0, -1][i], + 'device_id': '', 'skin_temp_raw': 30000 + i, + }]; + final rr = [{'rec_ts': _start, 'rr_ms': 850, + 'rr_ts_ms': _start * 1000, 'beat_ts_ms': _start * 1000 + 123}]; + final message = await _worker(frames, rr); + for (final key in _doubleKeys) { expect(message[key], isA(), reason: key); } + for (final key in _intKeys) { expect(message[key], isA>(), reason: key); } + expect(message.values.any((v) => v is List && v.any((e) => e is Map)), isFalse, + reason: 'worker substrate contains columns, never per-row maps'); + final got = Substrate.fromTransfer(message); + _same(got, substrateFromDecodedPage(frames, rr)); + expect(got.bandSleepState, [1, 0, -1]); + expect(got.rrTsMs, [_start * 1000.0 + 123]); + }); + + test('actual worker result adopts its received arrays', () async { + final message = await _worker([{'rec_ts': _start, 'hr': 60, 'az': 1.0}], []); + final adopted = Substrate.fromTransfer(message).toJson(); + for (final key in [..._intKeys, ..._doubleKeys]) { + expect(identical(adopted[key], message[key]), isTrue, reason: '$key copied after worker receive'); + } + }); + + test('a worker error does not prevent the next transfer', () async { + await expectLater(_worker([], [], page: {'type': 'page', 'frames': 'malformed transport'}), throwsStateError); + final good = await _worker([{'rec_ts': _start, 'hr': 60}], []); + expect(Substrate.fromTransfer(good).hr, [60]); + }); + + test('GUARD existing golden is byte-equal with transfer off, on, and across the worker', () async { + // Same timezone condition as the existing golden test; no regeneration. + if (DateTime.fromMillisecondsSinceEpoch(_start * 1000).timeZoneOffset != Duration.zero) { + markTestSkipped('committed golden is UTC; run this test with TZ=UTC'); + return; + } + expect(kAlgoVersion, 119, reason: 'ED-2a changes no output'); + final fixture = _fixture(); + final baseline = substrateFromDecodedPage(fixture.frames, fixture.rr); + final transferred = Substrate.fromTransfer(baseline.toJson()); + final workerMessage = await _worker(fixture.frames, fixture.rr); + final acrossWorker = Substrate.fromTransfer(workerMessage); + _same(transferred, baseline); + _same(acrossWorker, baseline); + final golden = jsonDecode(File('test/fixtures/two_device_day_expected.json').readAsStringSync()) as Map; + final expected = jsonEncode(golden['single_device']); + expect(jsonEncode(_bundle(baseline)), expected, reason: 'transfer off'); + expect(jsonEncode(_bundle(transferred)), expected, reason: 'transfer on'); + expect(jsonEncode(_bundle(acrossWorker)), expected, reason: 'actual worker round trip'); + }, timeout: const Timeout(Duration(minutes: 2))); +} diff --git a/test/guards/db_parts_test.dart b/test/guards/db_parts_test.dart new file mode 100644 index 000000000..86e1f6e45 --- /dev/null +++ b/test/guards/db_parts_test.dart @@ -0,0 +1,62 @@ +import 'dart:io'; + +import 'package:analyzer/dart/analysis/utilities.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/ast/visitor.dart'; +import 'package:flutter_test/flutter_test.dart'; + +class _ExecuteCalls extends RecursiveAstVisitor { + final arguments = []; + + @override + void visitMethodInvocation(MethodInvocation node) { + if (node.methodName.name == 'execute') { + arguments.add(node.argumentList.arguments.first); + } + super.visitMethodInvocation(node); + } +} + +void main() { + final parts = Directory('lib/data').listSync().whereType() + .where((f) => f.path.endsWith('.dart') && + f.readAsStringSync().contains("part of 'db.dart';")).toList(); + + test('DB parts contain only _ldb functions', () { + expect(parts.map((f) => f.path), contains('lib/data/db_input_revisions.dart')); + for (final file in parts) { + final unit = parseString(content: file.readAsStringSync()).unit; + expect(unit.declarations, isNotEmpty, reason: file.path); + for (final declaration in unit.declarations) { + expect(declaration, isA(), reason: file.path); + expect((declaration as FunctionDeclaration).name.lexeme, + startsWith('_ldb'), reason: file.path); + } + } + }); + + test('DB part execute calls carry literal SQL', () { + var calls = 0; + for (final file in parts) { + final visitor = _ExecuteCalls(); + parseString(content: file.readAsStringSync()).unit.accept(visitor); + calls += visitor.arguments.length; + for (final argument in visitor.arguments) { + expect(argument, isA(), reason: '${file.path}: $argument'); + } + } + expect(calls, greaterThanOrEqualTo(9)); + // Positive control: a variable SQL argument would fail the predicate. + final probe = _ExecuteCalls(); + parseString(content: 'void probe(dynamic db, String sql) { db.execute(sql); }') + .unit.accept(probe); + expect(probe.arguments.single, isNot(isA())); + }); + + test('DB parts stay below 6000 source lines', () { + for (final file in parts) { + expect(file.readAsLinesSync().length, lessThanOrEqualTo(6000), + reason: file.path); + } + }); +} From a6fbecb8e28dc49cea95d9f8a92a647bb1637a9e Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 05:36:49 -0600 Subject: [PATCH 2/5] derive: reuse staged sleep per day while its inputs are unchanged; walk only the days the target can use - A per-engine cache (14 days, LRU) keeps the staging worker's candidate and observation JSON. The key covers the search bounds, each local date's input revisions and store generation, resolved ownership and priority, the substrate shape, zone offsets over the window and at stored midsleeps, overrides, the warmed profile, stored sleep windows that can reach the history threshold, and vendor nights. A result computed without folding never serves a folding pass. - A pass that cannot fold the profile skips earlier days while the history could not reach 14 distinct nights (the habitual-midsleep prior is null either way) and stops after the target. Folding passes keep the full walk. Output is unchanged; kAlgoVersion stays 113. --- lib/compute/derivation_engine.dart | 157 +++++++++- lib/compute/derive_prepare.dart | 5 + lib/compute/inputs/canonical.dart | 18 ++ lib/compute/inputs/colmi_inputs.dart | 1 + lib/compute/inputs/garmin_inputs.dart | 1 + lib/compute/inputs/oura_inputs.dart | 2 + lib/compute/inputs/ultrahuman_inputs.dart | 1 + lib/compute/staged_sleep_cache.dart | 132 +++++++++ lib/compute/substrate.dart | 37 ++- lib/data/db.dart | 16 + test/ed2a_e12_staging_cache_test.dart | 275 ++++++++++++++++++ .../ed2a_e15_restricted_target_walk_test.dart | 166 +++++++++++ test/support/ed2a_recalc_fixture.dart | 184 ++++++++++++ 13 files changed, 986 insertions(+), 9 deletions(-) create mode 100644 lib/compute/staged_sleep_cache.dart create mode 100644 test/ed2a_e12_staging_cache_test.dart create mode 100644 test/ed2a_e15_restricted_target_walk_test.dart create mode 100644 test/support/ed2a_recalc_fixture.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index e3032aaec..6a03dad90 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -70,6 +70,7 @@ import 'hr_max.dart' show estimatedMaxHr, kHrFloorBpm, smoothedMaxHr, smoothedMinHr; import 'movement_floor_policy.dart' as mfp; import 'sleep_profile_policy.dart'; +import 'staged_sleep_cache.dart'; import 'derive_prepare.dart'; import 'onehz_pipeline.dart'; import 'step_cadence.dart'; @@ -3092,6 +3093,59 @@ class DerivationEngine { DerivationEngine({this.log, this.background = false, this.offloadActive}); final void Function(String)? log; + // ED-2a RED seams. Empty cache views and no-op controls describe the current + // uncached engine. Counters below observe real worker work, never results. + // E12 now backs the staging controls; later steps' controls remain inert. + final _stagedSleepCache = StagedSleepCache(); + @visibleForTesting + bool debugDisableStagedSleepCache = false; + @visibleForTesting + int get debugStagedSleepCacheMaxDays => _stagedSleepCache.maxDays; + @visibleForTesting + set debugStagedSleepCacheMaxDays(int value) => _stagedSleepCache.maxDays = value; + @visibleForTesting + int get debugStagedSleepCacheSize => _stagedSleepCache.size; + @visibleForTesting + void debugStagedSleepCacheRekey(int version) => _stagedSleepCache.rekey(version); + @visibleForTesting + bool debugDisableSubstrateRetention = false; + @visibleForTesting + int debugRetainedSubstrateMaxDays = 2; + @visibleForTesting + Set get debugRetainedSubstrateDays => const {}; + @visibleForTesting + void debugRetainedSubstrateRekey(int version) {} + @visibleForTesting + void trimForBackground() {} + @visibleForTesting + void handleMemoryPressure() {} + @visibleForTesting + void dispose() {} + @visibleForTesting + bool debugDisableSessionReuse = false; + @visibleForTesting + int debugCeilingComputed = 0; + @visibleForTesting + void debugSessionCacheRekey(int version) {} + @visibleForTesting + int debugStagingComputations = 0; + @visibleForTesting + void Function(bool mayFold, int segmentedDays, String? observation)? debugOnStagingWalk; + @visibleForTesting + bool debugDisableTargetDaySkip = false; + @visibleForTesting + Map debugRetainedSubstrate(String day) => const {}; + @visibleForTesting + Future debugSleepCandidate(String day) => + _sleepCandidateForDay(day); + @visibleForTesting + Future debugPrepareDay(String day) => + _prepareTargetDay(day); + @visibleForTesting + Future debugLoadSubstrate(int from, int to, {required String day, + Map> ownership = const {}}) => + _loadSubstrateRange(from, to, dayId: day, ownership: ownership); + /// True while a band offload is draining. The scheduler never STARTS a /// derive during one, but one can start mid-pass; the raw prune checks this /// before archiving each substrate bucket and stops for the pass, so a drain @@ -3151,9 +3205,11 @@ class DerivationEngine { static Future _withRunLock(T busy, Future Function() body) async { if (_running) return busy; _running = true; + beginWearableDayIndexPass(); try { return await body(); } finally { + endWearableDayIndexPass(); _running = false; } } @@ -3389,6 +3445,7 @@ class DerivationEngine { }) async { if (_running) return 0; _running = true; + beginWearableDayIndexPass(); final startedAt = DateTime.now().millisecondsSinceEpoch; _diag ..['running'] = true @@ -3837,6 +3894,7 @@ class DerivationEngine { } catch (e) { _log('[derive] freshness refresh failed: $e'); } + endWearableDayIndexPass(); _running = false; final finishedAt = DateTime.now().millisecondsSinceEpoch; _diag @@ -3859,6 +3917,7 @@ class DerivationEngine { if (days.isEmpty) return 0; if (_running) return 0; _running = true; + beginWearableDayIndexPass(); final startedAt = DateTime.now().millisecondsSinceEpoch; _diag ..['running'] = true @@ -3979,6 +4038,7 @@ class DerivationEngine { ..['stage'] = 'idle' ..['finished_at'] = finishedAt ..['duration_ms'] = finishedAt - startedAt; + endWearableDayIndexPass(); _running = false; } } @@ -4241,6 +4301,7 @@ class DerivationEngine { // A user sleep override is the source of truth — never serve the cached auto // candidate, and don't cache the override result (so a later edit / clear is // not shadowed by a stale artifact). The auto path keeps its finalized cache. + // The RAM worker cache below keys the override itself, including edits. final overrideRow = await LocalDb.getSleepOverride(dayId); final override = overrideRow == null ? null @@ -4282,7 +4343,24 @@ class DerivationEngine { // exists — and it has to be: staging the night off both devices' rows // while `napSub`/`sleepSub` score it off one device's is a window found in // a night that was never scored. - final (searchOwnership, _) = await _resolveOwnership(range.$1, range.$2); + final (searchOwnership, searchPriority) = + await _resolveOwnership(range.$1, range.$2); + List>? revisions; + if (!debugDisableStagedSleepCache) { + try { + revisions = []; + final first = DateTime.fromMillisecondsSinceEpoch(range.$1 * 1000); + for (var d = DateTime(first.year, first.month, first.day); + d.millisecondsSinceEpoch ~/ 1000 <= range.$2; + d = DateTime(d.year, d.month, d.day + 1)) { + final label = dayLabelOf(d); + final fp = await LocalDb.inputDayFingerprint(label); + revisions.add([label, fp.revisionSum, fp.revisionMax, fp.generation]); + } + } catch (_) { + revisions = null; // Unavailable provenance costs a staging, never a hit. + } + } final searchSub = await _loadSubstrateRange( range.$1, range.$2, @@ -4354,12 +4432,36 @@ class DerivationEngine { ? await wearableNight(range.$1, range.$2) : null; if (ownNight != null) vendorNights = [...vendorNights, ownNight]; + String? stagingKey; + if (revisions != null) { + try { + stagingKey = stagingInputKey( + fromSec: range.$1, + toSec: range.$2, + revisions: revisions, + sub: searchSub, + ownership: searchOwnership, + priority: searchPriority, + override: override, + profileJson: profileJson, + priorSleep: priorSleep, + vendorNights: vendorNights, + activeWearable: override == null ? await activeWearable() : null, + ); + } catch (_) { + // A context that cannot be keyed still takes the ordinary worker path. + } + } // Cancellable + TIMED OUT. This site previously used a bare `Isolate.run` // with no timeout at all, so a hung staging pass never completed its future // — `_running` stayed true and `DeriveScheduler._drain` never returned, i.e. // all derivation was dead until app restart. - final (candidateJson, observationJson) = - await _runIsolateCancellable(() { + final skipUnusedDays = !mayFold && !debugDisableTargetDaySkip; + // Read here: a `_perDayTimeout` inside `stage` would capture `this` into + // the context the worker closure ships, and the engine's callbacks (the + // app's own fields) are not sendable. + final stageTimeout = _perDayTimeout; + Future<(String, String?, int)> stage() => _runIsolateCancellable(() { try { final p = profileJson == null ? null @@ -4380,9 +4482,11 @@ class DerivationEngine { } ana.cardioRecordObservations = true; ana.resetCardioObservations(); + debugSegmentedDayCount = 0; final candidate = prepareSleepSessionCandidate( searchSub, targetDay: dayId, + skipUnusedDays: skipUnusedDays, override: override, priorSleep: priorSleep, vendorNights: vendorNights, @@ -4427,8 +4531,32 @@ class DerivationEngine { } } } - return (jsonEncode(candidate.toJson()), observationJson); - }, _perDayTimeout, label: 'sleep-staging $dayId'); + return (jsonEncode(candidate.toJson()), observationJson, debugSegmentedDayCount); + }, stageTimeout, label: 'sleep-staging $dayId'); + final hit = stagingKey == null ? null : _stagedSleepCache.lookup( + dayId, kAlgoVersion, stagingKey, mayFold: mayFold); + final String candidateJson; + final String? observationJson; + // Reuse worker output only; banked-night and fold checks below still run. + if (hit != null) { + candidateJson = hit.candidateJson; + observationJson = mayFold ? hit.observationJson : null; + } else { + debugStagingComputations++; + final result = await stage(); + candidateJson = result.$1; + observationJson = result.$2; + debugOnStagingWalk?.call(mayFold, result.$3, observationJson); + if (stagingKey != null) { + _stagedSleepCache.store(dayId, StagedSleepEntry( + algoVersion: kAlgoVersion, + key: stagingKey, + mayFold: mayFold, + candidateJson: candidateJson, + observationJson: observationJson, + )); + } + } final candidate = SleepSessionCandidate.fromJson( (jsonDecode(candidateJson) as Map).cast()); for (final f in candidate.flags) { @@ -5311,6 +5439,7 @@ class DerivationEngine { }) async { if (_running) return 0; _running = true; + beginWearableDayIndexPass(); try { await refreshSessionSensorSources(); // as [run] does // Baseline gate: compute the CURRENT signature and compare to the stored @@ -5439,6 +5568,7 @@ class DerivationEngine { return 0; } finally { await _runStorageHousekeeping(); + endWearableDayIndexPass(); _running = false; } } @@ -5753,6 +5883,7 @@ class DerivationEngine { Future refreshActivityReviews(Profile profile) async { if (_running) return false; _running = true; + beginWearableDayIndexPass(); try { final revisions = await LocalDb.applyPendingActivityReviews(); if (revisions.isEmpty) return true; @@ -5763,6 +5894,7 @@ class DerivationEngine { await LocalDb.finishActivityReviews(revisions); return (await (await LocalDb.instance).query('activity_review_days', limit: 1)).isEmpty; } finally { + endWearableDayIndexPass(); _running = false; } } @@ -6324,6 +6456,8 @@ class DerivationEngine { final blocks = await _runDayBlocksCancellable(blocksInput, _perDayTimeout); + debugCeilingComputed += blocks.ceilingComputations; + // Merge the computed blocks back into the isolate-1 bundle. scMap is the // CastMap view over bundle['scalars'], so addAll writes through — nap_min / // hrr_bpm reach the persisted series map below. @@ -10572,7 +10706,9 @@ class DerivationEngine { bundlePatch['workout_suggestions'] = wc.boutJson; if (wc.hrrBpm != null) scMap['hrr_bpm'] = wc.hrrBpm; if (wc.hrrTauS != null) scMap['hrr_tau_s'] = wc.hrrTauS; - final ceiling = _dayHrCeiling(daySub, inp.savedSessions); + var ceilingComputations = 0; + final ceiling = _dayHrCeiling(daySub, inp.savedSessions, + onCompute: () => ceilingComputations++); bundlePatch['hr_ceiling'] = ceiling; // TS-03 — the day's observed ceiling as a SCALAR too, so the app-wide // "highest we've seen" is a max over `metric_series` (one small query the @@ -10629,6 +10765,7 @@ class DerivationEngine { // than no diagnostic. return _DayBlocksOutput( + ceilingComputations: ceilingComputations, bundlePatch: bundlePatch, seriesPatch: seriesPatch, scalarPatch: scMap, @@ -10821,8 +10958,9 @@ class DerivationEngine { static Map _dayHrCeiling( Substrate s, - List> saved, - ) { + List> saved, { + void Function()? onCompute, + }) { ana.Metric? best; String? bestId; String? bestType; @@ -10843,6 +10981,7 @@ class DerivationEngine { valid: s.accelPresentAt(i))); } if (hr.isEmpty) continue; + onCompute?.call(); final m = ana.sessionHrCeiling(hr, accel, deviceFamily: s.deviceFamily); if (!m.present) { lastAbsent = m; @@ -11360,6 +11499,7 @@ class _DayBlocksInput { /// isolate; [wake] is persisted; candidates are reconciled and session HRR /// writes are applied by the database-owning caller. class _DayBlocksOutput { + final int ceilingComputations; final List> napSuggestions; final Map bundlePatch; final Map seriesPatch; @@ -11368,6 +11508,7 @@ class _DayBlocksOutput { final List> suggestionsToPersist; final List<(String, double)> sessionHrrWrites; const _DayBlocksOutput({ + this.ceilingComputations = 0, required this.bundlePatch, this.napSuggestions = const [], required this.seriesPatch, diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index 316077e2b..e3c01a277 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -408,6 +408,7 @@ PreparedDerivationPayload prepareDerivationPayload( SleepWindowOverride? override, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + bool skipUnusedDays = false, }) { if (sub.isEmpty || sub.lastTs == null) { return const PreparedDerivationPayload(dataNowSec: 0, days: []); @@ -416,6 +417,8 @@ PreparedDerivationPayload prepareDerivationPayload( for (final day in calendarDays( sub, override: override, + targetDay: targetDay, + skipUnusedDays: skipUnusedDays, priorSleep: priorSleep, vendorNights: vendorNights, )) { @@ -470,6 +473,7 @@ SleepSessionCandidate prepareSleepSessionCandidate( SleepWindowOverride? override, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + bool skipUnusedDays = false, }) { // An UNCLAIMED device night (our rows saw under half of it) ending today IS // the night: above our detection, below the user's word. It never reaches @@ -515,6 +519,7 @@ SleepSessionCandidate prepareSleepSessionCandidate( final payload = prepareDerivationPayload(sub, targetDay: targetDay, override: override, + skipUnusedDays: skipUnusedDays, priorSleep: priorSleep, vendorNights: claimed); if (payload.days.isEmpty) return SleepSessionCandidate.absent(targetDay); diff --git a/lib/compute/inputs/canonical.dart b/lib/compute/inputs/canonical.dart index ff4b2ca8b..e21a3a4c6 100644 --- a/lib/compute/inputs/canonical.dart +++ b/lib/compute/inputs/canonical.dart @@ -390,6 +390,13 @@ Future refreshSessionSensorSources() async { LocalDb.sessionWearableSource = (await activeWearable())?.$2; } +bool debugDisableWearableWorker = false; +void Function(String)? debugOnWearableConstruction; +// Called beside the actual row loops. A worker extraction must return its +// execution marker to this observer, not label a dispatch on the caller. +void debugRecordWearableConstruction() => + debugOnWearableConstruction?.call(Isolate.current.debugName ?? 'unnamed'); + /// The active wearable's rows over `[from, to]` as a substrate, for a day the /// primary band left empty, with a strap's session windows laid over them /// ([withStrapSessions]). Empty for an adapter with no inputs mapping. A day @@ -460,6 +467,7 @@ Future withStrapSessions(Substrate sub, int from, int to) async { 'AND d.rec_ts >= ? AND d.rec_ts <= ? ORDER BY d.rec_ts', [from, to], ); + debugRecordWearableConstruction(); final owner = ownerBy([for (final r in all) (r['sid'], r['src'])]); final rows = [ for (final r in all) @@ -616,6 +624,15 @@ Future> flagOnOrPrimary(List ids, return out; } +// RED pass-lifetime seams: no index is retained yet. The hook runs after an +// actual scan, so a test can count scans and inject a post-scan failure. +bool debugDisableWearableDayIndex = false; +void beginWearableDayIndexPass() {} +void endWearableDayIndexPass() {} +int get debugWearableDayIndexSize => 0; +void debugWearableDayIndexRekey(int version) {} +void Function((String, String), Map)? debugOnWearableDayIndexScan; + /// `{local day -> last second}` the active wearable has rows for, and the /// days a flagged-on strap recorded a session in while the wearable was off /// or charging ([wearableSubstrate] lays the session over the empty day). @@ -637,6 +654,7 @@ Future> wearableRecTsMaxByDay( in (await _strapSessionRecTsMaxByDay()).entries) { out[day] = math.max(out[day] ?? t, t); } + debugOnWearableDayIndexScan?.call(wearable, out); return out; } diff --git a/lib/compute/inputs/colmi_inputs.dart b/lib/compute/inputs/colmi_inputs.dart index b9823df78..70d499b68 100644 --- a/lib/compute/inputs/colmi_inputs.dart +++ b/lib/compute/inputs/colmi_inputs.dart @@ -162,6 +162,7 @@ Future colmiSubstrate(String deviceId, int from, int to) async { final sub = await colmiHr(deviceId, from, to); if (sub.isEmpty) return sub; final temps = await colmiSkinTemps(deviceId, from, to); + debugRecordWearableConstruction(); final skin = []; var j = 0; for (final t in sub.tsSec) { diff --git a/lib/compute/inputs/garmin_inputs.dart b/lib/compute/inputs/garmin_inputs.dart index 47c771e2b..49258b2cc 100644 --- a/lib/compute/inputs/garmin_inputs.dart +++ b/lib/compute/inputs/garmin_inputs.dart @@ -107,6 +107,7 @@ Future sparseHrSubstrate( [deviceId, family, from, to], ); if (rows.isEmpty) return Substrate.empty; + debugRecordWearableConstruction(); final n = rows.length; final zeros = List.filled(n, 0); final none = List.filled(n, -1); diff --git a/lib/compute/inputs/oura_inputs.dart b/lib/compute/inputs/oura_inputs.dart index 2eed4d12f..a00b6673f 100644 --- a/lib/compute/inputs/oura_inputs.dart +++ b/lib/compute/inputs/oura_inputs.dart @@ -68,6 +68,7 @@ Future ouraSubstrate(String deviceId, int from, int to) async { if (sub.isEmpty) return _ouraTempSubstrate(deviceId, from, to); final c = kHrCadenceSec[kOuraFamily]!; final temps = await _ouraTempRows(deviceId, from - c, to + c); + debugRecordWearableConstruction(); final skin = []; var j = 0; for (final t in sub.tsSec) { @@ -104,6 +105,7 @@ Future ouraSubstrate(String deviceId, int from, int to) async { Future _ouraTempSubstrate(String deviceId, int from, int to) async { final rows = await _ouraTempRows(deviceId, from, to); if (rows.isEmpty) return Substrate.empty; + debugRecordWearableConstruction(); final n = rows.length; final zeros = List.filled(n, 0); final none = List.filled(n, -1); diff --git a/lib/compute/inputs/ultrahuman_inputs.dart b/lib/compute/inputs/ultrahuman_inputs.dart index dd467784d..023305ec3 100644 --- a/lib/compute/inputs/ultrahuman_inputs.dart +++ b/lib/compute/inputs/ultrahuman_inputs.dart @@ -77,6 +77,7 @@ Future ultrahumanSubstrate(String deviceId, int from, int to) async { final sub = await sparseHrSubstrate(kUltrahumanFamily, deviceId, from, to); if (sub.isEmpty) return sub; + debugRecordWearableConstruction(); final temp = { for (final r in await ultrahumanRecords(deviceId, from, to)) if (_skinC(r) case final c?) r.tsA: (c * 100).round(), diff --git a/lib/compute/staged_sleep_cache.dart b/lib/compute/staged_sleep_cache.dart new file mode 100644 index 000000000..30fe3d638 --- /dev/null +++ b/lib/compute/staged_sleep_cache.dart @@ -0,0 +1,132 @@ +import 'dart:convert'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'sleep_profile_policy.dart'; +import 'substrate.dart'; +import 'vendor_sleep.dart'; + +class StagedSleepEntry { + StagedSleepEntry({ + required this.algoVersion, + required this.key, + required this.mayFold, + required this.candidateJson, + required this.observationJson, + }); + + int algoVersion; + final String key; + final bool mayFold; + final String candidateJson; + final String? observationJson; +} + +/// Per-engine RAM only. One worker result per day, least recently used first. +class StagedSleepCache { + final _byDay = {}; + int _maxDays = 14; + + int get size => _byDay.length; + int get maxDays => _maxDays; + set maxDays(int value) { + _maxDays = value < 0 ? 0 : value; + while (_byDay.length > _maxDays) { + _byDay.remove(_byDay.keys.first); + } + } + + StagedSleepEntry? lookup(String day, int algoVersion, String key, + {required bool mayFold}) { + final entry = _byDay[day]; + if (entry == null || entry.algoVersion != algoVersion || entry.key != key || + (mayFold && !entry.mayFold)) { + return null; + } + _byDay.remove(day); + _byDay[day] = entry; + return entry; + } + + void store(String day, StagedSleepEntry entry) { + _byDay.remove(day); + if (_maxDays == 0) return; + _byDay[day] = entry; + maxDays = _maxDays; + } + + void rekey(int version) { + for (final entry in _byDay.values) { + entry.algoVersion = version; + } + } +} + +/// Revisions are captured before loading rows. A concurrent write then makes +/// the next request miss. No sample columns are walked on the UI isolate. +String stagingInputKey({ + required int fromSec, + required int toSec, + required List> revisions, + required Substrate sub, + required Map> ownership, + required Map> priority, + required SleepWindowOverride? override, + required String? profileJson, + required List<({int startSec, int endSec, String dayKey})> priorSleep, + required List vendorNights, + required (String, String)? activeWearable, +}) { + // Count local midnights, including DST days, rather than dividing by 86400. + var walked = 0; + if (sub.tsSec.isNotEmpty) { + final first = DateTime.fromMillisecondsSinceEpoch(sub.tsSec.first * 1000); + for (var d = DateTime(first.year, first.month, first.day); + d.millisecondsSinceEpoch ~/ 1000 <= sub.tsSec.last; + d = DateTime(d.year, d.month, d.day + 1)) { + walked++; + } + } + final effectiveHistory = + {for (final h in priorSleep) h.dayKey}.length + walked < 14 + ? const <({int startSec, int endSec, String dayKey})>[] + : priorSleep; + + // Fold bookkeeping is re-read by the caller. Only a warmed-up profile can + // reach staging, but all of its fields (including updated_at_ms) are keyed. + Map? blendedProfile; + if (profileJson != null) { + final p = (jsonDecode(profileJson) as Map).cast(); + if (SleepProfilePolicy.shouldBlend((p['nights'] as num?)?.toInt())) { + blendedProfile = {...p}..remove(SleepProfilePolicy.foldedDaysKey); + } + } + + // Local day boundaries and historical midsleeps read offsets at their own + // instants. Key those too: equal current zone names/offsets are insufficient. + final zoneOffsets = >[]; + for (var t = fromSec - 86400; t <= toSec + 86400; t += 900) { + zoneOffsets.add([t, tzOffsetSecondsAt(t)]); + } + for (final h in effectiveHistory) { + final mid = (h.startSec + h.endSec) ~/ 2; + zoneOffsets.add([mid, tzOffsetSecondsAt(mid)]); + } + final now = DateTime.now(); + return jsonEncode([ + fromSec, toSec, revisions, + [for (final sig in InputSignal.values) [sig.name, + [for (final o in ownership[sig] ?? const []) + [o.start, o.end, o.deviceId]], priority[sig]]], + sub.deviceFamily, (sub.deviceIds.toList()..sort()), + sub.length, sub.firstTs, sub.lastTs, sub.rrMs.length, + now.timeZoneName, now.timeZoneOffset.inSeconds, zoneOffsets, + override == null ? null : + [override.dayId, override.onsetSec, override.offsetSec, override.source], + blendedProfile, + [for (final h in effectiveHistory) [h.startSec, h.endSec, h.dayKey]], + activeWearable == null ? null : [activeWearable.$1, activeWearable.$2], + [for (final n in vendorNights) [n.deviceId, n.source, n.decodedAtSec, + [for (final e in n.epochs) [e.startSec, e.endSec, e.stage]]]], + ]); +} diff --git a/lib/compute/substrate.dart b/lib/compute/substrate.dart index 4b153d870..08b534fe9 100644 --- a/lib/compute/substrate.dart +++ b/lib/compute/substrate.dart @@ -1220,6 +1220,11 @@ class SleepWindowOverride { String localDateLabel(int epochSec) => dayLabelOf(DateTime.fromMillisecondsSinceEpoch(epochSec * 1000)); +// RED controls: the walk still segments every day. The counter observes its +// actual body; GREEN must preserve the full walk for folding callers. +int debugSegmentedDayCount = 0; +bool debugStageEveryWalkedDay = false; + /// Split the substrate into CALENDAR days (local midnight → next local midnight). /// /// Each day owns its 24 h of data. A day's SLEEP is the main sleep that ENDED @@ -1249,12 +1254,19 @@ String localDateLabel(int epochSec) => /// always fell back to the fixed 03:30 cold-start anchor, and a night-shift /// sleeper's 4 h main block lost the alignment bonus to a shorter block nearer /// 03:30. Days found in THIS call still win (a full restage sees them all). +/// +/// With [skipUnusedDays], only [targetDay] is needed. Earlier days can be +/// skipped when even one new history key per day cannot reach the prior's +/// 14-night threshold. Callers needing profile observations must keep the +/// full walk: its longest observation can belong to an earlier day. List calendarDays( Substrate sub, { SleepWindowOverride? override, int Function(int epochSec)? tzOffsetAt, List<({int startSec, int endSec, String dayKey})> priorSleep = const [], List vendorNights = const [], + String? targetDay, + bool skipUnusedDays = false, }) { final tzOffset = tzOffsetAt ?? tzOffsetSecondsAt; if (sub.isEmpty) return const []; @@ -1268,6 +1280,20 @@ List calendarDays( ...priorSleep, ]; var dayStart = _localMidnight(dataStart); + final prunable = targetDay != null && + skipUnusedDays && + !debugStageEveryWalkedDay && + (override == null || override.dayId == targetDay); + var skipBefore = false; + if (prunable) { + var earlier = 0; + for (var d = dayStart, g = 0; + d < dataEnd && g++ < 400 && localDateLabel(d).compareTo(targetDay) < 0; + d = _nextLocalMidnight(d)) { + earlier++; + } + skipBefore = {for (final h in sleepHistory) h.dayKey}.length + earlier < 14; + } var guard = 0; while (dayStart < dataEnd && guard++ < 400) { final dayEnd = _nextLocalMidnight(dayStart); @@ -1278,6 +1304,16 @@ List calendarDays( continue; } + final dayLabel = localDateLabel(dayStart); + if (prunable) { + final order = dayLabel.compareTo(targetDay); + if (order > 0) break; + if (order < 0 && skipBefore) { + dayStart = dayEnd; + continue; + } + } + debugSegmentedDayCount++; // The main sleep that ENDS in this calendar day: search from the previous // local noon through this local midnight, then let the sleep selector pick // the overnight main block from any naps / split fragments it sees. The old @@ -1293,7 +1329,6 @@ List calendarDays( var sleepLo = 0, sleepHi = 0; var sleepSource = 'none'; final vendorIgnored = []; - final dayLabel = localDateLabel(dayStart); // Does the user have an override (manual / confirmed) for THIS day? final ov = (override != null && override.dayId == dayLabel) ? override : null; diff --git a/lib/data/db.dart b/lib/data/db.dart index 4715651a0..b71e72556 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -14979,6 +14979,21 @@ class LocalDb { ]; } + @visibleForTesting + static bool debugDisableSessionWindowsWorker = false; + @visibleForTesting + static void Function(String)? debugOnSessionWindowsConstruction; + + // RED forwarding seam: still performs the two existing reads. GREEN combines + // these at the repository call site and prepares both results in one worker. + @visibleForTesting + static Future<({Map> stats, + Map> samples})> sessionHrReadsForTest(int from, int to, + {int maxHrCeiling = 0, int minHrFloor = 0}) async => + (stats: await sessionHrStats(from, to, + maxHrCeiling: maxHrCeiling, minHrFloor: minHrFloor), + samples: await sessionHrSamplesBySession(from, to)); + /// Every session in [fromTs, toTs] (epoch SECONDS) with the worn rows of /// its window a session read keeps ([sessionWindowRows]), ascending. Only /// with a workout sensor flagged on; a session with none is absent, as from @@ -14997,6 +15012,7 @@ class LocalDb { 'ORDER BY s.id, d.rec_ts ASC', [fromTs, toTs], ); + debugOnSessionWindowsConstruction?.call(Isolate.current.debugName ?? 'unnamed'); final by = >>{}; for (final r in rows) { final id = r['id'] as String?; diff --git a/test/ed2a_e12_staging_cache_test.dart b/test/ed2a_e12_staging_cache_test.dart new file mode 100644 index 000000000..692f34b8c --- /dev/null +++ b/test/ed2a_e12_staging_cache_test.dart @@ -0,0 +1,275 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/sleep_profile_policy.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +Future _profile({String field = 'hr_floor_p5', int delta = 1}) async { + final p = {...const ana.SleepUserProfile(nights: 14, hrFloorP5: 48, + hrFloorP25: 51, hrSleepMedian: 55, hrArousal: 70, rmssdMed: 40, + rmssdMad: 5, enmoStillCut: 0.02, enmoMoveCut: 0.1, + lfhfMed: 1, rkMed: 2).toJson(), 'folded_days': [edDay]}; + p[field] = (p[field] as num) + delta; + await LocalDb.putBaseline('sleep_user_profile', jsonEncode(p)); +} + +Future _history({int shift = 0}) async { + final db = await LocalDb.instance; + for (var i = 1; i <= 14; i++) { + final d = DateTime(2020, 1, 2 - i); + final start = DateTime(d.year, d.month, d.day, 1).millisecondsSinceEpoch; + await db.insert('day_result', {'day_id': dayLabelOf(d), + 'algo_version': kAlgoVersion, 'payload_json': '{}', 'computed_at': 1, + 'window_json': jsonEncode({'onset_ms': start + shift * 1000, + 'offset_ms': start + (6 * 3600 + shift) * 1000})}, + conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +Future _vendor({String stage = 'light', String id = ''}) async { + final db = await LocalDb.instance; + for (final (offset, duration, label) in [(0, 7200, stage), + (7200, 3600, 'deep'), (10800, 3600, 'rem')]) { + await db.insert('vendor_sleep_epoch', { + 'device_id': id, 'night_onset_ts': edStart, 'start_ts': edStart + offset, + 'end_ts': edStart + offset + duration, 'stage': label, + 'source': 'gen4', 'decoded_at': 1, + }, conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +void main() { + final originalZone = Platform.environment['TZ'] ?? 'UTC'; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e12.db'); + await edRows(edStart, edStart + 1200, hr: 60, az: 1); + }); + tearDown(() async { edSetZone(originalZone); await edDrop(); }); + + test('unchanged staging input is computed once, including absent sleep', () async { + final e = DerivationEngine(); + final first = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, 1); + final again = await e.debugSleepCandidate(edDay); + expect(jsonEncode(again.toJson()), jsonEncode(first.toJson())); + expect(e.debugStagingComputations, 1, + reason: 'the second call must reuse the worker result, even absence'); + }); + + test('the staging worker closure does not drag the engine into the isolate', () async { + // AppState hands its engine a callback over its own fields (BLE phase + // futures among them). If the worker closure captured the engine, that + // callback would ride into Isolate.run and the send would be refused. + final unsendable = Completer(); + final e = DerivationEngine(offloadActive: () => unsendable.isCompleted); + final c = await e.debugSleepCandidate(edDay); + expect(c.toJson(), isNotNull); + expect(e.debugStagingComputations, 1); + }); + + final changes = Function(DerivationEngine)>{ + 'identical input replacement': (_) => edRows(edStart, edStart + 1, hr: 60, az: 1), + 'late RR revision': (_) async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': edStart * 1000, 'rec_ts': edStart, 'rr_ts_ms': edStart * 1000, 'beat_index': 0, 'rr_ms': 900}); + }, + 'search lookback revision on previous local day': (_) => + edRows(DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000, + DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000 + 1), + 'store generation with unchanged revisions': (_) async { + final before = await LocalDb.inputDayFingerprint(edDay); + await LocalDb.close(); await LocalDb.instance; + final after = await LocalDb.inputDayFingerprint(edDay); + expect(after.revisionSum, before.revisionSum); + expect(after.generation, greaterThan(before.generation)); + }, + 'algorithm': (e) async { e.debugStagedSleepCacheRekey(kAlgoVersion - 1); }, + 'HR ownership': (_) => edOwnership(signals: ['hr1Hz']), + 'accel ownership': (_) => edOwnership(signals: ['accel1Hz']), + 'PPG ownership': (_) => edOwnership(signals: ['ppgRedIr']), + 'skin-temperature ownership': (_) => edOwnership(signals: ['skinTempRaw']), + 'RR ownership': (_) async { + await (await LocalDb.instance).insert('device_coverage', { + 'device_id': 'rr-owner', 'signal': 'rrIntervals', 'start_ts': edStart, + 'end_ts': edStart + 600}); + await (await LocalDb.instance).insert('signal_priority', { + 'device_id': 'rr-owner', 'signal': 'rrIntervals', 'rank': 0, 'user_set': 1}); + }, + 'priority': (_) async { + await (await LocalDb.instance).insert('signal_priority', {'signal': 'hr1Hz', + 'device_id': '', 'rank': 4, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'family': (_) async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}); }, + 'history at 14 distinct nights': (_) => _history(shift: 60), + 'manual override': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'manual'), + 'confirmed wake override': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'confirmed'), + 'override onset edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart + 30, offsetTs: edStart + 900, source: 'manual'), + 'override offset edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 960, source: 'manual'), + 'override source edit': (_) => LocalDb.putSleepOverride(dayId: edDay, + onsetTs: edStart, offsetTs: edStart + 900, source: 'rejected'), + 'override removal': (_) => LocalDb.deleteSleepOverride(edDay), + 'vendor source': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'source': 'miband234'}); }, + 'vendor decode revision': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'decoded_at': 2}); }, + 'vendor window': (_) async { await (await LocalDb.instance).update( + 'vendor_sleep_epoch', {'end_ts': edStart + 14460}, + where: 'start_ts = ?', whereArgs: [edStart + 10800]); }, + 'vendor stages': (_) => _vendor(stage: 'deep'), + 'vendor device': (_) => _vendor(id: 'vendor-other'), + 'vendor flag': (_) => LocalDb.setCursor('wearable_enabled:garmin', '1'), + 'active wearable': (_) => LocalDb.setCursor('active_wearable', 'garmin-y'), + 'zone': (_) async { edSetZone('Pacific/Auckland'); }, + // Denver and Phoenix agree in winter, but disagree in the summer search + // window. The separate historical-zone case below pins that distinction. + for (final field in ['nights', 'hr_floor_p5', 'hr_floor_p25', + 'hr_sleep_median', 'hr_arousal', 'rmssd_med', 'rmssd_mad', + 'enmo_still_cut', 'enmo_move_cut', 'lfhf_med', 'rk_med', 'updated_at_ms']) + 'profile $field': (_) => _profile(field: field), + }; + for (final entry in changes.entries) { + test('${entry.key}: miss, exact output, then hit on the new key', () async { + if (entry.key == 'override removal' || entry.key.startsWith('override ') && entry.key.endsWith(' edit')) { + await LocalDb.putSleepOverride(dayId: edDay, onsetTs: edStart, + offsetTs: edStart + 900, source: 'manual'); + } + if (entry.key == 'priority') await edPriorityOrder(['', 'other']); + if (entry.key.startsWith('profile ')) await _profile(delta: 0); + if (entry.key.startsWith('history ')) await _history(); + if (entry.key.startsWith('vendor ') || entry.key == 'active wearable') { + await _vendor(); + await LocalDb.upsertDevice(id: 'garmin-x', adapterId: 'garmin', label: 'watch'); + await LocalDb.setCursor('active_wearable', 'garmin-x'); + await _vendor(id: 'garmin-x'); + } + if (entry.key == 'vendor flag' || entry.key == 'active wearable') { + await (await LocalDb.instance).delete('decoded_onehz'); + await (await LocalDb.instance).delete('vendor_sleep_epoch', + where: 'device_id = ?', whereArgs: ['']); + } + if (entry.key == 'vendor device') { + await LocalDb.upsertDevice(id: 'vendor-other', adapterId: 'garmin', label: 'new vendor'); + await LocalDb.setCursor('active_wearable', 'vendor-other'); + await LocalDb.setCursor('wearable_enabled:garmin', '1'); + await (await LocalDb.instance).delete('decoded_onehz'); + } + if (entry.key == 'active wearable') { + await LocalDb.upsertDevice(id: 'garmin-y', adapterId: 'garmin', label: 'watch2'); + await _vendor(id: 'garmin-y'); + await LocalDb.setCursor('wearable_enabled:garmin', '1'); + } + final e = DerivationEngine(); + await e.debugSleepCandidate(edDay); + final count = e.debugStagingComputations; + await entry.value(e); + final changed = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count + 1, + reason: 'changed ${entry.key} must invalidate the staged result'); + final cold = DerivationEngine()..debugDisableStagedSleepCache = true; + expect(jsonEncode(changed.toJson()), + jsonEncode((await cold.debugSleepCandidate(edDay)).toJson())); + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count + 1, + reason: 'the replacement entry must now hit'); + }); + } + + test('same present-day zone name/offset cannot hide historical night offsets', () async { + edSetZone('America/Chihuahua'); + final nowOffset = DateTime.now().timeZoneOffset; + final nowName = DateTime.now().timeZoneName; + final historical = DateTime.fromMillisecondsSinceEpoch(edStart * 1000).timeZoneOffset; + final e = DerivationEngine(); + await e.debugSleepCandidate(edDay); + final before = e.debugStagingComputations; + edSetZone('America/Guatemala'); + expect(DateTime.now().timeZoneOffset, nowOffset); + expect(DateTime.now().timeZoneName, nowName); + expect(DateTime.fromMillisecondsSinceEpoch(edStart * 1000).timeZoneOffset, + isNot(historical), reason: 'fixture zones agree now but differed in January 2020'); + final changed = await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, before + 1); + final cold = await DerivationEngine().debugSleepCandidate(edDay); + expect(jsonEncode(changed.toJson()), jsonEncode(cold.toJson())); + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, before + 1, reason: 'new historical-offset key must hit'); + }); + + test('selected derive followed by baseline rescan reuses actual staging work', () async { + final e = DerivationEngine(); + expect(await e.runDays(edProfile, {edDay}), 1); + final before = e.debugStagingComputations; + expect(before, greaterThan(0)); + await LocalDb.setCursor('baseline_sig', 'force-ed2a-rescan'); + expect(await e.rescanRecent(edProfile), greaterThan(0)); + expect(e.debugStagingComputations, before, + reason: 'baseline recompute must not rerun the unchanged staging worker'); + }); + + test('a nonfolding entry cannot satisfy a later mayFold request', () async { + final e = DerivationEngine(); + final modes = []; + e.debugOnStagingWalk = (fold, _, _) => modes.add(fold); + await e.debugSleepCandidate(edDay); + expect(modes, [false]); + final edge = DateTime(2020, 1, 3, 12).millisecondsSinceEpoch ~/ 1000; + await edRows(edge, edge + 1); + final changed = await e.debugSleepCandidate(edDay); + expect(modes, [false, true], reason: 'a hit without an observation cannot suppress a fold'); + final cold = await DerivationEngine().debugSleepCandidate(edDay); + expect(jsonEncode(changed.toJson()), jsonEncode(cold.toJson())); + final count = e.debugStagingComputations; + await e.debugSleepCandidate(edDay); + expect(e.debugStagingComputations, count, reason: 'fold-capable result is reusable even when observation is absent'); + }); + + test('bounded staging cache retains only the most recent two days', () async { + final e = DerivationEngine()..debugStagedSleepCacheMaxDays = 2; + for (final day in ['2020-01-01', edDay, '2020-01-03']) { + await e.debugSleepCandidate(day); + expect(e.debugStagedSleepCacheSize, lessThanOrEqualTo(2)); + } + expect(e.debugStagedSleepCacheSize, 2, + reason: 'three distinct staging keys must populate the bounded cache'); + final count = e.debugStagingComputations; + await e.debugSleepCandidate('2020-01-03'); + expect(e.debugStagingComputations, count); + await e.debugSleepCandidate('2020-01-01'); + expect(e.debugStagingComputations, count + 1, reason: 'oldest day was evicted'); + }); + + test('existing fixture: cached and uncached derive JSON stays exact after invalidation', () async { + await edExistingFixture(); + final cached = DerivationEngine(); + final uncached = DerivationEngine()..debugDisableStagedSleepCache = true; + await edPreparedOutput(cached); + expect(await edPreparedOutput(cached), await edPreparedOutput(uncached)); + await edRows(edStart, edStart + 1, hr: 87); + expect(await edPreparedOutput(cached), await edPreparedOutput(uncached)); + }); + + test('profile fold bookkeeping remains idempotent (passing guard)', () { + final p = SleepProfilePolicy.withFoldedDays( + const ana.SleepUserProfile(nights: 3).toJson(), {}, edDay); + expect(SleepProfilePolicy.shouldFold(alreadyFolded: {edDay}, + dayId: edDay, hasOverride: false), isFalse); + expect(SleepProfilePolicy.foldedDays(jsonEncode(p)), {edDay}); + }); +} diff --git a/test/ed2a_e15_restricted_target_walk_test.dart b/test/ed2a_e15_restricted_target_walk_test.dart new file mode 100644 index 000000000..331d4a43a --- /dev/null +++ b/test/ed2a_e15_restricted_target_walk_test.dart @@ -0,0 +1,166 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_analytics/onehz.dart' as ana; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +List<({int startSec, int endSec, String dayKey})> _history(int distinct, + {bool duplicates = false}) => [ + for (var i = 0; i < (duplicates ? 30 : distinct); i++) + (startSec: DateTime(2019, 12, 1 + i % distinct, 1).millisecondsSinceEpoch ~/ 1000, + endSec: DateTime(2019, 12, 1 + i % distinct, 7).millisecondsSinceEpoch ~/ 1000, + dayKey: dayLabelOf(DateTime(2019, 12, 1 + i % distinct))), +]; + +PreparedDerivationPayload _walk(Substrate sub, {required bool skip, + List<({int startSec, int endSec, String dayKey})> history = const [], + SleepWindowOverride? override}) { + debugSegmentedDayCount = 0; + return prepareDerivationPayload(sub, targetDay: edDay, + priorSleep: history, override: override, skipUnusedDays: skip); +} + +void main() { + setUpAll(edInit); + setUp(() => edFresh("ed2a_e15.db")); + tearDown(() async { + debugStageEveryWalkedDay = false; + debugSegmentedDayCount = 0; + ana.cardioRecordObservations = false; + ana.cardioUserProfile = null; + ana.resetCardioObservations(); + await edDrop(); + }); + + // Two local days, with enough seconds for the segmentation body on each. + final sub = edSmallSub(start: DateTime(2020, 1, 1, 22).millisecondsSinceEpoch ~/ 1000, + count: 10 * 3600); + for (final h in [0, 1, 12]) { + test('$h stored nights cannot reach 14: skip earlier day, equal target JSON', () { + final history = _history(h); + final full = _walk(sub, skip: false, history: history); + expect(debugSegmentedDayCount, 2); + final fast = _walk(sub, skip: true, history: history); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), + jsonEncode(edBundle(full.days.single))); + expect(debugSegmentedDayCount, 1, + reason: '$h distinct stored nights + one earlier day is below 14'); + }); + } + + test('30 rows with only 2 distinct night keys still permit the skip', () { + final history = _history(2, duplicates: true); + final full = _walk(sub, skip: false, history: history); + final fast = _walk(sub, skip: true, history: history); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(debugSegmentedDayCount, 1, reason: 'count distinct nights, not rows'); + }); + + for (final h in [13, 14, 15]) { + test('$h stored nights may reach 14: preserve full earlier walk (guard)', () { + final full = _walk(sub, skip: false, history: _history(h)); + final fast = _walk(sub, skip: true, history: _history(h)); + expect(debugSegmentedDayCount, 2, + reason: '13 prior nights plus one walked night can reach the prior threshold'); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), jsonEncode(edBundle(full.days.single))); + }); + } + + test('override on an earlier day preserves its history contribution (guard)', () { + final override = SleepWindowOverride(dayId: '2020-01-01', + onsetSec: sub.tsSec.first, offsetSec: sub.tsSec.first + 3600, source: 'manual'); + final full = _walk(sub, skip: false, override: override); + final fast = _walk(sub, skip: true, override: override); + expect(debugSegmentedDayCount, 2); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + }); + + test('full-walk control preserves upstream observations (guard)', () { + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final full = _walk(sub, skip: false); + final fullEpochs = [for (final o in ana.takeCardioObservations()) o.epochs]; + ana.resetCardioObservations(); + debugStageEveryWalkedDay = true; + final forced = _walk(sub, skip: true); + expect(debugSegmentedDayCount, 2); + expect(jsonEncode(forced.toJson()), jsonEncode(full.toJson())); + expect([for (final o in ana.takeCardioObservations()) o.epochs], fullEpochs); + }); + + test('production nonfolding worker actually requests the restricted skip', () async { + await edFresh('ed2a_e15_nonfold.db'); + await edRows(sub.tsSec.first, sub.tsSec.last + 1, hr: 55, az: 1); + final walked = <(bool, int)>[]; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, count, _) => walked.add((fold, count)); + final got = await e.debugSleepCandidate(edDay); + final full = DerivationEngine()..debugDisableTargetDaySkip = true; + expect(jsonEncode(got.toJson()), + jsonEncode((await full.debugSleepCandidate(edDay)).toJson())); + expect(walked, [(false, 1)], + reason: 'nonfolding production staging must skip the irrelevant earlier day'); + }); + + test('mayFold always walks earlier days and folds once (guard; D36 excluded)', () async { + await edFresh('ed2a_e15_fold.db'); + await edRows(sub.tsSec.first, sub.tsSec.last + 1, hr: 55, az: 1); + final edge = DateTime(2020, 1, 3, 12).millisecondsSinceEpoch ~/ 1000; + await edRows(edge, edge + 1); + final search = await DerivationEngine().debugLoadSubstrate( + sub.tsSec.first, sub.tsSec.last, day: edDay); + ana.cardioUserProfile = null; + ana.cardioRecordObservations = true; + ana.resetCardioObservations(); + final full = prepareSleepSessionCandidate(search, targetDay: edDay, skipUnusedDays: false); + final observations = ana.takeCardioObservations() + ..sort((a, b) => b.epochs.compareTo(a.epochs)); + expect(observations, isNotEmpty); + final o = observations.first; + final fullObservation = jsonEncode({'epochs': o.epochs, + 'hr_floor_p5': o.hrFloorP5, 'hr_floor_p25': o.hrFloorP25, + 'hr_sleep_median': o.hrSleepMedian, 'hr_arousal': o.hrArousal, + 'rmssd_med': o.rmssdMed, 'rmssd_mad': o.rmssdMad, + 'enmo_still_cut': o.enmoStillCut, 'enmo_move_cut': o.enmoMoveCut, + 'lfhf_med': o.lfhfMed, 'rk_med': o.rkMed}); + final walked = <(bool, int, String?)>[]; + final e = DerivationEngine()..debugOnStagingWalk = + (fold, count, obs) => walked.add((fold, count, obs)); + final candidate = await e.debugSleepCandidate(edDay); + expect(jsonEncode(candidate.toJson()), jsonEncode(full.toJson())); + expect(walked.single.$3, fullObservation, reason: "preserve the full walk's actual fold input"); + expect(walked.single.$1, isTrue, reason: 'fixture reaches a folding pass'); + expect(walked.single.$2, 2, + reason: 'the longest observation can still belong to the earlier day upstream'); + expect(walked.single.$3, isNotNull, reason: 'fixture must produce a real observation'); + final profile = (await LocalDb.baseline('sleep_user_profile'))!['payload_json']; + final p = jsonDecode(profile as String) as Map; + expect(p['nights'], 1); + expect(p['folded_days'], contains(edDay)); + await e.debugSleepCandidate(edDay); + expect((await LocalDb.baseline('sleep_user_profile'))!['payload_json'], profile, + reason: 'repeated staging must preserve every profile byte'); + }); + + test('existing fixture has exact target output with skip on and off (guard)', () async { + await edFresh('ed2a_e15_fixture.db'); + await edExistingFixture(); + final e = DerivationEngine(); + final sub = await e.debugLoadSubstrate(1577880000, 1578038400, day: edDay); + final full = _walk(sub, skip: false); + final fast = _walk(sub, skip: true); + expect(jsonEncode(fast.toJson()), jsonEncode(full.toJson())); + expect(jsonEncode(edBundle(fast.days.single)), jsonEncode(edBundle(full.days.single))); + }); +} diff --git a/test/support/ed2a_recalc_fixture.dart b/test/support/ed2a_recalc_fixture.dart new file mode 100644 index 000000000..0f0dfb125 --- /dev/null +++ b/test/support/ed2a_recalc_fixture.dart @@ -0,0 +1,184 @@ +// Shared ED-2a inputs. No cache or invalidation policy lives in this harness. +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:ffi/ffi.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/onehz_pipeline.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/day_label.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +const edProfile = Profile(ageYears: 35, weightKg: 75, heightCm: 178, + sex: 'male', restingHrManual: 54); +const edDay = '2020-01-02'; +final edStart = DateTime(2020, 1, 2, 10).millisecondsSinceEpoch ~/ 1000; + +void edInit() { + TestWidgetsFlutterBinding.ensureInitialized(); + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; +} + +Future edDrop() async { + await LocalDb.close(); + final dir = await databaseFactory.getDatabasesPath(); + await databaseFactory.deleteDatabase(p.join(dir, LocalDb.dbName)); + LocalDb.sessionSensorSources = {}; + LocalDb.sessionWearableSource = null; + DerivationEngine.debugSecondHalfHook = null; + DerivationEngine.debugRunning = false; +} + +Future edFresh(String name) async { + await LocalDb.close(); + LocalDb.dbName = name; + await edDrop(); + SharedPreferences.setMockInitialValues({}); + await LocalDb.instance; +} + +Future edRows(int from, int to, {int hr = 170, String id = '', + String family = 'gen4', String? source, double az = 1.3}) async { + final db = await LocalDb.instance; + final batch = db.batch(); + for (var ts = from; ts < to; ts++) { + batch.insert('decoded_onehz', {'device_id': id, 'ts_ms': ts * 1000, + 'rec_ts': ts, 'counter': ts, 'hr': hr, 'ax': 0.0, 'ay': 0.0, + 'az': az, 'device_family': family, 'source': source}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + await batch.commit(noResult: true); +} + +Future edSession({String id = 's1', String status = 'done', + int? start, int? end}) => LocalDb.putSession({'id': id, 'type': 'run', + 'status': status, 'start_ts': start ?? edStart, + 'end_ts': end ?? edStart + 1100, 'source': 'manual', + 'created_at': edStart * 1000}); + +// The committed two_device_day.json region table and formulas, also used by +// two_device_fixture_test.dart. Keep the fixture intact for exactness laws. +Future edExistingFixture() async { + final spec = jsonDecode(File('test/fixtures/two_device_day.json').readAsStringSync()) as Map; + const start = 1577923200; + int offset(String time) { + final c = time.split(':').map(int.parse).toList(); + return c[0] * 3600 + c[1] * 60 + c[2]; + } + final db = await LocalDb.instance; + final b = db.batch(); + for (final region in spec['regions'] as List) { + final r = region as Map; + final holes = {for (final h in (r['holes'] as List?) ?? []) offset(h as String)}; + for (var i = offset(r['start'] as String); i <= offset(r['end'] as String); i++) { + if (holes.contains(i)) continue; + final ts = start + i; + for (final family in r['devices'] as List) { + final primary = family == 'gen4'; + final id = primary ? '' : 'oura-a1b2c3d4'; + b.insert('decoded_onehz', {'device_id': id, 'ts_ms': ts * 1000, + 'rec_ts': ts, 'counter': ts, 'device_family': family, + 'hr': primary ? 60 + ts % 7 : null, + 'ax': primary ? 0.0 : null, 'ay': primary ? 0.0 : null, + 'az': primary ? 1.0 : null, + 'skin_temp_raw': primary ? 30000 + ts % 11 : null, + 'skin_temp_c': primary ? null : 33.0 + ts % 9 / 10}, + conflictAlgorithm: ConflictAlgorithm.replace); + if (primary) { + b.insert('decoded_rr', {'device_id': '', 'ts_ms': ts * 1000, + 'rec_ts': ts, 'rr_ts_ms': ts * 1000, 'beat_index': 0, 'rr_ms': 1000 - ts % 5 * 10}, + conflictAlgorithm: ConflictAlgorithm.replace); + } + } + } + } + await b.commit(noResult: true); +} + +Map edBundle(PreparedDerivationDay day) { + final d = day.daySub, s = day.sleepSub; + return deriveDayBundle(DayBundleInput(date: day.date, + dayTsSec: d.tsSec, dayHr: d.hr, dayRrTsMs: d.rrTsMs, dayRrMs: d.rrMs, + sleepTsSec: s.tsSec, sleepHr: s.hr, sleepRrTsMs: s.rrTsMs, + sleepRrMs: s.rrMs, sleepSkinTemp: s.skinTemp, sleepJson: day.sleepJson, + hypnoStages: day.hypnoStages, sleepOnsetSec: day.sleepOnsetSec, + sleepOffsetSec: day.sleepOffsetSec, profile: edProfile.toMap(), + dayConfidence: day.confidence, dayFlags: day.flags, + deviceFamily: d.deviceFamily, sleepSource: day.sleepSource).toJson()); +} + +Future edPreparedOutput(DerivationEngine engine) async { + final prepared = await engine.debugPrepareDay(edDay); + expect(prepared, isNotNull); + expect(prepared!.daySub.isEmpty, isFalse, reason: 'fixture must reach compute'); + return jsonEncode({'prepared': prepared.toJson(), 'bundle': edBundle(prepared), + 'profile': (await LocalDb.baseline('sleep_user_profile'))?['payload_json']}); +} + +// Strip only operational timestamps, never metric values, absence, notes, +// confidence, profile state or version. JSON property order stays observable. +Future edStored(String day) async { + final db = await LocalDb.instance; + final rows = await db.query('day_result', where: 'day_id = ?', + whereArgs: [day], orderBy: 'algo_version'); + expect(rows, isNotEmpty, reason: 'the real derive must have persisted a day'); + return jsonEncode({'day_result': [for (final row in rows) { + for (final e in row.entries) + if (e.key == 'payload_json') e.key: jsonEncode( + (jsonDecode(e.value as String) as Map)..remove('computed_at')) + else if (e.key != 'computed_at') e.key: e.value, + }], 'metric_series': await db.query('metric_series', where: 'date = ?', + whereArgs: [day], orderBy: 'key'), + 'profile': (await LocalDb.baseline('sleep_user_profile'))?['payload_json']}); +} + +Future edOwnership({String id = 'other', + List signals = const ['hr1Hz', 'rrIntervals']}) async { + final db = await LocalDb.instance; + for (final signal in signals) { + await db.insert('device_coverage', {'device_id': id, 'signal': signal, + 'start_ts': edStart, 'end_ts': edStart + 600}, + conflictAlgorithm: ConflictAlgorithm.replace); + await db.insert('signal_priority', {'signal': signal, 'device_id': id, + 'rank': 0, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + await db.insert('signal_priority', {'signal': signal, 'device_id': '', + 'rank': 1, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + } +} + +String edLabel(int second) => dayLabelOf(DateTime.fromMillisecondsSinceEpoch(second * 1000)); +Substrate edSmallSub({int? start, int count = 1200}) => Substrate.fromJson({ + 'ts_sec': [for (var i = 0; i < count; i++) (start ?? edStart) + i], + 'hr': List.filled(count, 60), + 'ax': List.filled(count, 0), 'ay': List.filled(count, 0), + 'az': List.filled(count, 1), 'device_family': 'gen4', +}); + +// Same libc seam used by the fork's zone regressions. No process spawning or +// Dart clock substitution: production DateTime resolves each instant's zone. +typedef _Setenv = Int32 Function(Pointer, Pointer, Int32); +typedef _SetenvDart = int Function(Pointer, Pointer, int); +void edSetZone(String zone) { + final libc = DynamicLibrary.process(); + final key = 'TZ'.toNativeUtf8(), value = zone.toNativeUtf8(); + try { + expect(libc.lookupFunction<_Setenv, _SetenvDart>('setenv')(key, value, 1), 0); + libc.lookupFunction('tzset')(); + } finally { calloc.free(key); calloc.free(value); } +} + +Future edPriorityOrder(List ids) async { + final db = await LocalDb.instance; + for (var rank = 0; rank < ids.length; rank++) { + await db.insert('signal_priority', {'signal': 'hr1Hz', 'device_id': ids[rank], + 'rank': rank, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + } +} From f693d588dcafa6e29fe524b98e31b0e6a8e5477f Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 05:36:50 -0600 Subject: [PATCH 3/5] derive: keep the last loaded substrate per day between passes while its inputs are unchanged The engine retains a bounded number of loaded day substrates, keyed like the staged-sleep cache by input revisions, store generation and ownership, and reuses an exact prefix or tail instead of reloading the day from SQLite. Raw and monotonized beat axes, starts and bucket provenance are kept as loaded. Retention is dropped when the app goes to the background, on memory pressure, on dispose, at finalization and eviction, and after a failed or unprepared pass. Output is unchanged; kAlgoVersion stays 113. --- lib/app.dart | 6 + lib/compute/derivation_engine.dart | 211 +++++++++++++++-- lib/compute/derive_prepare.dart | 105 ++++++++- lib/compute/substrate_retention.dart | 102 ++++++++ lib/state/app_state.dart | 9 +- test/ed2a_e13_substrate_retention_test.dart | 247 ++++++++++++++++++++ 6 files changed, 655 insertions(+), 25 deletions(-) create mode 100644 lib/compute/substrate_retention.dart create mode 100644 test/ed2a_e13_substrate_retention_test.dart diff --git a/lib/app.dart b/lib/app.dart index c4bfc9b4e..d2c32afbd 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -76,10 +76,16 @@ class _OpenStrapAppState extends State with WidgetsBindingObserver WidgetsBinding.instance.platformDispatcher.platformBrightness); } + @override + void didHaveMemoryPressure() { + context.read().handleMemoryPressure(); + } + @override void didChangeAppLifecycleState(AppLifecycleState state) { final app = context.read(); if (state == AppLifecycleState.resumed) { + app.resumeDerivationRetention(); // The user may have flipped our notification switch either way in OS // Settings while we were backgrounded. Drop the cached authorization // decision so the next present/schedule re-reads reality — a denial used diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 6a03dad90..51b4b0073 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -71,6 +71,7 @@ import 'hr_max.dart' import 'movement_floor_policy.dart' as mfp; import 'sleep_profile_policy.dart'; import 'staged_sleep_cache.dart'; +import 'substrate_retention.dart'; import 'derive_prepare.dart'; import 'onehz_pipeline.dart'; import 'step_cadence.dart'; @@ -3095,7 +3096,7 @@ class DerivationEngine { // ED-2a RED seams. Empty cache views and no-op controls describe the current // uncached engine. Counters below observe real worker work, never results. - // E12 now backs the staging controls; later steps' controls remain inert. + // E12/E13 now back staging and retention; later controls remain inert. final _stagedSleepCache = StagedSleepCache(); @visibleForTesting bool debugDisableStagedSleepCache = false; @@ -3108,19 +3109,63 @@ class DerivationEngine { @visibleForTesting void debugStagedSleepCacheRekey(int version) => _stagedSleepCache.rekey(version); @visibleForTesting - bool debugDisableSubstrateRetention = false; + bool get debugDisableSubstrateRetention => _disableSubstrateRetention; @visibleForTesting - int debugRetainedSubstrateMaxDays = 2; - @visibleForTesting - Set get debugRetainedSubstrateDays => const {}; + set debugDisableSubstrateRetention(bool value) { + _disableSubstrateRetention = value; + if (value) _clearRetainedSubstrate(); + } + bool _disableSubstrateRetention = false; @visibleForTesting - void debugRetainedSubstrateRekey(int version) {} + int get debugRetainedSubstrateMaxDays => _retained.maxDays; @visibleForTesting - void trimForBackground() {} + set debugRetainedSubstrateMaxDays(int value) => _retained.maxDays = value; @visibleForTesting - void handleMemoryPressure() {} + Set get debugRetainedSubstrateDays => _retained.days; @visibleForTesting - void dispose() {} + void debugRetainedSubstrateRekey(int version) => _retained.rekey(version); + void trimForBackground() { + _retentionInBackground = true; + _clearRetainedSubstrate(); + } + void resumeFromBackground() => _retentionInBackground = false; + void handleMemoryPressure() => _clearRetainedSubstrate(); + void dispose() { + _retentionDisposed = true; + _clearRetainedSubstrate(); + } + + final _retained = SubstrateRetention(); + int _retentionEpoch = 0; + int? _retentionPassEpoch; + bool _retentionInBackground = false; + bool _retentionDisposed = false; + final _retentionPreparedDays = {}; + bool get _canRetainSubstrate => !debugDisableSubstrateRetention && + !background && !_retentionInBackground && !_retentionDisposed && + (_retentionPassEpoch == null || _retentionPassEpoch == _retentionEpoch); + + void _clearRetainedSubstrate() { + _retentionEpoch++; + _retained.clear(); + } + + void _beginRetentionPass() { + _retentionPassEpoch = _retentionEpoch; + _retentionPreparedDays.clear(); + if (!_canRetainSubstrate) _retained.clear(); + } + + void _endRetentionPass() { + _retained.retainDays(_retentionPreparedDays); + _retentionPreparedDays.clear(); + _retentionPassEpoch = null; + } + + void _dropRetainedDay(String day) { + _retained.remove(day); + _retentionPreparedDays.remove(day); + } @visibleForTesting bool debugDisableSessionReuse = false; @visibleForTesting @@ -3134,7 +3179,7 @@ class DerivationEngine { @visibleForTesting bool debugDisableTargetDaySkip = false; @visibleForTesting - Map debugRetainedSubstrate(String day) => const {}; + Map debugRetainedSubstrate(String day) => _retained.extras(day); @visibleForTesting Future debugSleepCandidate(String day) => _sleepCandidateForDay(day); @@ -3445,6 +3490,7 @@ class DerivationEngine { }) async { if (_running) return 0; _running = true; + _beginRetentionPass(); beginWearableDayIndexPass(); final startedAt = DateTime.now().millisecondsSinceEpoch; _diag @@ -3808,6 +3854,7 @@ class DerivationEngine { } catch (e, st) { _diag['last_error'] = '$e'; lastRunError = '$e'; + _clearRetainedSubstrate(); _log('derive ERROR: $e\n$st'); return 0; } finally { @@ -3824,6 +3871,7 @@ class DerivationEngine { // `_running`. See _runStorageHousekeeping — this is the only place every // entry path and every early return actually reaches. _diag['stage'] = 'housekeeping'; + _endRetentionPass(); await _runStorageHousekeeping(long: heavy || force); // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days // inside the raw window re-derive from substrate above; everything older @@ -3917,6 +3965,7 @@ class DerivationEngine { if (days.isEmpty) return 0; if (_running) return 0; _running = true; + _beginRetentionPass(); beginWearableDayIndexPass(); final startedAt = DateTime.now().millisecondsSinceEpoch; _diag @@ -4028,9 +4077,11 @@ class DerivationEngine { } return done; } catch (e, st) { + _clearRetainedSubstrate(); _log('derive selected ERROR: $e\n$st'); return 0; } finally { + _endRetentionPass(); await _runStorageHousekeeping(); final finishedAt = DateTime.now().millisecondsSinceEpoch; _diag @@ -4048,6 +4099,21 @@ class DerivationEngine { static const int _maxDayRawPages = 300; Future _prepareTargetDay(String dayId) async { + try { + final prepared = await _prepareTargetDayPayload(dayId); + if (prepared == null || prepared.daySub.isEmpty) { + _dropRetainedDay(dayId); + } else if (_retentionPassEpoch != null) { + _retentionPreparedDays.add(dayId); + } + return prepared; + } catch (_) { + _dropRetainedDay(dayId); + rethrow; + } + } + + Future _prepareTargetDayPayload(String dayId) async { // Per-day page/row totals used to live in the shared `_diag` map (reset // then accumulated across this day's 2-3 substrate loads). Under // concurrent per-day processing (see `run()`), multiple days resetting/ @@ -4094,6 +4160,7 @@ class DerivationEngine { dayId: dayId, stats: stats, ownership: ownership, + load: 'day', ); // A day the primary band never saw is the ACTIVE WEARABLE's, when one is // chosen and flagged on (inputs/canonical.dart). Never mixed: a day with @@ -4122,6 +4189,7 @@ class DerivationEngine { dayId: dayId, stats: stats, ownership: ownership, + load: 'sleep', ); } // Single safe merge into the shared max-tracking diagnostics — one @@ -4367,6 +4435,7 @@ class DerivationEngine { dayId: dayId, stats: stats, ownership: searchOwnership, + load: 'search', ); // PERSONALIZED STAGER (v42): stage on a WORKER isolate, NOT the main/UI // thread. cardioStager reads analytics "ambient" globals — the rolling sleep @@ -4785,6 +4854,62 @@ class DerivationEngine { } Future _loadSubstrateRange( + int fromRecTs, + int toRecTs, { + required String dayId, + _PrepareStats? stats, + Map> ownership = const {}, + String load = 'direct', + }) async { + if (toRecTs < fromRecTs) { + _dropRetainedDay(dayId); + return Substrate.empty; + } + final epoch = _retentionEpoch; + String? context; + Map? revisions; + if (_canRetainSubstrate) { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [fromRecTs ~/ 900, toRecTs ~/ 900]); + revisions = {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}; + context = substrateLoadContext(fromRecTs, toRecTs, generation, ownership); + } catch (_) { + // Missing revision provenance always takes the ordinary cold path. + _dropRetainedDay(dayId); + } + } else { + _dropRetainedDay(dayId); + } + final old = context == null ? null : _retained.lookup(dayId, load); + final readFrom = old?.readFrom(kAlgoVersion, context!, revisions!, toRecTs); + if (readFrom != null && readFrom > toRecTs) return old!.sub; + try { + final (sub, extras) = await _readSubstrateRange( + readFrom ?? fromRecTs, toRecTs, dayId: dayId, stats: stats, + ownership: ownership, seed: readFrom == null ? null : old, + retain: context != null); + if (_canRetainSubstrate && epoch == _retentionEpoch && + context != null && extras.isNotEmpty) { + _retained.store(dayId, load, RetainedSubstrate( + version: kAlgoVersion, context: context, revisions: revisions!, + sub: sub, extras: extras, + cut: math.max(fromRecTs, ((sub.lastTs ?? fromRecTs) ~/ 900) * 900))); + } else { + _dropRetainedDay(dayId); + } + return sub; + } catch (_) { + _dropRetainedDay(dayId); + rethrow; + } + } + + Future<(Substrate, Map)> _readSubstrateRange( int fromRecTs, int toRecTs, { required String dayId, @@ -4795,8 +4920,10 @@ class DerivationEngine { // single-device install resolves to anyway and what the direct-load tests // pass. Map> ownership = const {}, + RetainedSubstrate? seed, + bool retain = false, }) async { - if (toRecTs < fromRecTs) return Substrate.empty; + if (toRecTs < fromRecTs) return (Substrate.empty, const {}); final port = ReceivePort(); // onError/onExit are LOAD-BEARING. Without them, an uncaught throw inside // the worker (a malformed SQLite row reaching one of the numeric reads in @@ -4814,6 +4941,7 @@ class DerivationEngine { ); final ready = Completer(); final result = Completer(); + Map extras = const {}; // A failure completes BOTH completers, but we may bail out via `ready` and // never await `result` — register a listener so that error is never an // unobserved async error. (The real error still propagates via `ready`.) @@ -4834,6 +4962,7 @@ class DerivationEngine { if (kind == 'substrate') { final payload = ((message['payload'] as Map?) ?? const {}) .cast(); + extras = ((message['retain'] as Map?) ?? const {}).cast(); if (!result.isCompleted) result.complete(Substrate.fromTransfer(payload)); } return; @@ -4865,11 +4994,17 @@ class DerivationEngine { } try { final worker = await ready.future; - worker.send(const {'type': 'config', 'mode': 'substrate'}); + worker.send({'type': 'config', 'mode': 'substrate', 'retain': retain}); + if (seed != null) worker.send(seed.seed(fromRecTs)); int? afterRecTs; int? afterCursor; - var rangePages = 0; - var rangeRows = 0; + final rawRowsByBucket = { + if (seed != null) + for (final e in (seed.extras['raw_rows_by_bucket'] as Map).entries) + if (e.key < fromRecTs ~/ 900) e.key: e.value, + }; + var rangeRows = rawRowsByBucket.values.fold(0, (int n, v) => n + v); + var rangePages = rangeRows ~/ _rawDecodeBatchSize; // Low water mark for the BEAT read, which is NOT the same window as the // frame read. Paging is driven by `decoded_onehz`, but a beat can exist // for a second with no 1 Hz row at all (gen4 R10-lite is hr-only and @@ -4901,6 +5036,25 @@ class DerivationEngine { ); if (decodedRows.isNotEmpty) { _trackPrepareBatch(decodedRows.length); + if (retain) { + // Count sorted bucket runs, including rows ownership will drop. + // This keeps the existing full-load row budget exact on a tail read. + var i = 0; + while (i < decodedRows.length) { + final bucket = (decodedRows[i]['rec_ts'] as num).toInt() ~/ 900; + var lo = i + 1, hi = decodedRows.length; + while (lo < hi) { + final mid = (lo + hi) >> 1; + if ((decodedRows[mid]['rec_ts'] as num).toInt() < (bucket + 1) * 900) { + lo = mid + 1; + } else { + hi = mid; + } + } + rawRowsByBucket[bucket] = (rawRowsByBucket[bucket] ?? 0) + lo - i; + i = lo; + } + } rangePages += 1; rangeRows += decodedRows.length; if (stats != null) { @@ -4999,12 +5153,15 @@ class DerivationEngine { // BOUNDED. `result.future` had no timeout at all, so any path that left // the worker unable to answer hung this call — and with it the whole // engine — permanently. - return await result.future.timeout( + final loaded = await result.future.timeout( _perDayTimeout, onTimeout: () => throw TimeoutException( 'substrate prepare for $dayId timed out after $_perDayTimeout', ), ); + return (loaded, { + ...extras, if (extras.isNotEmpty) 'raw_rows_by_bucket': rawRowsByBucket, + }); } finally { // ALWAYS tear down: on success, on error, and on timeout. The isolate is // killed rather than abandoned so a wedged worker can never outlive the @@ -5439,6 +5596,7 @@ class DerivationEngine { }) async { if (_running) return 0; _running = true; + _beginRetentionPass(); beginWearableDayIndexPass(); try { await refreshSessionSensorSources(); // as [run] does @@ -5564,9 +5722,11 @@ class DerivationEngine { await LocalDb.refreshComputeFreshness(); return done; } catch (e, st) { + _clearRetainedSubstrate(); _log('rescan ERROR: $e\n$st'); return 0; } finally { + _endRetentionPass(); await _runStorageHousekeeping(); endWearableDayIndexPass(); _running = false; @@ -5983,6 +6143,23 @@ class DerivationEngine { _BaselineHistoryCache history, { bool forceFinalize = false, bool suppliedSubstrate = false, + }) async { + try { + return await _derivePreparedDayPayload(day, profile, dataNowSec, history, + forceFinalize: forceFinalize, suppliedSubstrate: suppliedSubstrate); + } catch (_) { + _dropRetainedDay(day.date); + rethrow; + } + } + + Future _derivePreparedDayPayload( + PreparedDerivationDay day, + Profile profile, + int dataNowSec, + _BaselineHistoryCache history, { + bool forceFinalize = false, + bool suppliedSubstrate = false, }) async { final daySub = day.daySub; final sleepSub = day.sleepSub; @@ -6175,6 +6352,7 @@ class DerivationEngine { if (_isRealDayResult(existing)) { _log('derive ${day.date}: no substrate (raw pruned) — kept the ' 'existing result rather than blanking it'); + _dropRetainedDay(day.date); return false; } } @@ -6250,6 +6428,7 @@ class DerivationEngine { "under a day that already had real night scalars — kept the " 'existing result rather than overwriting the readiness baseline ' '(edge#305)'); + _dropRetainedDay(day.date); return false; } } @@ -6592,6 +6771,7 @@ class DerivationEngine { await _persistWakeDayFeatures(dayId: day.date, wake: blocks.wake); } catch (e, st) { secondHalfOk = false; + _dropRetainedDay(day.date); _log('day-blocks (offloaded second half) failed for ${day.date} — ' 'persisting headline day (partial): $e'); TelemetryService.instance.recordNonFatal(e, st, reason: 'day_blocks_failed'); @@ -6858,6 +7038,7 @@ class DerivationEngine { 'hr_ceiling_bpm': sc('hr_ceiling_bpm'), }, ); + if (effectiveFinalized || effectivePartial) _dropRetainedDay(day.date); // AFTER the row: retired only once the row that proves it is durable. if (quietSettled) await LocalDb.deleteComputeFreshness(quietKey); // NOTE: the sweep's `history` snapshot is deliberately NOT updated here. diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index e3c01a277..b008c9473 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -347,6 +347,11 @@ void derivationPrepareWorker(SendPort mainSendPort) { targetDay = message['target_day']?.toString(); final cfgMode = message['mode']?.toString(); if (cfgMode != null && cfgMode.isNotEmpty) mode = cfgMode; + state.retain = message['retain'] == true; + return; + } + if (type == 'seed') { + state.seed(message); return; } if (type == 'finish') { @@ -357,6 +362,7 @@ void derivationPrepareWorker(SendPort mainSendPort) { 'type': 'result', 'kind': 'substrate', 'payload': substrate.toJson(), + if (state.retainable) 'retain': state.retainExtras(), }); } else { final payload = prepareDerivationPayload( @@ -640,10 +646,68 @@ class _PrepareAccumulator { /// worker used to hang the whole engine (see [derivationPrepareWorker]). static num? _num(Object? v) => v is num ? v : null; + // Retention is recorded only for decoded loads requested by the engine. + bool retain = false; + bool _retainBroken = false; + bool get retainable => retain && !_retainBroken; + final List _rrStart = []; + final Map _buckets = {}; + Float64List? _rrTsRaw; + + void seed(Map message) { + List ints(String k) => message[k] as List; + List dbls(String k) => message[k] as List; + final ts = ints('ts_sec'); + final cut = message['cut'] as int; + var n = 0; + while (n < ts.length && ts[n] < cut) { n++; } + final starts = ints('rr_start'); + final beats = n < ts.length ? starts[n] : dbls('rr_ms').length; + void take(List into, String k) => into.addAll(ints(k).getRange(0, n)); + take(tsSec, 'ts_sec'); + take(hr, 'hr'); + take(spo2Red, 'spo2_red'); + take(spo2Ir, 'spo2_ir'); + take(skinTemp, 'skin_temp'); + take(skinContact, 'skin_contact'); + take(stepCount, 'step_count'); + take(hrValid, 'hr_valid'); + take(bandSleepState, 'band_sleep_state'); + ax.addAll(dbls('ax').getRange(0, n)); + ay.addAll(dbls('ay').getRange(0, n)); + az.addAll(dbls('az').getRange(0, n)); + // Start from the raw axis: buildSubstrate applies the running max once + // across the entire rebuilt load, including the prefix/tail seam. + rrTsMs.addAll(dbls('rr_ts_raw').getRange(0, beats)); + rrMs.addAll(dbls('rr_ms').getRange(0, beats)); + _rrStart.addAll(starts.getRange(0, n)); + for (final wire in message['meta'] as List) { + final m = wire as List; + final bucket = m[0] as int; + if (bucket >= cut ~/ 900) continue; + final acc = _buckets[bucket] = _BucketAcc() + ..families.addAll((m[1] as List).cast()) + ..deviceIds.addAll((m[2] as List).cast()) + ..skinUnit = m[3] as String?; + _families.addAll(acc.families); + _deviceIds.addAll(acc.deviceIds); + _skinTempUnit ??= acc.skinUnit; + } + } + + Map retainExtras() => { + 'rr_start': Int32List.fromList(_rrStart), + 'rr_ts_raw': _rrTsRaw!, + 'meta': [for (final e in _buckets.entries) + [e.key, e.value.families.toList(), e.value.deviceIds.toList(), + e.value.skinUnit]], + }; + void addRawPage(List hexes) { if (hexes.isEmpty) return; final sub = decodeSubstrate(hexes); if (sub.isEmpty) return; + _retainBroken = true; tsSec.addAll(sub.tsSec); hr.addAll(sub.hr); rrTsMs.addAll(sub.rrTsMs); @@ -712,6 +776,10 @@ class _PrepareAccumulator { final row = frameByRecTs[recTs]; _noteFamily(row?['device_family']); _noteDeviceId(row?['device_id']); + if (retain) { + _rrStart.add(rrMs.length); + _buckets.putIfAbsent(recTs ~/ 900, _BucketAcc.new).note(row); + } tsSec.add(recTs); // NULL (absent), a beat-only second (no row at all) and an impossible // byte all land on the same 0 — the array's one "no usable HR this @@ -799,6 +867,7 @@ class _PrepareAccumulator { // `addRawPage` each place their own beats correctly and then APPEND, so an // overlap between the last record of one page and the first of the next is // only visible once every page is in. See [monotonizeBeatAxis]. + if (retainable) _rrTsRaw = Float64List.fromList(rrTsMs); monotonizeBeatAxis(rrTsMs); // `_skinTempUnitDrops` (see [_skinTempFor]) is structurally 0 today — // `derivableSourceSql()` renders `source IS NULL` (db.dart:116), so a @@ -812,23 +881,41 @@ class _PrepareAccumulator { // which M0 must not take; M5 — which is what makes two-device admission // reachable for real — turns this into a refusal note on // `wellness.skin_temp` instead. + List packed(List values) => + retainable ? Int64List.fromList(values) : values; return Substrate( - tsSec: tsSec, - hr: hr, + tsSec: packed(tsSec), + hr: packed(hr), rrTsMs: rrTsMs, rrMs: rrMs, ax: ax, ay: ay, az: az, - spo2Red: spo2Red, - spo2Ir: spo2Ir, - skinTemp: skinTemp, - skinContact: skinContact, - stepCount: stepCount, - hrValid: hrValid, - bandSleepState: bandSleepState, + spo2Red: packed(spo2Red), + spo2Ir: packed(spo2Ir), + skinTemp: packed(skinTemp), + skinContact: packed(skinContact), + stepCount: packed(stepCount), + hrValid: packed(hrValid), + bandSleepState: packed(bandSleepState), deviceFamily: deviceFamily, deviceIds: _deviceIds, ); } } + +class _BucketAcc { + final Set families = {}; + final Set deviceIds = {}; + String? skinUnit; + + void note(Map? row) { + if (row == null) return; + final family = row['device_family']; + if (family is String && family.isNotEmpty) families.add(family); + final id = row['device_id']; + if (id is String) deviceIds.add(id); + skinUnit ??= row['skin_temp_raw'] is num + ? 'raw' : (row['skin_temp_c'] is num ? 'c' : null); + } +} diff --git a/lib/compute/substrate_retention.dart b/lib/compute/substrate_retention.dart new file mode 100644 index 000000000..af77a2b6a --- /dev/null +++ b/lib/compute/substrate_retention.dart @@ -0,0 +1,102 @@ +import 'dart:convert'; + +import '../ble/adapters/signals.dart'; +import '../data/coverage_resolver.dart'; +import 'substrate.dart'; + +/// One exact load, plus the worker metadata needed to rebuild its open bucket. +class RetainedSubstrate { + RetainedSubstrate({required this.version, required this.context, + required this.revisions, required this.sub, required this.extras, + required this.cut}); + + int version; + final String context; + final Map revisions; + final Substrate sub; + final Map extras; + final int cut; + + /// A changed closed bucket requires a cold load. Otherwise rebuild the tail + /// from [cut], or return the whole load when every bucket is unchanged. + int? readFrom(int version, String context, Map revisions, int to) { + if (version != this.version || context != this.context) return null; + var changed = false; + for (final b in {...this.revisions.keys, ...revisions.keys}) { + if ((revisions[b] ?? 0) == (this.revisions[b] ?? 0)) continue; + if (b < cut ~/ 900) return null; + changed = true; + } + return changed ? cut : to + 1; + } + + Map seed(int cut) => { + ...sub.toJson(), ...extras, 'type': 'seed', 'cut': cut, + }; +} + +/// At most three loads per day: search, buffered day and selected sleep. +/// Both days and loads are ordered least recently used first. +class SubstrateRetention { + final _days = >{}; + int _maxDays = 2; + + int get maxDays => _maxDays; + set maxDays(int value) { + _maxDays = value < 0 ? 0 : value; + while (_days.length > _maxDays) { + _days.remove(_days.keys.first); + } + } + Set get days => _days.keys.toSet(); + + RetainedSubstrate? lookup(String day, String load) { + final loads = _days.remove(day); + if (loads == null) return null; + _days[day] = loads; + final entry = loads.remove(load); + if (entry != null) loads[load] = entry; + return entry; + } + + void store(String day, String load, RetainedSubstrate entry) { + if (_maxDays == 0) return; + final loads = _days.remove(day) ?? {}; + loads.remove(load); + loads[load] = entry; + while (loads.length > 3) { + loads.remove(loads.keys.first); + } + _days[day] = loads; + maxDays = _maxDays; + } + + Map extras(String day) => + _days[day]?.values.last.extras ?? const {}; + void remove(String day) => _days.remove(day); + void clear() => _days.clear(); + void retainDays(Set days) => + _days.removeWhere((day, _) => !days.contains(day)); + void rekey(int version) { + for (final loads in _days.values) { + for (final entry in loads.values) { + entry.version = version; + } + } + } +} + +/// Only inputs consumed by the decoded loader belong here. Resolved spans +/// already carry priority, coverage and device eligibility decisions. +String substrateLoadContext(int from, int to, int generation, + Map> ownership) { + final now = DateTime.now(); + return jsonEncode([ + from, to, generation, now.timeZoneName, now.timeZoneOffset.inSeconds, + [for (var t = from - 86400; t <= to + 86400; t += 900) + [t, tzOffsetSecondsAt(t)]], + [for (final sig in InputSignal.values) [sig.name, + [for (final s in ownership[sig] ?? const []) + [s.start, s.end, s.deviceId]]]], + ]); +} diff --git a/lib/state/app_state.dart b/lib/state/app_state.dart index b581d5318..83c947fef 100644 --- a/lib/state/app_state.dart +++ b/lib/state/app_state.dart @@ -2030,6 +2030,7 @@ class AppState extends ChangeNotifier { } _sync.cancelQuietTimer(); _disposed = true; + _derive.dispose(); _ecg?.dispose(); _ecgTransport?.dispose(); // EVERY timer this object owns, not just three of them. @@ -3486,7 +3487,13 @@ class AppState extends ChangeNotifier { /// drops (band out of range / app jettisoned) — see [_onEngineState] / [_armRecovery]. /// /// On Android the Edge Tracking foreground service keeps the process + connection alive. - Future pauseForBackground() => _sync.pauseForBackground(); + Future pauseForBackground() { + _derive.trimForBackground(); + return _sync.pauseForBackground(); + } + + void resumeDerivationRetention() => _derive.resumeFromBackground(); + void handleMemoryPressure() => _derive.handleMemoryPressure(); /// A feature session (workout, breathing, ECG capture) is running — the /// "nothing else in flight" bar the one-off VACUUM waits for. ECG matters diff --git a/test/ed2a_e13_substrate_retention_test.dart b/test/ed2a_e13_substrate_retention_test.dart new file mode 100644 index 000000000..8a72ee80a --- /dev/null +++ b/test/ed2a_e13_substrate_retention_test.dart @@ -0,0 +1,247 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/ble/adapters/signals.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/coverage_resolver.dart'; +import 'package:openstrap_edge/compute/substrate.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +int _reads(DerivationEngine e) => e.snapshot()['raw_rows'] as int; +final _lo = (edStart ~/ 900) * 900; +final _hi = _lo + 5 * 900 - 1; +Future _load(DerivationEngine e, {int? lo, int? hi, + Map> ownership = const {}}) => + e.debugLoadSubstrate(lo ?? _lo, hi ?? _hi, day: edDay, ownership: ownership); + +void main() { + final originalZone = Platform.environment["TZ"] ?? "UTC"; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e13.db'); + await edRows(_lo, _lo + 4 * 900 + 200); + }); + tearDown(() async { edSetZone(originalZone); await edDrop(); }); + + test('unchanged load reuses retained arrays without rereading rows', () async { + final e = DerivationEngine(); + final first = await _load(e); + final reads = _reads(e); + expect(reads, greaterThan(0)); + final next = await _load(e); + expect(jsonEncode(next.toJson()), jsonEncode(first.toJson())); + expect(_reads(e), reads, reason: 'unchanged load must not decode rows twice'); + }); + + test('append rereads at most the open revision bucket and new tail', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + await edRows(_lo + 4 * 900 + 200, _hi + 1); + final warm = await _load(e); + final cold = await _load(DerivationEngine()..debugDisableSubstrateRetention = true); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e) - reads, lessThanOrEqualTo(900), + reason: 'four closed buckets must survive the tail append'); + }); + + final changes = Function()>{ + 'identical old-bucket revision': () => edRows(_lo + 30, _lo + 31), + 'old-bucket replacement': () => edRows(_lo + 30, _lo + 31, hr: 83), + 'delete seconds and beats': () async { + final db = await LocalDb.instance; + for (final table in ['decoded_onehz', 'decoded_rr']) { + await db.delete(table, where: 'rec_ts >= ? AND rec_ts < ?', + whereArgs: [_lo + 10, _lo + 20]); + } + }, + 'late beat in retained bucket': () async { + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': (_lo + 30) * 1000, 'rec_ts': _lo + 30, + 'rr_ts_ms': (_lo + 30) * 1000, 'beat_index': 0, 'rr_ms': 850}); + }, + 'zone': () async { edSetZone('Pacific/Auckland'); }, + 'generation': () async { await LocalDb.close(); await LocalDb.instance; }, + 'family': () async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}, + where: 'rec_ts = ?', whereArgs: [_lo + 30]); }, + 'device provenance': () => edRows(_lo + 30, _lo + 31, id: 'second'), + 'skin unit bucket': () async { await (await LocalDb.instance).update( + 'decoded_onehz', {'skin_temp_c': 33.1}, + where: 'rec_ts = ?', whereArgs: [_lo + 30]); }, + }; + for (final change in changes.entries) { + test('${change.key}: invalidate, preserve every byte, then retain again', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + await change.value(); + final warm = await _load(e); + expect(_reads(e), greaterThan(reads), reason: 'closed prefix must be reloaded'); + final cold = await _load(DerivationEngine()..debugDisableSubstrateRetention = true); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + final afterReload = _reads(e); + await _load(e); + expect(_reads(e), afterReload, reason: 'newly retained key must hit'); + }); + } + + test('algorithm rekey rejects retained work and next load hits', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + e.debugRetainedSubstrateRekey(kAlgoVersion - 1); + final warm = await _load(e); + expect(_reads(e), greaterThan(reads)); + expect(jsonEncode(warm.toJson()), jsonEncode((await _load(DerivationEngine())).toJson())); + final after = _reads(e); + await _load(e); + expect(_reads(e), after, reason: 'current-version retention should now hit'); + }); + + for (final signal in [InputSignal.hr1Hz, InputSignal.rrIntervals]) { + test('$signal ownership miss even if revisions are unchanged', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + final ownership = >{signal: [ + (start: _lo, end: _hi, deviceId: 'other')]}; + final warm = await _load(e, ownership: ownership); + final cold = await _load(DerivationEngine(), ownership: ownership); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e), greaterThan(reads)); + final again = _reads(e); + await _load(e, ownership: ownership); + expect(_reads(e), again, reason: 'ownership belongs to the retained key'); + }); + } + + for (final window in ['from', 'to']) { + test('$window window change reloads then hits', () async { + final e = DerivationEngine(); + await _load(e); + final reads = _reads(e); + final lo = window == 'from' ? _lo + 1 : _lo; + final hi = window == 'to' ? _hi - 1 : _hi; + final warm = await _load(e, lo: lo, hi: hi); + final cold = await _load(DerivationEngine(), lo: lo, hi: hi); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + expect(_reads(e), greaterThan(reads)); + final after = _reads(e); + await _load(e, lo: lo, hi: hi); + expect(_reads(e), after); + }); + } + + test('raw beat times, per-second beat offsets and bucket metadata survive retention', () async { + final db = await LocalDb.instance; + final rawTimes = []; + final intervals = []; + final starts = []; + final b = db.batch(); + b.delete('decoded_onehz', where: 'rec_ts = ?', whereArgs: [_lo + 1]); + // Include beat-only seconds, missing beats, multiple beats and a backward + // raw timestamp at the cut. Monotonizing a retained prefix twice loses it. + for (var t = _lo; t < _lo + 4 * 900 + 200; t++) { + starts.add(rawTimes.length); + for (var i = 0; i < (t % 3 == 0 ? 0 : 1 + t % 2); i++) { + final raw = t * 1000 - (i == 0 && t % 11 == 0 ? 1500 : i * 400); + rawTimes.add(raw.toDouble()); + intervals.add((700 + t % 90).toDouble()); + b.insert('decoded_rr', {'device_id': '', 'ts_ms': t * 1000, + 'rec_ts': t, 'beat_index': i, 'rr_ts_ms': t * 1000, + 'beat_ts_ms': raw, 'rr_ms': 700 + t % 90}); + } + } + await b.commit(noResult: true); + final e = DerivationEngine(); + final before = await _load(e); + var runningMax = double.negativeInfinity; + final axis = []; + for (final raw in rawTimes) { + if (raw > runningMax) runningMax = raw; + axis.add(runningMax); + } + expect(before.rrTsMs, axis, reason: 'hold backward placements; never sort or shift beats'); + expect(before.rrMs, intervals, reason: 'interval order is the recorded rhythm'); + final extras = e.debugRetainedSubstrate(edDay); + expect(extras['rr_ts_raw'], rawTimes, + reason: 'the raw axis must be retained separately from its running max'); + expect(extras['rr_start'], starts); + expect((extras['meta'] as List).map((m) => (m as List).first), + [for (var i = 0; i < 5; i++) _lo ~/ 900 + i]); + expect(before.rrTsMs, isNot(rawTimes)); + await edRows(_lo + 4 * 900 + 200, _hi + 1); + final warm = await _load(e); + final cold = await _load(DerivationEngine()); + expect(jsonEncode(warm.toJson()), jsonEncode(cold.toJson())); + }); + + test('retention respects its day bound and evicts the oldest day', () async { + final e = DerivationEngine()..debugRetainedSubstrateMaxDays = 2; + for (var i = 0; i < 3; i++) { + final lo = DateTime(2020, 1, 2 + i, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(lo, lo + 1200); + await e.debugLoadSubstrate(lo, lo + 1200, day: edLabel(lo)); + expect(e.debugRetainedSubstrateDays.length, lessThanOrEqualTo(2)); + } + expect(e.debugRetainedSubstrateDays, {'2020-01-03', '2020-01-04'}, + reason: 'a zero-sized stub must not satisfy the bound'); + }); + + for (final boundary in ['background', 'memory pressure', 'dispose']) { + test('$boundary clears populated retention and next load stays exact', () async { + final e = DerivationEngine(); + final before = await _load(e); + expect(e.debugRetainedSubstrateDays, contains(edDay), reason: 'populated before boundary'); + switch (boundary) { + case 'background': e.trimForBackground(); + case 'memory pressure': e.handleMemoryPressure(); + case 'dispose': e.dispose(); + } + expect(e.debugRetainedSubstrateDays, isEmpty); + final receiver = boundary == 'dispose' ? DerivationEngine() : e; + expect(jsonEncode((await _load(receiver)).toJson()), jsonEncode(before.toJson())); + }); + } + + test('real day finalization drops retained day and beat data', () async { + final e = DerivationEngine(); + await e.runDays(edProfile, {edDay}); + expect(e.debugRetainedSubstrateDays, contains(edDay)); + final later = DateTime(2020, 1, 6, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1200); + await e.runDays(edProfile, {edDay}); + expect((await LocalDb.dayResult(edDay))!['finalized'], 1); + expect(e.debugRetainedSubstrateDays, isNot(contains(edDay))); + expect(e.debugRetainedSubstrate(edDay), isEmpty); + }); + + test('a pass with no preparable day releases previously retained rows', () async { + final e = DerivationEngine(); + await _load(e); + expect(e.debugRetainedSubstrateDays, contains(edDay)); + final db = await LocalDb.instance; + await db.delete('decoded_onehz'); + await db.delete('decoded_rr'); + expect(await e.runDays(edProfile, {edDay}), 0); + expect(e.debugRetainedSubstrateDays, isEmpty); + expect(e.debugRetainedSubstrate(edDay), isEmpty); + }); + + test('existing fixture: retained and cold derive output exact before and after revision', () async { + await edExistingFixture(); + final warm = DerivationEngine(); + final cold = DerivationEngine()..debugDisableSubstrateRetention = true; + await edPreparedOutput(warm); + expect(await edPreparedOutput(warm), await edPreparedOutput(cold)); + await edRows(edStart, edStart + 1, hr: 89); + expect(await edPreparedOutput(warm), await edPreparedOutput(cold)); + }); +} From 11b944ab3289453da4a3854aae718fd1aea44c59 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 05:36:50 -0600 Subject: [PATCH 4/5] derive: reuse finished-session HR ceilings while their inputs are unchanged; index wearable days once per pass - A finished session's HR-ceiling envelope and unrounded winning bpm are reused while the day's input revisions, store generation, device family, priority and HR/acceleration ownership are unchanged. Session identity and window writes advance a session_ceiling_rev counter (rung 59, beside input_rev); derived score writes do not. Live sessions always recompute. - Wearable days are indexed once per pass instead of rescanned per day; the index is cleared in finally on every exit. Readiness deferral unchanged. - The E18 HR-ownership test gives the primary coverage as well, so the edit moves part of the window instead of removing all HR input. Output is unchanged; kAlgoVersion stays 113. --- lib/compute/derivation_engine.dart | 401 ++++++++++++++------- lib/compute/inputs/canonical.dart | 27 +- lib/data/db.dart | 18 +- lib/data/db_session_ceiling_revisions.dart | 29 ++ lib/data/recent_slice.dart | 1 + lib/data/restore_classification.dart | 1 + test/ed2a_e18_session_reuse_test.dart | 262 ++++++++++++++ test/ed2a_e19_wearable_day_index_test.dart | 201 +++++++++++ 8 files changed, 805 insertions(+), 135 deletions(-) create mode 100644 lib/data/db_session_ceiling_revisions.dart create mode 100644 test/ed2a_e18_session_reuse_test.dart create mode 100644 test/ed2a_e19_wearable_day_index_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 51b4b0073..c1eca5f69 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3096,7 +3096,7 @@ class DerivationEngine { // ED-2a RED seams. Empty cache views and no-op controls describe the current // uncached engine. Counters below observe real worker work, never results. - // E12/E13 now back staging and retention; later controls remain inert. + // E12/E13 back staging and retention; E18 backs finished-session reuse. final _stagedSleepCache = StagedSleepCache(); @visibleForTesting bool debugDisableStagedSleepCache = false; @@ -3171,7 +3171,86 @@ class DerivationEngine { @visibleForTesting int debugCeilingComputed = 0; @visibleForTesting - void debugSessionCacheRekey(int version) {} + void debugSessionCacheRekey(int version) { + for (final e in _ceilingCache.entries.toList()) { + _ceilingCache[e.key] = (version: version, sig: e.value.sig, + result: e.value.result); + } + } + + // One complete envelope and the unrounded bpm used to choose the winner. + // Entries survive passes on this engine; oldest entries are evicted first. + final _ceilingCache = <(String, String), + ({int version, String sig, _SessionCeiling result})>{}; + static const _ceilingCacheMax = 128; + final _ceilingInput = Expando<_CeilingInput>(); + + Future<_CeilingInput?> _ceilingInputRevision(String day) async { + try { + final db = await LocalDb.instance; + final generation = LocalDb.storeGeneration; + final lo = _localDayLabelToSec(day); + final hi = _localNextDayLabelToSec(day) - 1; + final rows = await db.rawQuery( + 'SELECT bucket, rev FROM input_rev WHERE bucket >= ? AND bucket <= ? ' + 'ORDER BY bucket', [lo ~/ 900, hi ~/ 900]); + return (generation: generation, revisions: {for (final r in rows) + (r['bucket'] as num).toInt(): (r['rev'] as num).toInt()}); + } catch (_) { + return null; // Unavailable provenance always recomputes. + } + } + + Future _sessionCeilingRevision() async { + try { + return await LocalDb.sessionCeilingRevision(); + } catch (_) { + return null; + } + } + + Future> _ceilingSignatures( + PreparedDerivationDay day, List> sessions, + int? sessionRevision) async { + final input = _ceilingInput[day]; + final sub = day.daySub; + if (input == null || sub.isEmpty || sessionRevision == null) return const {}; + final current = await _ceilingInputRevision(day.date); + if (current == null || input.generation != current.generation || + sessionRevision != await _sessionCeilingRevision()) { + return const {}; + } + final out = {}; + for (final row in sessions) { + final id = row['id']?.toString(); + final start = (row['start_ts'] as num?)?.toInt(); + final end = (row['end_ts'] as num?)?.toInt(); + if (id == null || row['status'] != 'done' || start == null || + end == null || end <= start) { + continue; + } + final lo = math.max(start, sub.tsSec.first); + final hi = math.min(end, sub.tsSec.last); + if (hi < lo) continue; + // E4 buckets are conservative at the clipped window's edges. Compare + // before/after the load so a racing decoded write cannot certify a hit. + String windowKey(_CeilingInput fp) => jsonEncode([ + fp.generation, for (final e in fp.revisions.entries) + if (e.key >= lo ~/ 900 && e.key <= hi ~/ 900) [e.key, e.value], + ]); + final revision = windowKey(input); + if (revision != windowKey(current)) continue; + out[id] = jsonEncode([ + revision, sessionRevision, id, row['type'], row['status'], start, end, + lo, hi, sub.deviceFamily, priorityKey(day.priority), + for (final signal in const [InputSignal.hr1Hz, InputSignal.accel1Hz]) + [signal.name, for (final span in day.ownership[signal] ?? const []) + if (span.end > lo && span.start <= hi) + [span.start, span.end, span.deviceId]], + ]); + } + return out; + } @visibleForTesting int debugStagingComputations = 0; @visibleForTesting @@ -3754,7 +3833,8 @@ class DerivationEngine { } else if (prepared != null) { _diag['prepared_days'] = (_diag['prepared_days'] as int) + 1; final complete = await _derivePreparedDay( - prepared, profile, dataNowSec, history); + prepared, profile, dataNowSec, history, + allowSessionReuse: !heavy && !force); done++; _diag['done_days'] = done; // Stored only when the derive was a fixed point; see @@ -3867,92 +3947,95 @@ class DerivationEngine { } catch (e) { _log('[derive] outcome callback threw: $e'); } - // Storage housekeeping runs here, after everything, still holding - // `_running`. See _runStorageHousekeeping — this is the only place every - // entry path and every early return actually reaches. - _diag['stage'] = 'housekeeping'; - _endRetentionPass(); - await _runStorageHousekeeping(long: heavy || force); - // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days - // inside the raw window re-derive from substrate above; everything older - // has none, so its headline is rebuilt from the stored TRIMP + wake - // window instead. No-ops after the first successful pass - // (`compute_freshness`), and never fatal — a failed rescale must not take - // the derive cycle down with it. - // - // AFTER the sweep, not before it. It used to be the first statement of - // run(), so the one launch where it actually fires — the first after the - // version bump, the launch the user is watching — spent a DB round trip - // and a bundle decode+encode PER DAY OF HISTORY before today's data was - // touched at all. Nothing in the sweep reads what it rewrites (it only - // ever touches days older than `rawRetentionDays`, which no pass - // re-derives), so the only thing the old ordering bought was that the - // FIRST cross-day rollup after the bump saw the rescaled history rather - // than the next one. Still holds `_running`, so nothing else can be - // reading day_result while it rewrites. - _diag['stage'] = 'strain_rescale'; try { - final rescaled = await backfillStrainScale( - female: workoutSex(profile.sex) == 'female', - ); - if (rescaled.didWork) { - _log('[derive] strain rescale: ${rescaled.bundleDays} day(s) ' - 'rebuilt, ${rescaled.skipped} skipped (no TRIMP or no wake ' - 'window)'); + // Storage housekeeping runs here, after everything, still holding + // `_running`. See _runStorageHousekeeping — this is the only place every + // entry path and every early return actually reaches. + _diag['stage'] = 'housekeeping'; + _endRetentionPass(); + await _runStorageHousekeeping(long: heavy || force); + // ONE-SHOT: rescale stored strain onto the recalibrated scale. Days + // inside the raw window re-derive from substrate above; everything older + // has none, so its headline is rebuilt from the stored TRIMP + wake + // window instead. No-ops after the first successful pass + // (`compute_freshness`), and never fatal — a failed rescale must not take + // the derive cycle down with it. + // + // AFTER the sweep, not before it. It used to be the first statement of + // run(), so the one launch where it actually fires — the first after the + // version bump, the launch the user is watching — spent a DB round trip + // and a bundle decode+encode PER DAY OF HISTORY before today's data was + // touched at all. Nothing in the sweep reads what it rewrites (it only + // ever touches days older than `rawRetentionDays`, which no pass + // re-derives), so the only thing the old ordering bought was that the + // FIRST cross-day rollup after the bump saw the rescaled history rather + // than the next one. Still holds `_running`, so nothing else can be + // reading day_result while it rewrites. + _diag['stage'] = 'strain_rescale'; + try { + final rescaled = await backfillStrainScale( + female: workoutSex(profile.sex) == 'female', + ); + if (rescaled.didWork) { + _log('[derive] strain rescale: ${rescaled.bundleDays} day(s) ' + 'rebuilt, ${rescaled.skipped} skipped (no TRIMP or no wake ' + 'window)'); + } + } catch (e) { + _log('[derive] strain rescale failed (kept old values): $e'); } - } catch (e) { - _log('[derive] strain rescale failed (kept old values): $e'); - } - // ONE-SHOT, after the sweep for the same reason as the strain rescale: - // the rest of the readiness_z history, then a re-score of every day - // below v112 that has no raw substrate left (substrate days were just - // re-derived, or are retried next pass). A re-score rewrites stored - // days, so the baselines and the cross-day rollup are refreshed here — - // `finally` is reached by every early return, including "nothing to - // derive" and "no decoded data". - _diag['stage'] = 'readiness_calibration'; - try { - final r = await backfillReadinessCalibration( - // Both substrate sources, as `_deriveScope` reads them: a wearable - // day with rows is the derive's, not the re-score's. - rawDays: await substrateDays(), - zHistoryLoader: () async { - final h = await _BaselineHistoryCache.load(); - return (day, fam) => h.valuesBefore('readiness_z', day, - family: _baselineFamily('readiness_z', fam)); - }, - ); - if (r.seeded > 0 || r.rescored > 0) { - _log('[derive] readiness calibration: ${r.seeded} z seeded, ' - '${r.rescored} day(s) re-scored' - '${r.forced > 0 ? ' (${r.forced} with substrate whose derive ' - 'never landed, from stored inputs)' : ''}'); - await _refreshBaselines(); + // ONE-SHOT, after the sweep for the same reason as the strain rescale: + // the rest of the readiness_z history, then a re-score of every day + // below v112 that has no raw substrate left (substrate days were just + // re-derived, or are retried next pass). A re-score rewrites stored + // days, so the baselines and the cross-day rollup are refreshed here — + // `finally` is reached by every early return, including "nothing to + // derive" and "no decoded data". + _diag['stage'] = 'readiness_calibration'; + try { + final r = await backfillReadinessCalibration( + // Both substrate sources, as `_deriveScope` reads them: a wearable + // day with rows is the derive's, not the re-score's. + rawDays: await substrateDays(), + zHistoryLoader: () async { + final h = await _BaselineHistoryCache.load(); + return (day, fam) => h.valuesBefore('readiness_z', day, + family: _baselineFamily('readiness_z', fam)); + }, + ); + if (r.seeded > 0 || r.rescored > 0) { + _log('[derive] readiness calibration: ${r.seeded} z seeded, ' + '${r.rescored} day(s) re-scored' + '${r.forced > 0 ? ' (${r.forced} with substrate whose derive ' + 'never landed, from stored inputs)' : ''}'); + await _refreshBaselines(); + } + if (r.rescored > 0) await _runCrossDay(profile); + } catch (e) { + _log('[derive] readiness calibration failed (will retry): $e'); } - if (r.rescored > 0) await _runCrossDay(profile); - } catch (e) { - _log('[derive] readiness calibration failed (will retry): $e'); - } - // Restamp Today's freshness here, not in each caller: the iOS - // background pass has no caller to do it, and getToday reads the - // night from `overnight_day`, so a night settled in the background - // kept serving the previous one. - try { - await LocalDb.refreshComputeFreshness(); - } catch (e) { - _log('[derive] freshness refresh failed: $e'); - } - endWearableDayIndexPass(); - _running = false; - final finishedAt = DateTime.now().millisecondsSinceEpoch; - _diag - ..['running'] = false - ..['stage'] = 'idle' - ..['active_days'] = const [] - ..['finished_at'] = finishedAt - ..['duration_ms'] = finishedAt - startedAt; + // Restamp Today's freshness here, not in each caller: the iOS + // background pass has no caller to do it, and getToday reads the + // night from `overnight_day`, so a night settled in the background + // kept serving the previous one. + try { + await LocalDb.refreshComputeFreshness(); + } catch (e) { + _log('[derive] freshness refresh failed: $e'); + } + } finally { + endWearableDayIndexPass(); + _running = false; + final finishedAt = DateTime.now().millisecondsSinceEpoch; + _diag + ..['running'] = false + ..['stage'] = 'idle' + ..['active_days'] = const [] + ..['finished_at'] = finishedAt + ..['duration_ms'] = finishedAt - startedAt; - try { await runTrace?.stop(); } catch (_) {} + try { await runTrace?.stop(); } catch (_) {} + } } } @@ -4081,16 +4164,19 @@ class DerivationEngine { _log('derive selected ERROR: $e\n$st'); return 0; } finally { - _endRetentionPass(); - await _runStorageHousekeeping(); - final finishedAt = DateTime.now().millisecondsSinceEpoch; - _diag - ..['running'] = false - ..['stage'] = 'idle' - ..['finished_at'] = finishedAt - ..['duration_ms'] = finishedAt - startedAt; - endWearableDayIndexPass(); - _running = false; + try { + _endRetentionPass(); + await _runStorageHousekeeping(); + final finishedAt = DateTime.now().millisecondsSinceEpoch; + _diag + ..['running'] = false + ..['stage'] = 'idle' + ..['finished_at'] = finishedAt + ..['duration_ms'] = finishedAt - startedAt; + } finally { + endWearableDayIndexPass(); + _running = false; + } } } @@ -4147,6 +4233,7 @@ class DerivationEngine { ].reduce(math.max); final (ownership, priority) = await _resolveOwnership(unionFrom, unionTo); + final ceilingInput = await _ceilingInputRevision(dayId); // Load the day PLUS the nap boundary buffer in ONE pass (each // _loadSubstrateRange spawns its own isolate, so a second load would @@ -4216,7 +4303,7 @@ class DerivationEngine { dayEndSec: dayEnd, sleepOffsetSec: candidate.present ? candidate.sleepOffsetSec : 0, ); - return candidate.toPreparedDay( + final prepared = candidate.toPreparedDay( daySub: daySub, napSub: napSub, sleepSub: sleepSub, @@ -4230,6 +4317,12 @@ class DerivationEngine { ownership: ownership, priority: priority, ); + // Wearable construction and supplied imports have other inputs; only the + // canonical decoded load above can carry this revision provenance. + if (wearable == null && ceilingInput != null) { + _ceilingInput[prepared] = ceilingInput; + } + return prepared; } /// [next] (the following day's main sleep) when it is TONIGHT's from this @@ -5726,10 +5819,13 @@ class DerivationEngine { _log('rescan ERROR: $e\n$st'); return 0; } finally { - _endRetentionPass(); - await _runStorageHousekeeping(); - endWearableDayIndexPass(); - _running = false; + try { + _endRetentionPass(); + await _runStorageHousekeeping(); + } finally { + endWearableDayIndexPass(); + _running = false; + } } } @@ -6143,10 +6239,12 @@ class DerivationEngine { _BaselineHistoryCache history, { bool forceFinalize = false, bool suppliedSubstrate = false, + bool allowSessionReuse = false, }) async { try { return await _derivePreparedDayPayload(day, profile, dataNowSec, history, - forceFinalize: forceFinalize, suppliedSubstrate: suppliedSubstrate); + forceFinalize: forceFinalize, suppliedSubstrate: suppliedSubstrate, + allowSessionReuse: allowSessionReuse); } catch (_) { _dropRetainedDay(day.date); rethrow; @@ -6160,6 +6258,7 @@ class DerivationEngine { _BaselineHistoryCache history, { bool forceFinalize = false, bool suppliedSubstrate = false, + bool allowSessionReuse = false, }) async { final daySub = day.daySub; final sleepSub = day.sleepSub; @@ -6470,10 +6569,24 @@ class DerivationEngine { // sample, so a completed session the band's PPG lost contact for // entirely (zero HR samples at all, e.g. a wrist-gripping lift) can // fall outside them and never reach the zero-coverage credit below. + final sessionRevision = await _sessionCeilingRevision(); final savedSessions = await LocalDb.sessionsInRange( _localDayLabelToSec(day.date), localNextMidnightSecForDayLabel(day.date), ); + final ceilingSigs = debugDisableSessionReuse ? const {} + : await _ceilingSignatures(day, savedSessions, sessionRevision); + final ceilingReuse = {}; + if (allowSessionReuse && !debugDisableSessionReuse) { + for (final e in ceilingSigs.entries) { + final hit = _ceilingCache[(day.date, e.key)]; + if (hit != null && hit.version == kAlgoVersion && hit.sig == e.value) { + ceilingReuse[e.key] = hit.result; + } + } + } + // Removed, live or changed sessions cannot resurrect a former result. + _ceilingCache.removeWhere((key, _) => key.$1 == day.date); // Off-wrist / charging spans over the NAP window (which runs past this // day's end), read here because the isolate has no DB handle. These are @@ -6572,6 +6685,7 @@ class DerivationEngine { } final napEdits = await ActivityStore.napEdits(reviewDb, day.date); final blocksInput = _DayBlocksInput( + ceilingReuse: ceilingReuse, daySub: daySub, napSub: day.napSub, napEdits: napEdits, @@ -6636,6 +6750,17 @@ class DerivationEngine { await _runDayBlocksCancellable(blocksInput, _perDayTimeout); debugCeilingComputed += blocks.ceilingComputations; + if (!debugDisableSessionReuse) { + for (final e in {...ceilingReuse, ...blocks.ceilingComputed}.entries) { + final sig = ceilingSigs[e.key]; + if (sig == null) continue; + _ceilingCache[(day.date, e.key)] = + (version: kAlgoVersion, sig: sig, result: e.value); + while (_ceilingCache.length > _ceilingCacheMax) { + _ceilingCache.remove(_ceilingCache.keys.first); + } + } + } // Merge the computed blocks back into the isolate-1 bundle. scMap is the // CastMap view over bundle['scalars'], so addAll writes through — nap_min / @@ -10888,7 +11013,9 @@ class DerivationEngine { if (wc.hrrBpm != null) scMap['hrr_bpm'] = wc.hrrBpm; if (wc.hrrTauS != null) scMap['hrr_tau_s'] = wc.hrrTauS; var ceilingComputations = 0; + final ceilingComputed = {}; final ceiling = _dayHrCeiling(daySub, inp.savedSessions, + reuse: inp.ceilingReuse, computed: ceilingComputed, onCompute: () => ceilingComputations++); bundlePatch['hr_ceiling'] = ceiling; // TS-03 — the day's observed ceiling as a SCALAR too, so the app-wide @@ -10947,6 +11074,7 @@ class DerivationEngine { return _DayBlocksOutput( ceilingComputations: ceilingComputations, + ceilingComputed: ceilingComputed, bundlePatch: bundlePatch, seriesPatch: seriesPatch, scalarPatch: scMap, @@ -11137,52 +11265,66 @@ class DerivationEngine { ) => _dayHrCeiling(s, saved); + // Reuse carries the full Metric JSON, including absence, and its unrounded + // bpm. Comparing serialized/rounded values could choose a different session. static Map _dayHrCeiling( Substrate s, List> saved, { + Map reuse = const {}, + Map? computed, void Function()? onCompute, }) { - ana.Metric? best; + Map? bestJson; + double? bestBpm; String? bestId; String? bestType; - ana.Metric? lastAbsent; + Map? lastAbsentJson; for (final row in saved) { final start = (row['start_ts'] as num?)?.toInt(); final end = (row['end_ts'] as num?)?.toInt(); if (start == null || end == null || end <= start) continue; - final hr = []; - final accel = []; - for (var i = 0; i < s.length; i++) { - final t = s.tsSec[i]; - if (t < start) continue; - if (t > end) break; - final tsMs = t * 1000.0; - hr.add(ana.HrSample(tsMs, s.hr[i].toDouble())); - accel.add(ana.AccelSample(tsMs, s.ax[i], s.ay[i], s.az[i], - valid: s.accelPresentAt(i))); - } - if (hr.isEmpty) continue; - onCompute?.call(); - final m = ana.sessionHrCeiling(hr, accel, deviceFamily: s.deviceFamily); - if (!m.present) { - lastAbsent = m; + final id = row['id']?.toString(); + var result = row['status'] == 'done' ? reuse[id] : null; + if (result == null) { + final hr = []; + final accel = []; + for (var i = 0; i < s.length; i++) { + final t = s.tsSec[i]; + if (t < start) continue; + if (t > end) break; + final tsMs = t * 1000.0; + hr.add(ana.HrSample(tsMs, s.hr[i].toDouble())); + accel.add(ana.AccelSample(tsMs, s.ax[i], s.ay[i], s.az[i], + valid: s.accelPresentAt(i))); + } + if (hr.isEmpty) continue; + onCompute?.call(); + final m = ana.sessionHrCeiling(hr, accel, deviceFamily: s.deviceFamily); + result = (json: m.toJson((v) => v.toJson()), + bpm: m.present ? m.value!.bpm : null); + if (id != null && row['status'] == 'done') computed?[id] = result; + } + final bpm = result.bpm; + if (bpm == null) { + lastAbsentJson = result.json; continue; } - if (best == null || m.value!.bpm > best.value!.bpm) { - best = m; - bestId = row['id']?.toString(); + if (bestBpm == null || bpm > bestBpm) { + bestJson = result.json; + bestBpm = bpm; + bestId = id; bestType = row['type']?.toString(); } } - final m = best ?? - lastAbsent ?? + final json = bestJson ?? + lastAbsentJson ?? const ana.Metric.absent( tier: ana.Tier.high, inputs_used: ['hr_1hz', 'accel_1hz', 'device_family'], note: 'no saved session on this day to observe a ceiling in', - ); + ).toJson((v) => v.toJson()); return { - ...m.toJson((v) => v.toJson()), + ...json, 'session_id': ?bestId, 'session_type': ?bestType, }; @@ -11542,11 +11684,15 @@ Future runCancellableIsolate( }) => DerivationEngine._runIsolateCancellable(compute, timeout, label: label); +typedef _SessionCeiling = ({Map json, double? bpm}); +typedef _CeilingInput = ({int generation, Map revisions}); + /// Sendable input for [DerivationEngine._computeDayBlocks] — crosses the /// `Isolate.run` boundary, so every field is plain data (Substrate is int/double /// lists; Profile is a primitive data class). DB reads that the /// pure compute needs are performed by the caller and passed in here. class _DayBlocksInput { + final Map ceilingReuse; final Substrate daySub; final Substrate napSub; final Substrate sleepSub; @@ -11641,6 +11787,7 @@ class _DayBlocksInput { /// `[startSec, endSec]`. Nap suggestions overlapping one are dropped. final List> rejectedSleepSpans; const _DayBlocksInput({ + this.ceilingReuse = const {}, required this.daySub, required this.napSub, required this.sleepSub, @@ -11680,6 +11827,7 @@ class _DayBlocksInput { /// isolate; [wake] is persisted; candidates are reconciled and session HRR /// writes are applied by the database-owning caller. class _DayBlocksOutput { + final Map ceilingComputed; final int ceilingComputations; final List> napSuggestions; final Map bundlePatch; @@ -11689,6 +11837,7 @@ class _DayBlocksOutput { final List> suggestionsToPersist; final List<(String, double)> sessionHrrWrites; const _DayBlocksOutput({ + this.ceilingComputed = const {}, this.ceilingComputations = 0, required this.bundlePatch, this.napSuggestions = const [], diff --git a/lib/compute/inputs/canonical.dart b/lib/compute/inputs/canonical.dart index e21a3a4c6..74b03d5bf 100644 --- a/lib/compute/inputs/canonical.dart +++ b/lib/compute/inputs/canonical.dart @@ -624,19 +624,32 @@ Future> flagOnOrPrimary(List ids, return out; } -// RED pass-lifetime seams: no index is retained yet. The hook runs after an -// actual scan, so a test can count scans and inject a post-scan failure. -bool debugDisableWearableDayIndex = false; -void beginWearableDayIndexPass() {} -void endWearableDayIndexPass() {} -int get debugWearableDayIndexSize => 0; -void debugWearableDayIndexRekey(int version) {} +// Pass-lifetime index. The hook runs after an actual scan, so a test can +// count scans and inject a post-scan failure. +Map<(String, String), Future>>? _passDayIndex; +bool _disableWearableDayIndex = false; +bool get debugDisableWearableDayIndex => _disableWearableDayIndex; +set debugDisableWearableDayIndex(bool value) { + _disableWearableDayIndex = value; + if (value) _passDayIndex?.clear(); +} +void beginWearableDayIndexPass() => _passDayIndex = {}; +void endWearableDayIndexPass() => _passDayIndex = null; +int get debugWearableDayIndexSize => _passDayIndex?.length ?? 0; +void debugWearableDayIndexRekey(int version) => _passDayIndex?.clear(); void Function((String, String), Map)? debugOnWearableDayIndexScan; /// `{local day -> last second}` the active wearable has rows for, and the /// days a flagged-on strap recorded a session in while the wearable was off /// or charging ([wearableSubstrate] lays the session over the empty day). Future> wearableRecTsMaxByDay( + (String, String) wearable) { + final cache = debugDisableWearableDayIndex ? null : _passDayIndex; + if (cache == null) return _scanWearableRecTsMaxByDay(wearable); + return cache.putIfAbsent(wearable, () => _scanWearableRecTsMaxByDay(wearable)); +} + +Future> _scanWearableRecTsMaxByDay( (String, String) wearable) async { final out = { ...await switch (wearable.$2) { diff --git a/lib/data/db.dart b/lib/data/db.dart index b71e72556..481fe48fe 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -65,6 +65,7 @@ import 'series_codec.dart'; import 'substrate_archive.dart'; part 'db_input_revisions.dart'; +part 'db_session_ceiling_revisions.dart'; /// The outcome of a database rebuild: why the old file would not open, where it /// was parked, and how many rows came back per table. @@ -511,6 +512,13 @@ class LocalDb { ); } + /// Session identity/window writes invalidate ceiling reuse, including an + /// identical REPLACE or UPDATE. Derived score writes do not affect it. + static Future sessionCeilingRevision() async { + final rows = await (await instance).query('session_ceiling_rev'); + return rows.isEmpty ? 0 : (rows.single['rev'] as num).toInt(); + } + @visibleForTesting static Future repairOpenSchemaForTest(Database db) => _repairOpenSchema(db); @@ -1287,6 +1295,7 @@ class LocalDb { // Additive revision counters only: no decoded scan or backfill. // Attach after any earlier rung that rebuilds decoded tables. await _createInputRev(db); + await _createSessionCeilingRev(db); } }, onOpen: (db) async { @@ -1409,6 +1418,7 @@ class LocalDb { await _ensureLiveStepCalibrationCounterVersion(db); // A decoded-table rebuild above drops its triggers; attach them last. await _createInputRev(db); + await _createSessionCeilingRev(db); // Views LAST — they depend on metric_series / day_result / baselines / sessions // / notifications all existing. DROP+CREATE so a shape change takes effect. await _ensureCoachViews(db); @@ -5615,6 +5625,10 @@ class LocalDb { '''); } + // Schema 59 and every-open repair; no session scan or backfill. + static Future _createSessionCeilingRev(Database db) => + _ldbCreateSessionCeilingRev(db); + static Future _createSyncState(DatabaseExecutor db) async { await db.execute(''' CREATE TABLE IF NOT EXISTS sync_ledger ( @@ -11248,8 +11262,8 @@ class LocalDb { final rows = await db.rawQuery( "SELECT name FROM sqlite_master WHERE type = 'table' " "AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata' " - // Clear revisions after the decoded DELETE triggers have finished. - "ORDER BY name = 'input_rev'", + // Clear revisions after the decoded/session DELETE triggers have finished. + "ORDER BY name IN ('input_rev', 'session_ceiling_rev')", ); var deleted = 0; await db.transaction((txn) async { diff --git a/lib/data/db_session_ceiling_revisions.dart b/lib/data/db_session_ceiling_revisions.dart new file mode 100644 index 000000000..cb3d07d8c --- /dev/null +++ b/lib/data/db_session_ceiling_revisions.dart @@ -0,0 +1,29 @@ +part of 'db.dart'; + +Future _ldbCreateSessionCeilingRev(Database db) async { + await db.execute('CREATE TABLE IF NOT EXISTS session_ceiling_rev (' + 'id INTEGER PRIMARY KEY CHECK (id = 1), rev INTEGER NOT NULL)'); + // Sparse legacy schemas get their sessions table during onOpen repair. + final present = await db.rawQuery( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'sessions'", + ); + if (present.isEmpty) return; + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_ai + AFTER INSERT ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_au + AFTER UPDATE OF id, type, status, start_ts, end_ts ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); + await db.execute('''CREATE TRIGGER IF NOT EXISTS sessions_ceiling_ad + AFTER DELETE ON sessions BEGIN + INSERT INTO session_ceiling_rev(id, rev) SELECT 1, 0 + WHERE NOT EXISTS (SELECT 1 FROM session_ceiling_rev WHERE id = 1); + UPDATE session_ceiling_rev SET rev = rev + 1 WHERE id = 1; + END'''); +} diff --git a/lib/data/recent_slice.dart b/lib/data/recent_slice.dart index 4ac058cae..99721b721 100644 --- a/lib/data/recent_slice.dart +++ b/lib/data/recent_slice.dart @@ -83,6 +83,7 @@ const Map kSliceTables = { 'compute_freshness': (SliceKeep.all, null, 'compute bookkeeping'), 'derive_fingerprint': (SliceKeep.all, null, 'which days changed since they were last scored'), 'input_rev': (SliceKeep.all, null, 'per-bucket input revisions'), + 'session_ceiling_rev': (SliceKeep.all, null, 'session ceiling revisions'), 'sleep_nap': (SliceKeep.all, null, 'naps'), 'sleep_override': (SliceKeep.all, null, 'sleep times the user edited'), // Workouts and activity: one row each. diff --git a/lib/data/restore_classification.dart b/lib/data/restore_classification.dart index 1e6167745..8551fb0f1 100644 --- a/lib/data/restore_classification.dart +++ b/lib/data/restore_classification.dart @@ -39,6 +39,7 @@ const kRestoreClassification = { 'compute_freshness': RestoreClass.localOnly, 'derive_fingerprint': RestoreClass.rebuildable, 'input_rev': RestoreClass.rebuildable, + 'session_ceiling_rev': RestoreClass.rebuildable, 'sleep_nap': RestoreClass.merge, 'sleep_override': RestoreClass.merge, 'sessions': RestoreClass.merge, diff --git a/test/ed2a_e18_session_reuse_test.dart b/test/ed2a_e18_session_reuse_test.dart new file mode 100644 index 000000000..cc57c6b1e --- /dev/null +++ b/test/ed2a_e18_session_reuse_test.dart @@ -0,0 +1,262 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/recent_slice.dart'; +import 'package:openstrap_edge/data/restore_classification.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +Future _light(DerivationEngine e) async { + expect(await e.run(edProfile), greaterThan(0), reason: 'light pass must reach the real derive'); + expect(e.snapshot()['last_error'], isNull); +} + +Future _sessionEdit(Map patch) async { + await (await LocalDb.instance).update('sessions', patch, + where: 'id = ?', whereArgs: ['s1']); +} + +void main() { + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e18.db'); + await edRows(edStart, edStart + 1200); + await edSession(); + // These worker-level tests must reach the worker even after main's + // day fingerprint settles. The integration test below keeps both enabled. + DerivationEngine.debugDeriveEveryDay = true; + }); + tearDown(() async { + DerivationEngine.debugDeriveEveryDay = false; + await edDrop(); + }); + + test('session revisions upgrade from 58 and repair without backfilling sessions', () async { + var db = await LocalDb.instance; + final sessions = await db.query('sessions'); + for (final event in ['ai', 'au', 'ad']) { + await db.execute('DROP TRIGGER sessions_ceiling_$event'); + } + await db.execute('DROP TABLE session_ceiling_rev'); + await db.setVersion(58); + await LocalDb.close(); + db = await LocalDb.instance; + expect(await db.getVersion(), 59); + expect(await db.query('sessions'), sessions); + expect(await db.query('session_ceiling_rev'), isEmpty, + reason: 'upgrade never scans or backfills existing sessions'); + await _sessionEdit({'type': 'run'}); + expect(await LocalDb.sessionCeilingRevision(), 1); + await LocalDb.repairOpenSchemaForTest(db); + await LocalDb.repairOpenSchemaForTest(db); + expect(await LocalDb.sessionCeilingRevision(), 1); + await LocalDb.setSessionHrr('s1', 20); + expect(await LocalDb.sessionCeilingRevision(), 1, + reason: 'derived-only writes do not invalidate input identity'); + await expectLater(db.transaction((txn) async { + await txn.update('sessions', {'type': 'bike'}); + throw StateError('rollback'); + }), throwsStateError); + expect(await LocalDb.sessionCeilingRevision(), 1); + expect((await db.query('sessions')).single['type'], 'run'); + }); + + test('session revisions travel in slices, rebuild on restore, and empty on wipe', () async { + final db = await LocalDb.instance; + expect(kRestoreClassification['session_ceiling_rev'], RestoreClass.rebuildable); + expect(kSliceTables['session_ceiling_rev']?.$1, SliceKeep.all); + expect(LocalDb.restoreTablesForTest, isNot(contains('session_ceiling_rev'))); + expect(await db.query('session_ceiling_rev'), hasLength(1)); + expect(await db.query('input_rev'), isNotEmpty); + final generation = LocalDb.storeGeneration; + await LocalDb.wipeAll(); + for (final table in await LocalDb.tableNames()) { + expect(await db.query(table, limit: 1), isEmpty, reason: table); + } + expect(LocalDb.storeGeneration, greaterThan(generation)); + await edSession(); + expect(await LocalDb.sessionCeilingRevision(), 1, + reason: 'wipe preserves working triggers'); + }); + + test('main skips settled days and changed samples still replace reused ceilings', () async { + DerivationEngine.debugDeriveEveryDay = false; + final e = DerivationEngine(); + Future settle() async { + for (var i = 0; i < 6; i++) { + final derived = await e.run(edProfile); + expect(e.snapshot()['last_error'], isNull); + if (derived == 0 && e.lastRunUnchangedDays == 1) return; + } + fail('main fingerprint never settled'); + } + await settle(); + final count = e.debugCeilingComputed; + expect(count, 1); + final first = await edStored(edDay); + expect(await e.run(edProfile), 0); + expect(e.lastRunUnchangedDays, 1); + expect(e.debugCeilingComputed, count); + expect(await edStored(edDay), first); + await edRows(edStart, edStart + 1200, hr: 160); + expect(await e.run(edProfile), 1, + reason: 'changed input must defeat main skip and the old ceiling key'); + expect(e.debugCeilingComputed, count + 1); + final changed = await edStored(edDay); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['scalars']['hr_ceiling_bpm'], 160); + await settle(); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + expect(await cold.runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), changed); + }); + + test('finished session ceiling is scanned once across unchanged light passes', () async { + final e = DerivationEngine(); + await _light(e); + expect(e.debugCeilingComputed, 1); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['scalars']['hr_ceiling_bpm'], 170, + reason: 'motion fixture must contain a real held ceiling'); + final first = await edStored(edDay); + await _light(e); + expect(await edStored(edDay), first); + expect(e.debugCeilingComputed, 1, reason: 'a hit must bypass sessionHrCeiling'); + }); + + final edits = Function(DerivationEngine)>{ + 'decoded sample revision (identical REPLACE)': (_) => edRows(edStart + 30, edStart + 31), + 'decoded sample revision (changed HR)': (_) => edRows(edStart + 30, edStart + 300, hr: 160), + 'late RR input revision': (_) async { + final t = edStart + 30; + await (await LocalDb.instance).insert('decoded_rr', {'device_id': '', + 'ts_ms': t * 1000, 'rec_ts': t, 'rr_ts_ms': t * 1000, 'beat_index': 0, 'rr_ms': 900}); + }, + 'session revision (identical REPLACE)': (_) => edSession(), + 'session revision (identical UPDATE)': (_) => _sessionEdit({'type': 'run'}), + 'session identity': (_) => _sessionEdit({'id': 's2'}), + 'session type': (_) => _sessionEdit({'type': 'bike'}), + 'session start': (_) => _sessionEdit({'start_ts': edStart + 1}), + 'session end': (_) => _sessionEdit({'end_ts': edStart + 1000}), + 'clipped sample window': (_) async { + await (await LocalDb.instance).delete('decoded_onehz', + where: 'rec_ts < ?', whereArgs: [edStart + 10]); + }, + 'family': (_) async { await (await LocalDb.instance).update( + 'decoded_onehz', {'device_family': 'gen5'}); }, + 'HR ownership': (_) async { + // The primary needs coverage too: as the only covering candidate, + // `other` would own the whole window and leave the day without HR. + await (await LocalDb.instance).insert('device_coverage', {'device_id': '', + 'signal': 'hr1Hz', 'start_ts': edStart, 'end_ts': edStart + 1200}); + await edOwnership(signals: ['hr1Hz']); + }, + 'accel ownership': (_) async { + final db = await LocalDb.instance; + await db.insert('device_coverage', {'device_id': 'other', 'signal': 'accel1Hz', + 'start_ts': edStart, 'end_ts': edStart + 600}); + await db.insert('signal_priority', {'device_id': 'other', 'signal': 'accel1Hz', + 'rank': 0, 'user_set': 1}); + }, + 'priority without changed input revisions': (_) async { + await (await LocalDb.instance).insert('signal_priority', {'signal': 'hr1Hz', + 'device_id': '', 'rank': 2, 'user_set': 1}, conflictAlgorithm: ConflictAlgorithm.replace); + }, + 'store generation': (_) async { await LocalDb.close(); await LocalDb.instance; }, + 'algorithm': (e) async { e.debugSessionCacheRekey(kAlgoVersion - 1); }, + }; + for (final edit in edits.entries) { + test('${edit.key}: miss, exact result, then reuse new key', () async { + if (edit.key.startsWith('priority ')) await edPriorityOrder(['', 'other']); + final e = DerivationEngine(); + await _light(e); + final count = e.debugCeilingComputed; + expect(count, 1); + await edit.value(e); + await _light(e); + expect(e.debugCeilingComputed, count + 1, reason: 'changed ${edit.key} must rescan'); + final warm = await edStored(edDay); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + expect(await cold.runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), warm, + reason: 'includes the full metric envelope and unrounded winning session'); + final after = e.debugCeilingComputed; + await _light(e); + expect(e.debugCeilingComputed, after, reason: 'new key should now hit'); + }); + } + + test('an absent ceiling is reusable without inventing a number', () async { + await edRows(edStart, edStart + 1200, az: 1); + final e = DerivationEngine(); + await _light(e); + final first = await edStored(edDay); + final payload = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(payload['hr_ceiling']['value'], '—'); + expect(payload['scalars']['hr_ceiling_bpm'], isNull); + await _light(e); + expect(await edStored(edDay), first); + expect(e.debugCeilingComputed, 1, reason: 'absence is a cached result too'); + }); + + test('live sessions always recompute, including done to live transition (guard)', () async { + final e = DerivationEngine(); + await _light(e); + await _sessionEdit({'status': 'live'}); + final before = e.debugCeilingComputed; + await _light(e); + expect(e.debugCeilingComputed, before + 1); + await _light(e); + expect(e.debugCeilingComputed, before + 2); + await _sessionEdit({'status': 'done'}); + await _light(e); + expect(e.debugCeilingComputed, before + 3, reason: 'return to done must not resurrect stale entry'); + }); + + for (final mode in ['heavy', 'force', 'selected']) { + test('$mode pass always recomputes (guard)', () async { + final e = DerivationEngine(); + await _light(e); + final count = e.debugCeilingComputed; + final done = switch (mode) { + 'heavy' => await e.run(edProfile, heavy: true), + 'force' => await e.run(edProfile, force: true), + _ => await e.runDays(edProfile, {edDay}), + }; + expect(done, greaterThan(0)); + expect(e.debugCeilingComputed, greaterThan(count)); + }); + } + + test('session removal does not keep its former ceiling (guard)', () async { + final e = DerivationEngine(); + await _light(e); + await (await LocalDb.instance).delete('sessions', where: 'id = ?', whereArgs: ['s1']); + await _light(e); + final got = jsonDecode((await LocalDb.dayResult(edDay))!['payload_json'] as String) as Map; + expect(got['hr_ceiling']['value'], '—'); + expect(got['scalars']['hr_ceiling_bpm'], isNull); + }); + + test('existing fixture: reuse and forced output byte equality after invalidation (guard)', () async { + await edExistingFixture(); + final e = DerivationEngine(); + final cold = DerivationEngine()..debugDisableSessionReuse = true; + await _light(e); + await _light(e); + final before = await edStored(edDay); + await cold.runDays(edProfile, {edDay}); + expect(await edStored(edDay), before); + await edRows(edStart + 30, edStart + 300, hr: 160); + await _light(e); + final changed = await edStored(edDay); + await cold.runDays(edProfile, {edDay}); + expect(await edStored(edDay), changed); + }); +} diff --git a/test/ed2a_e19_wearable_day_index_test.dart b/test/ed2a_e19_wearable_day_index_test.dart new file mode 100644 index 000000000..e96b0dff7 --- /dev/null +++ b/test/ed2a_e19_wearable_day_index_test.dart @@ -0,0 +1,201 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/inputs/canonical.dart'; +import 'package:openstrap_edge/compute/readiness_calibration_backfill.dart'; +import 'package:openstrap_edge/data/db.dart'; + +import 'support/ed2a_recalc_fixture.dart'; +import 'readiness_calibration_backfill_test.dart' show bundle; + +const _watch = ('garmin-a', 'garmin'); +Future _wearable() async { + await LocalDb.upsertDevice(id: _watch.$1, adapterId: _watch.$2, label: 'Watch A'); + await LocalDb.setCursor(kActiveWearableCursor, _watch.$1); + await LocalDb.setCursor(wearableEnabledCursor(_watch.$2), '1'); + await edRows(edStart, edStart + 1200, id: _watch.$1, family: _watch.$2, + source: _watch.$2, hr: 70, az: 0); +} + +void main() { + setUpAll(edInit); + setUp(() async { await edFresh('ed2a_e19.db'); await _wearable(); }); + tearDown(() async { + endWearableDayIndexPass(); + debugOnWearableDayIndexScan = null; + debugDisableWearableDayIndex = false; + await edDrop(); + }); + + test('one real wearable scan per pass; new pass sees newly appended days', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + beginWearableDayIndexPass(); + try { + final first = await wearableRecTsMaxByDay(_watch); + expect(first, {edDay: edStart + 1199}); + expect(await wearableRecTsMaxByDay(_watch), first); + expect(scans, 1, reason: 'second request in the same pass must hit'); + } finally { endWearableDayIndexPass(); } + expect(debugWearableDayIndexSize, 0); + final later = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(later, later + 1, id: _watch.$1, family: _watch.$2, source: _watch.$2); + beginWearableDayIndexPass(); + try { + expect((await wearableRecTsMaxByDay(_watch)).keys, contains('2020-01-03')); + expect(scans, 2, reason: 'an index belongs to one pass, not to the engine lifetime'); + } finally { endWearableDayIndexPass(); } + }); + + for (final change in ['input revision', 'generation', 'algorithm']) { + test('$change cannot lend a stale index to the next request', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + beginWearableDayIndexPass(); + try { await wearableRecTsMaxByDay(_watch); } + finally { endWearableDayIndexPass(); } + switch (change) { + case 'input revision': await edRows(edStart, edStart + 1, + id: _watch.$1, family: _watch.$2, source: _watch.$2, hr: 80); + case 'generation': await LocalDb.close(); await LocalDb.instance; + case 'algorithm': debugWearableDayIndexRekey(kAlgoVersion - 1); + } + beginWearableDayIndexPass(); + try { + final fresh = await wearableRecTsMaxByDay(_watch); + expect(fresh, {edDay: edStart + 1199}); + expect(scans, 2, reason: 'the changed context belongs to a fresh pass'); + expect(await wearableRecTsMaxByDay(_watch), fresh); + expect(scans, 2, reason: 'unchanged second request in that pass must hit'); + } finally { endWearableDayIndexPass(); } + }); + } + + for (final key in ['device', 'family']) { + test('different wearable $key gets its own pass key', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + final other = key == 'device' ? ('garmin-b', 'garmin') : ('garmin-a', 'oura'); + beginWearableDayIndexPass(); + try { + final original = await wearableRecTsMaxByDay(_watch); + final changed = await wearableRecTsMaxByDay(other); + expect(changed, isEmpty); + expect(original, isNotEmpty); + expect(scans, 2, reason: 'a different $key must miss'); + expect(await wearableRecTsMaxByDay(_watch), original); + expect(await wearableRecTsMaxByDay(other), changed); + expect(scans, 2, reason: 'both keys, including empty index, should now hit'); + } finally { endWearableDayIndexPass(); } + }); + } + + test('index is discarded when body throws after a populated lookup', () async { + var scans = 0; + var populated = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + Future failedPass() async { + beginWearableDayIndexPass(); + try { + expect(await wearableRecTsMaxByDay(_watch), isNotEmpty); + populated = debugWearableDayIndexSize; + throw StateError('after index fill'); + } finally { endWearableDayIndexPass(); } + } + await expectLater(failedPass(), throwsStateError); + expect(debugWearableDayIndexSize, 0); + beginWearableDayIndexPass(); + try { await wearableRecTsMaxByDay(_watch); } + finally { endWearableDayIndexPass(); } + expect(scans, 2, reason: 'the failed pass cannot lend its index to a retry'); + expect(populated, 1, reason: 'exception cleanup must be tested with an occupied index'); + }); + + test('real run clears the index and running latch on an injected scan failure', () async { + var scans = 0; + var occupiedDuringFailure = 0; + debugOnWearableDayIndexScan = (_, _) { + scans++; + if (scans == 1) { + occupiedDuringFailure = debugWearableDayIndexSize; + throw StateError('injected after wearable scan'); + } + }; + final e = DerivationEngine(); + await e.run(edProfile); + expect(e.running, isFalse, reason: 'failure must release the process-wide latch'); + expect(e.snapshot()['last_error'], contains('injected after wearable scan')); + expect(debugWearableDayIndexSize, 0, reason: 'outer finally must release the pass index'); + final beforeRetry = scans; + expect(await e.run(edProfile), greaterThan(0)); + expect(scans, greaterThan(beforeRetry), reason: 'retry must build a fresh index'); + expect(occupiedDuringFailure, 1, reason: 'the fault must occur after population'); + }); + + test('actual light run scans the wearable index once through readiness backfill', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + final e = DerivationEngine(); + expect(await e.run(edProfile), greaterThan(0)); + expect(scans, 1, reason: 'scope, edge and readiness backfill share the pass index'); + expect(debugWearableDayIndexSize, 0); + }); + + test('lookups outside a pass stay fresh (passing guard)', () async { + var scans = 0; + debugOnWearableDayIndexScan = (_, _) => scans++; + await wearableRecTsMaxByDay(_watch); + final t = DateTime(2020, 1, 3, 10).millisecondsSinceEpoch ~/ 1000; + await edRows(t, t + 1, id: _watch.$1, family: _watch.$2, source: _watch.$2); + expect((await wearableRecTsMaxByDay(_watch)).keys, contains('2020-01-03')); + expect(scans, 2); + expect(debugWearableDayIndexSize, 0); + }); + + test('readiness backfill defers wearable substrate and still rescores pruned days (guard)', () async { + for (final day in [edDay, '2020-01-01']) { + final payload = bundle(0.5, 0.5, 0); + await LocalDb.putDayResult(dayId: day, algoVersion: 111, + payloadJson: jsonEncode(payload), windowJson: '{}', finalized: true, + readiness: (payload['scalars'] as Map)['readiness'] as double); + } + final before = await edStored(edDay); + beginWearableDayIndexPass(); + late final ({int seeded, int rescored, int forced}) result; + try { + final raw = await substrateDays(); + expect(raw, contains(edDay)); + result = await backfillReadinessCalibration(rawDays: raw, + zHistoryLoader: () async => (day, family) => []); + } finally { endWearableDayIndexPass(); } + expect(result.rescored, 1, reason: 'pruned history still needs calibration'); + expect(result.forced, 0); + expect((await LocalDb.dayResult('2020-01-01'))!['algo_version'], kAlgoVersion); + expect((await LocalDb.dayResult(edDay))!['algo_version'], 111, + reason: 'wearable substrate must wait for its own derive'); + // readiness_z seeding is intentional; the day bundle must remain untouched. + final row = await LocalDb.dayResult(edDay); + expect(jsonDecode(before)['day_result'][0]['payload_json'], row!['payload_json']); + }); + + test('existing fixture: pass index enabled/disabled derive JSON remains exact (guard)', () async { + await edExistingFixture(); + final e = DerivationEngine(); + expect(await e.runDays(edProfile, {edDay}), 1); + final indexed = await edStored(edDay); + debugDisableWearableDayIndex = true; + expect(await DerivationEngine().runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), indexed); + debugDisableWearableDayIndex = false; + await edRows(edStart, edStart + 1, hr: 89); + expect(await e.runDays(edProfile, {edDay}), 1); + final changed = await edStored(edDay); + debugDisableWearableDayIndex = true; + expect(await DerivationEngine().runDays(edProfile, {edDay}), 1); + expect(await edStored(edDay), changed); + }); +} From 2ae2847d7329359f22e2ed0764b022a845f4e7e7 Mon Sep 17 00:00:00 2001 From: MikenRafter Date: Sat, 24 Oct 2026 05:36:50 -0600 Subject: [PATCH 5/5] derive: build wearable inputs and prepare session windows in a worker - Wearable construction and overlay loops for Garmin, Oura, Colmi and Ultrahuman run through the existing cancellable isolate with typed results instead of on the UI isolate. - getWorkouts reads session windows once and prepares the HR stats and samples together in one worker, instead of two reads. Windows stay inclusive; two-device behavior is unchanged. Output is unchanged; kAlgoVersion stays 113. --- lib/compute/derivation_engine.dart | 2 +- lib/compute/inputs/canonical.dart | 36 +++- lib/compute/inputs/colmi_inputs.dart | 21 ++- lib/compute/inputs/garmin_inputs.dart | 6 +- lib/compute/inputs/oura_inputs.dart | 25 ++- lib/compute/inputs/ultrahuman_inputs.dart | 19 +- lib/data/db.dart | 136 +++++++++----- lib/data/local_repository_impl.dart | 6 +- test/derive_unchanged_skip_test.dart | 11 ++ test/ed2a_e20_worker_preparation_test.dart | 204 +++++++++++++++++++++ 10 files changed, 389 insertions(+), 77 deletions(-) create mode 100644 test/ed2a_e20_worker_preparation_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index c1eca5f69..695e546a3 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -11676,7 +11676,7 @@ class _IsolateValue { /// Test seam for [DerivationEngine._runIsolateCancellable] — the isolate /// lifecycle guarantees (value / error / killed-on-timeout) are what the engine /// depends on to never hang, so they're pinned directly. -@visibleForTesting +/// Also used by wearable and session-window preparation in production. Future runCancellableIsolate( FutureOr Function() compute, Duration timeout, { diff --git a/lib/compute/inputs/canonical.dart b/lib/compute/inputs/canonical.dart index 74b03d5bf..17950e9fc 100644 --- a/lib/compute/inputs/canonical.dart +++ b/lib/compute/inputs/canonical.dart @@ -18,7 +18,7 @@ import '../../ble/adapters/_registry.dart' import '../../data/day_label.dart'; import '../../data/db.dart'; import '../crossday_pipeline.dart' show buildCrossDayBundle; -import '../derivation_engine.dart' show DerivationEngine, kAlgoVersion; +import '../derivation_engine.dart' show DerivationEngine, kAlgoVersion, runCancellableIsolate; import '../substrate.dart'; import '../vendor_sleep.dart'; import 'colmi_inputs.dart'; @@ -394,8 +394,31 @@ bool debugDisableWearableWorker = false; void Function(String)? debugOnWearableConstruction; // Called beside the actual row loops. A worker extraction must return its // execution marker to this observer, not label a dispatch on the caller. -void debugRecordWearableConstruction() => - debugOnWearableConstruction?.call(Isolate.current.debugName ?? 'unnamed'); +void debugRecordWearableConstruction([String? isolateName]) => + debugOnWearableConstruction?.call( + isolateName ?? Isolate.current.debugName ?? 'unnamed'); + +// Only plain rows, columns and bounds are captured by these builders. Return +// the execution marker with the value; observers stay on the calling isolate. +Future prepareWearable(R Function() build) async { + (R, String) construct() => (build(), Isolate.current.debugName ?? 'unnamed'); + final result = debugDisableWearableWorker + ? construct() + : await runCancellableIsolate(construct, const Duration(seconds: 30), + label: 'wearable construction'); + debugRecordWearableConstruction(result.$2); + return result.$1; +} + +Future constructWearable(Substrate Function() build, + {Substrate? unchanged}) async { + final columns = await prepareWearable(() { + final value = build(); + return identical(value, unchanged) ? null : value.toJson(); + }); + // A no-op overlay preserves even a direct caller's empty optional columns. + return columns == null ? unchanged! : Substrate.fromTransfer(columns); +} /// The active wearable's rows over `[from, to]` as a substrate, for a day the /// primary band left empty, with a strap's session windows laid over them @@ -467,7 +490,12 @@ Future withStrapSessions(Substrate sub, int from, int to) async { 'AND d.rec_ts >= ? AND d.rec_ts <= ? ORDER BY d.rec_ts', [from, to], ); - debugRecordWearableConstruction(); + return constructWearable(() => _overlayStrapSessions(sub, beats, all), + unchanged: sub); +} + +Substrate _overlayStrapSessions(Substrate sub, + List> beats, List> all) { final owner = ownerBy([for (final r in all) (r['sid'], r['src'])]); final rows = [ for (final r in all) diff --git a/lib/compute/inputs/colmi_inputs.dart b/lib/compute/inputs/colmi_inputs.dart index 70d499b68..cc7ebb2a7 100644 --- a/lib/compute/inputs/colmi_inputs.dart +++ b/lib/compute/inputs/colmi_inputs.dart @@ -51,10 +51,9 @@ Future> colmiSkinTemps( int to, ) async { final db = await LocalDb.instance; - final out = {}; // A reply is banked on or after the day it describes and at most the // ring's history (kColmiHistoryDays) later: scan only those captures. - for (final r in await db.rawQuery( + final rows = await db.rawQuery( 'SELECT hex, COALESCE(rec_ts * 1000, captured_at) AS anchor_ms ' 'FROM ${LocalDb.rawArchiveView} WHERE device_id = ? ' "AND reason = 'colmi_big_0x${kColmiBigTemperature.toRadixString(16)}' " @@ -64,7 +63,14 @@ Future> colmiSkinTemps( (from - 86400) * 1000, (to + (kColmiHistoryDays + 1) * 86400) * 1000, ], - )) { + ); + return prepareWearable(() => _colmiSkinTemps(rows, from, to)); +} + +List<(int, int, int)> _colmiSkinTemps( + List> rows, int from, int to) { + final out = {}; + for (final r in rows) { final hex = r['hex'] as String; final bytes = [ for (var i = 0; i + 1 < hex.length; i += 2) @@ -99,6 +105,10 @@ Future> colmiSkinTemps( Future colmiHr(String deviceId, int from, int to) async { final sub = await sparseHrSubstrate(kColmiFamily, deviceId, from, to); if (sub.length < 2) return sub; + return constructWearable(() => _colmiHr(sub)); +} + +Substrate _colmiHr(Substrate sub) { final gaps = [ for (var i = 1; i < sub.length; i++) sub.tsSec[i] - sub.tsSec[i - 1], ]..sort(); @@ -162,7 +172,10 @@ Future colmiSubstrate(String deviceId, int from, int to) async { final sub = await colmiHr(deviceId, from, to); if (sub.isEmpty) return sub; final temps = await colmiSkinTemps(deviceId, from, to); - debugRecordWearableConstruction(); + return constructWearable(() => _colmiSubstrate(sub, temps)); +} + +Substrate _colmiSubstrate(Substrate sub, List<(int, int, int)> temps) { final skin = []; var j = 0; for (final t in sub.tsSec) { diff --git a/lib/compute/inputs/garmin_inputs.dart b/lib/compute/inputs/garmin_inputs.dart index 49258b2cc..ebe695dce 100644 --- a/lib/compute/inputs/garmin_inputs.dart +++ b/lib/compute/inputs/garmin_inputs.dart @@ -107,7 +107,11 @@ Future sparseHrSubstrate( [deviceId, family, from, to], ); if (rows.isEmpty) return Substrate.empty; - debugRecordWearableConstruction(); + return constructWearable(() => _sparseHrSubstrate(family, deviceId, rows)); +} + +Substrate _sparseHrSubstrate(String family, String deviceId, + List> rows) { final n = rows.length; final zeros = List.filled(n, 0); final none = List.filled(n, -1); diff --git a/lib/compute/inputs/oura_inputs.dart b/lib/compute/inputs/oura_inputs.dart index a00b6673f..aaa578aaf 100644 --- a/lib/compute/inputs/oura_inputs.dart +++ b/lib/compute/inputs/oura_inputs.dart @@ -68,7 +68,18 @@ Future ouraSubstrate(String deviceId, int from, int to) async { if (sub.isEmpty) return _ouraTempSubstrate(deviceId, from, to); final c = kHrCadenceSec[kOuraFamily]!; final temps = await _ouraTempRows(deviceId, from - c, to + c); - debugRecordWearableConstruction(); + final db = await LocalDb.instance; + final beats = await db.rawQuery( + 'SELECT COALESCE(beat_ts_ms, rr_ts_ms) AS t, rr_ms AS rr FROM decoded_rr ' + 'WHERE device_id = ? AND device_family = ? AND rec_ts >= ? AND rec_ts <= ? ' + 'ORDER BY t, beat_index', + [deviceId, kOuraFamily, from, to], + ); + return constructWearable(() => _ouraSubstrate(sub, temps, beats, c)); +} + +Substrate _ouraSubstrate(Substrate sub, List> temps, + List> beats, int c) { final skin = []; var j = 0; for (final t in sub.tsSec) { @@ -81,13 +92,6 @@ Future ouraSubstrate(String deviceId, int from, int to) async { ((temps[j]['rec_ts'] as num) - t).abs() <= c; skin.add(near ? ((temps[j]['skin_temp_c'] as num) * 100).round() : 0); } - final db = await LocalDb.instance; - final beats = await db.rawQuery( - 'SELECT COALESCE(beat_ts_ms, rr_ts_ms) AS t, rr_ms AS rr FROM decoded_rr ' - 'WHERE device_id = ? AND device_family = ? AND rec_ts >= ? AND rec_ts <= ? ' - 'ORDER BY t, beat_index', - [deviceId, kOuraFamily, from, to], - ); final rr = [ for (final b in beats) if (plausibleRrOrNull(b['rr'] as num) case final ms?) @@ -105,7 +109,10 @@ Future ouraSubstrate(String deviceId, int from, int to) async { Future _ouraTempSubstrate(String deviceId, int from, int to) async { final rows = await _ouraTempRows(deviceId, from, to); if (rows.isEmpty) return Substrate.empty; - debugRecordWearableConstruction(); + return constructWearable(() => _ouraTempColumns(deviceId, rows)); +} + +Substrate _ouraTempColumns(String deviceId, List> rows) { final n = rows.length; final zeros = List.filled(n, 0); final none = List.filled(n, -1); diff --git a/lib/compute/inputs/ultrahuman_inputs.dart b/lib/compute/inputs/ultrahuman_inputs.dart index 023305ec3..300c022e8 100644 --- a/lib/compute/inputs/ultrahuman_inputs.dart +++ b/lib/compute/inputs/ultrahuman_inputs.dart @@ -47,13 +47,18 @@ Future> ultrahumanRecords( int to, ) async { final db = await LocalDb.instance; - final out = []; - for (final r in await db.rawQuery( + final rows = await db.rawQuery( "SELECT hex FROM ${LocalDb.rawArchiveView} WHERE device_id = ? " "AND reason = 'ultrahuman_record' AND rec_ts >= ? AND rec_ts <= ? " 'ORDER BY rec_ts', [deviceId, from, to], - )) { + ); + return prepareWearable(() => _ultrahumanRecords(rows)); +} + +List _ultrahumanRecords(List> rows) { + final out = []; + for (final r in rows) { final hex = r['hex'] as String; final rec = parseUltrahumanRecord([ for (var i = 0; i + 1 < hex.length; i += 2) @@ -77,9 +82,13 @@ Future ultrahumanSubstrate(String deviceId, int from, int to) async { final sub = await sparseHrSubstrate(kUltrahumanFamily, deviceId, from, to); if (sub.isEmpty) return sub; - debugRecordWearableConstruction(); + final records = await ultrahumanRecords(deviceId, from, to); + return constructWearable(() => _ultrahumanSubstrate(sub, records)); +} + +Substrate _ultrahumanSubstrate(Substrate sub, List records) { final temp = { - for (final r in await ultrahumanRecords(deviceId, from, to)) + for (final r in records) if (_skinC(r) case final c?) r.tsA: (c * 100).round(), }; return Substrate( diff --git a/lib/data/db.dart b/lib/data/db.dart index 481fe48fe..b940f179b 100644 --- a/lib/data/db.dart +++ b/lib/data/db.dart @@ -33,6 +33,7 @@ import '../coach/coach_db.dart' show CoachDb; import '../compute/derivation_engine.dart' show kAlgoVersion, + runCancellableIsolate, kOvernightGiveUpSec, kPinWakeToleranceSec, pinnableNight, @@ -75,6 +76,11 @@ typedef DbRebuild = ({ Map salvaged, }); +typedef SessionHrReads = ({ + Map> stats, + Map> samples, +}); + // ── SUBSTRATE ADMISSION — the one predicate that decides what may become a // number ─────────────────────────────────────────────────────────────────── // @@ -169,7 +175,13 @@ String derivableSourceSql([String col = 'source']) => kDerivableSources.isEmpty /// ponytail: on that gap the workout screen and the day disagree; give the /// band day the strap's window too (or drop the override there) once the /// owner decides whether a strap may fill a band gap. -List sessionWindowRows>(List rows) { +List sessionWindowRows>(List rows) => + _sessionWindowRowsFor(rows, sensors: LocalDb.sessionSensorSources, + wearable: LocalDb.sessionWearableSource); + +// Workers receive the flags explicitly; isolate-local statics start empty. +List _sessionWindowRowsFor>( + List rows, {required Set sensors, required String? wearable}) { bool derivable(Object? s) => s == null || kDerivableSources.contains(s); List plain() => [ for (final r in rows) @@ -180,7 +192,7 @@ List sessionWindowRows>(List rows) { for (final r in rows) { final s = r['source']; if (s == null) return plain(); - if (LocalDb.sessionSensorSources.contains(s) && worn(r)) { + if (sensors.contains(s) && worn(r)) { n[s] = (n[s] ?? 0) + 1; } } @@ -198,7 +210,6 @@ List sessionWindowRows>(List rows) { if (lo == null || t < lo) lo = t; if (hi == null || t > hi) hi = t; } - final wearable = LocalDb.sessionWearableSource; return [ for (final r in rows) if (r['source'] == owner || @@ -14998,23 +15009,33 @@ class LocalDb { @visibleForTesting static void Function(String)? debugOnSessionWindowsConstruction; - // RED forwarding seam: still performs the two existing reads. GREEN combines - // these at the repository call site and prepares both results in one worker. + // Test forwarding seam for the production combined read. @visibleForTesting - static Future<({Map> stats, - Map> samples})> sessionHrReadsForTest(int from, int to, - {int maxHrCeiling = 0, int minHrFloor = 0}) async => - (stats: await sessionHrStats(from, to, - maxHrCeiling: maxHrCeiling, minHrFloor: minHrFloor), - samples: await sessionHrSamplesBySession(from, to)); + static Future sessionHrReadsForTest(int from, int to, + {int maxHrCeiling = 0, int minHrFloor = 0}) => + sessionHrReads(from, to, + maxHrCeiling: maxHrCeiling, minHrFloor: minHrFloor); + + /// Both HR reads for the workout list. With a sensor enabled, fetch each + /// window once and construct its stats and samples together in a worker. + static Future sessionHrReads(int from, int to, + {int maxHrCeiling = 0, int minHrFloor = 0}) async { + if (sessionSensorSources.isEmpty) { + return (stats: await _sessionHrStatsSql(from, to, maxHrCeiling, minHrFloor), + samples: await _sessionHrSamplesSql(from, to)); + } + return _sessionWindows(from, to, maxHrCeiling, minHrFloor); + } /// Every session in [fromTs, toTs] (epoch SECONDS) with the worn rows of /// its window a session read keeps ([sessionWindowRows]), ascending. Only /// with a workout sensor flagged on; a session with none is absent, as from /// the band-only joins. - static Future>>> _sessionWindows( + static Future _sessionWindows( int fromTs, int toTs, + int maxHrCeiling, + int minHrFloor, ) async { final db = await instance; final rows = await db.rawQuery( @@ -15026,21 +15047,53 @@ class LocalDb { 'ORDER BY s.id, d.rec_ts ASC', [fromTs, toTs], ); - debugOnSessionWindowsConstruction?.call(Isolate.current.debugName ?? 'unnamed'); + final sensors = {...sessionSensorSources}; + final wearable = sessionWearableSource; + (SessionHrReads, String) construct() => _prepareSessionWindows( + rows, sensors, wearable, maxHrCeiling, minHrFloor); + final result = debugDisableSessionWindowsWorker + ? construct() + : await runCancellableIsolate(construct, const Duration(seconds: 30), + label: 'session windows'); + debugOnSessionWindowsConstruction?.call(result.$2); + return result.$1; + } + + static (SessionHrReads, String) _prepareSessionWindows( + List> rows, Set sensors, String? wearable, + int maxHrCeiling, int minHrFloor) { final by = >>{}; for (final r in rows) { final id = r['id'] as String?; if (id != null) (by[id] ??= []).add(r); } - return { - for (final MapEntry(key: id, value: rs) in by.entries) - if ([ - for (final r in sessionWindowRows(rs)) - if (((r['hr'] as num?) ?? 0) > 0) r, - ] - case final kept when kept.isNotEmpty) - id: kept, - }; + final stats = >{}; + final samples = >{}; + for (final MapEntry(key: id, value: rs) in by.entries) { + final hr = [ + for (final r in _sessionWindowRowsFor(rs, + sensors: sensors, wearable: wearable)) + if (((r['hr'] as num?) ?? 0) > 0) (r['hr'] as num).toInt(), + ]; + if (hr.isEmpty) continue; + samples[id] = hr; + final low = [ + for (final h in hr) + if (minHrFloor <= 0 || h >= minHrFloor) h, + ]; + final high = [ + for (final h in hr) + if (maxHrCeiling <= 0 || h <= maxHrCeiling) h, + ]; + stats[id] = { + 'n': hr.length, + 'avg_hr': hr.fold(0, (a, b) => a + b) / hr.length, + 'min_hr': low.isEmpty ? 0 : low.reduce((a, b) => a < b ? a : b), + 'max_hr': high.isEmpty ? 0 : high.reduce((a, b) => a > b ? a : b), + }; + } + return ((stats: stats, samples: samples), + Isolate.current.debugName ?? 'unnamed'); } /// Per-session HR aggregates over the 1 Hz substrate for every session in @@ -15060,28 +15113,14 @@ class LocalDb { int minHrFloor = 0, }) async { if (sessionSensorSources.isNotEmpty) { - return { - for (final MapEntry(key: id, value: rs) - in (await _sessionWindows(fromTs, toTs)).entries) - id: () { - final hr = [for (final r in rs) (r['hr'] as num).toInt()]; - final low = [ - for (final h in hr) - if (minHrFloor <= 0 || h >= minHrFloor) h, - ]; - final high = [ - for (final h in hr) - if (maxHrCeiling <= 0 || h <= maxHrCeiling) h, - ]; - return { - 'n': hr.length, - 'avg_hr': hr.fold(0, (a, b) => a + b) / hr.length, - 'min_hr': low.isEmpty ? 0 : low.reduce((a, b) => a < b ? a : b), - 'max_hr': high.isEmpty ? 0 : high.reduce((a, b) => a > b ? a : b), - }; - }(), - }; + return (await sessionHrReads(fromTs, toTs, + maxHrCeiling: maxHrCeiling, minHrFloor: minHrFloor)).stats; } + return _sessionHrStatsSql(fromTs, toTs, maxHrCeiling, minHrFloor); + } + + static Future>> _sessionHrStatsSql( + int fromTs, int toTs, int maxHrCeiling, int minHrFloor) async { final db = await instance; final ceilClause = maxHrCeiling > 0 ? 'AND d.hr <= $maxHrCeiling ' : ''; final floorClause = minHrFloor > 0 ? 'AND d.hr >= $minHrFloor ' : ''; @@ -15123,12 +15162,13 @@ class LocalDb { int toTs, ) async { if (sessionSensorSources.isNotEmpty) { - return { - for (final MapEntry(key: id, value: rs) - in (await _sessionWindows(fromTs, toTs)).entries) - id: [for (final r in rs) (r['hr'] as num).toInt()], - }; + return (await sessionHrReads(fromTs, toTs)).samples; } + return _sessionHrSamplesSql(fromTs, toTs); + } + + static Future>> _sessionHrSamplesSql( + int fromTs, int toTs) async { final db = await instance; final rows = await db.rawQuery( 'SELECT s.id AS id, d.hr AS hr ' diff --git a/lib/data/local_repository_impl.dart b/lib/data/local_repository_impl.dart index ceb77017d..caeb96ee1 100644 --- a/lib/data/local_repository_impl.dart +++ b/lib/data/local_repository_impl.dart @@ -2405,7 +2405,7 @@ class LocalRepositoryImpl extends LocalRepository { // "no data" (avg_hr == 0) even when the window is full of worn HR. try { final age = _profileAge(); - final stats = await LocalDb.sessionHrStats( + final (:stats, samples: rawBySession) = await LocalDb.sessionHrReads( fromTs, nowSec, maxHrCeiling: hrCeilingForAge(age), @@ -2414,10 +2414,6 @@ class LocalRepositoryImpl extends LocalRepository { // Spike-suppressed max/min per session (issue #127): smooth the raw 1 Hz // over one batched join so the list agrees with getWorkout's on-read // recompute. - final rawBySession = await LocalDb.sessionHrSamplesBySession( - fromTs, - nowSec, - ); for (final w in workouts) { final s = stats[w['id']]; final raw = rawBySession[w['id']]; diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart index 89194293e..9e49ccf66 100644 --- a/test/derive_unchanged_skip_test.dart +++ b/test/derive_unchanged_skip_test.dart @@ -49,6 +49,15 @@ Future _freshDb(String name) async { } } +// The fixture starts at 18:00 on the previous date. That short first day +// finalizes on its first derive, while its daytime summaries read the NEXT +// day's sleep window. Bank that real window before either parity run so the +// foreground worker order cannot decide whether the first day's evening is +// excluded as tonight's sleep. Every stored output is still compared below. +Future _primeFirstNight() async { + expect(await DerivationEngine().runDays(_profile, {'2026-07-13'}), 1); +} + Future<({int derived, int unchanged})> _pass({Profile profile = _profile}) async { final e = DerivationEngine(); final n = await e.run(profile, heavy: true); @@ -160,6 +169,7 @@ void main() { () async { const passes = 5; await _freshDb('unchanged_skip_on.db'); + await _primeFirstNight(); final log = <({int derived, int unchanged})>[]; for (var i = 0; i < passes; i++) { log.add(await _pass()); @@ -167,6 +177,7 @@ void main() { final withSkip = await _snapshot(); await _freshDb('unchanged_skip_off.db'); + await _primeFirstNight(); DerivationEngine.debugDeriveEveryDay = true; for (var i = 0; i < passes; i++) { final r = await _pass(); diff --git a/test/ed2a_e20_worker_preparation_test.dart b/test/ed2a_e20_worker_preparation_test.dart new file mode 100644 index 000000000..76b8a2641 --- /dev/null +++ b/test/ed2a_e20_worker_preparation_test.dart @@ -0,0 +1,204 @@ +@Timeout(Duration(minutes: 10)) +library; + +import 'dart:convert'; +import 'dart:isolate'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derive_prepare.dart'; +import 'package:openstrap_edge/compute/inputs/canonical.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:openstrap_edge/data/local_repository_impl.dart'; +import 'package:sqflite_common/sqflite_logger.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/ed2a_recalc_fixture.dart'; + +String get _mainIsolate => Isolate.current.debugName ?? 'unnamed'; +Future> _sessionReads() async { + final r = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200); + return {'stats': r.stats, 'samples': r.samples}; +} + +void main() { + final sql = []; + setUpAll(edInit); + setUp(() async { + await edFresh('ed2a_e20.db'); + // Log the real join, rather than counting calls to a mock query function. + await LocalDb.close(); + // ignore: experimental_member_use + databaseFactoryOrNull = SqfliteDatabaseFactoryLogger(databaseFactoryFfi, + options: SqfliteLoggerOptions(type: SqfliteDatabaseFactoryLoggerType.invoke, + log: (event) { + if (event is! SqfliteLoggerInvokeEvent) return; + void collect(Object? v) { + if (v is Map) { + if (v['sql'] case final String s) sql.add(s); + for (final x in v.values) { collect(x); } + } else if (v is List) { + for (final x in v) { collect(x); } + } + } + collect(event.arguments); + })); + await LocalDb.instance; + sql.clear(); + }); + tearDown(() async { + debugDisableWearableWorker = false; + debugOnWearableConstruction = null; + LocalDb.debugDisableSessionWindowsWorker = false; + LocalDb.debugOnSessionWindowsConstruction = null; + await edDrop(); + databaseFactory = databaseFactoryFfi; + }); + + for (final family in ['garmin', 'colmi', 'oura', 'ultrahuman', 'pebble', 'miband234']) { + test('$family construction runs off the caller isolate with exact output', () async { + final id = '$family-a'; + await edRows(edStart, edStart + 1200, id: id, family: family, source: family, hr: 71, az: 0); + debugDisableWearableWorker = true; + final inline = await wearableSubstrate((id, family), edStart, edStart + 1199); + expect(inline.isEmpty, isFalse, reason: 'fixture must enter a real builder'); + final locations = []; + debugOnWearableConstruction = locations.add; + debugDisableWearableWorker = false; + final worker = await wearableSubstrate((id, family), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + final a = prepareDerivationPayload(inline, targetDay: edDay).days.single; + final b = prepareDerivationPayload(worker, targetDay: edDay).days.single; + expect(jsonEncode(edBundle(b)), jsonEncode(edBundle(a))); + expect(locations, isNotEmpty, reason: 'actual construction must report execution location'); + expect(locations, everyElement(isNot(_mainIsolate)), + reason: 'row construction still executes on the UI/caller isolate'); + }); + } + + test('two wearable devices at one second remain separate (passing guard)', () async { + await edRows(edStart, edStart + 1200, id: 'a', family: 'garmin', source: 'garmin', hr: 65, az: 0); + await edRows(edStart, edStart + 1200, id: 'b', family: 'garmin', source: 'garmin', hr: 95, az: 0); + for (final id in ['a', 'b']) { + debugDisableWearableWorker = true; + final inline = await wearableSubstrate((id, 'garmin'), edStart, edStart + 1199); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate((id, 'garmin'), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.hr.toSet(), {id == 'a' ? 65 : 95}); + expect(worker.deviceIds, {id}); + } + }); + + test('two straps keep owner, beat timing and session overlay exact (guard)', () async { + LocalDb.sessionSensorSources = {'ble_hrs', 'polar'}; + LocalDb.sessionWearableSource = 'garmin'; + await edRows(edStart, edStart + 1200, id: 'watch', family: 'garmin', source: 'garmin', hr: 70, az: 0); + await edRows(edStart + 100, edStart + 700, id: 'strap-a', family: 'ble_hrs', source: 'ble_hrs', hr: 150, az: 0); + await edRows(edStart + 100, edStart + 300, id: 'strap-b', family: 'polar', source: 'polar', hr: 180, az: 0); + await edSession(start: edStart, end: edStart + 800); + final db = await LocalDb.instance; + for (final (id, source, rr) in [('strap-a', 'ble_hrs', 850), ('strap-b', 'polar', 900)]) { + for (var i = 0; i < (id == 'strap-a' ? 3 : 1); i++) { + final t = edStart + 150 + i; + await db.insert('decoded_rr', {'device_id': id, 'source': source, + 'ts_ms': t * 1000, 'rec_ts': t, 'rr_ts_ms': t * 1000, + 'beat_ts_ms': t * 1000 - 123, 'rr_ms': rr, 'beat_index': 0}); + } + } + debugDisableWearableWorker = true; + final inline = await wearableSubstrate(('watch', 'garmin'), edStart, edStart + 1199); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate(('watch', 'garmin'), edStart, edStart + 1199); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.hr, isNot(contains(180)), reason: 'minority source must not mix into a session'); + expect(worker.rrMs, [850, 850, 850]); + expect(worker.rrTsMs.first, (edStart + 150) * 1000 - 123); + }); + + test('existing two-device fixture: on/off worker derive JSON is byte-identical (guard)', () async { + await edExistingFixture(); + debugDisableWearableWorker = true; + final inline = await wearableSubstrate(('oura-a1b2c3d4', 'oura'), 1577923200, 1577952000); + debugDisableWearableWorker = false; + final worker = await wearableSubstrate(('oura-a1b2c3d4', 'oura'), 1577923200, 1577952000); + expect(inline.isEmpty, isFalse); + expect(jsonEncode(worker.toJson()), jsonEncode(inline.toJson())); + expect(worker.skinTemp.where((v) => v > 0), isNotEmpty); + expect(jsonEncode(edBundle(prepareDerivationPayload(worker, targetDay: edLabel(worker.tsSec.last)).days.single)), + jsonEncode(edBundle(prepareDerivationPayload(inline, targetDay: edLabel(inline.tsSec.last)).days.single))); + }); + + Future sessionFixture() async { + LocalDb.sessionSensorSources = {'ble_hrs', 'polar'}; + LocalDb.sessionWearableSource = 'garmin'; + await edSession(); + await edRows(edStart, edStart + 1200, id: 'watch', family: 'garmin', source: 'garmin', hr: 70); + await edRows(edStart + 100, edStart + 700, id: 'strap-a', family: 'ble_hrs', source: 'ble_hrs', hr: 150); + await edRows(edStart + 100, edStart + 300, id: 'strap-b', family: 'polar', source: 'polar', hr: 180); + } + + test('combined session stats and samples use one SELECT', () async { + await sessionFixture(); + sql.clear(); + final r = await _sessionReads(); + expect((r['samples'] as Map)['s1'], isNotEmpty); + final joins = sql.where((s) => s.contains('JOIN decoded_onehz') && s.contains('FROM sessions')); + expect(joins, hasLength(1), reason: 'stats and samples currently reread the same window twice'); + }); + + test('session grouping executes off the caller isolate with exact JSON', () async { + await sessionFixture(); + LocalDb.debugDisableSessionWindowsWorker = true; + final inline = await _sessionReads(); + final locations = []; + LocalDb.debugOnSessionWindowsConstruction = locations.add; + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await _sessionReads(); + expect(jsonEncode(worker), jsonEncode(inline)); + expect(locations, isNotEmpty); + expect(locations, everyElement(isNot(_mainIsolate)), + reason: 'grouping and owner selection still loop over rows on the caller'); + }); + + test('workout repository uses the combined read in its actual list flow', () async { + await sessionFixture(); + final repo = LocalRepositoryImpl(getProfileMap: () => edProfile.toMap()); + sql.clear(); + final got = await repo.getWorkouts(range: 'all'); + expect(got['workouts'], isNotEmpty); + final joins = sql.where((s) => s.contains('JOIN decoded_onehz') && s.contains('FROM sessions')); + expect(joins, hasLength(1), reason: 'GREEN must wire the real getWorkouts consumer too'); + }); + + test('bounded session aggregates and empty windows stay exact (guard)', () async { + await sessionFixture(); + LocalDb.debugDisableSessionWindowsWorker = true; + final inline = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200, + maxHrCeiling: 140, minHrFloor: 80); + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await LocalDb.sessionHrReadsForTest(edStart, edStart + 1200, + maxHrCeiling: 140, minHrFloor: 80); + expect(jsonEncode({'stats': worker.stats, 'samples': worker.samples}), + jsonEncode({'stats': inline.stats, 'samples': inline.samples})); + expect(worker.stats['s1']!['min_hr'], 150); + expect(worker.stats['s1']!['max_hr'], 70); + expect(worker.samples['s1'], containsAll([70, 150])); + final empty = await LocalDb.sessionHrReadsForTest(edStart - 100, edStart - 1); + expect(empty.stats, isEmpty); + expect(empty.samples, isEmpty); + }); + + test('session windows keep two-device ownership and last-second samples (guard)', () async { + await sessionFixture(); + final inline = await _sessionReads(); + LocalDb.debugDisableSessionWindowsWorker = false; + final worker = await _sessionReads(); + expect(jsonEncode(worker), jsonEncode(inline)); + final samples = (worker['samples'] as Map)['s1'] as List; + expect(samples, isNot(contains(180))); + expect(samples.first, 70); + expect(samples.last, 70, reason: 'inclusive session end must survive preparation'); + final stats = (worker['stats'] as Map)['s1'] as Map; + expect(stats['n'], 1101); + }); +}