Derive: stop recalculating unchanged days from scratch on every pass

lib/app.dart

@@ -76,10 +76,16 @@ class _OpenStrapAppState extends State<OpenStrapApp> with WidgetsBindingObserver
         WidgetsBinding.instance.platformDispatcher.platformBrightness);
   }
 
+  @override
+  void didHaveMemoryPressure() {
+    context.read<AppState>().handleMemoryPressure();
+  }
+
   @override
   void didChangeAppLifecycleState(AppLifecycleState state) {
     final app = context.read<AppState>();
     if (state == AppLifecycleState.resumed) {
+      app.resumeDerivationRetention();
       // The user may have flipped our notification switch either way in OS
       // Settings while we were backgrounded. Drop the cached authorization
       // decision so the next present/schedule re-reads reality — a denial used

lib/compute/derive_prepare.dart

@@ -347,6 +347,11 @@ void derivationPrepareWorker(SendPort mainSendPort) {
       targetDay = message['target_day']?.toString();
       final cfgMode = message['mode']?.toString();
       if (cfgMode != null && cfgMode.isNotEmpty) mode = cfgMode;
+      state.retain = message['retain'] == true;
+      return;
+    }
+    if (type == 'seed') {
+      state.seed(message);
       return;
     }
     if (type == 'finish') {
@@ -357,6 +362,7 @@ void derivationPrepareWorker(SendPort mainSendPort) {
             'type': 'result',
             'kind': 'substrate',
             'payload': substrate.toJson(),
+            if (state.retainable) 'retain': state.retainExtras(),
           });
         } else {
           final payload = prepareDerivationPayload(
@@ -408,6 +414,7 @@ PreparedDerivationPayload prepareDerivationPayload(
   SleepWindowOverride? override,
   List<({int startSec, int endSec, String dayKey})> priorSleep = const [],
   List<VendorNight> vendorNights = const [],
+  bool skipUnusedDays = false,
 }) {
   if (sub.isEmpty || sub.lastTs == null) {
     return const PreparedDerivationPayload(dataNowSec: 0, days: []);
@@ -416,6 +423,8 @@ PreparedDerivationPayload prepareDerivationPayload(
   for (final day in calendarDays(
     sub,
     override: override,
+    targetDay: targetDay,
+    skipUnusedDays: skipUnusedDays,
     priorSleep: priorSleep,
     vendorNights: vendorNights,
   )) {
@@ -470,6 +479,7 @@ SleepSessionCandidate prepareSleepSessionCandidate(
   SleepWindowOverride? override,
   List<({int startSec, int endSec, String dayKey})> priorSleep = const [],
   List<VendorNight> vendorNights = const [],
+  bool skipUnusedDays = false,
 }) {
   // An UNCLAIMED device night (our rows saw under half of it) ending today IS
   // the night: above our detection, below the user's word. It never reaches
@@ -515,6 +525,7 @@ SleepSessionCandidate prepareSleepSessionCandidate(
   final payload = prepareDerivationPayload(sub,
       targetDay: targetDay,
       override: override,
+      skipUnusedDays: skipUnusedDays,
       priorSleep: priorSleep,
       vendorNights: claimed);
   if (payload.days.isEmpty) return SleepSessionCandidate.absent(targetDay);
@@ -635,10 +646,68 @@ class _PrepareAccumulator {
   /// worker used to hang the whole engine (see [derivationPrepareWorker]).
   static num? _num(Object? v) => v is num ? v : null;
 
+  // Retention is recorded only for decoded loads requested by the engine.
+  bool retain = false;
+  bool _retainBroken = false;
+  bool get retainable => retain && !_retainBroken;
+  final List<int> _rrStart = [];
+  final Map<int, _BucketAcc> _buckets = {};
+  Float64List? _rrTsRaw;
+
+  void seed(Map message) {
+    List<int> ints(String k) => message[k] as List<int>;
+    List<double> dbls(String k) => message[k] as List<double>;
+    final ts = ints('ts_sec');
+    final cut = message['cut'] as int;
+    var n = 0;
+    while (n < ts.length && ts[n] < cut) { n++; }
+    final starts = ints('rr_start');
+    final beats = n < ts.length ? starts[n] : dbls('rr_ms').length;
+    void take(List<int> into, String k) => into.addAll(ints(k).getRange(0, n));
+    take(tsSec, 'ts_sec');
+    take(hr, 'hr');
+    take(spo2Red, 'spo2_red');
+    take(spo2Ir, 'spo2_ir');
+    take(skinTemp, 'skin_temp');
+    take(skinContact, 'skin_contact');
+    take(stepCount, 'step_count');
+    take(hrValid, 'hr_valid');
+    take(bandSleepState, 'band_sleep_state');
+    ax.addAll(dbls('ax').getRange(0, n));
+    ay.addAll(dbls('ay').getRange(0, n));
+    az.addAll(dbls('az').getRange(0, n));
+    // Start from the raw axis: buildSubstrate applies the running max once
+    // across the entire rebuilt load, including the prefix/tail seam.
+    rrTsMs.addAll(dbls('rr_ts_raw').getRange(0, beats));
+    rrMs.addAll(dbls('rr_ms').getRange(0, beats));
+    _rrStart.addAll(starts.getRange(0, n));
+    for (final wire in message['meta'] as List) {
+      final m = wire as List;
+      final bucket = m[0] as int;
+      if (bucket >= cut ~/ 900) continue;
+      final acc = _buckets[bucket] = _BucketAcc()
+        ..families.addAll((m[1] as List).cast<String>())
+        ..deviceIds.addAll((m[2] as List).cast<String>())
+        ..skinUnit = m[3] as String?;
+      _families.addAll(acc.families);
+      _deviceIds.addAll(acc.deviceIds);
+      _skinTempUnit ??= acc.skinUnit;
+    }
+  }
+
+  Map<String, Object?> retainExtras() => {
+    'rr_start': Int32List.fromList(_rrStart),
+    'rr_ts_raw': _rrTsRaw!,
+    'meta': [for (final e in _buckets.entries)
+      [e.key, e.value.families.toList(), e.value.deviceIds.toList(),
+        e.value.skinUnit]],
+  };
+
   void addRawPage(List<String> hexes) {
     if (hexes.isEmpty) return;
     final sub = decodeSubstrate(hexes);
     if (sub.isEmpty) return;
+    _retainBroken = true;
     tsSec.addAll(sub.tsSec);
     hr.addAll(sub.hr);
     rrTsMs.addAll(sub.rrTsMs);
@@ -707,6 +776,10 @@ class _PrepareAccumulator {
       final row = frameByRecTs[recTs];
       _noteFamily(row?['device_family']);
       _noteDeviceId(row?['device_id']);
+      if (retain) {
+        _rrStart.add(rrMs.length);
+        _buckets.putIfAbsent(recTs ~/ 900, _BucketAcc.new).note(row);
+      }
       tsSec.add(recTs);
       // NULL (absent), a beat-only second (no row at all) and an impossible
       // byte all land on the same 0 — the array's one "no usable HR this
@@ -794,6 +867,7 @@ class _PrepareAccumulator {
     // `addRawPage` each place their own beats correctly and then APPEND, so an
     // overlap between the last record of one page and the first of the next is
     // only visible once every page is in. See [monotonizeBeatAxis].
+    if (retainable) _rrTsRaw = Float64List.fromList(rrTsMs);
     monotonizeBeatAxis(rrTsMs);
     // `_skinTempUnitDrops` (see [_skinTempFor]) is structurally 0 today —
     // `derivableSourceSql()` renders `source IS NULL` (db.dart:116), so a
@@ -807,23 +881,41 @@ class _PrepareAccumulator {
     // which M0 must not take; M5 — which is what makes two-device admission
     // reachable for real — turns this into a refusal note on
     // `wellness.skin_temp` instead.
+    List<int> packed(List<int> values) =>
+        retainable ? Int64List.fromList(values) : values;
     return Substrate(
-      tsSec: tsSec,
-      hr: hr,
+      tsSec: packed(tsSec),
+      hr: packed(hr),
       rrTsMs: rrTsMs,
       rrMs: rrMs,
       ax: ax,
       ay: ay,
       az: az,
-      spo2Red: spo2Red,
-      spo2Ir: spo2Ir,
-      skinTemp: skinTemp,
-      skinContact: skinContact,
-      stepCount: stepCount,
-      hrValid: hrValid,
-      bandSleepState: bandSleepState,
+      spo2Red: packed(spo2Red),
+      spo2Ir: packed(spo2Ir),
+      skinTemp: packed(skinTemp),
+      skinContact: packed(skinContact),
+      stepCount: packed(stepCount),
+      hrValid: packed(hrValid),
+      bandSleepState: packed(bandSleepState),
       deviceFamily: deviceFamily,
       deviceIds: _deviceIds,
     );
   }
 }
+
+class _BucketAcc {
+  final Set<String> families = {};
+  final Set<String> deviceIds = {};
+  String? skinUnit;
+
+  void note(Map<String, dynamic>? row) {
+    if (row == null) return;
+    final family = row['device_family'];
+    if (family is String && family.isNotEmpty) families.add(family);
+    final id = row['device_id'];
+    if (id is String) deviceIds.add(id);
+    skinUnit ??= row['skin_temp_raw'] is num
+        ? 'raw' : (row['skin_temp_c'] is num ? 'c' : null);
+  }
+}

lib/compute/inputs/canonical.dart

@@ -18,7 +18,7 @@ import '../../ble/adapters/_registry.dart'
 import '../../data/day_label.dart';
 import '../../data/db.dart';
 import '../crossday_pipeline.dart' show buildCrossDayBundle;
-import '../derivation_engine.dart' show DerivationEngine, kAlgoVersion;
+import '../derivation_engine.dart' show DerivationEngine, kAlgoVersion, runCancellableIsolate;
 import '../substrate.dart';
 import '../vendor_sleep.dart';
 import 'colmi_inputs.dart';
@@ -390,6 +390,36 @@ Future<void> refreshSessionSensorSources() async {
   LocalDb.sessionWearableSource = (await activeWearable())?.$2;
 }
 
+bool debugDisableWearableWorker = false;
+void Function(String)? debugOnWearableConstruction;
+// Called beside the actual row loops. A worker extraction must return its
+// execution marker to this observer, not label a dispatch on the caller.
+void debugRecordWearableConstruction([String? isolateName]) =>
+    debugOnWearableConstruction?.call(
+        isolateName ?? Isolate.current.debugName ?? 'unnamed');
+
+// Only plain rows, columns and bounds are captured by these builders. Return
+// the execution marker with the value; observers stay on the calling isolate.
+Future<R> prepareWearable<R>(R Function() build) async {
+  (R, String) construct() => (build(), Isolate.current.debugName ?? 'unnamed');
+  final result = debugDisableWearableWorker
+      ? construct()
+      : await runCancellableIsolate(construct, const Duration(seconds: 30),
+          label: 'wearable construction');
+  debugRecordWearableConstruction(result.$2);
+  return result.$1;
+}
+
+Future<Substrate> constructWearable(Substrate Function() build,
+    {Substrate? unchanged}) async {
+  final columns = await prepareWearable(() {
+    final value = build();
+    return identical(value, unchanged) ? null : value.toJson();
+  });
+  // A no-op overlay preserves even a direct caller's empty optional columns.
+  return columns == null ? unchanged! : Substrate.fromTransfer(columns);
+}
+
 /// The active wearable's rows over `[from, to]` as a substrate, for a day the
 /// primary band left empty, with a strap's session windows laid over them
 /// ([withStrapSessions]). Empty for an adapter with no inputs mapping. A day
@@ -460,6 +490,12 @@ Future<Substrate> withStrapSessions(Substrate sub, int from, int to) async {
     'AND d.rec_ts >= ? AND d.rec_ts <= ? ORDER BY d.rec_ts',
     [from, to],
   );
+  return constructWearable(() => _overlayStrapSessions(sub, beats, all),
+      unchanged: sub);
+}
+
+Substrate _overlayStrapSessions(Substrate sub,
+    List<Map<String, Object?>> beats, List<Map<String, Object?>> all) {
   final owner = ownerBy([for (final r in all) (r['sid'], r['src'])]);
   final rows = [
     for (final r in all)
@@ -616,10 +652,32 @@ Future<List<String>> flagOnOrPrimary(List<String> ids,
   return out;
 }
 
+// Pass-lifetime index. The hook runs after an actual scan, so a test can
+// count scans and inject a post-scan failure.
+Map<(String, String), Future<Map<String, int>>>? _passDayIndex;
+bool _disableWearableDayIndex = false;
+bool get debugDisableWearableDayIndex => _disableWearableDayIndex;
+set debugDisableWearableDayIndex(bool value) {
+  _disableWearableDayIndex = value;
+  if (value) _passDayIndex?.clear();
+}
+void beginWearableDayIndexPass() => _passDayIndex = {};
+void endWearableDayIndexPass() => _passDayIndex = null;
+int get debugWearableDayIndexSize => _passDayIndex?.length ?? 0;
+void debugWearableDayIndexRekey(int version) => _passDayIndex?.clear();
+void Function((String, String), Map<String, int>)? debugOnWearableDayIndexScan;
+
 /// `{local day -> last second}` the active wearable has rows for, and the
 /// days a flagged-on strap recorded a session in while the wearable was off
 /// or charging ([wearableSubstrate] lays the session over the empty day).
 Future<Map<String, int>> wearableRecTsMaxByDay(
+    (String, String) wearable) {
+  final cache = debugDisableWearableDayIndex ? null : _passDayIndex;
+  if (cache == null) return _scanWearableRecTsMaxByDay(wearable);
+  return cache.putIfAbsent(wearable, () => _scanWearableRecTsMaxByDay(wearable));
+}
+
+Future<Map<String, int>> _scanWearableRecTsMaxByDay(
     (String, String) wearable) async {
   final out = {
     ...await switch (wearable.$2) {
@@ -637,6 +695,7 @@ Future<Map<String, int>> wearableRecTsMaxByDay(
       in (await _strapSessionRecTsMaxByDay()).entries) {
     out[day] = math.max(out[day] ?? t, t);
   }
+  debugOnWearableDayIndexScan?.call(wearable, out);
   return out;
 }
 

lib/compute/inputs/colmi_inputs.dart

@@ -51,10 +51,9 @@ Future<List<(int, int, int)>> colmiSkinTemps(
   int to,
 ) async {
   final db = await LocalDb.instance;
-  final out = <int, (int, int, int)>{};
   // A reply is banked on or after the day it describes and at most the
   // ring's history (kColmiHistoryDays) later: scan only those captures.
-  for (final r in await db.rawQuery(
+  final rows = await db.rawQuery(
     'SELECT hex, COALESCE(rec_ts * 1000, captured_at) AS anchor_ms '
     'FROM ${LocalDb.rawArchiveView} WHERE device_id = ? '
     "AND reason = 'colmi_big_0x${kColmiBigTemperature.toRadixString(16)}' "
@@ -64,7 +63,14 @@ Future<List<(int, int, int)>> colmiSkinTemps(
       (from - 86400) * 1000,
       (to + (kColmiHistoryDays + 1) * 86400) * 1000,
     ],
-  )) {
+  );
+  return prepareWearable(() => _colmiSkinTemps(rows, from, to));
+}
+
+List<(int, int, int)> _colmiSkinTemps(
+    List<Map<String, Object?>> rows, int from, int to) {
+  final out = <int, (int, int, int)>{};
+  for (final r in rows) {
     final hex = r['hex'] as String;
     final bytes = [
       for (var i = 0; i + 1 < hex.length; i += 2)
@@ -99,6 +105,10 @@ Future<List<(int, int, int)>> colmiSkinTemps(
 Future<Substrate> colmiHr(String deviceId, int from, int to) async {
   final sub = await sparseHrSubstrate(kColmiFamily, deviceId, from, to);
   if (sub.length < 2) return sub;
+  return constructWearable(() => _colmiHr(sub));
+}
+
+Substrate _colmiHr(Substrate sub) {
   final gaps = [
     for (var i = 1; i < sub.length; i++) sub.tsSec[i] - sub.tsSec[i - 1],
   ]..sort();
@@ -162,6 +172,10 @@ Future<Substrate> colmiSubstrate(String deviceId, int from, int to) async {
   final sub = await colmiHr(deviceId, from, to);
   if (sub.isEmpty) return sub;
   final temps = await colmiSkinTemps(deviceId, from, to);
+  return constructWearable(() => _colmiSubstrate(sub, temps));
+}
+
+Substrate _colmiSubstrate(Substrate sub, List<(int, int, int)> temps) {
   final skin = <int>[];
   var j = 0;
   for (final t in sub.tsSec) {

lib/compute/inputs/garmin_inputs.dart

@@ -107,6 +107,11 @@ Future<Substrate> sparseHrSubstrate(
     [deviceId, family, from, to],
   );
   if (rows.isEmpty) return Substrate.empty;
+  return constructWearable(() => _sparseHrSubstrate(family, deviceId, rows));
+}
+
+Substrate _sparseHrSubstrate(String family, String deviceId,
+    List<Map<String, Object?>> rows) {
   final n = rows.length;
   final zeros = List<double>.filled(n, 0);
   final none = List<int>.filled(n, -1);