From b718f470da3f6ec980be020b0c1017bc0aa06b32 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 04:30:04 +0530 Subject: [PATCH 1/8] name the first difference when the skip test's snapshots disagree a mismatch printed both multi-megabyte snapshots, and on CI the log was cut off right after the failure line, so it looked like the test process had died. it now names the table, the row and where the values part. --- test/derive_unchanged_skip_test.dart | 58 +++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart index 647b20922..4da930982 100644 --- a/test/derive_unchanged_skip_test.dart +++ b/test/derive_unchanged_skip_test.dart @@ -88,6 +88,59 @@ Future _snapshot() async { }); } +/// Where two [_snapshot]s first part: the table, the row, and each column +/// that differs, values cut short. +String _firstDifference(String a, String b) { + String cut(Object? v) { + final t = '$v'; + return t.length <= 400 ? t : '${t.substring(0, 400)}… (${t.length} chars)'; + } + + final x = jsonDecode(a) as Map; + final y = jsonDecode(b) as Map; + for (final table in x.keys) { + final rx = x[table] as List, ry = y[table] as List; + if (rx.length != ry.length) { + return '$table: ${rx.length} rows with the skip, ${ry.length} without'; + } + for (var i = 0; i < rx.length; i++) { + final m = rx[i] as Map, n = ry[i] as Map; + final cols = {...m.keys, ...n.keys} + .where((k) => jsonEncode(m[k]) != jsonEncode(n[k])) + .toList(); + if (cols.isEmpty) continue; + final key = m['day_id'] ?? m['date'] ?? m['key'] ?? m['id'] ?? i; + return [ + '$table row $i ($key) differs in ${cols.join(', ')}:', + for (final c in cols) + if (m[c] is String && n[c] is String) + _stringDifference(c as String, m[c] as String, n[c] as String) + else ...[ + ' $c with the skip: ${cut(m[c])}', + ' $c without: ${cut(n[c])}', + ], + ].join('\n'); + } + } + return 'snapshots differ outside the compared tables'; +} + +String _stringDifference(String col, String a, String b) { + var i = 0; + while (i < a.length && i < b.length && a.codeUnitAt(i) == b.codeUnitAt(i)) { + i++; + } + String around(String t) { + final from = i < 200 ? 0 : i - 200; + final to = i + 200 > t.length ? t.length : i + 200; + return t.substring(from, to); + } + + return ' $col first differs at char $i\n' + ' with the skip: …${around(a)}…\n' + ' without: …${around(b)}…'; +} + void main() { setUpAll(() { sqfliteFfiInit(); @@ -121,7 +174,10 @@ void main() { // The skip actually happened: by the last pass nothing was derived. expect(log.last.derived, 0, reason: '$log'); expect(log.last.unchanged, greaterThan(0), reason: '$log'); - expect(withSkip, everyDay); + // Not expect(withSkip, everyDay): on a mismatch that prints both + // multi-megabyte snapshots, and the CI log was cut off right there, so a + // failure read as a silent crash. Name the first difference instead. + if (withSkip != everyDay) fail(_firstDifference(withSkip, everyDay)); }, timeout: const Timeout(Duration(minutes: 10))); test('an override, new data, a profile change and a version bump re-derive', From ac63cc76a9e0d570d0646e6c6032d1c520d04c51 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 04:30:04 +0530 Subject: [PATCH 2/8] probe: repeat the skip equivalence test on CI --- .github/workflows/skip-flake-probe.yml | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/skip-flake-probe.yml diff --git a/.github/workflows/skip-flake-probe.yml b/.github/workflows/skip-flake-probe.yml new file mode 100644 index 000000000..477985b9c --- /dev/null +++ b/.github/workflows/skip-flake-probe.yml @@ -0,0 +1,37 @@ +name: skip-flake-probe + +on: + push: + branches: [fix/derive-skip-test-flake] + +permissions: + contents: read + +jobs: + probe: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: subosito/flutter-action@v2 + with: + flutter-version: 3.41.6 + channel: stable + cache: true + - run: flutter pub get + - name: Equivalence test, repeated + run: | + nproc + fails=0 + for i in 1 2 3 4 5 6; do + echo "=== run $i" + if ! flutter test --concurrency=1 --reporter=expanded \ + test/derive_unchanged_skip_test.dart \ + --plain-name "stores exactly what deriving them does" \ + 2>&1 | grep -v '^\[' | tail -40; then :; fi + done From 34c67afa36b586e250bd92c9feb185d832dfbc0a Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 05:12:29 +0530 Subject: [PATCH 3/8] don't finalize a day while its next day is still deriving a day's waking hours end at tonight's sleep, which comes from the next day's stored window. in a pass that derives several days at once, a day old enough to finalize could read that window before the next day had stored it, keep its evening to midnight and lock that in for good, depending only on which day finished first. such a day now stays pending for one more pass, which reads the window and finalizes it. --- lib/compute/derivation_engine.dart | 44 ++++++++++++++++++++++++------ lib/compute/derive_prepare.dart | 10 +++++++ 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 2ba99a93f..1cea2b857 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -3681,7 +3681,7 @@ class DerivationEngine { onDayDone?.call(dayId, completed, orderedDays.length); } - await runWithConcurrency(orderedDays, _deriveConcurrency, processDay); + await _runDayPool(orderedDays, processDay); // BEFORE the prune below: a day still owed a strain score is re-scored // while its raw exists (and the prune holds it if this fails). _diag['stage'] = 'quiet_repass'; @@ -3928,7 +3928,7 @@ class DerivationEngine { onDayDone?.call(dayId, completed, orderedDays.length); } - await runWithConcurrency(orderedDays, _deriveConcurrency, processDay); + await _runDayPool(orderedDays, processDay); done += await _quietLevelRepass(profile, dataNowSec); // A SELECTED re-analyze that happens to cover the whole raw history, with // every day resolved, is a full restage by any other name — it re-derived @@ -4060,9 +4060,14 @@ class DerivationEngine { } // Tonight's sleep is the NEXT day's main sleep; it is only known once // that day has derived, and this day keeps re-deriving until it - // finalizes, so it converges on the following pass. + // finalizes, so it converges on the following pass. A day that would + // finalize in the same pass its next day is still deriving is held back + // ([PreparedDerivationDay.tonightPending]), or which of the two finished + // first would decide its stored result for good. + final nextDayId = localDateLabel(dayEnd); + final nextWindow = await LocalDb.sleepWindowJsonFor(nextDayId); final tonight = tonightSleep( - storedWindowSpan(await LocalDb.sleepWindowJsonFor(localDateLabel(dayEnd))), + storedWindowSpan(nextWindow), dayStartSec: dayStart, dayEndSec: dayEnd, sleepOffsetSec: candidate.present ? candidate.sleepOffsetSec : 0, @@ -4073,6 +4078,7 @@ class DerivationEngine { sleepSub: sleepSub, tonightSleepOnsetSec: tonight?.startSec ?? 0, tonightSleepOffsetSec: tonight?.endSec ?? 0, + tonightPending: nextWindow == null && _passUnfinished.contains(nextDayId), ownership: ownership, priority: priority, ); @@ -5247,7 +5253,7 @@ class DerivationEngine { )..sort((a, b) => b.compareTo(a)); // newest first, like the sweep if (days.isEmpty) return 0; _log('derive: re-scoring ${days.length} day(s) owed a quiet level'); - await runWithConcurrency(days, _deriveConcurrency, (dayId) async { + await _runDayPool(days, (dayId) async { try { await debugQuietRepassHook?.call(dayId); final prepared = await _prepareTargetDay(dayId); @@ -5394,7 +5400,7 @@ class DerivationEngine { onDayDone?.call(dayId, completed, orderedDays.length); } - await runWithConcurrency(orderedDays, _deriveConcurrency, processDay); + await _runDayPool(orderedDays, processDay); if (orderedDays.isNotEmpty) { await _refreshBaselines(); @@ -5603,6 +5609,27 @@ class DerivationEngine { /// real wall-clock speedup from the device's other cores — in the FOREGROUND. /// A headless background slot has no spare cores to soak up, so it takes one /// lane; [DerivePacing] owns that decision and explains it. + /// The days this pass's pool has not finished yet ([_runDayPool]). + final Set _passUnfinished = {}; + + /// [runWithConcurrency] over [days], tracking which are still unfinished so + /// a day can tell its next day is mid-derive ([_prepareTargetDay]). + Future _runDayPool( + List days, Future Function(String dayId) body) async { + _passUnfinished.addAll(days); + try { + await runWithConcurrency(days, _deriveConcurrency, (dayId) async { + try { + await body(dayId); + } finally { + _passUnfinished.remove(dayId); + } + }); + } finally { + _passUnfinished.removeAll(days); + } + } + int get _deriveConcurrency { try { return _pacing.concurrency(Platform.numberOfProcessors); @@ -6424,8 +6451,9 @@ class DerivationEngine { // forceFinalize. Locking an all-absent row is what made the destructive // re-analyze permanent; leaving it unlocked means a later pass (or a // restored/backfilled substrate) can still fill the day in. - final finalized = - !producedNothing && (forceFinalize || (ageFinalized && secondHalfOk)); + final finalized = !producedNothing && + (forceFinalize || + (ageFinalized && secondHalfOk && !day.tonightPending)); // A failed/timed-out second half yields a headline-only bundle, and // putDayResult replaces the row wholesale — so re-deriving an already diff --git a/lib/compute/derive_prepare.dart b/lib/compute/derive_prepare.dart index 6e37321f6..316077e2b 100644 --- a/lib/compute/derive_prepare.dart +++ b/lib/compute/derive_prepare.dart @@ -38,6 +38,11 @@ class PreparedDerivationDay { final int tonightSleepOnsetSec; final int tonightSleepOffsetSec; + /// The next day is being derived in this same pass and has no stored + /// window yet, so tonight's sleep is unknown only for now. Such a day is not + /// finalized this pass: the next pass reads the window and locks it then. + final bool tonightPending; + /// M5: the resolved exclusive-owner spans, one entry per anchor signal /// (`hr1Hz`, `rrIntervals`), over the union window `_prepareTargetDay` /// resolved once. Empty (never populated) on the import path — a bulk @@ -73,6 +78,7 @@ class PreparedDerivationDay { this.sleepSource = 'auto', this.tonightSleepOnsetSec = 0, this.tonightSleepOffsetSec = 0, + this.tonightPending = false, this.ownership = const {}, this.priority = const {}, }) : napSub = napSub ?? daySub; @@ -92,6 +98,7 @@ class PreparedDerivationDay { 'nap_sub': napSub.toJson(), 'tonight_sleep_onset_sec': tonightSleepOnsetSec, 'tonight_sleep_offset_sec': tonightSleepOffsetSec, + 'tonight_pending': tonightPending, // ponytail: `ownership` is NOT round-tripped here. Grepped for real // callers of PreparedDerivationDay.toJson()/fromJson() outside this file // — none exist (only `candidate.toPreparedDay(...)` at @@ -120,6 +127,7 @@ class PreparedDerivationDay { (m['tonight_sleep_onset_sec'] as num?)?.toInt() ?? 0, tonightSleepOffsetSec: (m['tonight_sleep_offset_sec'] as num?)?.toInt() ?? 0, + tonightPending: m['tonight_pending'] == true, daySub: daySub, sleepSub: Substrate.fromJson( ((m['sleep_sub'] as Map?) ?? const {}).cast(), @@ -260,6 +268,7 @@ class SleepSessionCandidate { Substrate? napSub, int tonightSleepOnsetSec = 0, int tonightSleepOffsetSec = 0, + bool tonightPending = false, Map> ownership = const {}, Map> priority = const {}, }) => PreparedDerivationDay( @@ -286,6 +295,7 @@ class SleepSessionCandidate { sleepSub: sleepSub, tonightSleepOnsetSec: tonightSleepOnsetSec, tonightSleepOffsetSec: tonightSleepOffsetSec, + tonightPending: tonightPending, ownership: ownership, priority: priority, ); From 326c08853ac0e6d9c7c28330be06710057892886 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 05:54:05 +0530 Subject: [PATCH 4/8] drop the baselines' build date from the skip test's snapshot built_for_day is the wall-clock day, so a run whose two halves straddle midnight stored different dates for the same baselines. --- test/derive_unchanged_skip_test.dart | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/derive_unchanged_skip_test.dart b/test/derive_unchanged_skip_test.dart index 4da930982..89194293e 100644 --- a/test/derive_unchanged_skip_test.dart +++ b/test/derive_unchanged_skip_test.dart @@ -66,7 +66,10 @@ Future _snapshot() async { if (!e.key.endsWith('_at') && e.key != 'updated_ts') e.key: e.value is String ? (e.value as String).replaceAll( - RegExp(r'"(updated|built)[A-Za-z_]*":\s*\d+'), '') + // Numbers and dates: built_for_day is the wall-clock + // day, so two halves either side of midnight differ. + RegExp(r'"(updated|built)[A-Za-z_]*":\s*(\d+|"[^"]*")'), + '') : e.value, }, ]; From b408427eaa605e7b104484c452823aa7df0bd722 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 06:31:19 +0530 Subject: [PATCH 5/8] drop the repeat-run workflow --- .github/workflows/skip-flake-probe.yml | 37 -------------------------- 1 file changed, 37 deletions(-) delete mode 100644 .github/workflows/skip-flake-probe.yml diff --git a/.github/workflows/skip-flake-probe.yml b/.github/workflows/skip-flake-probe.yml deleted file mode 100644 index 477985b9c..000000000 --- a/.github/workflows/skip-flake-probe.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: skip-flake-probe - -on: - push: - branches: [fix/derive-skip-test-flake] - -permissions: - contents: read - -jobs: - probe: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: subosito/flutter-action@v2 - with: - flutter-version: 3.41.6 - channel: stable - cache: true - - run: flutter pub get - - name: Equivalence test, repeated - run: | - nproc - fails=0 - for i in 1 2 3 4 5 6; do - echo "=== run $i" - if ! flutter test --concurrency=1 --reporter=expanded \ - test/derive_unchanged_skip_test.dart \ - --plain-name "stores exactly what deriving them does" \ - 2>&1 | grep -v '^\[' | tail -40; then :; fi - done From 9132dec1eff3af037fafe816c78142c73c41100d Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 06:48:35 +0530 Subject: [PATCH 6/8] hold on a missing window span, and never fingerprint a held day a next day whose stored window is empty has no span either, so it holds the day the same as a missing row. a held day also no longer records its inputs as unchanged, or a second held pass could store a fingerprint and skip the day before it ever read tonight's window. --- lib/compute/derivation_engine.dart | 21 ++++++-- test/derive_tonight_pending_test.dart | 77 +++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 test/derive_tonight_pending_test.dart diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 1cea2b857..888c59a5f 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -4065,9 +4065,10 @@ class DerivationEngine { // ([PreparedDerivationDay.tonightPending]), or which of the two finished // first would decide its stored result for good. final nextDayId = localDateLabel(dayEnd); - final nextWindow = await LocalDb.sleepWindowJsonFor(nextDayId); + final nextSpan = + storedWindowSpan(await LocalDb.sleepWindowJsonFor(nextDayId)); final tonight = tonightSleep( - storedWindowSpan(nextWindow), + nextSpan, dayStartSec: dayStart, dayEndSec: dayEnd, sleepOffsetSec: candidate.present ? candidate.sleepOffsetSec : 0, @@ -4078,7 +4079,11 @@ class DerivationEngine { sleepSub: sleepSub, tonightSleepOnsetSec: tonight?.startSec ?? 0, tonightSleepOffsetSec: tonight?.endSec ?? 0, - tonightPending: nextWindow == null && _passUnfinished.contains(nextDayId), + // No span, not just no row: a stored '{}' is no window either. Not + // `tonight == null`: a next night that starts after midnight has a + // window but no tonight, and holding on it would hold every such day. + tonightPending: (nextSpan == null && _passUnfinished.contains(nextDayId)) || + debugHoldTonight.contains(dayId), ownership: ownership, priority: priority, ); @@ -5612,6 +5617,12 @@ class DerivationEngine { /// The days this pass's pool has not finished yet ([_runDayPool]). final Set _passUnfinished = {}; + /// Test seam: days prepared as [PreparedDerivationDay.tonightPending] + /// whatever the next day's state, so a hold can be exercised without + /// racing two workers. Empty in production. + @visibleForTesting + static Set debugHoldTonight = {}; + /// [runWithConcurrency] over [days], tracking which are still unfinished so /// a day can tell its next day is mid-derive ([_prepareTargetDay]). Future _runDayPool( @@ -6721,7 +6732,9 @@ class DerivationEngine { ], ); } - return secondHalfOk; + // A held day is not settled either: it must derive again next pass to + // read tonight's window, so it may not be recorded as unchanged. + return secondHalfOk && !day.tonightPending; } /// Bank one day's (phone steps, counter ticks) pair for the (family, diff --git a/test/derive_tonight_pending_test.dart b/test/derive_tonight_pending_test.dart new file mode 100644 index 000000000..13fef5c39 --- /dev/null +++ b/test/derive_tonight_pending_test.dart @@ -0,0 +1,77 @@ +// A day old enough to finalize, whose next day has not stored its window yet +// in the same pass, is held: not finalized and not recorded as unchanged, so +// the next pass reads tonight's window and locks the day then. The hold is +// forced through the test seam rather than by racing two workers. + +import 'package:flutter_test/flutter_test.dart'; +import 'package:openstrap_edge/compute/derivation_engine.dart'; +import 'package:openstrap_edge/compute/profile.dart'; +import 'package:openstrap_edge/data/db.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'support/synthetic_day.dart'; + +const _profile = Profile(ageYears: 32, weightKg: 72, heightCm: 176, sex: 'm'); + +// Three days of data: the day before the first is old enough to finalize on +// the first pass. +final _days = [ + for (var i = 0; i < 3; i++) SyntheticDay(DateTime(2026, 7, 13 + i)), +]; +const _held = '2026-07-12'; + +void main() { + setUpAll(() { + sqfliteFfiInit(); + databaseFactory = databaseFactoryFfi; + }); + + tearDown(() async { + DerivationEngine.debugHoldTonight = {}; + await LocalDb.close(); + }); + + test('a held day is neither finalized nor fingerprinted, then settles', + () async { + const name = 'derive_tonight_pending.db'; + SharedPreferences.setMockInitialValues({}); + await LocalDb.close(); + LocalDb.dbName = name; + await databaseFactory + .deleteDatabase(p.join(await databaseFactory.getDatabasesPath(), name)); + var committed = 0; + for (final d in _days) { + for (final (raws, samples) in d.primaryBatches()) { + final keep = [ + for (var j = 0; j < raws.length; j++) + if (raws[j].recTs! > committed) j, + ]; + if (keep.isEmpty) continue; + await LocalDb.commitSyncBatch( + [for (final j in keep) raws[j]], + [for (final j in keep) samples[j]], + deviceFamily: 'gen4', + ); + committed = raws[keep.last].recTs!; + } + } + + // Two held passes: the second derives from unchanged inputs, which is + // where a fingerprint would be recorded and the day skipped for good. + DerivationEngine.debugHoldTonight = {_held}; + for (var i = 0; i < 2; i++) { + await DerivationEngine().run(_profile, heavy: true); + expect(await LocalDb.dayResult(_held), isNotNull); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), isNot(contains(_held)), + reason: 'held pass $i'); + expect(await LocalDb.deriveFingerprint(_held), isNull, + reason: 'held pass $i'); + } + + DerivationEngine.debugHoldTonight = {}; + await DerivationEngine().run(_profile, heavy: true); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), contains(_held)); + }, timeout: const Timeout(Duration(minutes: 5))); +} From cf7caac20105a26ab95db5443947a527816e6c79 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 07:06:26 +0530 Subject: [PATCH 7/8] keep the tonight hold when a failed second half carries a row forward the recovery path for a failed second half restored the previous row's finalized flag, so a held day that had been finalized before stayed locked anyway. imports still force-finalize. --- lib/compute/derivation_engine.dart | 5 +- test/derive_tonight_pending_test.dart | 109 +++++++++++++++++--------- 2 files changed, 77 insertions(+), 37 deletions(-) diff --git a/lib/compute/derivation_engine.dart b/lib/compute/derivation_engine.dart index 888c59a5f..89b359a57 100644 --- a/lib/compute/derivation_engine.dart +++ b/lib/compute/derivation_engine.dart @@ -6493,7 +6493,10 @@ class DerivationEngine { finalizedByAge: finalized, ); effectivePartial = outcome.partial; - effectiveFinalized = outcome.finalized; + // The tonight hold applies here too: carrying a finalized row + // forward would otherwise lock a day the hold kept open. + effectiveFinalized = + outcome.finalized && (forceFinalize || !day.tonightPending); _log('derive ${day.date}: second half failed — carried the previous ' "result's detail blocks forward (v$prevVersion -> v$kAlgoVersion, " 'partial=$effectivePartial)'); diff --git a/test/derive_tonight_pending_test.dart b/test/derive_tonight_pending_test.dart index 13fef5c39..a3922b492 100644 --- a/test/derive_tonight_pending_test.dart +++ b/test/derive_tonight_pending_test.dart @@ -22,6 +22,31 @@ final _days = [ ]; const _held = '2026-07-12'; +Future _seed(String name) async { + SharedPreferences.setMockInitialValues({}); + await LocalDb.close(); + LocalDb.dbName = name; + await databaseFactory.deleteDatabase( + p.join(await databaseFactory.getDatabasesPath(), name), + ); + var committed = 0; + for (final d in _days) { + for (final (raws, samples) in d.primaryBatches()) { + final keep = [ + for (var j = 0; j < raws.length; j++) + if (raws[j].recTs! > committed) j, + ]; + if (keep.isEmpty) continue; + await LocalDb.commitSyncBatch( + [for (final j in keep) raws[j]], + [for (final j in keep) samples[j]], + deviceFamily: 'gen4', + ); + committed = raws[keep.last].recTs!; + } + } +} + void main() { setUpAll(() { sqfliteFfiInit(); @@ -30,48 +55,60 @@ void main() { tearDown(() async { DerivationEngine.debugHoldTonight = {}; + DerivationEngine.debugSecondHalfHook = null; await LocalDb.close(); }); - test('a held day is neither finalized nor fingerprinted, then settles', - () async { - const name = 'derive_tonight_pending.db'; - SharedPreferences.setMockInitialValues({}); - await LocalDb.close(); - LocalDb.dbName = name; - await databaseFactory - .deleteDatabase(p.join(await databaseFactory.getDatabasesPath(), name)); - var committed = 0; - for (final d in _days) { - for (final (raws, samples) in d.primaryBatches()) { - final keep = [ - for (var j = 0; j < raws.length; j++) - if (raws[j].recTs! > committed) j, - ]; - if (keep.isEmpty) continue; - await LocalDb.commitSyncBatch( - [for (final j in keep) raws[j]], - [for (final j in keep) samples[j]], - deviceFamily: 'gen4', + test( + 'a held day is neither finalized nor fingerprinted, then settles', + () async { + await _seed('derive_tonight_pending.db'); + + // Two held passes: the second derives from unchanged inputs, which is + // where a fingerprint would be recorded and the day skipped for good. + DerivationEngine.debugHoldTonight = {_held}; + for (var i = 0; i < 2; i++) { + await DerivationEngine().run(_profile, heavy: true); + expect(await LocalDb.dayResult(_held), isNotNull); + expect( + await LocalDb.finalizedDayIds(kAlgoVersion), + isNot(contains(_held)), + reason: 'held pass $i', + ); + expect( + await LocalDb.deriveFingerprint(_held), + isNull, + reason: 'held pass $i', ); - committed = raws[keep.last].recTs!; } - } - // Two held passes: the second derives from unchanged inputs, which is - // where a fingerprint would be recorded and the day skipped for good. - DerivationEngine.debugHoldTonight = {_held}; - for (var i = 0; i < 2; i++) { + DerivationEngine.debugHoldTonight = {}; await DerivationEngine().run(_profile, heavy: true); - expect(await LocalDb.dayResult(_held), isNotNull); - expect(await LocalDb.finalizedDayIds(kAlgoVersion), isNot(contains(_held)), - reason: 'held pass $i'); - expect(await LocalDb.deriveFingerprint(_held), isNull, - reason: 'held pass $i'); - } + expect(await LocalDb.finalizedDayIds(kAlgoVersion), contains(_held)); + }, + timeout: const Timeout(Duration(minutes: 5)), + ); - DerivationEngine.debugHoldTonight = {}; - await DerivationEngine().run(_profile, heavy: true); - expect(await LocalDb.finalizedDayIds(kAlgoVersion), contains(_held)); - }, timeout: const Timeout(Duration(minutes: 5))); + test( + 'a failed second half does not carry a finalized row past the hold', + () async { + await _seed('derive_tonight_pending_recovery.db'); + await DerivationEngine().run(_profile, heavy: true); + expect(await LocalDb.finalizedDayIds(kAlgoVersion), contains(_held)); + + // Re-derived (forced) while held, with its second half failing: the + // recovery path carries the complete finalized row forward, and must not + // carry the lock with it. + DerivationEngine.debugHoldTonight = {_held}; + DerivationEngine.debugSecondHalfHook = (day) async { + if (day == _held) throw StateError('second half down'); + }; + await DerivationEngine().runDays(_profile, {_held}, force: true); + expect( + await LocalDb.finalizedDayIds(kAlgoVersion), + isNot(contains(_held)), + ); + }, + timeout: const Timeout(Duration(minutes: 5)), + ); } From 9ddd3cce6f29427625f6cc7f578e490111ebdc02 Mon Sep 17 00:00:00 2001 From: abdulsaheel Date: Sun, 1 Nov 2026 11:54:50 +0530 Subject: [PATCH 8/8] start the recovery test from a finalized day on every run a cold first pass can hold the day when the next day's window lands after it, which is the case the hold exists for, so the test now takes a second pass before it checks the finalized row. --- test/derive_tonight_pending_test.dart | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/derive_tonight_pending_test.dart b/test/derive_tonight_pending_test.dart index a3922b492..59b1ed092 100644 --- a/test/derive_tonight_pending_test.dart +++ b/test/derive_tonight_pending_test.dart @@ -93,6 +93,10 @@ void main() { 'a failed second half does not carry a finalized row past the hold', () async { await _seed('derive_tonight_pending_recovery.db'); + // A cold first pass may legitimately hold the day (the next day's + // window can land after it in the same pass), so a second pass is what + // guarantees the finalized row this test starts from. + await DerivationEngine().run(_profile, heavy: true); await DerivationEngine().run(_profile, heavy: true); expect(await LocalDb.finalizedDayIds(kAlgoVersion), contains(_held));