diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift index 99430ddda3..1bb7afc133 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/AnalyticsEngine.swift @@ -506,11 +506,19 @@ public enum AnalyticsEngine { } else { let rrSorted = rr.sortedByTsStable() let enrichedProvided: [SleepSession] = providedSleep.map { s in + // #1801: an HR-only night keeps its NIL restingHR/avgHRV. This is the ONE call site that + // can undo the display-only guarantee — enriching here would hand Charge and the baselines + // exactly the two values that decision withholds, silently and by default. + if s.hrOnly { return s } guard s.restingHR == nil || s.avgHRV == nil else { return s } let rhr = s.restingHR ?? SleepStager.sessionRestingHR(start: s.start, end: s.end, hr: hr) let hrv = s.avgHRV ?? SleepStager.sessionAvgHRV(start: s.start, end: s.end, rr: rrSorted) + // `hrOnly` carried explicitly: unlike Kotlin's `copy`, this rebuilds the struct field by + // field, so a new flag is dropped by DEFAULT unless named here. The guard above means an + // HR-only night never reaches this line today; passing it anyway keeps that a belt rather + // than the only thing standing between the flag and silent loss. return SleepSession(start: s.start, end: s.end, efficiency: s.efficiency, - stages: s.stages, restingHR: rhr, avgHRV: hrv) + stages: s.stages, restingHR: rhr, avgHRV: hrv, hrOnly: s.hrOnly) } let keptDetected = refinedSessions.filter { d in !enrichedProvided.contains { $0.start < d.end && d.start < $0.end } @@ -665,7 +673,18 @@ public enum AnalyticsEngine { // negligible shift. The Rest/sleep-quality term is main-night; the recovery physiology is // day-best-resting, night-dominated. Keep these two definitions distinct on purpose. // Daily resting HR = lowest per-session resting HR across matched sessions. - let restingHRDaily = matched.compactMap { $0.restingHR }.min() + // #1801: the sessions whose PHYSIOLOGY may be folded into the day's aggregates. An HR-only night + // is excluded — it may describe its own duration, stages and Rest, but resting HR, HRV and SDNN + // are what Charge and the baselines are built from, and a baseline is the one thing a false + // positive cannot be unwound from. + // + // Named once rather than filtered at each use, because the nil restingHR/avgHRV an HR-only + // session carries only protects the aggregates that READ those fields. The deep-window HRV pool + // and the SDNN index below re-derive from `rr` over the session's own stages and would have + // folded one in regardless — which is precisely the "one forgotten call site" a scattered filter + // invites. + let physiologySessions = matched.filter { !$0.hrOnly } + let restingHRDaily = physiologySessions.compactMap { $0.restingHR }.min() // Daily avg HRV = in-bed-weighted mean of per-session avg HRV. let avgHRVDaily: Double? = { if deepHrvWindow { @@ -675,13 +694,13 @@ public enum AnalyticsEngine { // = successive diffs). nil when no deep sleep is detected (WHOOP-4.0 staging can be sparse) — // the caller shows calibrating, never a fabricated number. let rrSorted = rr.sortedByTsStable() - let deep = matched.flatMap { s in + let deep = physiologySessions.flatMap { s in SleepStager.sessionHrvWindows(start: s.start, end: s.end, rr: rrSorted, stages: s.stages) .filter { $0.stage == "deep" }.compactMap { $0.rmssd } } return deep.isEmpty ? nil : deep.reduce(0, +) / Double(deep.count) } - let pairs = matched.compactMap { s -> (Double, Double)? in + let pairs = physiologySessions.compactMap { s -> (Double, Double)? in s.avgHRV.map { ($0, Double(s.end - s.start)) } } guard !pairs.isEmpty else { return nil } @@ -698,7 +717,7 @@ public enum AnalyticsEngine { // against a watch meaningless. The 5-min index is window-comparable to those. nil when no segment has // enough clean beats (HRVAnalyzer's own gate). Keeps the R-R timestamps (segmentation needs them). let avgSDNNDaily: Double? = { - let inBed = rr.filter { r in matched.contains { r.ts >= $0.start && r.ts < $0.end } } + let inBed = rr.filter { r in physiologySessions.contains { r.ts >= $0.start && r.ts < $0.end } } return inBed.isEmpty ? nil : HRVAnalyzer.sdnnIndex(inBed, segmentSec: 300) }() diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift index 8a5b5b237a..6432437438 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/SleepStager.swift @@ -50,11 +50,22 @@ public struct SleepSession: Equatable, Sendable { public let restingHR: Int? /// Mean RMSSD over 5-min windows across the session (ms), or nil. public let avgHRV: Double? + /// Staged WITHOUT a motion spine, from heart rate alone (#1801). + /// + /// True only for a strap that streams HR but banks no motion, where Stage 0's gravity-stillness + /// spine has nothing to work with. Such a night is weaker by construction, not by tuning: with + /// motion gone a quiet evening at rest can sit in the sleep band. It is allowed to describe itself + /// — duration, stages, Rest — and must NOT reach anything it cannot be unwound from, which is why + /// `restingHR` and `avgHRV` are left nil on one rather than filtered out downstream. + /// + /// Kotlin twin: `DetectedSleep.hrOnly` (the model names diverge, `DetectedSleep`/`SleepSession`). + public let hrOnly: Bool public init(start: Int, end: Int, efficiency: Double, stages: [StageSegment], - restingHR: Int?, avgHRV: Double?) { + restingHR: Int?, avgHRV: Double?, hrOnly: Bool = false) { self.start = start; self.end = end; self.efficiency = efficiency self.stages = stages; self.restingHR = restingHR; self.avgHRV = avgHRV + self.hrOnly = hrOnly } } @@ -472,6 +483,43 @@ public enum SleepStager { return periods } + /// Percentile of the window's bpm that anchors the HR-only sleep band. + /// + /// NOT `hrBaseline`. That is the window MEDIAN, and it is the right anchor where it is used — as a + /// CONFIRMATION gate on a run gravity stillness already found, where being permissive is deliberate. + /// As a PRIMARY threshold it is disqualified by arithmetic rather than by tuning: a median splits the + /// samples in half by definition, so a band of `median * 1.05` admits strictly more than half of any + /// window whatever the data. Measured on a realistic 24 h (16 h awake 74-96, 8 h night 59-70) it + /// called 14.4 hours sleep against a truth of 8. + /// + /// A tenth percentile sits in the night's trough instead, which is what a sleep band should be + /// anchored to, and cannot admit half the window however the day is shaped. + public static let hrOnlyAnchorPercentile: Double = 0.10 + + /// Multiplier above `hrOnlyAnchorPercentile` that still counts as asleep. + /// + /// Numerically equal to `hrSleepBandMult` today, and deliberately a SEPARATE constant: that one is a + /// confirmation gate's tolerance and this one is a detector's, and a future change to either has no + /// business silently moving the other. + /// + /// Chosen conservatively from a sweep over anchor x multiplier against windows with known truth, + /// because the two failure directions are not symmetric. Over-detection puts a wrong Rest number on + /// screen; under-detection leaves "No data", which is the state this feature is trying to improve on + /// and therefore a safe place to fail. p10 x 1.05 measured 8.1 h against a truth of 8 on a + /// field-shaped 24 h window, and under-reads a long multi-night window rather than over-reading it + /// (8.7 h of 16 h across two nights in 54 h) — pinned in `SleepStagerHrOnlyAnchorTests`. + public static let hrOnlyBandMult: Double = 1.05 + + /// The `hrOnlyAnchorPercentile` of `hr` by bpm, or nil when empty. Nearest-rank (no interpolation), so + /// the value is always one the wearer actually recorded and the two platforms cannot disagree on a + /// rounding rule. + static func hrOnlyBaseline(_ hr: [HRSample]) -> Double? { + if hr.isEmpty { return nil } + let sorted = hr.map { Double($0.bpm) }.sorted() + let idx = min(max(Int(Double(sorted.count - 1) * hrOnlyAnchorPercentile), 0), sorted.count - 1) + return sorted[idx] + } + /// Epoch for the HR-only spine, in seconds. public static let hrOnlyEpochS: Int = 60 @@ -523,7 +571,7 @@ public enum SleepStager { // understatement would then be weighed against the caller's minimum-duration gate. let times = keys.map { $0 * epochS } let ends = keys.map { lastTs[$0]! } - let flags = keys.map { HRVAnalyzer.median(byEpoch[$0]!) <= baseline * hrSleepBandMult } + let flags = keys.map { HRVAnalyzer.median(byEpoch[$0]!) <= baseline * hrOnlyBandMult } let maxGapS = maxGapMinutes * 60 var periods: [Period] = [] var runStart = 0 @@ -544,6 +592,52 @@ public enum SleepStager { return periods } + /// Whole sleep SESSIONS from heart rate alone, for a strap that banks no motion (#1801). + /// + /// `hrOnlySleepRuns` supplies the spine this normally gets from gravity stillness; `SleepStagerV2` + /// then stages each surviving run from HR and R-R with an EMPTY gravity array. That is not a + /// degenerate call: V2's epoch features read HR and R-R directly and only its motion-quiescence terms + /// go quiet, so it returns a real hypnogram rather than one flat stage. A stageless session would be + /// dropped by `sleepSessionFromProvided` anyway, so a night that fails to stage is correctly omitted + /// here rather than passed on hollow. + /// + /// The anchor is `hrOnlyBaseline` — the `hrOnlyAnchorPercentile` of the window — and NOT the + /// `hrBaseline` median the motion path derives. Reusing that median looked like parity and was a bug: + /// as a confirmation gate on an already-detected run it is deliberately permissive, but as a primary + /// threshold it admits over half of any window by definition. See `hrOnlyAnchorPercentile`. + /// + /// `restingHR` and `avgHRV` are left NIL deliberately, and that is the whole display-only guarantee. + /// An HR-only night may describe itself — duration, stages, Rest — but the resting HR and HRV it + /// would contribute are exactly what Charge and the baselines fold in, and a baseline is the one + /// thing a false positive cannot be unwound from. Withholding the values is structural; a downstream + /// filter would be one forgotten call site away from failing open. + /// `public` because the app target calls it: `Strand/Data/IntelligenceEngine.swift` is the day scan, + /// and it lives outside this package. The spine and the anchor below it stay `internal` — the tests + /// reach them with `@testable`, and nothing outside should be building its own spine. + public static func hrOnlySessions(hr: [HRSample], rr: [RRInterval], resp: [RespSample], + minMinutes: Int = minSleepMin) -> [SleepSession] { + let hrS = hr.sorted { $0.ts < $1.ts } + guard let baseline = hrOnlyBaseline(hrS) else { return [] } + let rrS = rr.sorted { $0.ts < $1.ts } + var out: [SleepSession] = [] + // mergePeriods for the same reason the motion path calls it: a run boundary is a threshold + // crossing, and a sleeping heart rate oscillates across the band all night. Without this the + // spine returns the night's minutes correctly but shredded into sub-mergeMin fragments, every one + // of which then fails the minimum-duration gate below — 8 h of detected sleep yielding zero + // sessions. Absorbing the short runs first is what turns a spine into a night. + for p in mergePeriods(hrOnlySleepRuns(hrS, baseline: baseline)) { + if p.stage != "sleep" { continue } + if (p.end - p.start) < minMinutes * 60 { continue } + let stages = SleepStagerV2.stageSession(start: p.start, end: p.end, grav: [], + hr: hrS, rr: rrS, resp: resp) + if stages.isEmpty { continue } + out.append(SleepSession(start: p.start, end: p.end, + efficiency: efficiency(start: p.start, end: p.end, stages: stages), + stages: stages, restingHR: nil, avgHRV: nil, hrOnly: true)) + } + return out + } + /// Absorb runs shorter than mergeMin minutes into their neighbours. static func mergePeriods(_ periods: [Period], mergeMinutes: Int = mergeMin) -> [Period] { if periods.isEmpty { return [] } diff --git a/Packages/StrandAnalytics/Sources/StrandAnalytics/WakeMotionRefinement.swift b/Packages/StrandAnalytics/Sources/StrandAnalytics/WakeMotionRefinement.swift index d8703124b0..26c8bcd0c1 100644 --- a/Packages/StrandAnalytics/Sources/StrandAnalytics/WakeMotionRefinement.swift +++ b/Packages/StrandAnalytics/Sources/StrandAnalytics/WakeMotionRefinement.swift @@ -135,8 +135,14 @@ public enum WakeMotionRefinement { let newStages = refine(session.stages, grav: grav, steps: steps) guard newStages != session.stages else { return session } let newEfficiency = SleepStager.efficiency(start: session.start, end: session.end, stages: newStages) + // `hrOnly` carried explicitly. The Kotlin twin is a `copy`, which preserves a new field for + // free; this rebuilds the struct field by field and drops one BY DEFAULT. Refinement only ever + // sees motion-detected sessions today, so an HR-only night never reaches here — but the two + // platforms would have disagreed silently the moment it did, which is the shape of divergence + // the parity contract exists to catch. return SleepSession(start: session.start, end: session.end, efficiency: newEfficiency, - stages: newStages, restingHR: session.restingHR, avgHRV: session.avgHRV) + stages: newStages, restingHR: session.restingHR, avgHRV: session.avgHRV, + hrOnly: session.hrOnly) } /// Toggle-shaped convenience for the session-level overload (see `apply(_:grav:steps:enabled:)`). diff --git a/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerHrOnlyAnchorTests.swift b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerHrOnlyAnchorTests.swift new file mode 100644 index 0000000000..68731afed1 --- /dev/null +++ b/Packages/StrandAnalytics/Tests/StrandAnalyticsTests/SleepStagerHrOnlyAnchorTests.swift @@ -0,0 +1,53 @@ +import XCTest +import WhoopProtocol +@testable import StrandAnalytics + +/// Twin of the Kotlin `SleepStagerHrOnlyAnchorTest`: what anchors the HR-only band, and why it is not +/// the median (#1801). +final class SleepStagerHrOnlyAnchorTests: XCTestCase { + + private func window(_ aC: Int, _ aA: Int, _ aH: Int, _ nC: Int, _ nA: Int, _ nH: Int) -> [HRSample] { + let t0 = 1_788_300_000 + var hr: [HRSample] = [] + for i in 0..<(aH * 3600) { hr.append(HRSample(ts: t0 + i, bpm: aC + Int(sin(Double(i) / 500.0) * Double(aA)))) } + for j in 0..<(nH * 3600) { + hr.append(HRSample(ts: t0 + aH * 3600 + j, bpm: nC + Int(sin(Double(j) / 900.0) * Double(nA)))) + } + return hr + } + + private func sleepHours(_ hr: [HRSample], _ baseline: Double) -> Double { + Double(SleepStager.hrOnlySleepRuns(hr, baseline: baseline) + .filter { $0.stage == "sleep" } + .reduce(0) { $0 + ($1.end - $1.start) }) / 3600.0 + } + + func testAnchorIsTenthPercentileNearestRank() { + let hr = (1...100).map { HRSample(ts: 1_788_300_000 + $0, bpm: $0) } + XCTAssertEqual(SleepStager.hrOnlyAnchorPercentile, 0.10, accuracy: 1e-9) + XCTAssertEqual(SleepStager.hrOnlyBaseline(hr)!, 10.0, accuracy: 1e-9) + } + + /// The regression this file exists for: the median anchor admits over half of any window by + /// definition, and measured 14.4 h against a truth of 8 on a field-shaped day. + func testMedianOverDetectsAndPercentileDoesNot() { + let hr = window(74, 11, 16, 64, 5, 8) + let median = sleepHours(hr, SleepStager.hrBaseline(hr)!) + let percentile = sleepHours(hr, SleepStager.hrOnlyBaseline(hr)!) + XCTAssertGreaterThan(median, 13.0) + XCTAssertTrue((7.0...9.5).contains(percentile), "was \(percentile)") + XCTAssertLessThan(percentile, median - 4.0) + } + + /// Errs toward UNDER-detection: a missed night leaves "No data", an invented one shows a wrong Rest. + func testLongMultiNightWindowIsUnderReadNotOverRead() { + let h = sleepHours(window(78, 12, 38, 60, 6, 16), SleepStager.hrOnlyBaseline(window(78, 12, 38, 60, 6, 16))!) + XCTAssertLessThanOrEqual(h, 16.0, "was \(h)") + XCTAssertGreaterThanOrEqual(h, 6.0, "was \(h)") + } + + func testDetectorBandIsASeparateConstantFromTheConfirmationGate() { + XCTAssertEqual(SleepStager.hrOnlyBandMult, 1.05, accuracy: 1e-9) + XCTAssertEqual(SleepStager.hrSleepBandMult, 1.05, accuracy: 1e-9) + } +} diff --git a/Strand/Data/IntelligenceEngine.swift b/Strand/Data/IntelligenceEngine.swift index 97a7893cd7..b7dcbbc8f5 100644 --- a/Strand/Data/IntelligenceEngine.swift +++ b/Strand/Data/IntelligenceEngine.swift @@ -940,8 +940,10 @@ final class IntelligenceEngine: ObservableObject { // since we last scored it THIS session, skipping the 7 stream reads + `analyzeDay`. Gated to a // registered WHOOP owner (4.0 or 5/MG) via the UN-coalesced `forRegistryDevice` — which // returns nil for a ring/import/unknown, so the cache never treats one as a WHOOP (a ring's - // `providedSleep` could change a day without an HR move; a WHOOP always streams gravity, so - // its `providedSleep` is empty and the reuse is byte-identical). The per-day key folds the + // `providedSleep` could change a day without an HR move; a WHOOP normally streams gravity, so + // its `providedSleep` is empty and the reuse is byte-identical — and since #1801 a WHOOP with + // NO gravity gets an HR-only `providedSleep` instead, which is derived from the very HR the + // key already fingerprints, so the reuse stays sound by a different route). The per-day key folds the // night's HR fingerprint (a narrower per-day witness than the whole-pass raw-input gate) and, for a // 4.0, the window-wide skin anchor (a re-anchor from another night shifts that night's skin // conversion without moving its HR). A 5/MG banks skin-temp centidegrees directly — no @@ -1169,7 +1171,16 @@ final class IntelligenceEngine: ObservableObject { if owner != Repository.whoopSource, grav.count < 2 { let persisted = (try? await store.sleepSessions(deviceId: owner, from: from, to: to, limit: 4000)) ?? [] - providedSleep = persisted.compactMap { AnalyticsEngine.sleepSession(fromProvided: $0) } + let stored = persisted.compactMap { AnalyticsEngine.sleepSession(fromProvided: $0) } + // #1801: the comment above assumes a WHOOP always streams a gravity vector. A 5/MG that + // cannot bond does not — it is never clocked, so it banks nothing and persists no + // hypnogram either, leaving neither a spine nor a stored night however much HR it holds. + // Fall back to the HR-only spine ONLY when the device supplied nothing of its own: a + // hypnogram the device actually recorded is always better evidence than one inferred + // from heart rate. + providedSleep = stored.isEmpty + ? SleepStager.hrOnlySessions(hr: hr, rr: rr, resp: resp) + : stored } else { providedSleep = [] } diff --git a/android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt b/android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt index 22410d68a4..9082479b7a 100644 --- a/android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt +++ b/android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt @@ -443,7 +443,10 @@ object AnalyticsEngine { } else { val rrSorted = rr.sortedBy { it.ts } val enrichedProvided = providedSleep.map { s -> - if (s.restingHR != null && s.avgHRV != null) s + // #1801: an HR-only night keeps its NULL restingHR/avgHRV. This is the ONE call site that + // can undo the display-only guarantee — enriching here would hand Charge and the baselines + // exactly the two values that decision withholds, silently and by default. + if (s.hrOnly || (s.restingHR != null && s.avgHRV != null)) s else s.copy( restingHR = s.restingHR ?: SleepStager.sessionRestingHR(s.start, s.end, hr), avgHRV = s.avgHRV ?: SleepStager.sessionAvgHRV(s.start, s.end, rrSorted), @@ -547,7 +550,18 @@ object AnalyticsEngine { // negligible shift. The Rest/sleep-quality term is main-night; the recovery physiology is // day-best-resting, night-dominated. Mirrors the Swift note in AnalyticsEngine.swift. // Daily resting HR = lowest per-session resting HR across matched sessions. - val restingHRDaily: Int? = matched.mapNotNull { it.restingHR }.minOrNull() + // #1801: the sessions whose PHYSIOLOGY may be folded into the day's aggregates. An HR-only night + // is excluded — it may describe its own duration, stages and Rest, but resting HR, HRV and SDNN + // are what Charge and the baselines are built from, and a baseline is the one thing a false + // positive cannot be unwound from. + // + // Named once rather than filtered at each use, because the null restingHR/avgHRV an HR-only + // session carries only protects the aggregates that READ those fields. The deep-window HRV pool + // and the SDNN index below re-derive from `rr` over the session's own stages and would have + // folded one in regardless — which is precisely the "one forgotten call site" a scattered filter + // invites. + val physiologySessions = matched.filter { !it.hrOnly } + val restingHRDaily: Int? = physiologySessions.mapNotNull { it.restingHR }.minOrNull() // Daily avg HRV = in-bed-weighted mean of per-session avg HRV. val avgHRVDaily: Double? = if (deepHrvWindow) { // #141: WHOOP-style HRV — pool RMSSD over DEEP-stage 5-min windows only (slow-wave sleep), @@ -556,13 +570,13 @@ object AnalyticsEngine { // (RMSSD = successive diffs). null when the night has no detected deep sleep (WHOOP-4.0 staging // can be sparse) — the caller then shows calibrating, never a fabricated number. val rrSorted = rr.sortedBy { it.ts } - val deep = matched.flatMap { s -> + val deep = physiologySessions.flatMap { s -> SleepStager.sessionHrvWindows(s.start, s.end, rrSorted, s.stages) .filter { it.stage == "deep" }.mapNotNull { it.rmssd } } if (deep.isEmpty()) null else deep.sum() / deep.size } else run { - val pairs = matched.mapNotNull { s -> + val pairs = physiologySessions.mapNotNull { s -> s.avgHRV?.let { it to (s.end - s.start).toDouble() } } if (pairs.isEmpty()) { @@ -578,7 +592,7 @@ object AnalyticsEngine { // timestamps needed for segmentation and stays distinct from avgHrv (RMSSD). The half-open sleep // bounds match every other in-bed aggregate; no qualifying 20-clean-beat segment means null. val avgSDNNDaily = HrvAnalyzer.sdnnIndex( - rr.filter { sample -> matched.any { sample.ts >= it.start && sample.ts < it.end } }, + rr.filter { sample -> physiologySessions.any { sample.ts >= it.start && sample.ts < it.end } }, segmentSec = 300, ) diff --git a/android/app/src/main/java/com/noop/analytics/AnalyticsModels.kt b/android/app/src/main/java/com/noop/analytics/AnalyticsModels.kt index 04c4587414..f3d7fa5927 100644 --- a/android/app/src/main/java/com/noop/analytics/AnalyticsModels.kt +++ b/android/app/src/main/java/com/noop/analytics/AnalyticsModels.kt @@ -87,6 +87,16 @@ data class DetectedSleep( val restingHR: Int?, /** Mean RMSSD over 5-min windows across the session (ms), or null. */ val avgHRV: Double?, + /** + * Staged WITHOUT a motion spine, from heart rate alone (#1801). + * + * True only for a strap that streams HR but banks no motion, where Stage 0's gravity-stillness spine + * has nothing to work with. Such a night is weaker by construction, not by tuning: with motion gone a + * quiet evening at rest can sit in the sleep band. It is allowed to describe itself — duration, + * stages, Rest — and must NOT reach anything it cannot be unwound from, which is why + * [restingHR] and [avgHRV] are left null on one rather than filtered out downstream. + */ + val hrOnly: Boolean = false, ) /** diff --git a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt index 46c337640e..fc091dcc9b 100644 --- a/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt +++ b/android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt @@ -958,8 +958,15 @@ object IntelligenceEngine { // import namespace are untouched; analyzeDay still lets a DETECTED session win where they overlap. val providedSleep: List = if (owner != importedDeviceId && grav.size < 2) { - repo.sleepSessionsForDevice(owner, from, to, 4000) + val stored = repo.sleepSessionsForDevice(owner, from, to, 4000) .mapNotNull { AnalyticsEngine.sleepSessionFromProvided(it) } + // #1801: a ring persists its OWN hypnogram, so `stored` covers it. A 5/MG that cannot + // bond persists nothing (it is never clocked, so it banks nothing) and streams no motion + // either — neither a spine nor a stored night, which is why this day scored blank with + // `reason=no-motion` however much HR it held. Fall back to the HR-only spine ONLY when + // the device supplied nothing of its own: a hypnogram the device actually recorded is + // always better evidence than one inferred from heart rate. + if (stored.isNotEmpty()) stored else SleepStager.hrOnlySessions(hr, rr, resp) } else { emptyList() } @@ -2905,10 +2912,17 @@ object IntelligenceEngine { stepCount: Int, providedCount: Int, windowHours: Int, ): String { // `reason` names WHICH absence this is, because grav=0 is printed but its consequence is not. - // With no motion the stager has no HR-only fallback, so no quantity of HR can stage a night — a - // strap capability limit, not a coverage gap, and the two want completely different follow-ups. - // With motion present the inputs were there and staging still produced nothing, which is the case - // actually worth investigating. + // + // `no-motion` USED to mean "and therefore nothing further was attempted" — the stager had no + // HR-only fallback, so no quantity of HR could stage a night. Since #1801 it does: a day with no + // gravity now also runs [SleepStager.hrOnlySessions], so this line printing `no-motion` means the + // motion spine was absent AND heart rate alone did not yield a night either — too little of it in + // the sleep band, or a run that staged to nothing. That is a stronger statement than it used to + // be, and the follow-up it wants is different: no longer "this strap cannot", but "why did the + // HR-only spine find nothing here". + // + // With motion present the inputs were there and staging still produced nothing, which remains the + // case most worth investigating. val reason = if (gravCount == 0) "no-motion" else "staged-none" return "sleep-detect day=$day NO-NIGHT hr=$hrCount rr=$rrCount resp=$respCount " + "grav=$gravCount steps=$stepCount provided=$providedCount window=${windowHours}h " + diff --git a/android/app/src/main/java/com/noop/analytics/SleepStager.kt b/android/app/src/main/java/com/noop/analytics/SleepStager.kt index c2c9ff5efe..4c78641d16 100644 --- a/android/app/src/main/java/com/noop/analytics/SleepStager.kt +++ b/android/app/src/main/java/com/noop/analytics/SleepStager.kt @@ -524,6 +524,49 @@ object SleepStager { return medianHR <= baseline * hrSleepBandMult } + /** + * Percentile of the window's bpm that anchors the HR-only sleep band. + * + * NOT [hrBaseline]. That is the window MEDIAN, and it is the right anchor where it is used — as a + * CONFIRMATION gate on a run gravity stillness already found, where being permissive is deliberate. + * As a PRIMARY threshold it is disqualified by arithmetic rather than by tuning: a median splits the + * samples in half by definition, so a band of `median * 1.05` admits strictly more than half of any + * window whatever the data. Measured on a realistic 24 h (16 h awake 74-96, 8 h night 59-70) it + * called 14.4 hours sleep against a truth of 8. + * + * A tenth percentile sits in the night's trough instead, which is what a sleep band should be + * anchored to, and cannot admit half the window however the day is shaped. + */ + const val hrOnlyAnchorPercentile: Double = 0.10 + + /** + * Multiplier above [hrOnlyAnchorPercentile] that still counts as asleep. + * + * Numerically equal to [hrSleepBandMult] today, and deliberately a SEPARATE constant: that one is a + * confirmation gate's tolerance and this one is a detector's, and a future change to either has no + * business silently moving the other. + * + * Chosen conservatively from a sweep over anchor x multiplier against windows with known truth, + * because the two failure directions are not symmetric. Over-detection puts a wrong Rest number on + * screen; under-detection leaves "No data", which is the state this feature is trying to improve on + * and therefore a safe place to fail. p10 x 1.05 measured 8.1 h against a truth of 8 on a field-shaped + * 24 h window, and under-reads a long multi-night window rather than over-reading it (8.7 h of 16 h + * across two nights in 54 h) — pinned in `SleepStagerHrOnlyAnchorTest`. + */ + const val hrOnlyBandMult: Double = 1.05 + + /** + * The [hrOnlyAnchorPercentile] of `hr` by bpm, or null when empty. Nearest-rank (no interpolation), so + * the value is always one the wearer actually recorded and the two platforms cannot disagree on a + * rounding rule. + */ + internal fun hrOnlyBaseline(hr: List): Double? { + if (hr.isEmpty()) return null + val sorted = hr.map { it.bpm.toDouble() }.sorted() + val idx = ((sorted.size - 1) * hrOnlyAnchorPercentile).toInt().coerceIn(0, sorted.size - 1) + return sorted[idx] + } + /** Epoch for the HR-only spine, in seconds. */ const val hrOnlyEpochS: Long = 60 @@ -580,7 +623,7 @@ object SleepStager { // understatement would then be weighed against the caller's minimum-duration gate. val times = keys.map { it * epochS } val ends = keys.map { lastTs.getValue(it) } - val flags = keys.map { HrvAnalyzer.median(byEpoch.getValue(it)) <= baseline * hrSleepBandMult } + val flags = keys.map { HrvAnalyzer.median(byEpoch.getValue(it)) <= baseline * hrOnlyBandMult } val maxGapS = (maxGapMinutes * 60).toLong() val periods = ArrayList() var runStart = 0 @@ -605,6 +648,62 @@ object SleepStager { return periods } + /** + * Whole sleep SESSIONS from heart rate alone, for a strap that banks no motion (#1801). + * + * [hrOnlySleepRuns] supplies the spine this normally gets from gravity stillness; [SleepStagerV2] + * then stages each surviving run from HR and R-R with an EMPTY gravity list. That is not a + * degenerate call: V2's epoch features read HR and R-R directly and only its motion-quiescence terms + * go quiet, so it returns a real hypnogram rather than one flat stage. A stageless session would be + * dropped by `AnalyticsEngine.sleepSessionFromProvided` anyway, so a night that fails to stage is + * correctly omitted here rather than passed on hollow. + * + * The anchor is [hrOnlyBaseline] — the [hrOnlyAnchorPercentile] of the window — and NOT the + * [hrBaseline] median the motion path derives. Reusing that median looked like parity and was a bug: + * as a confirmation gate on an already-detected run it is deliberately permissive, but as a primary + * threshold it admits over half of any window by definition. See [hrOnlyAnchorPercentile]. + * + * [DetectedSleep.restingHR] and [DetectedSleep.avgHRV] are left NULL deliberately, and that is the + * whole display-only guarantee. An HR-only night may describe itself — duration, stages, Rest — but + * the resting HR and HRV it would contribute are exactly what Charge and the baselines fold in, and + * a baseline is the one thing a false positive cannot be unwound from. Withholding the values is + * structural; a downstream filter would be one forgotten call site away from failing open. + */ + internal fun hrOnlySessions( + hr: List, + rr: List, + resp: List, + minMinutes: Int = minSleepMin, + ): List { + val hrS = hr.sortedBy { it.ts } + val baseline = hrOnlyBaseline(hrS) ?: return emptyList() + val rrS = rr.sortedBy { it.ts } + val out = ArrayList() + // mergePeriods for the same reason the motion path calls it: a run boundary is a threshold + // crossing, and a sleeping heart rate oscillates across the band all night. Without this the + // spine returns the night's minutes correctly but shredded into sub-mergeMin fragments, every one + // of which then fails the minimum-duration gate below — 8 h of detected sleep yielding zero + // sessions. Absorbing the short runs first is what turns a spine into a night. + for (p in mergePeriods(hrOnlySleepRuns(hrS, baseline))) { + if (p.stage != "sleep") continue + if ((p.end - p.start) < minMinutes * 60L) continue + val stages = SleepStagerV2.stageSession(p.start, p.end, emptyList(), hrS, rrS, resp) + if (stages.isEmpty()) continue + out.add( + DetectedSleep( + start = p.start, + end = p.end, + efficiency = efficiency(start = p.start, end = p.end, stages = stages), + stages = stages, + restingHR = null, + avgHRV = null, + hrOnly = true, + ) + ) + } + return out + } + /** Per-record sleep flags from a rolling fraction of "still" samples. */ internal fun classifyStill(grav: List, deltas: List): List { val n = grav.size diff --git a/android/app/src/test/java/com/noop/analytics/HrOnlyPhysiologyIsolationTest.kt b/android/app/src/test/java/com/noop/analytics/HrOnlyPhysiologyIsolationTest.kt new file mode 100644 index 0000000000..7acebc9e54 --- /dev/null +++ b/android/app/src/test/java/com/noop/analytics/HrOnlyPhysiologyIsolationTest.kt @@ -0,0 +1,74 @@ +package com.noop.analytics + +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The display-only guarantee for an HR-only night (#1801), guarded at the level it actually fails. + * + * A null `restingHR`/`avgHRV` only protects an aggregate that READS those fields. Two of the day's + * physiological aggregates do not: the deep-window HRV pool and the SDNN index both re-derive from `rr` + * over each session's own stages, so an HR-only night — which has stages — would have been folded into + * both, and from there into Charge and the HRV baseline. `deepHrvWindow` is a user setting, so that path + * is reachable, not theoretical. + * + * Read from the source because `physiologySessions` is a local inside `analyzeDay` and cannot be + * observed directly, and because the failure this guards is a FUTURE aggregate reaching for `matched` + * out of habit. A behavioural test on one path would not catch that; this does. + */ +class HrOnlyPhysiologyIsolationTest { + + private fun analyzeDayBody(): String { + var root = java.io.File(System.getProperty("user.dir") ?: ".").canonicalFile + val src = run { + repeat(4) { + val f = java.io.File(root, "android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt") + if (f.isFile) return@run f.readText() + root = root.parentFile ?: root + } + error("AnalyticsEngine.kt not found — this test must not pass by default") + } + val decl = src.indexOf("val physiologySessions") + assertTrue("analyzeDay must name the physiology-session set", decl > 0) + // Start AFTER the declaration: that line reads `matched` legitimately, since defining the + // filtered set is the one place the unfiltered one belongs. + val from = src.indexOf("\n", decl) + 1 + val to = src.indexOf("val avgSDNNDaily", from) + assertTrue("expected the SDNN index to follow the physiology set", to > from) + // Through the end of the SDNN call, which is the last of the four aggregates. + return src.substring(from, src.indexOf(")", src.indexOf("segmentSec = 300", to)) + 1) + } + + @Test + fun `the physiology set excludes HR-only nights`() { + var root = java.io.File(System.getProperty("user.dir") ?: ".").canonicalFile + val src = run { + repeat(4) { + val f = java.io.File(root, "android/app/src/main/java/com/noop/analytics/AnalyticsEngine.kt") + if (f.isFile) return@run f.readText() + root = root.parentFile ?: root + } + error("AnalyticsEngine.kt not found — this test must not pass by default") + } + assertTrue("physiologySessions must filter out hrOnly", + src.contains("val physiologySessions = matched.filter { !it.hrOnly }")) + } + + /** + * The four aggregates built from a night's physiology must read the filtered set. `matched` still + * has legitimate uses in `analyzeDay` — duration, stage totals, Rest, the onset — which is exactly + * why this checks the physiology region rather than banning the name outright. + */ + @Test + fun `no physiological aggregate reads the unfiltered sessions`() { + val body = analyzeDayBody() + val offenders = Regex("matched\\.[a-zA-Z]").findAll(body).map { it.value }.toList() + assertTrue( + "physiological aggregates must use physiologySessions, found: $offenders", + offenders.isEmpty(), + ) + for (needle in listOf("restingHRDaily", "val deep", "val pairs", "avgSDNNDaily")) { + assertTrue("$needle must sit inside the guarded region", body.contains(needle)) + } + } +} diff --git a/android/app/src/test/java/com/noop/analytics/SleepStagerHrOnlyAnchorTest.kt b/android/app/src/test/java/com/noop/analytics/SleepStagerHrOnlyAnchorTest.kt new file mode 100644 index 0000000000..a421837e3d --- /dev/null +++ b/android/app/src/test/java/com/noop/analytics/SleepStagerHrOnlyAnchorTest.kt @@ -0,0 +1,107 @@ +package com.noop.analytics + +import com.noop.data.HrSample +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * What anchors the HR-only sleep band, and why it is not the median (#1801). + * + * The windows here are synthetic sinusoids with KNOWN sleep hours. They cannot show that staging is + * accurate on a real night — nothing offline can — but they do measure the one property that decides + * whether this spine is usable at all: how much of a window it calls sleep. + */ +class SleepStagerHrOnlyAnchorTest { + + /** `aH` hours awake around `aC`+/-`aA` bpm, then `nH` hours asleep around `nC`+/-`nA`. */ + private fun window(aC: Int, aA: Int, aH: Int, nC: Int, nA: Int, nH: Int): List { + val t0 = 1_788_300_000L + val hr = ArrayList() + var i = 0 + while (i < aH * 3600) { hr.add(HrSample("d", t0 + i, aC + (Math.sin(i / 500.0) * aA).toInt())); i++ } + var j = 0 + while (j < nH * 3600) { + hr.add(HrSample("d", t0 + aH * 3600L + j, nC + (Math.sin(j / 900.0) * nA).toInt())); j++ + } + return hr + } + + private fun sleepHours(hr: List, baseline: Double): Double = + SleepStager.hrOnlySleepRuns(hr, baseline) + .filter { it.stage == "sleep" }.sumOf { it.end - it.start } / 3600.0 + + /** Nearest-rank, so the anchor is always a bpm the wearer actually recorded. */ + @Test + fun `the anchor is the tenth percentile by nearest rank`() { + val hr = (1..100).map { HrSample("d", 1_788_300_000L + it, it) } + assertEquals(0.10, SleepStager.hrOnlyAnchorPercentile, 1e-9) + // (100-1)*0.10 = 9.9 -> index 9 -> the 10th smallest, bpm 10. + assertEquals(10.0, SleepStager.hrOnlyBaseline(hr)!!, 1e-9) + } + + /** + * The percentile index, pinned against the Swift twin's OWN output across the sizes where a + * nearest-rank rule can disagree — n below 1/p, and either side of an exact boundary. Every value + * below is stdout from `hrOnlyBaseline` compiled standalone, not read off the Kotlin. + */ + @Test + fun `the percentile index matches the Swift twin`() { + val cases = listOf( + 1 to 1.0, 2 to 1.0, 3 to 1.0, 7 to 1.0, 10 to 1.0, + 11 to 2.0, 99 to 10.0, 100 to 10.0, 101 to 11.0, 1000 to 100.0, + ) + for ((n, expected) in cases) { + val hr = (1..n).map { HrSample("d", 1_788_300_000L + it, it) } + assertEquals("n=$n", expected, SleepStager.hrOnlyBaseline(hr)!!, 1e-9) + } + } + + /** + * The regression this file exists for. The first draft anchored on [SleepStager.hrBaseline], the + * window MEDIAN — which admits over half of any window by definition, because a median splits the + * samples in half and the band then adds 5% on top. On a field-shaped day that called 14.4 h sleep + * against a truth of 8. The percentile anchor must stay far below it. + */ + @Test + fun `the median anchor over-detects and the percentile anchor does not`() { + val hr = window(74, 11, 16, 64, 5, 8) // 24 h, 8 h of it asleep + val median = sleepHours(hr, SleepStager.hrBaseline(hr)!!) + val percentile = sleepHours(hr, SleepStager.hrOnlyBaseline(hr)!!) + assertTrue("median anchor should over-detect badly, was $median", median > 13.0) + assertTrue("percentile anchor should land near the truth of 8, was $percentile", + percentile in 7.0..9.5) + assertTrue("percentile must be far tighter than median", percentile < median - 4.0) + } + + /** + * The chosen corner errs toward UNDER-detection, deliberately: a missed night leaves "No data", + * which is the state this feature improves on, while an invented night puts a wrong Rest on screen. + * Two nights inside one 54 h window are under-read rather than over-read. + */ + @Test + fun `a long multi-night window is under-read rather than over-read`() { + val hr = window(78, 12, 38, 60, 6, 16) // 54 h, 16 h of it asleep + val h = sleepHours(hr, SleepStager.hrOnlyBaseline(hr)!!) + assertTrue("must not exceed the truth of 16 h, was $h", h <= 16.0) + assertTrue("must still find a real night, was $h", h >= 6.0) + } + + /** A flat sleeper — a small awake/asleep separation — is under-read, never inflated. */ + @Test + fun `a flat sleeper is under-read never inflated`() { + val hr = window(70, 6, 16, 58, 3, 8) + val h = sleepHours(hr, SleepStager.hrOnlyBaseline(hr)!!) + assertTrue("must not exceed the truth of 8 h, was $h", h <= 8.5) + } + + /** + * The detector's tolerance is its OWN constant, not the confirmation gate's. Equal today; this + * pins that a future change to one cannot silently move the other. + */ + @Test + fun `the detector band is a separate constant from the confirmation gate`() { + assertEquals(1.05, SleepStager.hrOnlyBandMult, 1e-9) + assertEquals(1.05, SleepStager.hrSleepBandMult, 1e-9) + } +} diff --git a/android/app/src/test/java/com/noop/analytics/SleepStagerHrOnlySessionsTest.kt b/android/app/src/test/java/com/noop/analytics/SleepStagerHrOnlySessionsTest.kt new file mode 100644 index 0000000000..0b01999d89 --- /dev/null +++ b/android/app/src/test/java/com/noop/analytics/SleepStagerHrOnlySessionsTest.kt @@ -0,0 +1,111 @@ +package com.noop.analytics + +import com.noop.data.HrSample +import com.noop.data.RrInterval +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * Whole HR-only sessions (#1801), and the display-only guarantee that rides on them. + * + * The night is synthetic: a slow HR drift well under the window median, so the spine puts it in the + * sleep band. That is enough to exercise assembly and the null contract; it is NOT a claim that the + * staging is accurate on a real night, which no test here can establish. + */ +class SleepStagerHrOnlySessionsTest { + + /** + * A field-shaped window: `aH` hours awake around 74 +/- 11 bpm, then `nH` hours asleep around + * 64 +/- 5 — the shape the #1801 report shows. The same generator the anchor tests use, so the two + * files cannot drift into disagreeing about what a detectable night looks like. + */ + private fun window(aH: Int = 16, nH: Int = 8): Pair, List> { + val t0 = 1_788_300_000L + val hr = ArrayList() + val rr = ArrayList() + var i = 0 + while (i < aH * 3600) { + val bpm = 74 + (Math.sin(i / 500.0) * 11).toInt() + hr.add(HrSample("d", t0 + i, bpm)); rr.add(RrInterval("d", t0 + i, 60000 / bpm)); i++ + } + var j = 0 + while (j < nH * 3600) { + val bpm = 64 + (Math.sin(j / 900.0) * 5).toInt() + val t = t0 + aH * 3600L + j + hr.add(HrSample("d", t, bpm)); rr.add(RrInterval("d", t, 60000 / bpm)); j++ + } + return hr to rr + } + + @Test + fun `a low-HR night becomes at least one staged session`() { + val (hr, rr) = window() + val out = SleepStager.hrOnlySessions(hr, rr, emptyList()) + assertTrue("expected at least one night, got ${out.size}", out.isNotEmpty()) + assertTrue("every session must carry stages", out.all { it.stages.isNotEmpty() }) + assertTrue("every session must span at least minSleepMin", + out.all { (it.end - it.start) >= SleepStager.minSleepMin * 60L }) + // Conservative by design: it may under-read a night, but must never invent more sleep than the + // window holds. + assertTrue("total must not exceed the 8 h actually asleep", + out.sumOf { it.end - it.start } <= 8 * 3600L) + } + + /** + * The display-only guarantee at its source. restingHR and avgHRV are the two values Charge and the + * baselines fold in, and a baseline is the one thing a false positive cannot be unwound from. + */ + @Test + fun `an HR-only session withholds resting HR and HRV and marks itself`() { + val (hr, rr) = window() + val s = SleepStager.hrOnlySessions(hr, rr, emptyList()).first() + assertTrue("must be flagged hrOnly", s.hrOnly) + assertNull("restingHR must stay null", s.restingHR) + assertNull("avgHRV must stay null", s.avgHRV) + } + + /** A stretch below the minimum-duration gate is not a night. */ + @Test + fun `a short low-HR stretch is not a night`() { + val (hr, rr) = window(aH = 16, nH = 8) + val cut = 1_788_300_000L + 16 * 3600L + 1500L // ~25 min of night, well under minSleepMin + assertTrue( + SleepStager.hrOnlySessions(hr.filter { it.ts < cut }, rr.filter { it.ts < cut }, emptyList()) + .isEmpty() + ) + } + + /** + * A strap that DOES bank motion must never reach this path. A WHOOP 4.0 streams a gravity vector and + * stages its nights from the motion spine — it is reported working — so the HR-only fallback exists + * only for the case where that spine has nothing. Read from the source because the gate lives at the + * call site in IntelligenceEngine, and a fallback that quietly widened to every strap would replace a + * working detector with a weaker one. + */ + @Test + fun `the fallback is reachable only when gravity is absent`() { + var root = java.io.File(System.getProperty("user.dir") ?: ".").canonicalFile + val src = run { + repeat(4) { + val f = java.io.File(root, "android/app/src/main/java/com/noop/analytics/IntelligenceEngine.kt") + if (f.isFile) return@run f.readText() + root = root.parentFile ?: root + } + error("IntelligenceEngine.kt not found — this test must not pass by default") + } + val idx = src.indexOf("SleepStager.hrOnlySessions(") + assertTrue("IntelligenceEngine must call the HR-only fallback", idx > 0) + val guard = src.substring(maxOf(0, idx - 1200), idx) + assertTrue("the fallback must sit behind the absent-gravity gate", guard.contains("grav.size < 2")) + assertTrue("and must only run when the device supplied no hypnogram of its own", + guard.contains("stored.isNotEmpty()")) + } + + /** No HR at all cannot produce a night, and must not throw. */ + @Test + fun `no hr yields nothing`() { + assertTrue(SleepStager.hrOnlySessions(emptyList(), emptyList(), emptyList()).isEmpty()) + } +}