Keep the v26 optical window's absolute base, not just its deltas

Packages/WhoopProtocol/Sources/WhoopProtocol/HistoricalStreams.swift

@@ -258,7 +258,8 @@ public func extractHistoricalStreams(_ parsed: [ParsedFrame],
             if let samples = p["ppg_waveform"]?.intArrayValue, !samples.isEmpty {
                 ppgRecords.append((ts: ts, samples: samples))
                 out.ppgWaveform.append(PpgWaveformSample(ts: ts, samples: samples,
-                                                         burstIndex: p["burst_index"]?.intValue))
+                                                         burstIndex: p["burst_index"]?.intValue,
+                                                         baseCode: p["ppg_base_code"]?.intValue))
             }
             if let bpm = p["heart_rate"]?.intValue, bpm != 0 {  // skip startup hr=0
                 out.hr.append(HRSample(ts: ts, bpm: bpm))
@@ -433,3 +434,66 @@ public func extractHistoricalStreams(_ parsed: [ParsedFrame],
     out.droppedImplausibleNewestTs = droppedNewest
     return out
 }
+
+/// Reconstruct a WHOOP 5/MG v26 optical window from its stored parts. #2019.
+///
+/// The strap sends a 25-sample window as one absolute ADC code plus 24 deltas, so this is the only way to
+/// get back the signal the strap measured: `sample[0] = baseCode`, `sample[i+1] = sample[i] + delta[i]`.
+/// NOOP stores the two as they arrive rather than folding them together, because the delta blob is
+/// little-endian i16 and a real code (about 378,000 on the captured fixture) does not fit in one.
+///
+/// nil when `baseCode` is nil, which is the honest answer for a row written before the base was read: a
+/// delta series cannot be inverted without its starting point, and returning the deltas, or a window
+/// built from a fabricated zero, would present a signal nobody measured.
+///
+/// CAVEAT: the deltas are SATURATED, clamped at the i16 bounds by the encoder, so a window containing a
+/// clamped delta reconstructs only approximately. Nothing here can detect that after the fact; a delta at
+/// exactly ±32,768 is the signal to distrust, and the captured fixture's largest magnitude is 1,833.
+///
+/// Mirror EXACTLY in Kotlin (`ppgWaveformAbsolute`).
+public func ppgWaveformAbsolute(baseCode: Int?, deltas: [Int]) -> [Int]? {
+    guard let baseCode else { return nil }
+    var out: [Int] = [baseCode]
+    out.reserveCapacity(deltas.count + 1)
+    var acc = baseCode
+    for d in deltas {
+        acc += d
+        out.append(acc)
+    }
+    return out
+}
+
+/// A delta at either i16 rail: the encoder clamped it, so the window reconstructs only approximately.
+public func isSaturatedPpgDelta(_ delta: Int) -> Bool {
+    delta == Int(Int16.min) || delta == Int(Int16.max)
+}
+
+/// The per-session v26 optical census, or nil when the session carried no v26 windows (a 4.0, or a 5/MG
+/// that banked none) so a log with nothing to say stays quiet. #2019.
+///
+/// Three things a strap log could not previously answer, all of which decide whether the banked windows
+/// are usable for the channel-mapping work this stream exists for:
+///
+/// - how many windows arrived at all;
+/// - how many carried the absolute base. A window without one cannot be reconstructed, ever. On a
+///   well-formed record the base is always readable, so `withBase` below the window count means
+///   TRUNCATED records or a firmware that does not carry it at frame-abs 23, and either is worth knowing
+///   rather than silently banking un-reconstructable windows;
+/// - how many windows hold a SATURATED delta. Those reconstruct only approximately, and the caveat is
+///   worthless without a way to see whether it ever fires.
+///
+/// The base range is carried because it is the DC level over the session, which is the quantity the whole
+/// stream is banked for and the one that used to be discarded entirely.
+///
+/// Mirror EXACTLY in Kotlin (`ppgWaveformCensusLine`).
+public func ppgWaveformCensusLine(windows: Int, withBase: Int, saturatedWindows: Int,
+                                  baseMin: Int?, baseMax: Int?) -> String? {
+    guard windows > 0 else { return nil }
+    let range: String = {
+        guard let baseMin, let baseMax else { return " base n/a" }
+        return " base \(baseMin)..\(baseMax)"
+    }()
+    let note = saturatedWindows > 0 ? " (a saturated window reconstructs only approximately)" : ""
+    return "Backfill: v26 optical census: \(windows) window(s), \(withBase) with a base, "
+        + "\(saturatedWindows) saturated,\(range)\(note)"
+}

Packages/WhoopProtocol/Sources/WhoopProtocol/Interpreter.swift

@@ -643,14 +643,24 @@ private func decodeWhoop5HistoricalV26(_ frame: [UInt8], fb: FieldBuilder) {
     if let unix = readDType(frame, 15, "u32") {
         fb.add(15, 4, "unix", "time", value: .int(unix), note: "real unix seconds")
     }
+    // #2019: the ABSOLUTE optical code the 24 values below are deltas FROM. The window is 25 samples,
+    // not 24: sample 0 is this code and delta i produces sample i+1. Reading only the deltas and calling
+    // them the waveform stored a derivative as if it were a signal, and discarded the DC level, which is
+    // the half an SpO2 ratio-of-ratios needs. Verified on the captured frame in the waveform tests:
+    // 378,307 here, a valid 20-bit code, against 24 deltas that are every one NEGATIVE, which no
+    // absolute optical reading can be.
+    if let base = readDType(frame, 23, "u32") {
+        fb.add(23, 4, "ppg_base_code", "ppg", value: .int(base),
+               note: "absolute optical ADC code; ppg_waveform holds the deltas from it")
+    }
     var samples: [Int] = []
     for off in stride(from: 27, to: 75, by: 2) {
         guard let v = readI16(frame, off) else { break }
         samples.append(v)
     }
     if !samples.isEmpty {
         fb.add(27, samples.count * 2, "ppg_waveform", "ppg", value: .intArray(samples),
-               note: "optical PPG @24 Hz, LE-i16 ADC counts")
+               note: "optical PPG DELTAS, LE-i16; absolute sample i+1 = base + running sum")
         fb.parsed["ppg_sample_count"] = .int(samples.count)
     }
     // PR#563: the remaining per-record v26 bytes, surfaced as RAW NEUTRAL fields — read off the real

Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift

@@ -373,12 +373,20 @@ public struct SleepStateSample: Equatable, Codable {
 /// `decodeWhoop5HistoricalV26`); a truncated frame can yield fewer than 24.
 public struct PpgWaveformSample: Equatable, Codable, Sendable {
     public let ts: Int          // wall-clock unix seconds (one record per second)
-    public let samples: [Int]   // raw i16 ADC counts @24 Hz, verbatim from `ppg_waveform` (usually 24)
+    /// The window's 24 i16 DELTAS, verbatim from `ppg_waveform`. #2019: these are NOT absolute samples.
+    /// The strap sends a 25-sample window as one absolute code plus 24 deltas; reconstruct with
+    /// `ppgWaveformAbsolute(baseCode:deltas:)`.
+    public let samples: [Int]
     public let burstIndex: Int?  // raw per-burst counter @21; nil for legacy archives
-    public init(ts: Int, samples: [Int], burstIndex: Int? = nil) {
+    /// #2019: the absolute optical ADC code `samples` are deltas from, at frame-abs 23. nil on a row
+    /// written before it was read, and that nil is TRUE rather than merely missing: a delta series
+    /// cannot be inverted without its starting point, so those windows have no recoverable level.
+    public let baseCode: Int?
+    public init(ts: Int, samples: [Int], burstIndex: Int? = nil, baseCode: Int? = nil) {
         self.ts = ts
         self.samples = samples
         self.burstIndex = burstIndex
+        self.baseCode = baseCode
     }
 }
 

Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5PpgWaveformTests.swift

@@ -101,12 +101,37 @@ final class Whoop5PpgWaveformTests: XCTestCase {
         let f = parseFrame(bytes(v26Hex), family: .whoop5)
         let streams = extractHistoricalStreams([f], deviceClockRef: 1_780_917_232, wallClockRef: 1_780_917_232)
         XCTAssertEqual(streams.ppgWaveform,
-                       [PpgWaveformSample(ts: 1_780_917_232, samples: expectedWaveform, burstIndex: 1)])
+                       [PpgWaveformSample(ts: 1_780_917_232, samples: expectedWaveform, burstIndex: 1,
+                                          baseCode: expectedBaseCode)])
         XCTAssertTrue(streams.ppgHr.isEmpty, "a lone 1 s record is too short for a confident HR estimate")
         // Not "no rows at all" — the Backfiller's silent-data-loss diagnostic must see this as decoded.
         XCTAssertFalse(streams.isEmpty)
     }
 
+    /// #2019: the absolute optical code the stored values are deltas FROM, at frame-abs 23. It is what
+    /// makes the window reconstructable and what used to be discarded.
+    private let expectedBaseCode = 378_307
+
+    /// The reconstruction, on the real captured window. Every absolute sample must land inside the
+    /// 20-bit ADC domain, which is the check that says the base and the deltas belong together: read as
+    /// absolute values the stored deltas are all NEGATIVE, and no optical reading can be. The excursion
+    /// is 4.2% of the base, an ordinary PPG perfusion index. Mirrored in Kotlin.
+    func testTheWindowReconstructsIntoTheOpticalDomain() {
+        let f = parseFrame(bytes(v26Hex), family: .whoop5)
+        let streams = extractHistoricalStreams([f], deviceClockRef: 1_780_917_232,
+                                               wallClockRef: 1_780_917_232)
+        let row = try! XCTUnwrap(streams.ppgWaveform.first)
+        let absolute = try! XCTUnwrap(ppgWaveformAbsolute(baseCode: row.baseCode, deltas: row.samples))
+        XCTAssertEqual(absolute.count, 25, "one absolute code plus 24 deltas is a 25-sample window")
+        XCTAssertEqual(absolute.first, expectedBaseCode)
+        XCTAssertEqual(absolute.last, 362_532)
+        XCTAssertTrue(absolute.allSatisfy { $0 >= 0 && $0 < (1 << 20) },
+                      "every sample must sit in the 20-bit ADC domain")
+        // A legacy row, whose base was never stored, is honestly unreconstructable rather than silently
+        // reconstructed from a fabricated zero.
+        XCTAssertNil(ppgWaveformAbsolute(baseCode: nil, deltas: row.samples))
+    }
+
     func testExtractHistoricalStreamsCarriesEachBurstIndexWithItsWaveform() {
         let parsed = [v26Hex, v26HexChannel46].map { parseFrame(bytes($0), family: .whoop5) }
         let streams = extractHistoricalStreams(parsed,
@@ -146,3 +171,57 @@ final class Whoop5PpgWaveformTests: XCTestCase {
         XCTAssertEqual(round.ppgWaveform, withBurst.ppgWaveform)
     }
 }
+
+/// #2019: the per-session optical census. Mirrored by the Kotlin `PpgWaveformCensusTest` against the
+/// SAME literals, so the two platforms cannot report the same offload differently.
+final class PpgWaveformCensusTests: XCTestCase {
+
+    /// A session with no v26 windows says nothing at all: a 4.0, or a 5/MG that banked none, must not
+    /// print a census of zero.
+    func testNoWindowsIsSilent() {
+        XCTAssertNil(ppgWaveformCensusLine(windows: 0, withBase: 0, saturatedWindows: 0,
+                                           baseMin: nil, baseMax: nil))
+    }
+
+    /// The ordinary healthy session: every window carried a base, none saturated.
+    func testEveryWindowReconstructable() {
+        XCTAssertEqual(
+            ppgWaveformCensusLine(windows: 412, withBase: 412, saturatedWindows: 0,
+                                  baseMin: 361_204, baseMax: 379_881),
+            "Backfill: v26 optical census: 412 window(s), 412 with a base, 0 saturated, base 361204..379881")
+    }
+
+    /// `withBase` below the window count is the signal that matters: those windows can never be
+    /// reconstructed, and before this line existed they were banked with nothing said about it.
+    func testWindowsMissingABaseAreVisible() {
+        XCTAssertEqual(
+            ppgWaveformCensusLine(windows: 10, withBase: 7, saturatedWindows: 0,
+                                  baseMin: 100, baseMax: 200),
+            "Backfill: v26 optical census: 10 window(s), 7 with a base, 0 saturated, base 100..200")
+    }
+
+    /// A saturated window earns the caveat inline, because the caveat is the reason to distrust the
+    /// reconstruction and it is worthless if it only lives in a doc comment.
+    func testSaturationCarriesItsCaveat() {
+        let line = ppgWaveformCensusLine(windows: 5, withBase: 5, saturatedWindows: 2,
+                                         baseMin: 1, baseMax: 2)
+        XCTAssertEqual(line, "Backfill: v26 optical census: 5 window(s), 5 with a base, 2 saturated, "
+                       + "base 1..2 (a saturated window reconstructs only approximately)")
+    }
+
+    /// An absent range reads as absent rather than as a fabricated zero.
+    func testAbsentBaseRangeSaysSo() {
+        XCTAssertEqual(
+            ppgWaveformCensusLine(windows: 3, withBase: 0, saturatedWindows: 0,
+                                  baseMin: nil, baseMax: nil),
+            "Backfill: v26 optical census: 3 window(s), 0 with a base, 0 saturated, base n/a")
+    }
+
+    /// Both i16 rails count, and an ordinary delta does not.
+    func testSaturationIsBothRails() {
+        XCTAssertTrue(isSaturatedPpgDelta(-32_768))
+        XCTAssertTrue(isSaturatedPpgDelta(32_767))
+        XCTAssertFalse(isSaturatedPpgDelta(-1_833))
+        XCTAssertFalse(isSaturatedPpgDelta(0))
+    }
+}

Packages/WhoopStore/Sources/WhoopStore/Database.swift

@@ -919,6 +919,18 @@ extension WhoopStore {
             // No index: the table is capped at maxStoredMessages (40 rows), so a full scan + sort on
             // read is negligible and an index buys nothing worth the extra Room<->GRDB parity surface.
         }
+        // #2019: carry the v26 optical window's ABSOLUTE base code beside its deltas.
+        //
+        // The 25-sample window is one absolute ADC code plus 24 deltas, and only the deltas were read, so
+        // the stored `samples` blob is a derivative and the DC level was thrown away. Nullable and
+        // additive: an existing row keeps its deltas and gets a null base, which is the true statement
+        // about it. A delta series cannot be inverted without the base, so those windows have no
+        // recoverable absolute level and no backfill can invent one. Twin of Room's MIGRATION_37_38.
+        migrator.registerMigration("v44-ppg-waveform-base-code") { db in
+            try db.alter(table: "ppgWaveformSample") { t in
+                t.add(column: "baseCode", .integer)
+            }
+        }
         return migrator
     }
 }

Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift

@@ -360,12 +360,13 @@ extension WhoopStore {
             // `packPpgSamples`) rather than 24 scalar rows, so this insert is O(records), not O(samples).
             if !streams.ppgWaveform.isEmpty {
                 let stmt = try db.cachedStatement(sql: """
-                    INSERT INTO ppgWaveformSample (deviceId, ts, samples, burstIndex) VALUES (?, ?, ?, ?)
+                    INSERT INTO ppgWaveformSample (deviceId, ts, samples, burstIndex, baseCode)
+                    VALUES (?, ?, ?, ?, ?)
                     ON CONFLICT(deviceId, ts) DO NOTHING
                     """)
                 for s in streams.ppgWaveform {
                     try stmt.execute(arguments: [deviceId, s.ts, WhoopStore.packPpgSamples(s.samples),
-                                                 s.burstIndex])
+                                                 s.burstIndex, s.baseCode])
                     ppgWaveformWritten += 1
                 }
             }
@@ -690,13 +691,18 @@ extension WhoopStore {
         -> [PpgWaveformSample] {
         try syncRead { db in
             try Row.fetchAll(db, sql: """
-                SELECT ts, samples, burstIndex FROM ppgWaveformSample
+                SELECT ts, samples, burstIndex, baseCode FROM ppgWaveformSample
                 WHERE deviceId = ? AND ts >= ? AND ts <= ?
                 ORDER BY ts LIMIT ?
                 """, arguments: [deviceId, from, to, limit])
+                // #2019: baseCode is SELECTed explicitly. This projection names its columns, so a new one
+                // is invisible to it until it is listed — the write would have banked the base and every
+                // read would have handed back nil, which is the same answer a legacy row gives and would
+                // have looked like the column doing nothing.
                 .map { PpgWaveformSample(ts: $0["ts"],
                                          samples: WhoopStore.unpackPpgSamples($0["samples"]),
-                                         burstIndex: $0["burstIndex"]) }
+                                         burstIndex: $0["burstIndex"],
+                                         baseCode: $0["baseCode"]) }
         }
     }
 

Packages/WhoopStore/Tests/WhoopStoreTests/PpgWaveformSampleTests.swift

@@ -31,19 +31,27 @@ final class PpgWaveformSampleTests: XCTestCase {
     func testPpgWaveformTableShape() async throws {
         let store = try await WhoopStore.inMemory()
         let cols = try await store.columnNamesForTest(table: "ppgWaveformSample")
-        XCTAssertEqual(Set(cols), ["deviceId", "ts", "samples", "burstIndex"])
+        // #2019: `baseCode` is the absolute optical code `samples` are deltas from. Nullable and
+        // additive, so an existing row keeps its deltas and reads null, which is the true statement
+        // about it: a delta series cannot be inverted without its starting point.
+        XCTAssertEqual(Set(cols), ["deviceId", "ts", "samples", "burstIndex", "baseCode"])
     }
 
     func testPpgWaveformInsertRoundTripAndDedup() async throws {
         let store = try await WhoopStore.inMemory()
-        let streams = Streams(ppgWaveform: [PpgWaveformSample(ts: 1_780_917_232,
-                                                              samples: realSamples, burstIndex: 7)])
+        // #2019: with a baseCode, so the round trip proves the column is WRITTEN and READ. The read is a
+        // named projection, so a new column is invisible to it until it is listed, and a base that banked
+        // but never came back would read as nil, which is exactly what a legacy row reads as.
+        let streams = Streams(ppgWaveform: [PpgWaveformSample(ts: 1_780_917_232, samples: realSamples,
+                                                              burstIndex: 7, baseCode: 378_307)])
         _ = try await store.insert(streams, deviceId: "my-whoop")
         let n1 = try await store.ppgWaveformCountForTest()
         XCTAssertEqual(n1, 1)
         let read = try await store.ppgWaveformSamples(deviceId: "my-whoop",
                                                        from: 1_780_917_232, to: 1_780_917_232)
-        XCTAssertEqual(read, [PpgWaveformSample(ts: 1_780_917_232, samples: realSamples, burstIndex: 7)])
+        XCTAssertEqual(read, [PpgWaveformSample(ts: 1_780_917_232, samples: realSamples, burstIndex: 7,
+                                                baseCode: 378_307)])
+        XCTAssertEqual(read.first?.baseCode, 378_307, "the absolute base must survive the round trip")
         // Re-inserting the same (deviceId, ts) is idempotent, ON CONFLICT DO NOTHING (mirrors every
         // other per-second stream's dedupe rule).
         _ = try await store.insert(streams, deviceId: "my-whoop")