feat(store): persist the WHOOP 5.0 v26 raw PPG waveform (#156 follow-up)

Packages/WhoopProtocol/Sources/WhoopProtocol/HistoricalStreams.swift

@@ -55,8 +55,11 @@ public func isPlausibleHistoricalUnix(_ ts: Int, wallNow: Int,
 ///
 /// Console (type-50, `frame[typeIndex] == 0x32`) frames are strap-side debug-log text that decode to
 /// zero rows BY DESIGN and are never returned. 5/MG v26 (raw PPG block, hist_version 26) is also
-/// skipped: it is known-and-unstored by design, not lost biometric data. Only genuine type-47
-/// record frames whose payload would otherwise be silently dropped are returned.
+/// skipped unconditionally (even on a CRC failure): a v26 record's payload is the optical waveform,
+/// which `extractHistoricalStreams` now persists durably in its OWN stream (`Streams.ppgWaveform` /
+/// WhoopStore's `ppgWaveformSample` table, issue #156 follow-up) whenever it decodes — this reject
+/// archive exists for genuinely-undecodable records, and a decoded v26 record was never one of those.
+/// Only genuine type-47 record frames whose payload would otherwise be silently dropped are returned.
 ///
 /// Used by the Backfiller/BLEManager to archive undecodable history BEFORE acking the trim. Mirrors
 /// the Android rejectedHistoricalRecords so one mapping toolchain re-ingests both archives.
@@ -70,7 +73,7 @@ public func rejectedHistoricalRecords(_ rawFrames: [[UInt8]], family: DeviceFami
         // Only genuine HISTORICAL_DATA records (47). Console (50) and METADATA frames have a
         // different type byte, so they never pass this gate — they are excluded by construction.
         guard f.count > typeIndex, Int(f[typeIndex]) == 47 else { return false }
-        if family == .whoop5, f.count > versionIndex, Int(f[versionIndex]) == 26 { return false }  // v26 PPG: skipped by design
+        if family == .whoop5, f.count > versionIndex, Int(f[versionIndex]) == 26 { return false }  // v26 PPG: has its own durable stream (ppgWaveform), not this reject archive
         let p = parseFrame(f, family: family)
         // Envelope/CRC reject: parse failed outright or the CRC32 trailer mismatched.
         if !p.ok || p.crcOK == false { return true }
@@ -176,6 +179,8 @@ public func extractHistoricalStreams(_ parsed: [ParsedFrame],
     // v26 optical-PPG records (issue #156): no measured HR/motion, just the 24 Hz waveform. Collect
     // (corrected-wall ts, samples) here and derive a per-second HR after the loop (PpgHr.derivePpgHr),
     // so the timeline stays continuous through the v26-heavy stretches that have no v18 HR summary.
+    // The SAME (ts, samples) are also appended to `out.ppgWaveform` below (issue #156 follow-up) so the
+    // raw waveform is durable too, not just the derived estimate this local buffer exists to produce.
     var ppgRecords: [(ts: Int, samples: [Int])] = []
     for r in parsed {
         if !r.ok || r.crcOK == false { continue }
@@ -187,10 +192,13 @@ public func extractHistoricalStreams(_ parsed: [ParsedFrame],
             // `correctedWall` returns nil for an implausible ts (covers the v26 PPG baseTs too, since the
             // v26 waveform rides this same `unix`) — skip the whole record so no garbage-ts row is banked.
             guard let rawTs = p["unix"]?.intValue, let ts = correctedWall(rawTs) else { continue }
-            // v26 PPG buffer: stash the waveform for the post-loop HR estimator. A v26 record carries
-            // no heart_rate/spo2/gravity, so it adds nothing to the branches below — handled here only.
+            // v26 PPG buffer: stash the waveform for the post-loop HR estimator AND persist the raw
+            // samples themselves (issue #156 follow-up — previously ONLY the derived estimate survived,
+            // the waveform that produced it was discarded here). A v26 record carries no
+            // heart_rate/spo2/gravity, so it adds nothing to the branches below — handled here only.
             if let samples = p["ppg_waveform"]?.intArrayValue, !samples.isEmpty {
                 ppgRecords.append((ts: ts, samples: samples))
+                out.ppgWaveform.append(PpgWaveformSample(ts: ts, samples: samples))
             }
             if let bpm = p["heart_rate"]?.intValue, bpm != 0 {  // skip startup hr=0
                 out.hr.append(HRSample(ts: ts, bpm: bpm))

Packages/WhoopProtocol/Sources/WhoopProtocol/Streams.swift

@@ -202,6 +202,20 @@ public struct SleepStateSample: Equatable, Codable {
     public init(ts: Int, state: Int) { self.ts = ts; self.state = state }
 }
 
+/// The WHOOP 5.0 v26 optical PPG buffer's RAW waveform, one record per second (issue #156 follow-up).
+/// `PpgHr.derivePpgHr` already turns these into a per-second HR estimate (`ppgHr`), but until now the
+/// 24 Hz samples themselves were discarded the moment that estimate was taken — HistoricalStreams
+/// collected them into a transient `ppgRecords` buffer purely to feed the estimator, and nothing else
+/// ever saw them. Persisted here (and, in WhoopStore, its own table) so a future re-analysis — a better
+/// HR estimator, HRV-from-PPG, a waveform viewer — can run over the ORIGINAL samples, not just the
+/// derived bpm. `samples` are raw AC-coupled ADC counts, no invented scale (see
+/// `decodeWhoop5HistoricalV26`); a truncated frame can yield fewer than 24.
+public struct PpgWaveformSample: Equatable, Codable, Sendable {
+    public let ts: Int          // wall-clock unix seconds (one record per second)
+    public let samples: [Int]   // raw i16 ADC counts @24 Hz, verbatim from `ppg_waveform` (usually 24)
+    public init(ts: Int, samples: [Int]) { self.ts = ts; self.samples = samples }
+}
+
 public struct Streams: Equatable, Codable {
     public var hr: [HRSample]
     public var rr: [RRInterval]
@@ -217,6 +231,10 @@ public struct Streams: Equatable, Codable {
     /// PPG-derived per-second HR from the WHOOP 5.0 v26 optical buffer (issue #156). Kept separate from
     /// `hr` (the measured stream) so consumers can COALESCE without conflating the two sources.
     public var ppgHr: [PpgHrSample]
+    /// The RAW v26 optical PPG waveform itself (issue #156 follow-up), one record per second — the
+    /// samples `ppgHr` is derived FROM. Kept separate so a consumer that only wants the HR estimate
+    /// never pays for the 24x-larger raw stream, and so the two can be persisted/pruned independently.
+    public var ppgWaveform: [PpgWaveformSample]
     public var events: [WhoopEvent]
     public var battery: [BatterySample]
     /// #547 diagnostic: how many historical records `extractHistoricalStreams` DROPPED this chunk for an
@@ -239,11 +257,12 @@ public struct Streams: Equatable, Codable {
                 spo2: [SpO2Sample] = [], skinTemp: [SkinTempSample] = [],
                 resp: [RespSample] = [], gravity: [GravitySample] = [],
                 steps: [StepSample] = [], sleepState: [SleepStateSample] = [],
-                ppgHr: [PpgHrSample] = [],
+                ppgHr: [PpgHrSample] = [], ppgWaveform: [PpgWaveformSample] = [],
                 events: [WhoopEvent] = [], battery: [BatterySample] = []) {
         self.hr = hr; self.rr = rr
         self.spo2 = spo2; self.skinTemp = skinTemp; self.resp = resp; self.gravity = gravity
         self.steps = steps; self.sleepState = sleepState; self.ppgHr = ppgHr
+        self.ppgWaveform = ppgWaveform
         self.events = events; self.battery = battery
     }
 
@@ -253,13 +272,14 @@ public struct Streams: Equatable, Codable {
     public var isEmpty: Bool {
         hr.isEmpty && rr.isEmpty && spo2.isEmpty && skinTemp.isEmpty && resp.isEmpty
             && gravity.isEmpty && steps.isEmpty && sleepState.isEmpty && ppgHr.isEmpty
-            && events.isEmpty && battery.isEmpty
+            && ppgWaveform.isEmpty && events.isEmpty && battery.isEmpty
     }
 
     private enum CodingKeys: String, CodingKey {
         case hr, rr, spo2, skinTemp = "skin_temp", resp, gravity, steps
         case sleepState = "sleep_state"
         case ppgHr = "ppg_hr"
+        case ppgWaveform = "ppg_waveform"
         case events, battery
     }
 
@@ -276,6 +296,7 @@ public struct Streams: Equatable, Codable {
         steps = try c.decodeIfPresent([StepSample].self, forKey: .steps) ?? []
         sleepState = try c.decodeIfPresent([SleepStateSample].self, forKey: .sleepState) ?? []
         ppgHr = try c.decodeIfPresent([PpgHrSample].self, forKey: .ppgHr) ?? []
+        ppgWaveform = try c.decodeIfPresent([PpgWaveformSample].self, forKey: .ppgWaveform) ?? []
         events = try c.decodeIfPresent([WhoopEvent].self, forKey: .events) ?? []
         battery = try c.decodeIfPresent([BatterySample].self, forKey: .battery) ?? []
     }

Packages/WhoopProtocol/Tests/WhoopProtocolTests/Whoop5PpgWaveformTests.swift

@@ -86,4 +86,50 @@ final class Whoop5PpgWaveformTests: XCTestCase {
         let meanStep = zip(w, w.dropFirst()).map { abs($1 - $0) }.reduce(0, +) / (w.count - 1)
         XCTAssertLessThan(meanStep * 4, range)
     }
+
+    // MARK: - Durable waveform persistence (issue #156 follow-up)
+    //
+    // Until now `extractHistoricalStreams` collected a v26 record's waveform into a transient local
+    // buffer ONLY to derive a per-second HR estimate (`ppgHr`); the samples themselves were discarded
+    // once that estimate was taken. These tests prove the raw waveform now survives as its OWN stream
+    // (`Streams.ppgWaveform`), independently of whether the HR estimator had enough context to run.
+
+    /// A single v26 record is one second of samples — nowhere near the >=3-consecutive-second run
+    /// `PpgHr.derivePpgHr` requires, so `ppgHr` stays empty. Before this change that meant the ENTIRE
+    /// record vanished (nothing else read `ppg_waveform`); now the raw waveform is still captured.
+    func testExtractHistoricalStreamsPersistsRawPpgWaveform() {
+        let f = parseFrame(bytes(v26Hex), family: .whoop5)
+        let streams = extractHistoricalStreams([f], deviceClockRef: 1_780_917_232, wallClockRef: 1_780_917_232)
+        XCTAssertEqual(streams.ppgWaveform, [PpgWaveformSample(ts: 1_780_917_232, samples: expectedWaveform)])
+        XCTAssertTrue(streams.ppgHr.isEmpty, "a lone 1 s record is too short for a confident HR estimate")
+        // Not "no rows at all" — the Backfiller's silent-data-loss diagnostic must see this as decoded.
+        XCTAssertFalse(streams.isEmpty)
+    }
+
+    /// `Streams.isEmpty` must count a waveform-only decode as non-empty even when `ppgHr` (derived FROM
+    /// it) is empty — otherwise the Backfiller's #77 "this chunk carried no sensor records" diagnostic
+    /// would misfire on a chunk that in fact persisted a raw waveform.
+    func testStreamsIsEmptyConsidersPpgWaveform() {
+        var s = Streams()
+        XCTAssertTrue(s.isEmpty)
+        s.ppgWaveform = [PpgWaveformSample(ts: 1, samples: [1, 2, 3])]
+        XCTAssertFalse(s.isEmpty)
+    }
+
+    /// Streams decode tolerance: a JSON missing `ppg_waveform` still decodes (defaults to empty), and a
+    /// present `ppg_waveform` round-trips, including negative (AC-coupled) sample values. Mirrors the
+    /// decodeIfPresent guard for the other biometric keys (see PpgHrTests' ppg_hr analogue).
+    func testStreamsDecodeToleratesMissingAndPresentPpgWaveform() throws {
+        let dec = JSONDecoder()
+        // Missing key → empty.
+        let s1 = try dec.decode(Streams.self, from: Data(#"{"hr":[]}"#.utf8))
+        XCTAssertTrue(s1.ppgWaveform.isEmpty)
+        // Present key → decoded under the snake_case CodingKey.
+        let json = #"{"ppg_waveform":[{"ts":1780917232,"samples":[-1432,-1332,12]}]}"#
+        let s2 = try dec.decode(Streams.self, from: Data(json.utf8))
+        XCTAssertEqual(s2.ppgWaveform, [PpgWaveformSample(ts: 1_780_917_232, samples: [-1432, -1332, 12])])
+        // Round-trip encode → decode is identity.
+        let round = try dec.decode(Streams.self, from: JSONEncoder().encode(s2))
+        XCTAssertEqual(round.ppgWaveform, s2.ppgWaveform)
+    }
 }

Packages/WhoopStore/Sources/WhoopStore/Database.swift

@@ -525,6 +525,33 @@ extension WhoopStore {
                 WHERE efficiency > 1.5
                 """)
         }
+
+        // v27 (issue #156 follow-up): durable storage for the WHOOP 5.0 v26 optical PPG waveform. The
+        // strap's 24 Hz buffer was fully DECODED (`ppg_waveform`, 24 i16 ADC samples/record) but only
+        // ever used to derive a per-second HR estimate (`ppgHrSample`, v12) — the waveform itself was
+        // discarded right after, and `rejectedHistoricalRecords` explicitly excludes v26 from the
+        // undecodable-record reject archive ("known-and-unstored by design"), so it had no home at all.
+        //
+        // One row per (deviceId, ts) — the SAME shape as every other per-second decoded stream (hrSample,
+        // spo2Sample, ppgHrSample, …) — but the 24 samples are packed into a compact BLOB (2 bytes/sample,
+        // little-endian i16, `WhoopStore.packPpgSamples`/`unpackPpgSamples`) instead of 24 scalar rows.
+        // That keeps a v26-heavy night to roughly the same order of magnitude as ONE extra per-second
+        // stream (≈50 bytes/row), not 24x that. Additive only, a NEW table, no existing row touched.
+        //
+        // Retention: no pruning, matching every other durable per-second table (hrSample, spo2Sample, …
+        // are never pruned either) — this is decoded biometric history, not the transient raw outbox.
+        // `PrunePolicy`'s ~50 MB cap governs ONLY `rawBatch` (raw, pre-decode frames kept for re-decode /
+        // re-sync); it is untouched by and unrelated to this table. Growth here is bounded by how much
+        // v26 data a strap actually emits (firmware chooses v26 vs v18 per second, not every night is
+        // v26-heavy), not by an artificial cap.
+        migrator.registerMigration("v27-ppg-waveform") { db in
+            try db.create(table: "ppgWaveformSample") { t in
+                t.column("deviceId", .text).notNull()
+                t.column("ts", .integer).notNull()
+                t.column("samples", .blob).notNull()
+                t.primaryKey(["deviceId", "ts"])
+            }
+        }
         return migrator
     }
 }

Packages/WhoopStore/Sources/WhoopStore/DeviceRegistryStore.swift

@@ -90,6 +90,11 @@ public struct DeviceRegistryStore: Sendable {
         // v25-oura-raw: the opt-in Oura cloud-import raw archive is deviceId-keyed too, so "delete this
         // device's data" must clear it — else an imported Oura source's payloads would survive deletion.
         "ouraRaw",
+        // v27-ppg-waveform (issue #156 follow-up): the durable raw v26 optical PPG waveform is
+        // deviceId-keyed exactly like every other per-second stream above — must be cleared too, or a
+        // "delete all of this device's data" leaves the raw waveform behind (the same privacy defect
+        // this list exists to close).
+        "ppgWaveformSample",
     ]
 
     /// Permanently delete every recorded sample/derived row belonging to one device, across all

Packages/WhoopStore/Sources/WhoopStore/StreamStore.swift

@@ -12,6 +12,35 @@ extension WhoopStore {
         return String(decoding: data, as: UTF8.self)
     }
 
+    /// Pack a decoded v26 PPG waveform's samples as little-endian i16 (2 bytes/sample) — a single
+    /// compact BLOB per (deviceId, ts) row instead of 24 scalar rows (issue #156 follow-up, v27). Any
+    /// sample count is handled (a truncated frame can decode fewer than 24); each value is truncated to
+    /// Int16's range, matching the wire format it came from (`readI16` in the decoder never produces
+    /// anything wider).
+    static func packPpgSamples(_ samples: [Int]) -> Data {
+        var buf = Data(capacity: samples.count * 2)
+        for s in samples {
+            let v = Int16(truncatingIfNeeded: s)
+            buf.append(UInt8(truncatingIfNeeded: v))
+            buf.append(UInt8(truncatingIfNeeded: v >> 8))
+        }
+        return buf
+    }
+
+    /// Inverse of `packPpgSamples`. A trailing odd byte (a corrupt/truncated blob) is dropped rather
+    /// than thrown — a read path never crashes on a malformed row.
+    static func unpackPpgSamples(_ data: Data) -> [Int] {
+        let bytes = [UInt8](data)
+        var out = [Int](); out.reserveCapacity(bytes.count / 2)
+        var i = 0
+        while i + 1 < bytes.count {
+            let u = UInt16(bytes[i]) | (UInt16(bytes[i + 1]) << 8)
+            out.append(Int(Int16(bitPattern: u)))
+            i += 2
+        }
+        return out
+    }
+
     /// Insert or update a device row (natural key = id).
     public func upsertDevice(id: String, mac: String?, name: String?) async throws {
         let now = Int(Date().timeIntervalSince1970)
@@ -173,6 +202,20 @@ extension WhoopStore {
                     try stmt.execute(arguments: [deviceId, s.ts, s.bpm, s.conf])
                 }
             }
+            // RAW v26 optical PPG waveform (#156 follow-up) — the samples `ppgHr` above is derived FROM.
+            // Persist-only, same as steps/sleepState/ppgHr: not added to the 8-field return tuple. ON
+            // CONFLICT DO NOTHING keeps the FIRST-seen waveform for a second, matching every other
+            // per-second stream's dedupe rule. Packed into one compact BLOB per row (see
+            // `packPpgSamples`) rather than 24 scalar rows, so this insert is O(records), not O(samples).
+            if !streams.ppgWaveform.isEmpty {
+                let stmt = try db.cachedStatement(sql: """
+                    INSERT INTO ppgWaveformSample (deviceId, ts, samples) VALUES (?, ?, ?)
+                    ON CONFLICT(deviceId, ts) DO NOTHING
+                    """)
+                for s in streams.ppgWaveform {
+                    try stmt.execute(arguments: [deviceId, s.ts, WhoopStore.packPpgSamples(s.samples)])
+                }
+            }
             return (hr, rr, ev, bat, spo2, skin, resp, grav)
         }
     }
@@ -395,6 +438,26 @@ extension WhoopStore {
         try syncRead { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM ppgHrSample") ?? 0 }
     }
 
+    /// The RAW v26 optical PPG waveform (#156 follow-up), one record per second, in `[from, to]` for one
+    /// device, ascending by ts. `samples` are the raw i16 ADC counts the strap sent, unpacked from the
+    /// compact on-disk BLOB (`packPpgSamples`/`unpackPpgSamples`). Empty when the strap never emitted
+    /// v26 (the WHOOP 4.0 / v18-only common case) or the window has no v26-heavy stretch.
+    public func ppgWaveformSamples(deviceId: String, from: Int, to: Int, limit: Int = 200_000) async throws
+        -> [PpgWaveformSample] {
+        try syncRead { db in
+            try Row.fetchAll(db, sql: """
+                SELECT ts, samples FROM ppgWaveformSample
+                WHERE deviceId = ? AND ts >= ? AND ts <= ?
+                ORDER BY ts LIMIT ?
+                """, arguments: [deviceId, from, to, limit])
+                .map { PpgWaveformSample(ts: $0["ts"], samples: WhoopStore.unpackPpgSamples($0["samples"])) }
+        }
+    }
+
+    public func ppgWaveformCountForTest() async throws -> Int {
+        try syncRead { db in try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM ppgWaveformSample") ?? 0 }
+    }
+
     public func deviceRowForTest(id: String) async throws -> (mac: String?, name: String?)? {
         try syncRead { db in
             guard let row = try Row.fetchOne(db,