Add per-chunk historical clock + R-R packing diagnostic (#1008)

Packages/WhoopProtocol/Sources/WhoopProtocol/ChunkClockDiag.swift

@@ -0,0 +1,91 @@
+import Foundation
+
+/// #1008 diagnostic: the per-CHUNK view of the historical clock basis and how densely each decoded
+/// chunk packs its R-R intervals onto wall seconds.
+///
+/// Why per-chunk and not per-session. The session summary already logs the ONE `(device, wall)`
+/// correlation captured on the first chunk (`sessionClockDevice`/`sessionClockWall`, #67). That is
+/// enough to say whether the stale-RTC correction could engage at all, but it cannot answer the two
+/// questions #1008 actually turns on:
+///
+///  1. **Does the offset move within a single offload?** A strap RTC that drifts (measured at ~8 s/hour
+///     on a 4.0) has a different `wall - device` offset at the start and the end of a long session.
+///     One number per session hides the trajectory; one per chunk shows it.
+///  2. **Is the R-R overcount duplication, or packing?** `extractHistoricalStreams` stamps EVERY R-R
+///     interval inside one type-47 record with that record's single `unix` second, so a record carrying
+///     8 intervals emits 8 rows on one timestamp (`ord` 0...7). That inflates beats-per-second without
+///     any record being delivered twice. `pack` (intervals per STAMPED second) separates the two: packing
+///     shows `pack` well above 1 while `dens` (intervals per SPAN second) stays near the true heart rate;
+///     genuine duplication moves both together.
+///
+/// Log-only and allocation-light — it walks the chunk's timestamps once. Nothing here feeds a stored
+/// value or a gate. Twin of the Kotlin `ChunkClockDiag`.
+public enum ChunkClockDiag {
+
+    /// Build the per-chunk line, or `nil` when the chunk decoded no R-R at all (a motion/temp-only or
+    /// console-only chunk has no clock story to tell and would only add noise to a strap log).
+    ///
+    /// - Parameters:
+    ///   - chunk: 1-based index of this chunk within the session.
+    ///   - deviceClockRef: the strap-side half of the session correlation, as passed to the decoder.
+    ///   - wallClockRef: the phone-side half of the same correlation.
+    ///   - rrTimestamps: the resolved wall seconds of every R-R interval this chunk decoded, in
+    ///     emission order. Duplicates are meaningful (they ARE the packing) and must not be pre-uniqued.
+    public static func line(chunk: Int,
+                            deviceClockRef: Int,
+                            wallClockRef: Int,
+                            rrTimestamps: [Int]) -> String? {
+        guard !rrTimestamps.isEmpty else { return nil }
+
+        let offset = wallClockRef - deviceClockRef
+        // Mirrors the FIRST gate inside `extractHistoricalStreams`: below the threshold the offset is
+        // DISCARDED and each record keeps its own raw unix second, so a small drift (tens of seconds) never
+        // reaches the stored timestamps — the drift is baked into the strap's own stamps instead, which is
+        // a different problem with a different fix.
+        //
+        // Named for the GATE, not the outcome, and deliberately so: an open gate does NOT prove the
+        // correction was applied. Past it, the #471 overshoot guard re-checks EVERY record and keeps the
+        // raw ts whenever the corrected value would post-date wall time — which is exactly the field case
+        // that motivated it (a drained strap whose RTC reset to ~epoch has an offset of decades, so every
+        // record overshoots and none are corrected). That decision is per-record and depends on each raw
+        // stamp, so no single per-chunk flag can state it; reporting the gate is the honest half.
+        let staleGateOpen = abs(offset) > histStaleClockThresholdSec
+
+        var perSecond: [Int: Int] = [:]
+        var oldest = rrTimestamps[0], newest = rrTimestamps[0]
+        for ts in rrTimestamps {
+            perSecond[ts, default: 0] += 1
+            if ts < oldest { oldest = ts }
+            if ts > newest { newest = ts }
+        }
+        let stampedSeconds = perSecond.count
+        let maxPerSecond = perSecond.values.max() ?? 0
+        let spanSeconds = newest - oldest + 1          // inclusive; a single-second chunk spans 1
+
+        return "Backfill: hist clock chunk=\(chunk) offset=\(signed(offset))s staleGate=\(staleGateOpen ? "open" : "closed")"
+            + " rr=\(rrTimestamps.count) secs=\(stampedSeconds)"
+            + " pack=\(fixed2(rrTimestamps.count, stampedSeconds)) max=\(maxPerSecond)"
+            + " span=\(spanSeconds)s dens=\(fixed2(rrTimestamps.count, spanSeconds))"
+    }
+
+    /// Always-signed so a drift trajectory reads at a glance across chunks (`+15s` → `+48s` → `+200s`).
+    static func signed(_ v: Int) -> String { v >= 0 ? "+\(v)" : "\(v)" }
+
+    /// `numerator/denominator` to 2dp, by INTEGER half-up rounding — deliberately NOT `String(format:)`.
+    ///
+    /// Two traps this sidesteps, both of which would silently desync the Swift and Kotlin logs that the
+    /// tests assert are byte-identical:
+    ///  - **Rounding mode.** C/Swift `printf("%.2f")` rounds half-to-EVEN; Java's `String.format` rounds
+    ///    half-UP. They disagree on any exact tie, and ties are ordinary here — `pack` of 9/8 renders
+    ///    `1.12` on Swift and `1.13` on Kotlin.
+    ///  - **Locale.** A comma decimal separator on a German/French device would break a log parser.
+    ///
+    /// Integer math has neither problem: `+denominator` before the divide is the half-up bias, and the
+    /// operands are small (interval counts), nowhere near overflow.
+    static func fixed2(_ numerator: Int, _ denominator: Int) -> String {
+        guard denominator > 0 else { return "0.00" }
+        let hundredths = (numerator * 200 + denominator) / (denominator * 2)
+        let frac = hundredths % 100
+        return "\(hundredths / 100)." + (frac < 10 ? "0\(frac)" : "\(frac)")
+    }
+}

Packages/WhoopProtocol/Sources/WhoopProtocol/HistoricalStreams.swift

@@ -8,6 +8,12 @@ import Foundation
 /// Keep in lockstep with the Android `EXPECTED_UNHANDLED_HISTORICAL_TYPES`.
 let expectedUnhandledHistoricalTypes: Set<String> = ["METADATA", "CONSOLE_LOGS"]
 
+/// How stale the strap RTC must be before the `(wall - device)` offset is applied to a historical
+/// record's own unix second. BELOW this the offset is discarded and the raw stamp is kept verbatim, so
+/// an everyday drift of seconds-to-minutes never reaches stored timestamps. Shared with `ChunkClockDiag`
+/// so the strap log's `corr=on|off` cannot disagree with what the decoder actually did.
+public let histStaleClockThresholdSec = 86_400   // 1 day
+
 /// Shared plausibility bounds for a type-47 record's own unix timestamp (#547). A WHOOP strap with a
 /// bad clock/flash (repeated trim=0xFFFFFFFF no-cursor) emits records whose decoded unix is scattered
 /// garbage — far-past (2024/2029), a bogus 2027=1827642881, and even FUTURE dates. NOOP used to trust
@@ -159,7 +165,7 @@ public func extractHistoricalStreams(_ parsed: [ParsedFrame],
     // grid so the same record re-syncs to the SAME corrected ts (offloaded rows dedupe by (deviceId, ts);
     // an un-snapped, slightly-different offset on re-sync would duplicate every row). For a normal or
     // identity clockRef the offset is ~0 (< threshold) → rawTs is returned unchanged (current behavior).
-    let staleThreshold = 86_400          // 1 day
+    let staleThreshold = histStaleClockThresholdSec
     let snapGranularity = 300            // 5 min
     let clockOffset = wallClockRef - deviceClockRef
     // The wall "now" the plausibility gate's FUTURE bound measures against. A record genuinely can't

Packages/WhoopProtocol/Tests/WhoopProtocolTests/ChunkClockDiagTests.swift

@@ -0,0 +1,86 @@
+import XCTest
+@testable import WhoopProtocol
+
+/// #1008: the per-chunk clock/packing diag. These assert the two readings the line exists to
+/// distinguish — packing vs duplication — and that `corr` agrees with the decoder's own gate.
+final class ChunkClockDiagTests: XCTestCase {
+
+    func testNoRrYieldsNoLine() {
+        XCTAssertNil(ChunkClockDiag.line(chunk: 1, deviceClockRef: 1_000, wallClockRef: 1_000,
+                                         rrTimestamps: []))
+    }
+
+    /// One interval per second at a steady rate: `pack` and `dens` both sit at 1.00 — the honest case.
+    func testHonestStreamPacksAndDensifiesAtOne() {
+        let ts = Array(1_700_000_000 ..< 1_700_000_010)
+        let line = ChunkClockDiag.line(chunk: 1, deviceClockRef: 1_000, wallClockRef: 1_000,
+                                       rrTimestamps: ts)
+        XCTAssertEqual(line, "Backfill: hist clock chunk=1 offset=+0s staleGate=closed"
+                       + " rr=10 secs=10 pack=1.00 max=1 span=10s dens=1.00")
+    }
+
+    /// PACKING — the shape actually observed on 4.0 (`ord` 0...7 on one second). Every interval of a
+    /// record lands on that record's single stamp, so `pack`/`max` blow up while `dens` stays at the true
+    /// beat rate. This is what must NOT be misread as re-delivery.
+    func testPackedRecordShowsHighPackButHonestDensity() {
+        // 3 records, 10s apart, 8 intervals each — 24 beats over a 21s span (~1.14 beats/s).
+        var ts: [Int] = []
+        for record in 0 ..< 3 {
+            ts.append(contentsOf: Array(repeating: 1_700_000_000 + record * 10, count: 8))
+        }
+        let line = ChunkClockDiag.line(chunk: 4, deviceClockRef: 1_000, wallClockRef: 1_048,
+                                       rrTimestamps: ts)
+        XCTAssertEqual(line, "Backfill: hist clock chunk=4 offset=+48s staleGate=closed"
+                       + " rr=24 secs=3 pack=8.00 max=8 span=21s dens=1.14")
+    }
+
+    /// DUPLICATION — the same seconds delivered twice moves `pack` AND `dens` together (both double),
+    /// which is how a strap log tells the two apart at a glance.
+    func testDuplicatedDeliveryMovesPackAndDensityTogether() {
+        let once = Array(1_700_000_000 ..< 1_700_000_010)
+        let line = ChunkClockDiag.line(chunk: 2, deviceClockRef: 1_000, wallClockRef: 1_000,
+                                       rrTimestamps: once + once)
+        XCTAssertEqual(line, "Backfill: hist clock chunk=2 offset=+0s staleGate=closed"
+                       + " rr=20 secs=10 pack=2.00 max=2 span=10s dens=2.00")
+    }
+
+    /// `staleGate` must mirror `extractHistoricalStreams`: an everyday drift of minutes is DISCARDED by the
+    /// decoder, so it must not read as a correction the stored stamps never received.
+    func testStaleGateClosedBelowThresholdAndOpenAboveIt() {
+        let ts = [1_700_000_000]
+        let below = ChunkClockDiag.line(chunk: 1, deviceClockRef: 0,
+                                        wallClockRef: histStaleClockThresholdSec, rrTimestamps: ts)
+        XCTAssertTrue(below!.contains("staleGate=closed"), "at exactly the threshold the decoder keeps rawTs")
+        let above = ChunkClockDiag.line(chunk: 1, deviceClockRef: 0,
+                                        wallClockRef: histStaleClockThresholdSec + 1, rrTimestamps: ts)
+        XCTAssertTrue(above!.contains("staleGate=open"))
+    }
+
+    /// A strap RTC AHEAD of the phone must render as a negative offset, not an unsigned number — the
+    /// trajectory across chunks is only readable if the sign survives.
+    func testNegativeOffsetKeepsItsSign() {
+        let line = ChunkClockDiag.line(chunk: 7, deviceClockRef: 1_200, wallClockRef: 1_000,
+                                       rrTimestamps: [1_700_000_000])
+        XCTAssertTrue(line!.contains("offset=-200s"), line!)
+    }
+
+    /// PARITY REGRESSION. `pack` of 9/8 is an exact binary tie (1.125). `printf("%.2f")` rounds
+    /// half-to-EVEN and would render `1.12`, while Kotlin's `String.format` rounds half-UP and renders
+    /// `1.13` — the two strap logs would silently disagree. Both platforms now round half-UP by integer
+    /// math, so this asserts `1.13` and its Kotlin twin asserts the identical string.
+    func testExactTieRoundsHalfUpNotHalfEven() {
+        // 9 intervals over 8 stamped seconds: one second carries 2, the rest 1.
+        var ts = Array(1_700_000_000 ..< 1_700_000_008)
+        ts.append(1_700_000_000)
+        let line = ChunkClockDiag.line(chunk: 1, deviceClockRef: 0, wallClockRef: 0, rrTimestamps: ts)
+        XCTAssertTrue(line!.contains("pack=1.13"), line!)
+    }
+
+    /// Timestamps arrive in emission order, which is not guaranteed sorted across a record boundary;
+    /// the span must come from the true min/max, not the first/last element.
+    func testSpanUsesMinAndMaxNotFirstAndLast() {
+        let line = ChunkClockDiag.line(chunk: 1, deviceClockRef: 0, wallClockRef: 0,
+                                       rrTimestamps: [1_700_000_005, 1_700_000_000, 1_700_000_002])
+        XCTAssertTrue(line!.contains("span=6s"), line!)
+    }
+}

Strand/Collect/Backfiller.swift

@@ -132,6 +132,9 @@ final class Backfiller {
     private(set) var sessionClockDevice: Int?
     private(set) var sessionClockWall: Int?
     private(set) var sessionUsedIdentityRef = false
+    /// #1008: 1-based chunk counter for the per-chunk `hist clock` diag line, so a strap log can be read
+    /// as a trajectory across one offload. Reset per session alongside the clock ref above.
+    private var chunkIndex = 0
     /// Logged once per session when the strap reports trim=0xFFFFFFFF — the "no valid flash cursor"
     /// sentinel: it has no banked history to offload (a clock/charge state, not a decode bug).
     private var loggedNoCursor = false
@@ -261,6 +264,7 @@ final class Backfiller {
         sessionClockDevice = nil          // #67: re-capture the decode clock ref for this session
         sessionClockWall = nil
         sessionUsedIdentityRef = false
+        chunkIndex = 0
         loggedNoCursor = false
         loggedFutureRtc = false
         sessionDroppedImplausible = 0
@@ -481,6 +485,16 @@ final class Backfiller {
                 return DecodedChunk(parsed: parsed, decoded: decoded, rejected: rejected)
             }.value
             let parsed = d.parsed
+            // #1008: per-chunk clock basis + R-R packing. The session summary logs only the FIRST chunk's
+            // correlation, which cannot show the offset moving across a long offload nor separate "the same
+            // beats arrived twice" from "one record stamped 8 intervals on one second". Log-only.
+            chunkIndex += 1
+            if let l = ChunkClockDiag.line(chunk: chunkIndex,
+                                           deviceClockRef: ref.device,
+                                           wallClockRef: ref.wall,
+                                           rrTimestamps: d.decoded.rr.map(\.ts)) {
+                log?(l)
+            }
             // Observability (PR #241): log which layout this strap emits on a HEALTHY sync too — the
             // unmapped-version path below only fires for layouts NOOP can't decode, so a normal log
             // never revealed v18/v24/v25/v26. Once per distinct layout this session.

android/app/src/main/java/com/noop/ble/Backfiller.kt

@@ -6,6 +6,7 @@ import com.noop.data.InsertCounts
 import com.noop.data.StreamBatch
 import com.noop.data.WhoopRepository
 import com.noop.protocol.BadClockDiagnostics
+import com.noop.protocol.ChunkClockDiag
 import com.noop.protocol.DeviceFamily
 import com.noop.protocol.Framing
 import com.noop.protocol.HistoricalMeta
@@ -253,6 +254,9 @@ class Backfiller(
      * [begin]; each distinct layout is logged once per session. (PR #241, ryanbr.)
      */
     private val loggedLayoutVersions = HashSet<Int>()
+    /** #1008: 1-based chunk counter for the per-chunk `hist clock` diag line, so a strap log can be read
+     *  as a trajectory across one offload. Reset per session. Twin of the Swift `chunkIndex`. */
+    private var chunkIndex = 0
 
     /** SpO2 RE dump (PR #945, reimplemented): how many full-record dumps this session emitted, bounded by
      *  [com.noop.analytics.Spo2ReTrace.MAX_SAMPLES]. Session-scoped so the cap spans chunks; reset in begin. */
@@ -315,6 +319,7 @@ class Backfiller(
         loggedNoCursor = false
         loggedFutureRtc = false
         loggedLayoutVersions.clear()
+        chunkIndex = 0
         spo2Dumped = 0
         loggedImplausibleClock = false
         sessionDroppedImplausible = 0
@@ -397,6 +402,13 @@ class Backfiller(
                 sessionOldestUnix = sessionOldestUnix, sessionNewestUnix = sessionNewestUnix,
                 ppgHrSubLagInterp = ppgHrSubLagInterp(),
             )
+            // #1008: per-chunk clock basis + R-R packing. The session summary logs only the FIRST chunk's
+            // correlation, which cannot show the offset moving across a long offload nor separate "the same
+            // beats arrived twice" from "one record stamped 8 intervals on one second". Log-only. Twin of
+            // the Swift Backfiller emit.
+            chunkIndex += 1
+            ChunkClockDiag.line(chunkIndex, ref.device, ref.wall, decoded.rr.map { it.ts })
+                ?.let { log(it) }
             // Observability (PR #241): which historical layout does this strap emit? Only the unmapped/
             // reject path logged a version before, so a healthy sync never revealed v24/v25 (4.0) or
             // v18/v26 (5/MG). Sample the chunk's first genuine record (null ⇒ console/CRC-fail); log