diff --git a/src/selfhost/clock-skew.ts b/src/selfhost/clock-skew.ts index 90ef04d529..03f9385788 100644 --- a/src/selfhost/clock-skew.ts +++ b/src/selfhost/clock-skew.ts @@ -8,6 +8,10 @@ // outbound request, sampled at exactly the cadence the vulnerable code path itself runs. let lastSkewSeconds = 0; +// The wall-clock time (ms) of the last SUCCESSFUL sample, or null before the first one. Backs the staleness +// signal below so an old sample can't silently look current if token-mint activity — the only thing that +// refreshes lastSkewSeconds — stalls (#7000). +let lastSkewSampleAtMs: number | null = null; /** * Update the last-observed clock-skew sample from a GitHub response's `Date` header. Positive means @@ -20,7 +24,9 @@ export function recordClockSkewFromResponse(response: Response): void { if (!dateHeader) return; const remoteMs = Date.parse(dateHeader); if (!Number.isFinite(remoteMs)) return; - lastSkewSeconds = (Date.now() - remoteMs) / 1000; + const localMs = Date.now(); + lastSkewSeconds = (localMs - remoteMs) / 1000; + lastSkewSampleAtMs = localMs; } /** The most recently observed clock-skew sample in seconds (0 until the first successful sample). */ @@ -28,7 +34,18 @@ export function clockSkewSecondsSample(): number { return lastSkewSeconds; } +/** + * Seconds since the last successful clock-skew sample, or a -1 sentinel when none has landed yet — the same + * "never sampled" convention as {@link d1DatabaseSizeBytesSample} (src/selfhost/d1-size-probe.ts). Lets an + * operator tell a fresh reading apart from an old sample the token-mint path simply hasn't refreshed (#7000). + */ +export function clockSkewSampleAgeSeconds(): number { + if (lastSkewSampleAtMs === null) return -1; + return (Date.now() - lastSkewSampleAtMs) / 1000; +} + /** Test-only: reset the module-level sample between tests. */ export function resetClockSkewForTest(): void { lastSkewSeconds = 0; + lastSkewSampleAtMs = null; } diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index ed0b63af2d..7767cb9aa7 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -52,6 +52,7 @@ export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["loopover_github_rest_rate_limit_remaining", { help: "Newest observed GitHub REST rate-limit remaining count, by key scope.", type: "gauge" }], ["loopover_host_load_avg1_per_core", { help: "One-minute host load average normalized by CPU core count.", type: "gauge" }], ["loopover_clock_skew_seconds", { help: "Clock skew in seconds between this process and GitHub's server time (positive = ahead), sampled from GitHub App JWT-mint response Date headers.", type: "gauge" }], + ["loopover_clock_skew_sample_age_seconds", { help: "Seconds since the last successful clock-skew sample (loopover_clock_skew_seconds); -1 when no sample has landed yet, so a stale reading is distinguishable from a fresh one.", type: "gauge" }], ["loopover_uptime_seconds", { help: "Self-host process uptime in seconds.", type: "gauge" }], ["loopover_backup_acknowledged", { help: "1 when SQLite backup is acknowledged or Postgres is in use; 0 when the boot backup advisory would fire.", type: "gauge" }], ["loopover_config_dir_empty_acknowledged", { help: "1 when LOOPOVER_REPO_CONFIG_DIR is unset, has entries, or is acknowledged; 0 when it's configured but the mounted directory is empty.", type: "gauge" }], diff --git a/src/server.ts b/src/server.ts index 76554442d5..476d35a71b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -57,7 +57,7 @@ import { sqliteBackupAdvisory, type ReadinessProbe, } from "./selfhost/health"; -import { clockSkewSecondsSample } from "./selfhost/clock-skew"; +import { clockSkewSampleAgeSeconds, clockSkewSecondsSample } from "./selfhost/clock-skew"; import { d1DatabaseSizeBytesSample, d1SignalSnapshotsRowsPerKeySample, d1TableRowCountSamples, isD1SizeProbeEnabled, runD1SizeProbe } from "./selfhost/d1-size-probe"; import { gauge, gaugeVector, incr, observe, renderMetrics, setSelfHostedMetricsMode } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; @@ -828,6 +828,9 @@ async function main(): Promise { // from "no signal on this platform" (see host-pressure.ts). gauge("loopover_host_load_avg1_per_core", async () => (await maintenancePressure()).hostLoadAvg1PerCore ?? -1); gauge("loopover_clock_skew_seconds", () => clockSkewSecondsSample()); + // Companion staleness gauge (#7000): -1 until the first sample, then the sample's age in seconds, so an old + // clock-skew reading (token-mint activity stalled) is distinguishable from a fresh one on the dashboard. + gauge("loopover_clock_skew_sample_age_seconds", () => clockSkewSampleAgeSeconds()); // D1 size/row-count observability probe (#3810): opt-in Cloudflare Management API poll for the shared // cloud D1's file size and monitored-table row counts. Always registered (byte-identical -1/empty samples // when the probe is disabled or has never completed) so the metric names/HELP/TYPE lines are present on diff --git a/test/unit/clock-skew.test.ts b/test/unit/clock-skew.test.ts index b3ad6ff2a5..0cf72b347f 100644 --- a/test/unit/clock-skew.test.ts +++ b/test/unit/clock-skew.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { clockSkewSecondsSample, recordClockSkewFromResponse, resetClockSkewForTest } from "../../src/selfhost/clock-skew"; +import { clockSkewSampleAgeSeconds, clockSkewSecondsSample, recordClockSkewFromResponse, resetClockSkewForTest } from "../../src/selfhost/clock-skew"; beforeEach(() => resetClockSkewForTest()); afterEach(() => vi.useRealTimers()); @@ -52,3 +52,36 @@ describe("clock-skew", () => { expect(clockSkewSecondsSample()).toBe(0); }); }); + +describe("clock-skew sample age (#7000)", () => { + it("reports the -1 never-sampled sentinel before any successful sample", () => { + expect(clockSkewSampleAgeSeconds()).toBe(-1); + }); + + it("reports the sample age in seconds after a successful sample, growing as time passes", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-06T12:05:00.000Z")); + recordClockSkewFromResponse(new Response(null, { headers: { date: "Mon, 06 Jul 2026 12:00:00 GMT" } })); + expect(clockSkewSampleAgeSeconds()).toBe(0); // just sampled — no time has passed yet + vi.setSystemTime(new Date("2026-07-06T12:05:30.000Z")); + expect(clockSkewSampleAgeSeconds()).toBe(30); // 30s later, the same sample is now 30s old + }); + + it("does not advance the sample time when a response is ignored — age keeps growing from the last good sample", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-06T12:00:00.000Z")); + recordClockSkewFromResponse(new Response(null, { headers: { date: "Mon, 06 Jul 2026 12:00:00 GMT" } })); + vi.setSystemTime(new Date("2026-07-06T12:01:00.000Z")); + recordClockSkewFromResponse(new Response(null)); // no Date header — ignored, must not reset the sample time + expect(clockSkewSampleAgeSeconds()).toBe(60); // still measured from the 12:00:00 sample, not "just now" + }); + + it("resetClockSkewForTest restores the age to the -1 never-sampled sentinel", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-06T12:00:00.000Z")); + recordClockSkewFromResponse(new Response(null, { headers: { date: "Mon, 06 Jul 2026 12:00:00 GMT" } })); + expect(clockSkewSampleAgeSeconds()).not.toBe(-1); + resetClockSkewForTest(); + expect(clockSkewSampleAgeSeconds()).toBe(-1); + }); +});