From 9f10cf204ec4207be1f088b5dff22b4059fb94bf Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:36:17 -0700 Subject: [PATCH] fix(selfhost): phase-align the cron scheduler to wall-clock boundaries Cloudflare's own */2 * * * * cron trigger fires exactly on wall-clock 2-minute boundaries (:00, :02, :04, ...), which every minute-gated job in enqueueScheduledJobs (minute % 10 === 0, minute === 0, minute % 30 === 0 -- all even) depends on to ever run. The self-host entrypoint's plain setInterval instead ticked every CRON_INTERVAL_MS from whatever moment the container booted, with no relation to wall-clock boundaries -- and since the interval evenly divides an hour, that locks every tick to a FIXED minute parity for the container's entire lifetime. A container booting in an odd minute then ticks ONLY on odd minutes forever, so refresh-registry, ops-alerts, sweep-watchdog, backfill-registered-repos, and both reconciliation sweeps silently NEVER fire. Confirmed live on edge-nl-01: the app container booted at :49 (odd) and ran 3+ hours of on-schedule ~2-minute ticks with zero occurrences of any minute-gated job, while the unconditional every-tick sweep ran normally the whole time. Phase-align the first tick to the next true wall-clock boundary (computed from epoch, itself minute-aligned) with a one-shot setTimeout, then hand off to setInterval from that aligned moment. --- src/selfhost/cron-alignment.ts | 13 ++++++++++ src/server.ts | 24 +++++++++++++++--- test/unit/selfhost-cron-alignment.test.ts | 31 +++++++++++++++++++++++ 3 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 src/selfhost/cron-alignment.ts create mode 100644 test/unit/selfhost-cron-alignment.test.ts diff --git a/src/selfhost/cron-alignment.ts b/src/selfhost/cron-alignment.ts new file mode 100644 index 0000000000..a774766a42 --- /dev/null +++ b/src/selfhost/cron-alignment.ts @@ -0,0 +1,13 @@ +/** Milliseconds from `nowMs` until the next wall-clock boundary of `intervalMs`, so a self-host `setTimeout` + * can phase-align its first tick to the same instants Cloudflare's own cron trigger would fire on (e.g. the + * every-2-minutes trigger fires exactly at :00, :02, :04, … UTC). Computed against epoch -- itself minute-aligned -- + * rather than the caller's own boot time, since `nowMs % intervalMs` only lands on true minute boundaries + * (matching what `enqueueScheduledJobs`'s `getUTCMinutes()`-based gates check) when measured from a fixed, + * minute-aligned origin; measuring from an arbitrary boot moment would just reproduce the exact bug this + * exists to fix (see server.ts's cron setup). Exactly on a boundary already (`nowMs % intervalMs === 0`) + * waits a FULL intervalMs rather than firing immediately, matching `setInterval`'s own "no immediate first + * fire" semantics the caller is replacing. */ +export function delayToNextWallClockBoundaryMs(nowMs: number, intervalMs: number): number { + const msIntoCycle = nowMs % intervalMs; + return msIntoCycle === 0 ? intervalMs : intervalMs - msIntoCycle; +} diff --git a/src/server.ts b/src/server.ts index 47461d93c2..8b2b69ae08 100644 --- a/src/server.ts +++ b/src/server.ts @@ -60,6 +60,7 @@ import { import { clockSkewSampleAgeSeconds, clockSkewSecondsSample } from "./selfhost/clock-skew"; import { d1DatabaseSizeBytesSample, d1SignalSnapshotsRowsPerKeySample, d1TableRowCountSamples, isD1SizeProbeEnabled, runD1SizeProbe } from "./selfhost/d1-size-probe"; import { gauge, gaugeVector, incr, observe, renderMetrics, setSelfHostedMetricsMode } from "./selfhost/metrics"; +import { delayToNextWallClockBoundaryMs } from "./selfhost/cron-alignment"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter, tuneGithubRateLimitObservationsAutovacuum, widenGithubIdColumnsToBigint } from "./selfhost/pg-adapter"; import { createPgQueue } from "./selfhost/pg-queue"; @@ -1090,10 +1091,23 @@ async function main(): Promise { backend.queue.start(); - // Cron — loopover ticks ~every 2 minutes; drive the SAME scheduled handler. + // Cron — loopover ticks ~every 2 minutes; drive the SAME scheduled handler. Cloudflare's own `*/2 * * * *` + // trigger fires exactly on wall-clock 2-minute boundaries (:00, :02, :04, …), which is what + // enqueueScheduledJobs's minute-gated jobs (`minute % 10 === 0`, `minute === 0`, `minute % 30 === 0` — all + // even) rely on to ever run. A plain `setInterval(fn, intervalMs)` instead ticks every intervalMs FROM + // WHATEVER MOMENT THE CONTAINER BOOTED, with no relation to wall-clock boundaries — and since intervalMs + // evenly divides an hour, that locks the tick's minute value to a FIXED parity for the container's entire + // lifetime. A container that happens to boot in an odd minute then ticks ONLY on odd minutes forever, so + // every minute-gated job above silently NEVER fires — confirmed live on edge-nl-01 (booted at an odd + // minute: 3+ hours of ~2-min ticks with zero refresh-registry/ops-alerts/sweep-watchdog/reconciliation + // dispatches, while the unconditional every-tick sweep ran normally). Phase-align the FIRST tick to the + // next true wall-clock boundary — computed from epoch, which is itself minute-aligned, so `Date.now() % + // intervalMs` lands on the same boundaries Cloudflare's cron would for any intervalMs that evenly divides + // an hour (the default 120_000 included) — with a one-shot setTimeout, then hand off to setInterval from + // that aligned moment so every subsequent tick keeps landing on those boundaries. const intervalMs = Number(process.env.CRON_INTERVAL_MS ?? 120_000); /* v8 ignore start -- self-host entrypoint timers start a live server; monitor semantics are covered in selfhost tests. */ - const cron = setInterval(() => { + const runCronTick = (): void => { const controller = { scheduledTime: Date.now(), cron: "*/2 * * * *", @@ -1110,7 +1124,11 @@ async function main(): Promise { }), ), ); - }, intervalMs); + }; + let cron: NodeJS.Timeout = setTimeout(() => { + runCronTick(); + cron = setInterval(runCronTick, intervalMs); + }, delayToNextWallClockBoundaryMs(Date.now(), intervalMs)); /* v8 ignore stop */ // Orb fleet-telemetry export — ALWAYS ON (the fleet-calibration contract of self-hosting). Self-gates diff --git a/test/unit/selfhost-cron-alignment.test.ts b/test/unit/selfhost-cron-alignment.test.ts new file mode 100644 index 0000000000..7b21b4cd85 --- /dev/null +++ b/test/unit/selfhost-cron-alignment.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { delayToNextWallClockBoundaryMs } from "../../src/selfhost/cron-alignment"; + +const TWO_MINUTES_MS = 120_000; + +describe("delayToNextWallClockBoundaryMs (self-host cron phase alignment)", () => { + it("returns the delay to the next boundary when booting mid-cycle", () => { + // 2026-07-21T20:49:04.768Z -- the exact odd-minute boot moment observed live on edge-nl-01, which + // (with a plain, unaligned setInterval) locked every subsequent tick to odd minutes forever. + const bootMs = Date.parse("2026-07-21T20:49:04.768Z"); + const delay = delayToNextWallClockBoundaryMs(bootMs, TWO_MINUTES_MS); + const firstTickMs = bootMs + delay; + expect(new Date(firstTickMs).getUTCMinutes() % 2).toBe(0); + expect(delay).toBeGreaterThan(0); + expect(delay).toBeLessThanOrEqual(TWO_MINUTES_MS); + }); + + it("waits a full interval when already exactly on a boundary, matching setInterval's no-immediate-fire semantics", () => { + const onBoundaryMs = Date.parse("2026-07-21T20:50:00.000Z"); + expect(delayToNextWallClockBoundaryMs(onBoundaryMs, TWO_MINUTES_MS)).toBe(TWO_MINUTES_MS); + }); + + it("aligns to a minute-10/30 boundary regardless of which second within the minute it boots", () => { + const bootMs = Date.parse("2026-07-21T20:57:43.219Z"); + const delay = delayToNextWallClockBoundaryMs(bootMs, TWO_MINUTES_MS); + const firstTick = new Date(bootMs + delay); + expect(firstTick.getUTCMinutes()).toBe(58); + expect(firstTick.getUTCSeconds()).toBe(0); + expect(firstTick.getUTCMilliseconds()).toBe(0); + }); +});