Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/selfhost/cron-alignment.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
/** Milliseconds from `nowMs` until the next wall-clock boundary of `intervalMs`, so a self-host `setTimeout`
* can phase-align its first tick to the same instants Cloudflare's own cron trigger would fire on (e.g. the
* every-2-minutes trigger fires exactly at :00, :02, :04, … UTC). Computed against epoch -- itself minute-aligned --
* rather than the caller's own boot time, since `nowMs % intervalMs` only lands on true minute boundaries
* (matching what `enqueueScheduledJobs`'s `getUTCMinutes()`-based gates check) when measured from a fixed,
* minute-aligned origin; measuring from an arbitrary boot moment would just reproduce the exact bug this
* exists to fix (see server.ts's cron setup). Exactly on a boundary already (`nowMs % intervalMs === 0`)
* waits a FULL intervalMs rather than firing immediately, matching `setInterval`'s own "no immediate first
* fire" semantics the caller is replacing. */
export function delayToNextWallClockBoundaryMs(nowMs: number, intervalMs: number): number {
const msIntoCycle = nowMs % intervalMs;
return msIntoCycle === 0 ? intervalMs : intervalMs - msIntoCycle;
}
24 changes: 21 additions & 3 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ import {
import { clockSkewSampleAgeSeconds, clockSkewSecondsSample } from "./selfhost/clock-skew";
import { d1DatabaseSizeBytesSample, d1SignalSnapshotsRowsPerKeySample, d1TableRowCountSamples, isD1SizeProbeEnabled, runD1SizeProbe } from "./selfhost/d1-size-probe";
import { gauge, gaugeVector, incr, observe, renderMetrics, setSelfHostedMetricsMode } from "./selfhost/metrics";
import { delayToNextWallClockBoundaryMs } from "./selfhost/cron-alignment";
import { runSelfHostMigrations } from "./selfhost/migrate";
import { createPgAdapter, tuneGithubRateLimitObservationsAutovacuum, widenGithubIdColumnsToBigint } from "./selfhost/pg-adapter";
import { createPgQueue } from "./selfhost/pg-queue";
Expand Down Expand Up @@ -1090,10 +1091,23 @@ async function main(): Promise<void> {

backend.queue.start();

// Cron — loopover ticks ~every 2 minutes; drive the SAME scheduled handler.
// Cron — loopover ticks ~every 2 minutes; drive the SAME scheduled handler. Cloudflare's own `*/2 * * * *`
// trigger fires exactly on wall-clock 2-minute boundaries (:00, :02, :04, …), which is what
// enqueueScheduledJobs's minute-gated jobs (`minute % 10 === 0`, `minute === 0`, `minute % 30 === 0` — all
// even) rely on to ever run. A plain `setInterval(fn, intervalMs)` instead ticks every intervalMs FROM
// WHATEVER MOMENT THE CONTAINER BOOTED, with no relation to wall-clock boundaries — and since intervalMs
// evenly divides an hour, that locks the tick's minute value to a FIXED parity for the container's entire
// lifetime. A container that happens to boot in an odd minute then ticks ONLY on odd minutes forever, so
// every minute-gated job above silently NEVER fires — confirmed live on edge-nl-01 (booted at an odd
// minute: 3+ hours of ~2-min ticks with zero refresh-registry/ops-alerts/sweep-watchdog/reconciliation
// dispatches, while the unconditional every-tick sweep ran normally). Phase-align the FIRST tick to the
// next true wall-clock boundary — computed from epoch, which is itself minute-aligned, so `Date.now() %
// intervalMs` lands on the same boundaries Cloudflare's cron would for any intervalMs that evenly divides
// an hour (the default 120_000 included) — with a one-shot setTimeout, then hand off to setInterval from
// that aligned moment so every subsequent tick keeps landing on those boundaries.
const intervalMs = Number(process.env.CRON_INTERVAL_MS ?? 120_000);
/* v8 ignore start -- self-host entrypoint timers start a live server; monitor semantics are covered in selfhost tests. */
const cron = setInterval(() => {
const runCronTick = (): void => {
const controller = {
scheduledTime: Date.now(),
cron: "*/2 * * * *",
Expand All @@ -1110,7 +1124,11 @@ async function main(): Promise<void> {
}),
),
);
}, intervalMs);
};
let cron: NodeJS.Timeout = setTimeout(() => {
runCronTick();
cron = setInterval(runCronTick, intervalMs);
}, delayToNextWallClockBoundaryMs(Date.now(), intervalMs));
/* v8 ignore stop */

// Orb fleet-telemetry export — ALWAYS ON (the fleet-calibration contract of self-hosting). Self-gates
Expand Down
31 changes: 31 additions & 0 deletions test/unit/selfhost-cron-alignment.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import { describe, expect, it } from "vitest";
import { delayToNextWallClockBoundaryMs } from "../../src/selfhost/cron-alignment";

const TWO_MINUTES_MS = 120_000;

describe("delayToNextWallClockBoundaryMs (self-host cron phase alignment)", () => {
it("returns the delay to the next boundary when booting mid-cycle", () => {
// 2026-07-21T20:49:04.768Z -- the exact odd-minute boot moment observed live on edge-nl-01, which
// (with a plain, unaligned setInterval) locked every subsequent tick to odd minutes forever.
const bootMs = Date.parse("2026-07-21T20:49:04.768Z");
const delay = delayToNextWallClockBoundaryMs(bootMs, TWO_MINUTES_MS);
const firstTickMs = bootMs + delay;
expect(new Date(firstTickMs).getUTCMinutes() % 2).toBe(0);
expect(delay).toBeGreaterThan(0);
expect(delay).toBeLessThanOrEqual(TWO_MINUTES_MS);
});

it("waits a full interval when already exactly on a boundary, matching setInterval's no-immediate-fire semantics", () => {
const onBoundaryMs = Date.parse("2026-07-21T20:50:00.000Z");
expect(delayToNextWallClockBoundaryMs(onBoundaryMs, TWO_MINUTES_MS)).toBe(TWO_MINUTES_MS);
});

it("aligns to a minute-10/30 boundary regardless of which second within the minute it boots", () => {
const bootMs = Date.parse("2026-07-21T20:57:43.219Z");
const delay = delayToNextWallClockBoundaryMs(bootMs, TWO_MINUTES_MS);
const firstTick = new Date(bootMs + delay);
expect(firstTick.getUTCMinutes()).toBe(58);
expect(firstTick.getUTCSeconds()).toBe(0);
expect(firstTick.getUTCMilliseconds()).toBe(0);
});
});