From 730700eafd2316bebcb7aed6f58588b8b2ecf453 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:20:49 -0700 Subject: [PATCH 1/4] perf(relay): delete expired tunnels four at a time within a time budget Each deletion is a row-locked database transaction plus two Cloudflare calls. Run one at a time, a full budget of 100 deletions could outlast the cron's two-minute timeout, which discards the sweep's counters. Run deletions four at a time and stop starting new ones after 90 seconds. A rate limit still stops new deletions; ones already running finish. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/operations/release.md | 6 +- .../ManagedEndpointReaper.test.ts | 62 ++++++++++ .../src/environments/ManagedEndpointReaper.ts | 113 ++++++++++++------ 3 files changed, 144 insertions(+), 37 deletions(-) diff --git a/docs/operations/release.md b/docs/operations/release.md index c18752519960..e5ea4f8c9c5e 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -210,8 +210,10 @@ A deleted legacy tunnel keeps its allocation, so its hostname is kept. When the 3. Run the legacy steps of the disposable-host canary below. 4. Before enabling, confirm the web and mobile builds that show the "update T3 Code on that computer" message are live. Without them, a user whose older host lost its tunnel only sees it as offline. -5. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, so a backlog of - 20,000 takes about 17 hours. Watch `deletedLegacy`, `failed`, and `truncated`. +5. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, four at a time, and + stops starting new deletions after 90 seconds. A backlog of 20,000 takes about 17 hours if each + sweep finishes its 100. Watch `deletedLegacy`, `attempted`, `failed`, and `truncated`; an + `attempted` well under 100 means sweeps are running out of time. ### Disposable-host canary diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index 60481191da9e..af789efd5220 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -75,6 +75,8 @@ function harness(input?: { readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode; readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode; readonly legacyTunnelGraceMinutes?: number; + /** Simulated time each release takes, advanced on the test clock. */ + readonly releaseDelayMs?: number; }) { const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; const deleted: string[] = []; @@ -191,6 +193,9 @@ function harness(input?: { release: (request) => Effect.gen(function* () { releases.push(request); + if (input?.releaseDelayMs !== undefined) { + yield* TestClock.adjust(input.releaseDelayMs); + } if (request.expectedTunnelId === input?.skipTunnelId) { return false; } @@ -689,6 +694,63 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("starts no new deletion after a rate limit, even with deletions in flight", () => { + // Enough candidates to fill every concurrent slot several times over. + const entries = Array.from({ length: 12 }, (_, index) => + tunnel({ + id: index === 0 ? "limited" : `ok-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: recoverableOwners(entries), + rateLimitedTunnelId: "limited", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + const result = yield* reaper.sweep; + expect(result.truncated).toBe(true); + expect(result.failed).toBe(1); + // Releases already running may finish, but none start afterwards. + expect(result.attempted).toBeLessThanOrEqual( + ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY, + ); + expect(state.releases.length).toBe(result.attempted); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("stops starting deletions when the sweep's time budget runs out", () => { + const entries = Array.from({ length: 40 }, (_, index) => + tunnel({ + id: `slow-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: recoverableOwners(entries), + // Ten seconds each: the 90-second budget allows about 9 rounds. + releaseDelayMs: 10_000, + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + const result = yield* reaper.sweep; + expect(result.truncated).toBe(true); + expect(result.attempted).toBeGreaterThan(0); + expect(result.attempted).toBeLessThan(entries.length); + expect(result.deleted).toBe(result.attempted); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("continues past a page of older hosts to find recoverable tunnels", () => { const entries = Array.from({ length: 101 }, (_, index) => tunnel({ diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 3d3a2f2056c0..2e7dc2071396 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -1,3 +1,4 @@ +import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -27,6 +28,15 @@ const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const; // returning host that has updated recovers the tunnel at the same hostname; // one that has not sees the client's "update T3 Code" message instead. const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7; +// Deletions run a few at a time: each one is a row-locked database +// transaction plus two Cloudflare calls, so one at a time cannot finish a +// full attempt budget inside the cron's timeout. Each holds a Hyperdrive +// connection while it runs; keep this well under the 20-connection origin +// limit that request handlers share. +export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4; +// Stop starting deletions after this long, leaving room under the cron's +// two-minute timeout to finish in-flight ones and record the counters. +export const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000; export interface ManagedEndpointSweepResult { readonly mode: RelayConfiguration.ManagedEndpointCleanupMode; @@ -275,6 +285,16 @@ export const make = Effect.gen(function* () { let attempted = 0; let deleted = 0; let wouldDelete = 0; + const candidates: Array<{ + readonly owner: ManagedEndpointAllocations.ManagedEndpointTunnelAllocation; + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; + readonly status: "down" | "inactive"; + readonly legacy: boolean; + readonly inactiveBefore: string; + }> = []; let wouldDeleteLegacy = 0; let deletedLegacy = 0; let skippedLegacy = 0; @@ -330,46 +350,69 @@ export const make = Effect.gen(function* () { wouldDelete += 1; if (mode === "dry-run") continue; } - if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) { + if (candidates.length >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) { truncated = true; break; } - attempted += 1; - // The release re-reads the tunnel and deletes only if it is still in - // this status and inactive since before this tunnel's cutoff. - const result = yield* provider - .release({ - userId: owner.userId, - environmentId: owner.environmentId, - expectedTunnelId: tunnel.id, - expectedInactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff), - expectedStatus: status, - }) - .pipe(Effect.result); - if (result._tag === "Failure") { - failed += 1; - yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { - tunnelId: tunnel.id, - tunnelName: tunnel.name, - legacy, - cause: result.failure, - }); - if (isRateLimited(result.failure)) { - truncated = true; - break; - } - } else if (result.success) { - deleted += 1; - if (legacy) deletedLegacy += 1; - yield* Effect.logInfo("Deleted an inactive managed tunnel", { - tunnelId: tunnel.id, - tunnelName: tunnel.name, - status, - legacy, - }); - } + candidates.push({ + owner, + tunnel, + status, + legacy, + // The release re-reads the tunnel and deletes only if it is still in + // this status and inactive since before this cutoff. + inactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff), + }); } + const deleteDeadline = + (yield* Clock.currentTimeMillis) + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS; + let stopDeleting = false; + yield* Effect.forEach( + candidates, + (candidate) => + Effect.gen(function* () { + if (stopDeleting || (yield* Clock.currentTimeMillis) >= deleteDeadline) { + stopDeleting = true; + truncated = true; + return; + } + attempted += 1; + const result = yield* provider + .release({ + userId: candidate.owner.userId, + environmentId: candidate.owner.environmentId, + expectedTunnelId: candidate.tunnel.id, + expectedInactiveBefore: candidate.inactiveBefore, + expectedStatus: candidate.status, + }) + .pipe(Effect.result); + if (result._tag === "Failure") { + failed += 1; + yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { + tunnelId: candidate.tunnel.id, + tunnelName: candidate.tunnel.name, + legacy: candidate.legacy, + cause: result.failure, + }); + if (isRateLimited(result.failure)) { + stopDeleting = true; + truncated = true; + } + } else if (result.success) { + deleted += 1; + if (candidate.legacy) deletedLegacy += 1; + yield* Effect.logInfo("Deleted an inactive managed tunnel", { + tunnelId: candidate.tunnel.id, + tunnelName: candidate.tunnel.name, + status: candidate.status, + legacy: candidate.legacy, + }); + } + }), + { concurrency: MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY, discard: true }, + ); + return { mode, legacyMode, From 0c8c62eec16ae72d742944af1985c09593a89172 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:45:55 -0700 Subject: [PATCH 2/4] fix(relay): count the deletion budget from the start of the sweep The 90-second budget started after listing, so a slow listing could push the sweep past the cron's two-minute timeout, which drops its counters. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../ManagedEndpointReaper.test.ts | 40 ++++++++++++++++++- .../src/environments/ManagedEndpointReaper.ts | 9 +++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts index af789efd5220..c71811a553d0 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.test.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -77,6 +77,8 @@ function harness(input?: { readonly legacyTunnelGraceMinutes?: number; /** Simulated time each release takes, advanced on the test clock. */ readonly releaseDelayMs?: number; + /** Simulated time each Cloudflare list request takes. */ + readonly listDelayMs?: number; }) { const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; const deleted: string[] = []; @@ -121,7 +123,10 @@ function harness(input?: { return Effect.succeed(found); }), list: (request) => - Effect.sync(() => { + Effect.gen(function* () { + if (input?.listDelayMs !== undefined) { + yield* TestClock.adjust(input.listDelayMs); + } listRequests.push(request); const matching = remaining.filter((entry) => entry.status === request.status); const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100); @@ -751,6 +756,39 @@ describe("ManagedEndpointReaper", () => { }).pipe(Effect.provide(state.layer)); }); + it.effect("counts listing time against the deletion budget", () => { + const entries = Array.from({ length: 40 }, (_, index) => + tunnel({ + id: `slow-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const sweepWith = (listDelayMs: number) => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + return (yield* reaper.sweep).attempted; + }).pipe( + Effect.provide( + harness({ + tunnels: entries, + allocations: recoverableOwners(entries), + releaseDelayMs: 10_000, + listDelayMs, + }).layer, + ), + ); + + return Effect.gen(function* () { + const fastListing = yield* sweepWith(0); + // Two list requests of 20 seconds each use 40 of the 90-second budget. + const slowListing = yield* sweepWith(20_000); + expect(slowListing).toBeLessThan(fastListing); + }); + }); + it.effect("continues past a page of older hosts to find recoverable tunnels", () => { const entries = Array.from({ length: 101 }, (_, index) => tunnel({ diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 2e7dc2071396..42a3d3c65e46 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -34,8 +34,9 @@ const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7; // connection while it runs; keep this well under the 20-connection origin // limit that request handlers share. export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4; -// Stop starting deletions after this long, leaving room under the cron's -// two-minute timeout to finish in-flight ones and record the counters. +// Stop starting deletions this long after the sweep starts, leaving room under +// the cron's two-minute timeout to finish in-flight ones and record the +// counters. Counted from sweep start so slow listing eats into it. export const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000; export interface ManagedEndpointSweepResult { @@ -190,6 +191,7 @@ export const make = Effect.gen(function* () { if ((mode === "off" && legacyMode === "off") || !namespace) { return emptyResult(mode, legacyMode); } + const sweepStartedAtMillis = yield* Clock.currentTimeMillis; // The override exists for the disposable canary stage; prod always // waits the full grace period. const legacyGraceMinutes = @@ -365,8 +367,7 @@ export const make = Effect.gen(function* () { }); } - const deleteDeadline = - (yield* Clock.currentTimeMillis) + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS; + const deleteDeadline = sweepStartedAtMillis + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS; let stopDeleting = false; yield* Effect.forEach( candidates, From 5fdf58e25d0859b3a767c6d883bf6fa4b58bdc05 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Thu, 24 Sep 2026 23:21:59 -0700 Subject: [PATCH 3/4] docs(relay): say a short sweep can mean a rate limit or the time budget Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/operations/release.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/operations/release.md b/docs/operations/release.md index e5ea4f8c9c5e..4185f87b2a75 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -212,8 +212,10 @@ A deleted legacy tunnel keeps its allocation, so its hostname is kept. When the message are live. Without them, a user whose older host lost its tunnel only sees it as offline. 5. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, four at a time, and stops starting new deletions after 90 seconds. A backlog of 20,000 takes about 17 hours if each - sweep finishes its 100. Watch `deletedLegacy`, `attempted`, `failed`, and `truncated`; an - `attempted` well under 100 means sweeps are running out of time. + sweep finishes its 100. Watch `deletedLegacy`, `attempted`, `failed`, and `truncated`. An + `attempted` well under 100 with `truncated` set means the sweep stopped early: either the time + budget ran out or Cloudflare rate-limited a deletion. The counters don't say which; the relay + logs a warning with the Cloudflare error for each failed deletion. ### Disposable-host canary From f5d28aba3fe7dd21aee8578c48d27bae86e1e6f7 Mon Sep 17 00:00:00 2001 From: Julius Marminge <51714798+juliusmarminge@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:31:14 -0700 Subject: [PATCH 4/4] refactor(relay): keep the sweep delete budget module-private Co-Authored-By: Claude Opus 5.5 (1M context) --- infra/relay/src/environments/ManagedEndpointReaper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts index 42a3d3c65e46..f1c9ccc84e71 100644 --- a/infra/relay/src/environments/ManagedEndpointReaper.ts +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -37,7 +37,7 @@ export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4; // Stop starting deletions this long after the sweep starts, leaving room under // the cron's two-minute timeout to finish in-flight ones and record the // counters. Counted from sweep start so slow listing eats into it. -export const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000; +const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000; export interface ManagedEndpointSweepResult { readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;