diff --git a/.github/workflows/deploy-relay.yml b/.github/workflows/deploy-relay.yml index 6a55960738..0eab1c22fb 100644 --- a/.github/workflows/deploy-relay.yml +++ b/.github/workflows/deploy-relay.yml @@ -46,6 +46,7 @@ jobs: RELAY_DOMAIN: ${{ vars.RELAY_DOMAIN }} RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }} RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }} + RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }} CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }} CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }} APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }} diff --git a/apps/server/src/cloud/CliState.test.ts b/apps/server/src/cloud/CliState.test.ts index 39f904b47b..d59b89875e 100644 --- a/apps/server/src/cloud/CliState.test.ts +++ b/apps/server/src/cloud/CliState.test.ts @@ -8,6 +8,7 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { ServerConfig } from "../config.ts"; import * as CliState from "./CliState.ts"; import { + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, @@ -24,6 +25,7 @@ const persistedCloudLinkSecrets = [ RELAY_ENVIRONMENT_CREDENTIAL_SECRET, CLOUD_MINT_PUBLIC_KEY, CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, PUBLISH_AGENT_ACTIVITY_SECRET, ] as const; diff --git a/apps/server/src/cloud/CliState.ts b/apps/server/src/cloud/CliState.ts index d4abba7997..17b4bf6598 100644 --- a/apps/server/src/cloud/CliState.ts +++ b/apps/server/src/cloud/CliState.ts @@ -3,6 +3,7 @@ import * as Option from "effect/Option"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, @@ -67,6 +68,7 @@ export const clearPersistedCloudLink = Effect.gen(function* () { secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), secrets.remove(CLOUD_MINT_PUBLIC_KEY), secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), + secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), ], { concurrency: "unbounded" }, diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index d43db4c5c4..71677a43d6 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -5,11 +5,14 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as PlatformError from "effect/PlatformError"; +import * as Queue from "effect/Queue"; import * as Sink from "effect/Sink"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as RelayClient from "@t3tools/shared/relayClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; @@ -61,6 +64,7 @@ function makeHandle(input: { readonly onKill: () => void; readonly isRunning?: () => boolean; readonly exitCode?: Effect.Effect; + readonly output?: Stream.Stream; }) { return ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(input.pid), @@ -74,13 +78,70 @@ function makeHandle(input: { stdin: Sink.drain, stdout: Stream.empty, stderr: Stream.empty, - all: Stream.empty, + all: input.output ?? Stream.empty, getInputFd: () => Sink.drain, getOutputFd: () => Stream.empty, }); } describe("CloudManagedEndpointRuntime", () => { + it("retries connector startup failures but stops for unsupported runtimes", () => { + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + failure: "not-installed", + reason: "The relay client is not installed.", + }), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + failure: "spawn-failed", + reason: "spawn failed", + }), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ + status: "failed", + failure: "unsupported-platform", + reason: "Relay client is unsupported on linux-arm.", + }), + ).toBe(false); + expect( + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus({ status: "unsupported" }), + ).toBe(false); + }); + + it.effect("serializes updates to persisted cloud link state", () => + Effect.gen(function* () { + const firstEntered = yield* Deferred.make(); + const releaseFirst = yield* Deferred.make(); + const secondEntered = yield* Deferred.make(); + const runtime = yield* buildCloudManagedEndpointRuntime( + ChildProcessSpawner.make(() => Effect.die("unused")), + ); + + const first = yield* runtime + .withLinkStateLock( + Deferred.succeed(firstEntered, undefined).pipe( + Effect.andThen(Deferred.await(releaseFirst)), + ), + ) + .pipe(Effect.forkChild); + yield* Deferred.await(firstEntered); + + const second = yield* runtime + .withLinkStateLock(Deferred.succeed(secondEntered, undefined)) + .pipe(Effect.forkChild); + expect(yield* Deferred.isDone(secondEntered)).toBe(false); + + yield* Deferred.succeed(releaseFirst, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + expect(yield* Deferred.isDone(secondEntered)).toBe(true); + }), + ); + it("classifies Cloudflare connection and warning output", () => { expect( ManagedEndpointRuntime.classifyRelayClientOutput( @@ -108,6 +169,125 @@ describe("CloudManagedEndpointRuntime", () => { ).toBe("warning"); }); + it("recognizes tunnel authorization failures without matching ordinary transport errors", () => { + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-09-15T06:30:43Z ERR Register tunnel error from server side error="Failed to get tunnel" connIndex=0 event=0 ip=198.41.200.23', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Record for tunnel not found" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="Unauthorized: Invalid tunnel secret" connIndex=0', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0', + ), + ).toBe(false); + }); + + it.effect("keeps recovery requests sent before the server starts consuming them", () => + Effect.gen(function* () { + const runtime = yield* buildCloudManagedEndpointRuntime( + ChildProcessSpawner.make(() => Effect.die("unused")), + ); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "tunnel-1", + }; + + yield* runtime.requestRecovery(config); + + expect(Option.getOrNull(yield* Stream.runHead(runtime.recoveryRequests))).toEqual(config); + }), + ); + + it.effect("recovers a rejected tunnel without waiting for the connector to exit", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const firstBatchObserved = yield* Deferred.make(); + const secondBatchObserved = yield* Deferred.make(); + const recoveryRequested = yield* Deferred.make(); + const recoveryRetried = yield* Deferred.make(); + let recoveryRequestCount = 0; + const spawned: Array = []; + const encoder = new TextEncoder(); + const connectorOutput = Stream.fromQueue(output).pipe( + Stream.tap((chunk) => { + const line = new TextDecoder().decode(chunk); + if (line === "first checkpoint\n") { + return Deferred.succeed(firstBatchObserved, undefined).pipe(Effect.asVoid); + } + if (line === "second checkpoint\n") { + return Deferred.succeed(secondBatchObserved, undefined).pipe(Effect.asVoid); + } + return Effect.void; + }), + ); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const pid = 600; + spawned.push(pid); + const handle = makeHandle({ pid, onKill: () => {}, output: connectorOutput }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "deleted-tunnel", + }; + const rejectedLine = + '2026-09-15T06:30:43Z ERR Register tunnel error from server side error="Failed to get tunnel" connIndex=0 event=0 ip=198.41.200.23\n'; + + yield* runtime.recoveryRequests.pipe( + Stream.runForEach((requested) => { + recoveryRequestCount += 1; + return Deferred.succeed( + recoveryRequestCount === 1 ? recoveryRequested : recoveryRetried, + requested, + ).pipe(Effect.asVoid); + }), + Effect.forkChild, + ); + yield* runtime.applyConfig(config); + + yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(3))); + yield* Queue.offer(output, encoder.encode("first checkpoint\n")); + yield* Deferred.await(firstBatchObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer( + output, + encoder.encode( + "2026-06-17T02:00:00Z INF Registered tunnel connection connIndex=0\n" + + rejectedLine.repeat(3), + ), + ); + yield* Queue.offer(output, encoder.encode("second checkpoint\n")); + yield* Deferred.await(secondBatchObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer(output, encoder.encode(rejectedLine)); + + expect(yield* Deferred.await(recoveryRequested)).toEqual(config); + + yield* Queue.offer(output, encoder.encode(rejectedLine.repeat(4))); + + expect(yield* Deferred.await(recoveryRetried)).toEqual(config); + expect(spawned).toEqual([600]); + }), + ); + it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () => Effect.gen(function* () { const spawned: Array = []; @@ -155,8 +335,8 @@ describe("CloudManagedEndpointRuntime", () => { expect(spawned.map((command) => command.command)).toEqual(["cloudflared", "cloudflared"]); expect(spawned.map((command) => command.args)).toEqual([ - ["tunnel", "run"], - ["tunnel", "run"], + ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"], + ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"], ]); expect(spawned.map((command) => command.options.env?.TUNNEL_TOKEN)).toEqual([ "token-1", @@ -377,6 +557,37 @@ describe("CloudManagedEndpointRuntime", () => { }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("a recovery that returns the same config keeps the crash backoff", () => + Effect.gen(function* () { + const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(900, 4); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "same-token", + tunnelId: "same-tunnel", + }; + // The startup consumer re-applies whatever the relay hands back. When the + // relay confirms the current tunnel, that must not look like a config change. + yield* runtime.recoveryRequests.pipe( + Stream.runForEach((requested) => runtime.applyConfig(requested).pipe(Effect.asVoid)), + Effect.forkChild, + ); + + yield* runtime.applyConfig(config); + yield* Deferred.succeed(exits[0]!, ChildProcessSpawner.ExitCode(1)); + yield* Deferred.await(spawnSignals[1]!); + expect(spawned).toEqual([900, 901]); + + // Second rapid crash still waits out the base delay. + yield* Deferred.succeed(exits[1]!, ChildProcessSpawner.ExitCode(1)); + yield* TestClock.adjust(Duration.millis(999)); + expect(spawned).toEqual([900, 901]); + yield* TestClock.adjust(Duration.millis(1)); + yield* Deferred.await(spawnSignals[2]!); + expect(spawned).toEqual([900, 901, 902]); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("an explicit config change clears the backoff and preempts a delayed restart", () => Effect.gen(function* () { const { spawner, spawned, exits, spawnSignals } = yield* makeCrashLoopSpawner(800, 3); @@ -509,6 +720,7 @@ describe("CloudManagedEndpointRuntime", () => { expect(status).toEqual({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "not-installed", reason: "The relay client is not installed.", }); expect(spawn).not.toHaveBeenCalled(); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index cc657bdebf..21091c5c44 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -6,7 +6,7 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; -import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Result from "effect/Result"; import * as Semaphore from "effect/Semaphore"; @@ -15,22 +15,6 @@ import * as Stream from "effect/Stream"; import * as ChildProcess from "effect/unstable/process/ChildProcess"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; -import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG, decodeRuntimeConfig } from "./config.ts"; - -function bytesToString(bytes: Uint8Array): string { - return new TextDecoder().decode(bytes); -} - -const readRuntimeConfig = Effect.gen(function* () { - const secrets = yield* ServerSecretStore.ServerSecretStore; - const bytes = yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); - if (Option.isNone(bytes)) { - return null; - } - return Option.getOrNull(decodeRuntimeConfig(bytesToString(bytes.value))); -}); - export type CloudManagedEndpointRuntimeStatus = | { readonly status: "disabled"; @@ -38,6 +22,7 @@ export type CloudManagedEndpointRuntimeStatus = | { readonly status: "failed"; readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"]; + readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed"; readonly reason: string; readonly tunnelId?: string; readonly tunnelName?: string; @@ -60,6 +45,9 @@ export class CloudManagedEndpointRuntime extends Context.Service< readonly applyConfig: ( config: RelayManagedEndpointRuntimeConfig | null, ) => Effect.Effect; + readonly recoveryRequests: Stream.Stream; + readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect; + readonly withLinkStateLock: (effect: Effect.Effect) => Effect.Effect; } >()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {} @@ -79,6 +67,8 @@ interface ActiveConnector { const RELAY_RESTART_STABLE_UPTIME_MS = 30_000; const RELAY_RESTART_BACKOFF_BASE_MS = 1_000; const RELAY_RESTART_BACKOFF_MAX_MS = 60_000; +// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token. +const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4; export function classifyRelayClientOutput(line: string): "connected" | "warning" | "debug" { if (/\bRegistered tunnel connection\b/iu.test(line)) { @@ -90,6 +80,32 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning" return /\b(?:ERR|WRN|FTL|PNC)\b/u.test(line) ? "warning" : "debug"; } +/** + * Cloudflare's edge rejects a connector whose tunnel was deleted or whose + * token no longer matches. Current edge output is + * `error="Failed to get tunnel"` with no prefix; older edges prefixed the + * same messages with `Unauthorized:`. Match both so recovery fires on either. + */ +export function isRejectedRelayClientTunnelOutput(line: string): boolean { + return ( + /\bRegister tunnel error from server side\b/iu.test(line) && + /error="(?:Unauthorized:\s*)?(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test( + line, + ) + ); +} + +/** Connector startup failures can clear after installation or a later spawn attempt. */ +export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean { + if (typeof status !== "object" || status === null || !("status" in status)) { + return false; + } + if (status.status !== "failed" || !("failure" in status)) { + return false; + } + return status.failure === "not-installed" || status.failure === "spawn-failed"; +} + function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string { return JSON.stringify({ providerKind: config.providerKind, @@ -117,8 +133,10 @@ export const make = Effect.gen(function* () { const relayClient = yield* RelayClient.RelayClient; const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); + const recoveryRequests = yield* Queue.sliding(1); const reconcileSemaphore = yield* Semaphore.make(1); const restartDelayRef = yield* Ref.make(0); + const linkStateSemaphore = yield* Semaphore.make(1); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; const stopActive = Effect.gen(function* () { @@ -191,6 +209,7 @@ export const make = Effect.gen(function* () { tunnelId: connector.config.tunnelId, tunnelName: connector.config.tunnelName, }); + yield* Queue.offer(recoveryRequests, connector.config); yield* reconcileConfig(desiredConfig); }), ); @@ -198,8 +217,10 @@ export const make = Effect.gen(function* () { Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })), ); - const observeConnectorOutput = (connector: ActiveConnector) => - connector.child.all.pipe( + const observeConnectorOutput = (connector: ActiveConnector) => { + let rejectedRegistrations = 0; + + return connector.child.all.pipe( Stream.decodeText(), Stream.splitLines, Stream.map((line) => line.trim()), @@ -214,8 +235,22 @@ export const make = Effect.gen(function* () { }; switch (classifyRelayClientOutput(line)) { case "connected": + rejectedRegistrations = 0; return Effect.logInfo("Relay client tunnel connection registered", attributes); case "warning": + if (isRejectedRelayClientTunnelOutput(line)) { + rejectedRegistrations += 1; + if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) { + rejectedRegistrations = 0; + return Effect.logWarning( + "Relay client tunnel was rejected; requesting recovery", + attributes, + ).pipe( + Effect.andThen(Queue.offer(recoveryRequests, connector.config)), + Effect.asVoid, + ); + } + } return Effect.logWarning("Relay client reported a transport warning", attributes); case "debug": return Effect.logDebug("Relay client output", attributes); @@ -230,6 +265,7 @@ export const make = Effect.gen(function* () { }), ), ); + }; reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) { if (!config || config.providerKind !== "cloudflare_tunnel") { @@ -261,6 +297,7 @@ export const make = Effect.gen(function* () { return { status: "failed", providerKind: "cloudflare_tunnel", + failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed", reason: executable.status === "unsupported" ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.` @@ -273,16 +310,20 @@ export const make = Effect.gen(function* () { const connectorScope = yield* Scope.make("sequential"); const child = yield* spawner .spawn( - ChildProcess.make(executable.executablePath, ["tunnel", "run"], { - detached: false, - env: { - ...process.env, - TUNNEL_TOKEN: config.connectorToken, + ChildProcess.make( + executable.executablePath, + ["tunnel", "--no-autoupdate", "--loglevel", "info", "--output", "default", "run"], + { + detached: false, + env: { + ...process.env, + TUNNEL_TOKEN: config.connectorToken, + }, + shell: false, + stderr: "pipe", + stdout: "pipe", }, - shell: false, - stderr: "pipe", - stdout: "pipe", - }), + ), ) .pipe( Effect.provideService(Scope.Scope, connectorScope), @@ -303,6 +344,7 @@ export const make = Effect.gen(function* () { Effect.as({ status: "failed", providerKind: "cloudflare_tunnel", + failure: "spawn-failed", reason: String(cause), ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), @@ -338,6 +380,7 @@ export const make = Effect.gen(function* () { return { status: "failed", providerKind: "cloudflare_tunnel", + failure: "spawn-failed", reason: "Relay client did not start.", ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}), ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), @@ -347,26 +390,31 @@ export const make = Effect.gen(function* () { const applyConfig = Effect.fn("CloudManagedEndpointRuntime.applyConfig")( (config: RelayManagedEndpointRuntimeConfig | null) => reconcileSemaphore.withPermits(1)( - // An explicit config change starts over with a fresh backoff. - Ref.set(restartDelayRef, 0).pipe( - Effect.andThen(Ref.set(desiredConfigRef, config)), - Effect.andThen(reconcileConfig(config)), - ), + Effect.gen(function* () { + // A real config change starts over with a fresh backoff. Recovery + // that hands back the same tunnel and token must keep the delay, or + // a crash-looping connector respawns on every recovery round trip. + const desired = yield* Ref.get(desiredConfigRef); + const unchanged = + desired !== null && + config !== null && + runtimeConfigKey(desired) === runtimeConfigKey(config); + if (!unchanged) { + yield* Ref.set(restartDelayRef, 0); + } + yield* Ref.set(desiredConfigRef, config); + return yield* reconcileConfig(config); + }), ), ); const runtime = CloudManagedEndpointRuntime.of({ applyConfig, + recoveryRequests: Stream.fromQueue(recoveryRequests), + requestRecovery: (config) => Queue.offer(recoveryRequests, config).pipe(Effect.asVoid), + withLinkStateLock: linkStateSemaphore.withPermits(1), }); - const initialConfig = yield* readRuntimeConfig.pipe( - Effect.catch((cause) => - Effect.logWarning("Failed to read managed endpoint runtime config", { cause }).pipe( - Effect.as(null), - ), - ), - ); - yield* runtime.applyConfig(initialConfig); yield* Effect.addFinalizer(() => runtime.applyConfig(null)); return runtime; }); diff --git a/apps/server/src/cloud/config.ts b/apps/server/src/cloud/config.ts index 2eff693f61..9b1b281ba2 100644 --- a/apps/server/src/cloud/config.ts +++ b/apps/server/src/cloud/config.ts @@ -1,4 +1,7 @@ -import { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; +import { + RelayManagedEndpointOrigin, + RelayManagedEndpointRuntimeConfig, +} from "@t3tools/contracts/relay"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; @@ -7,6 +10,7 @@ import type * as ServerSecretStore from "../auth/ServerSecretStore.ts"; export const CLOUD_MINT_PUBLIC_KEY = "cloud-mint-ed25519-public-key"; export const CLOUD_ENDPOINT_RUNTIME_CONFIG = "cloud-endpoint-runtime-config"; +export const CLOUD_ENDPOINT_CONFIRMED_ORIGIN = "cloud-endpoint-confirmed-origin"; export const CLOUD_LINKED_USER_ID = "cloud-linked-user-id"; export const RELAY_URL_SECRET = "cloud-relay-url"; export const RELAY_ISSUER_SECRET = "cloud-relay-issuer"; @@ -21,6 +25,19 @@ export const decodeRuntimeConfig = Schema.decodeUnknownOption( Schema.fromJsonString(RelayManagedEndpointRuntimeConfig), ); +export const ManagedEndpointConfirmedOrigin = Schema.Struct({ + config: RelayManagedEndpointRuntimeConfig, + origin: RelayManagedEndpointOrigin, +}); + +export const encodeConfirmedOriginJson = Schema.encodeEffect( + Schema.fromJsonString(ManagedEndpointConfirmedOrigin), +); + +export const decodeConfirmedOrigin = Schema.decodeUnknownOption( + Schema.fromJsonString(ManagedEndpointConfirmedOrigin), +); + export function isAgentActivityPublishingEnabledValue(value: string | null): boolean { return value === "true"; } diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 481213e3a1..0ee0951ded 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -1,15 +1,19 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import type * as NodeOS from "node:os"; import { describe, expect, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; +import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; import * as Tracer from "effect/Tracer"; +import * as Stream from "effect/Stream"; import { HttpClient, HttpClientResponse, @@ -33,8 +37,19 @@ import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import { CLOUD_CLI_DESIRED_LINK_SECRET } from "./CliState.ts"; import * as CliTokenManager from "./CliTokenManager.ts"; -import type { RelayLinkProofRequest } from "@t3tools/contracts/relay"; -import { CLOUD_ENDPOINT_RUNTIME_CONFIG, RELAY_URL_SECRET } from "./config.ts"; +import { + RelayManagedEndpointRecoveryRegistrationRequest, + type RelayLinkProofRequest, +} from "@t3tools/contracts/relay"; +import { + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, + CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_LINKED_USER_ID, + decodeConfirmedOrigin, + decodeRuntimeConfig, + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + RELAY_URL_SECRET, +} from "./config.ts"; import { clearDesktopUpdateRestartMarker, consumeCloudReplayGuards, @@ -43,9 +58,19 @@ import { isSupportedLinkProviderKind, linkProofScopes, pendingServiceUpdateExists, + parseManagedEndpointLocalOrigin, reconcileDesiredCloudLink, + reconcileDesiredCloudLinkIfStillDesired, + recoverManagedCloudTunnel, + registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, + startManagedCloudTunnelIfOriginConfirmed, } from "./http.ts"; +import { + managedTunnelStartupAction, + retryManagedTunnelRegistration, +} from "./managedTunnelStartup.ts"; +import { shouldRetryCloudLink } from "./relayResponse.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; import { traceAuthenticatedRelayRequest, traceRelayRequest } from "./traceRelayRequest.ts"; @@ -67,6 +92,10 @@ const idleAwarenessRelay = AgentAwarenessRelay.AgentAwarenessRelay.of({ requestCatchUp: () => Effect.void, start: () => Effect.void, }); +const decodeManagedTunnelRecoveryRegistration = Schema.decodeUnknownEffect( + Schema.fromJsonString(RelayManagedEndpointRecoveryRegistrationRequest), +); + function makeSecretStore( create: ServerSecretStore.ServerSecretStore["Service"]["create"], ): ServerSecretStore.ServerSecretStore["Service"] { @@ -221,6 +250,9 @@ describe("reconcileDesiredCloudLink", () => { ManagedEndpointRuntime.CloudManagedEndpointRuntime, ManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ applyConfig: unusedSecretStoreOperation, + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"]), ), Effect.provideService( @@ -247,6 +279,39 @@ describe("reconcileDesiredCloudLink", () => { ); }); +describe("parseManagedEndpointLocalOrigin", () => { + it.each([ + { + input: "http://127.0.0.1:80", + httpBaseUrl: "http://127.0.0.1", + wsBaseUrl: "ws://127.0.0.1", + port: 80, + }, + { + input: "https://127.0.0.1:443", + httpBaseUrl: "https://127.0.0.1", + wsBaseUrl: "wss://127.0.0.1", + port: 443, + }, + ])("accepts an explicit default port in $input", ({ input, httpBaseUrl, wsBaseUrl, port }) => { + expect(parseManagedEndpointLocalOrigin(input)).toEqual({ + httpBaseUrl, + wsBaseUrl, + origin: { localHttpHost: "127.0.0.1", localHttpPort: port }, + }); + }); + + it.each([ + "ftp://127.0.0.1:3773", + "http://user:password@127.0.0.1:3773", + "http://127.0.0.1:3773/api", + "http://127.0.0.1:3773?mode=test", + "http://127.0.0.1:3773#fragment", + ])("rejects non-origin URL %s", (input) => { + expect(() => parseManagedEndpointLocalOrigin(input)).toThrow("Invalid local origin"); + }); +}); + describe("releaseManagedTunnelOnShutdown", () => { const cliToken: CliTokenManager.PersistedToken = { accessToken: "cli-access-token", @@ -264,7 +329,10 @@ describe("releaseManagedTunnelOnShutdown", () => { Effect.sync(() => { values.set(name, value); }), - create: unusedSecretStoreOperation, + create: (name, value) => + Effect.sync(() => { + values.set(name, value); + }), getOrCreateRandom: unusedSecretStoreOperation, remove: (name) => Effect.sync(() => { @@ -278,7 +346,9 @@ describe("releaseManagedTunnelOnShutdown", () => { readonly store: ServerSecretStore.ServerSecretStore["Service"]; readonly applyConfigCalls: Array; readonly requests: Array; + readonly onRequest?: (request: HttpClientRequest.HttpClientRequest) => Effect.Effect; readonly respond?: () => Response; + readonly respondEffect?: Effect.Effect; } // Writes the launcher's durable state file into this test's baseDir with @@ -331,10 +401,19 @@ describe("releaseManagedTunnelOnShutdown", () => { applyConfig: (config) => Effect.sync(() => { harness.applyConfigCalls.push(config); - return { - status: "disabled", - } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus; + return config === null + ? ({ + status: "disabled", + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus) + : ({ + status: "running", + providerKind: "cloudflare_tunnel", + pid: 123, + } satisfies ManagedEndpointRuntime.CloudManagedEndpointRuntimeStatus); }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, }), ), Effect.provideService( @@ -356,11 +435,14 @@ describe("releaseManagedTunnelOnShutdown", () => { HttpClient.make((request) => Effect.sync(() => { harness.requests.push(request); - return HttpClientResponse.fromWeb( - request, - (harness.respond ?? (() => Response.json({ ok: true })))(), - ); - }), + }).pipe( + Effect.andThen(harness.onRequest?.(request) ?? Effect.void), + Effect.andThen( + harness.respondEffect ?? + Effect.sync(() => (harness.respond ?? (() => Response.json({ ok: true })))()), + ), + Effect.map((response) => HttpClientResponse.fromWeb(request, response)), + ), ), ), // The release consults the launcher state file under the configured @@ -376,10 +458,27 @@ describe("releaseManagedTunnelOnShutdown", () => { // The persisted state of a CLI-managed link whose tunnel is releasable. const managedLinkSecrets = [ [CLOUD_ENDPOINT_RUNTIME_CONFIG, "runtime-config"], + [CLOUD_ENDPOINT_CONFIRMED_ORIGIN, "confirmed-origin"], [RELAY_URL_SECRET, "https://relay.example.test"], [CLOUD_CLI_DESIRED_LINK_SECRET, "managed"], ] as const; + it.effect("does not recreate a link that was unlinked while startup registration retried", () => { + const { store, values } = makeMemorySecretStore(managedLinkSecrets); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + // Registration started while this marker existed. Unlink removes it + // before startup receives the relay's final not_linked response. + values.delete(CLOUD_CLI_DESIRED_LINK_SECRET); + + expect(yield* reconcileDesiredCloudLinkIfStillDesired("http://127.0.0.1:3773")).toBeNull(); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + it.effect("stops the connector, releases the relay tunnel, and drops the dead token", () => { const { store, values } = makeMemorySecretStore(managedLinkSecrets); const applyConfigCalls: Array = []; @@ -398,6 +497,7 @@ describe("releaseManagedTunnelOnShutdown", () => { ); expect(request.headers.authorization).toBe("Bearer cli-access-token"); expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); + expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false); }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); @@ -654,6 +754,501 @@ describe("releaseManagedTunnelOnShutdown", () => { }), ); }); + + it.effect("registers an existing tunnel and starts the confirmed connector", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toMatchObject({ + status: "ready", + }); + expect(requests).toHaveLength(1); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.url).toBe( + "https://relay.example.test/v1/environments/env_123/tunnel/recovery", + ); + expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential"); + const body = requests[0]?.body; + expect(body?._tag).toBe("Uint8Array"); + if (body?._tag === "Uint8Array") { + expect( + yield* decodeManagedTunnelRecoveryRegistration(new TextDecoder().decode(body.body)), + ).toMatchObject({ + cloudUserId: "user-123", + tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + } + expect(applyConfigCalls).toHaveLength(1); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => Response.json({ status: "ready" }), + }), + ); + }); + + it.effect("reconciles a changed port after a relay outage outlasts the startup fallback", () => { + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "existing-token", + tunnelId: "existing-tunnel", + }; + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)], + [ + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, + JSON.stringify({ + config, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + let relayAvailable = false; + const localOrigin = "http://127.0.0.1:4884"; + + return Effect.gen(function* () { + const fallbackStarted = yield* Deferred.make(); + const firstFailure = yield* Deferred.make(); + expect(yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin)).toBe(false); + const registration = yield* Effect.forkChild( + retryManagedTunnelRegistration( + registerManagedCloudTunnelRecovery(localOrigin).pipe( + Effect.tapError(() => Deferred.succeed(firstFailure, undefined)), + ), + shouldRetryCloudLink, + startManagedCloudTunnelIfOriginConfirmed(localOrigin, { + requireConfirmedOrigin: false, + }).pipe( + Effect.orDie, + Effect.tap((started) => { + expect(started).toBe(true); + return Deferred.succeed(fallbackStarted, undefined); + }), + Effect.asVoid, + ), + ), + { startImmediately: true }, + ); + yield* Deferred.await(firstFailure); + yield* TestClock.adjust("15 minutes"); + yield* Effect.raceFirst( + Deferred.await(fallbackStarted), + Fiber.join(registration).pipe( + Effect.andThen(Effect.die("Registration ended before starting the fallback")), + ), + ); + expect(applyConfigCalls).toEqual([config]); + const attemptsBeforeRecovery = requests.length; + + relayAvailable = true; + yield* TestClock.adjust("1 minute"); + expect(yield* Fiber.join(registration)).toMatchObject({ status: "ready" }); + expect(requests.length).toBeGreaterThan(attemptsBeforeRecovery); + const marker = yield* store.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); + expect(Option.isSome(marker)).toBe(true); + if (Option.isSome(marker)) { + expect( + Option.getOrThrow(decodeConfirmedOrigin(new TextDecoder().decode(marker.value))), + ).toEqual({ + config, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 }, + }); + } + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => + relayAvailable + ? Response.json({ status: "ready" }) + : Response.json({ message: "relay unavailable" }, { status: 503 }), + }), + ); + }); + + it.effect( + "starts a connector with a marker for the current origin without contacting relay", + () => { + const configJson = + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}'; + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "existing-token", + tunnelId: "existing-tunnel", + }; + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, configJson], + [ + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, + `{"config":${configJson},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}`, + ], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773")).toBe(true); + expect(applyConfigCalls).toEqual([config]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }, + ); + + it.effect.each([ + { name: "missing", marker: undefined, origin: "http://127.0.0.1:3773" }, + { + name: "stale", + marker: + '{"config":{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"},"origin":{"localHttpHost":"127.0.0.1","localHttpPort":3773}}', + origin: "http://127.0.0.1:4884", + }, + ])("does not start a connector with a $name origin marker", ({ marker, origin }) => { + const entries: Array = [ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}', + ], + ]; + if (marker !== undefined) entries.push([CLOUD_ENDPOINT_CONFIRMED_ORIGIN, marker]); + const { store } = makeMemorySecretStore(entries); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* startManagedCloudTunnelIfOriginConfirmed(origin)).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect( + "starts the stored connector without a marker when confirmation is not required", + () => { + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "existing-token", + tunnelId: "existing-tunnel", + }; + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, JSON.stringify(config)], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect( + yield* startManagedCloudTunnelIfOriginConfirmed("http://127.0.0.1:3773", { + requireConfirmedOrigin: false, + }), + ).toBe(true); + expect(applyConfigCalls).toEqual([config]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }, + ); + + it.effect.each(["replaced", "removed"] as const)( + "does not activate a tunnel when its runtime config is %s during registration", + (mutation) => { + const originalConfig = + '{"providerKind":"cloudflare_tunnel","connectorToken":"existing-token","tunnelId":"existing-tunnel"}'; + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, originalConfig], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773")).toEqual({ + status: "superseded", + }); + expect(applyConfigCalls).toEqual([]); + expect(values.has(CLOUD_ENDPOINT_CONFIRMED_ORIGIN)).toBe(false); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => { + if (mutation === "replaced") { + values.set( + CLOUD_ENDPOINT_RUNTIME_CONFIG, + new TextEncoder().encode( + '{"providerKind":"cloudflare_tunnel","connectorToken":"fresh-token","tunnelId":"fresh-tunnel"}', + ), + ); + } else { + values.delete(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } + return Response.json({ status: "ready" }); + }, + }), + ); + }, + ); + + it.effect("requests startup recovery for a legacy config without a recorded tunnel ID", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"token"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const registration = yield* registerManagedCloudTunnelRecovery("http://127.0.0.1:3773"); + expect(registration).toEqual({ + status: "recovery_required", + config: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }); + expect( + managedTunnelStartupAction({ + wantsCliLink: false, + registration, + }), + ).toEqual({ + action: "request_recovery", + config: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("recovers a web-linked tunnel with its environment credential", () => { + const oldConfig = + '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}'; + const nextConfig = { + providerKind: "cloudflare_tunnel", + connectorToken: "new-token", + tunnelId: "new-tunnel", + } as const; + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(true); + expect(requests).toHaveLength(1); + expect(requests[0]?.method).toBe("POST"); + expect(requests[0]?.url).toBe("https://relay.example.test/v1/environments/env_123/tunnel"); + expect(requests[0]?.headers.authorization).toBe("Bearer environment-credential"); + expect(applyConfigCalls).toEqual([nextConfig]); + expect( + Option.getOrNull( + decodeRuntimeConfig(new TextDecoder().decode(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG))), + ), + ).toEqual(nextConfig); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => + Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: nextConfig, + }), + }), + ); + }); + + it.effect("allows managed tunnel provisioning to take longer than ten seconds", () => + Effect.gen(function* () { + const oldConfig = + '{"providerKind":"cloudflare_tunnel","connectorToken":"old-token","tunnelId":"old-tunnel"}'; + const nextConfig = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "new-token", + tunnelId: "new-tunnel", + }; + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, oldConfig], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + const requestStarted = yield* Deferred.make(); + const response = yield* Deferred.make(); + const recovery = yield* recoverManagedCloudTunnel("http://127.0.0.1:3773").pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + onRequest: () => Deferred.succeed(requestStarted, undefined), + respondEffect: Deferred.await(response), + }), + Effect.forkChild({ startImmediately: true }), + ); + + yield* Deferred.await(requestStarted); + expect(requests).toHaveLength(1); + yield* TestClock.adjust("11 seconds"); + yield* Effect.yieldNow; + yield* Deferred.succeed( + response, + Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: nextConfig, + }), + ); + + expect(yield* Fiber.join(recovery)).toBe(true); + expect(requests).toHaveLength(1); + expect(applyConfigCalls).toEqual([nextConfig]); + }), + ); + + it.effect("does not recover an environment without a managed tunnel credential", () => { + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false); + expect(applyConfigCalls).toEqual([]); + expect(requests).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect("ignores recovery requests for a tunnel that has already been replaced", () => { + const { store } = makeMemorySecretStore([ + [ + CLOUD_ENDPOINT_RUNTIME_CONFIG, + '{"providerKind":"cloudflare_tunnel","connectorToken":"current-token","tunnelId":"current-tunnel"}', + ], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + expect( + yield* recoverManagedCloudTunnel("http://127.0.0.1:3773", { + providerKind: "cloudflare_tunnel", + connectorToken: "old-token", + tunnelId: "old-tunnel", + }), + ).toBe(false); + expect(requests).toEqual([]); + expect(applyConfigCalls).toEqual([]); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + + it.effect.each([ + { status: 401, errorTag: "EnvironmentHttpUnauthorizedError" }, + { status: 403, errorTag: "EnvironmentHttpForbiddenError" }, + { status: 409, errorTag: "EnvironmentHttpBadRequestError" }, + ])("preserves a permanent $status relay recovery failure", ({ status, errorTag }) => { + const { store } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + const error = yield* Effect.flip(recoverManagedCloudTunnel("http://127.0.0.1:3773")); + + expect(error._tag).toBe(errorTag); + expect(requests).toHaveLength(1); + expect(applyConfigCalls).toEqual([]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => Response.json({}, { status }), + }), + ); + }); + + it.effect("keeps a tunnel configuration replaced during recovery", () => { + const { store, values } = makeMemorySecretStore([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, "old-config"], + [RELAY_URL_SECRET, "https://relay.example.test"], + [CLOUD_LINKED_USER_ID, "user-123"], + [RELAY_ENVIRONMENT_CREDENTIAL_SECRET, "environment-credential"], + ]); + const applyConfigCalls: Array = []; + const requests: Array = []; + const freshConfig = new TextEncoder().encode("fresh-config"); + + return Effect.gen(function* () { + expect(yield* recoverManagedCloudTunnel("http://127.0.0.1:3773")).toBe(false); + expect(values.get(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(freshConfig); + expect(applyConfigCalls).toEqual([]); + }).pipe( + provideReleaseHarness({ + store, + applyConfigCalls, + requests, + respond: () => { + values.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, freshConfig); + return Response.json({ + endpoint: { + httpBaseUrl: "https://environment.example.test/", + wsBaseUrl: "wss://environment.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + endpointRuntime: { + providerKind: "cloudflare_tunnel", + connectorToken: "replacement-token", + }, + }); + }, + }), + ); + }); }); describe("link proof provider kinds", () => { diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index 68324db22f..9be8e4f3d8 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -31,6 +31,10 @@ import { RelayEnvironmentLinkProofPayload, RelayLinkProofRequest, RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryProofPayload, + RelayManagedEndpointRecoveryRegistrationResponse, + RelayManagedEndpointRecoveryResponse, + type RelayManagedEndpointRuntimeConfig, RelayOkResponse, } from "@t3tools/contracts/relay"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; @@ -39,6 +43,7 @@ import { RELAY_HEALTH_REQUEST_TYP, RELAY_HEALTH_RESPONSE_TYP, RELAY_LINK_PROOF_TYP, + RELAY_MANAGED_TUNNEL_RECOVERY_TYP, RELAY_MINT_REQUEST_TYP, RELAY_MINT_RESPONSE_TYP, signRelayJwt, @@ -54,10 +59,12 @@ import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Schedule from "effect/Schedule"; import * as HttpEffect from "effect/unstable/http/HttpEffect"; import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; +import * as HttpServer from "effect/unstable/http/HttpServer"; import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; @@ -73,9 +80,13 @@ import { } from "./serviceProtocol.ts"; import { CLOUD_ENDPOINT_RUNTIME_CONFIG, + CLOUD_ENDPOINT_CONFIRMED_ORIGIN, + decodeConfirmedOrigin, CLOUD_LINKED_USER_ID, CLOUD_MINT_PUBLIC_KEY, + decodeRuntimeConfig, encodeEndpointRuntimeConfigJson, + encodeConfirmedOriginJson, PUBLISH_AGENT_ACTIVITY_SECRET, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, RELAY_ISSUER_SECRET, @@ -90,7 +101,7 @@ import { import * as CliTokenManager from "./CliTokenManager.ts"; import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts"; import { traceRelayRequest } from "./traceRelayRequest.ts"; -import { filterRelayResponse, relayRequestError } from "./relayResponse.ts"; +import { filterRelayResponse, relayRequestError, shouldRetryCloudLink } from "./relayResponse.ts"; const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-"; const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-"; @@ -107,6 +118,7 @@ const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60; const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60; // The desktop app stops its backends within seconds of writing the marker. const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1); +const MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT = Duration.minutes(2); const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]); const CLOUD_CREDENTIAL_RESPONSE_HEADERS = { "cache-control": "no-store", @@ -341,6 +353,33 @@ function endpointRequestPort(url: URL): number { return Number(url.port || (url.protocol === "https:" ? 443 : 80)); } +export function parseManagedEndpointLocalOrigin(localOrigin: string) { + const url = new URL(localOrigin); + if ( + localOrigin !== localOrigin.trim() || + (url.protocol !== "http:" && url.protocol !== "https:") || + url.username !== "" || + url.password !== "" || + url.pathname !== "/" || + url.search !== "" || + url.hash !== "" || + localOrigin.includes("?") || + localOrigin.includes("#") + ) { + throw new Error("Invalid local origin"); + } + const wsUrl = new URL(url.origin); + wsUrl.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + return { + httpBaseUrl: url.origin, + wsBaseUrl: wsUrl.origin, + origin: { + localHttpHost: url.hostname, + localHttpPort: endpointRequestPort(url), + } satisfies RelayManagedEndpointOrigin, + }; +} + // The proof pins the relay to a loopback port on this machine, so the request // asking for one has to have reached the server directly at that same port. export function isAllowedEndpointOrigin(input: { @@ -501,56 +540,250 @@ const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")( ), ); -const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( +function managedEndpointRuntimeConfigsMatch( + left: RelayManagedEndpointRuntimeConfig, + right: RelayManagedEndpointRuntimeConfig, +): boolean { + return ( + left.providerKind === right.providerKind && + left.connectorToken === right.connectorToken && + left.tunnelId === right.tunnelId && + left.tunnelName === right.tunnelName + ); +} + +const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")(function* ( dependencies: CloudHttpDependencies, - payload: RelayEnvironmentConfigRequest, + input: { + readonly config: RelayManagedEndpointRuntimeConfig; + readonly configJson: string; + readonly origin: RelayManagedEndpointOrigin; + }, ) { - yield* validateRelayConfigPayload(payload); - yield* validateLinkedCloudUser({ - secrets: dependencies.secrets, - cloudUserId: payload.cloudUserId, + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if (Option.isNone(currentConfig) || bytesToString(currentConfig.value) !== input.configJson) { + return null; + } + const status = yield* dependencies.endpointRuntime.applyConfig(input.config); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + const marker = yield* encodeConfirmedOriginJson({ + config: input.config, + origin: input.origin, + }); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker)); + return status; + }), + ); +}); + +const activateManagedTunnelWithRetry = ( + dependencies: CloudHttpDependencies, + input: { + readonly config: RelayManagedEndpointRuntimeConfig; + readonly configJson: string; + readonly origin: RelayManagedEndpointOrigin; + }, + retryRuntimeFailures: boolean, +) => { + const activate = activateManagedTunnel(dependencies, input); + return retryRuntimeFailures + ? activate.pipe( + Effect.retry({ + while: (error) => + error._tag === "EnvironmentCloudEndpointUnavailableError" && + ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( + error.endpointRuntimeStatus, + ), + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.jittered, + ), + }), + ) + : activate; +}; + +export const startManagedCloudTunnelIfOriginConfirmed = Effect.fn( + "environment.cloud.startManagedCloudTunnelIfOriginConfirmed", +)(function* (localOrigin: string, options?: { readonly requireConfirmedOrigin?: boolean }) { + const dependencies = yield* cloudHttpDependencies; + const requireConfirmedOrigin = options?.requireConfirmedOrigin ?? true; + const parsedOrigin = yield* Effect.try({ + try: () => parseManagedEndpointLocalOrigin(localOrigin), + catch: () => + new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }), }); - yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( - payload.endpointRuntime, + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const [runtimeBytes, markerBytes] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), + ]); + if (Option.isNone(runtimeBytes)) return false; + const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeBytes.value))); + if (config === null || config.providerKind !== "cloudflare_tunnel") return false; + // With the marker required, only a config the relay already confirmed on + // this port may start. Without it, startup is falling back after the + // relay stayed unreachable: an unconfirmed origin may send traffic to a + // stale port, but that beats no remote access at all. + if (requireConfirmedOrigin) { + if (Option.isNone(markerBytes)) return false; + const marker = Option.getOrNull(decodeConfirmedOrigin(bytesToString(markerBytes.value))); + if ( + marker === null || + !managedEndpointRuntimeConfigsMatch(marker.config, config) || + marker.origin.localHttpHost !== parsedOrigin.origin.localHttpHost || + marker.origin.localHttpPort !== parsedOrigin.origin.localHttpPort + ) { + return false; + } + } + const status = yield* dependencies.endpointRuntime.applyConfig(config); + if (status.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: status, + }); + } + return true; + }), ); - const ok = - endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running"; - if (!ok) { - return yield* new EnvironmentCloudEndpointUnavailableError({ - message: "Managed endpoint runtime could not be started.", - endpointRuntimeStatus, +}); + +const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* ( + dependencies: CloudHttpDependencies, + payload: RelayEnvironmentConfigRequest, + options?: { + readonly lockHeld?: boolean; + readonly confirmedOrigin?: RelayManagedEndpointOrigin; + }, +) { + const apply = Effect.gen(function* () { + yield* validateRelayConfigPayload(payload); + yield* validateLinkedCloudUser({ + secrets: dependencies.secrets, + cloudUserId: payload.cloudUserId, }); - } + yield* validateCloudMintPublicKey(payload.cloudMintPublicKey); + // Reject unsupported runtimes before touching the connector so a bad + // payload cannot stop a healthy tunnel on its way to a 503. + if ( + payload.endpointRuntime !== null && + payload.endpointRuntime.providerKind !== "cloudflare_tunnel" + ) { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus: { + status: "unsupported", + providerKind: payload.endpointRuntime.providerKind, + }, + }); + } + yield* dependencies.endpointRuntime.applyConfig(null); + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); - yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); - yield* dependencies.secrets.set( - RELAY_ISSUER_SECRET, - stringToBytes(payload.relayIssuer ?? payload.relayUrl), - ); - yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); - yield* dependencies.secrets.set( - RELAY_ENVIRONMENT_CREDENTIAL_SECRET, - stringToBytes(payload.environmentCredential), - ); - yield* dependencies.secrets.set(CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey)); - yield* dependencies.awarenessRelay.requestCatchUp(); - if (payload.endpointRuntime) { - const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); + yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl)); yield* dependencies.secrets.set( - CLOUD_ENDPOINT_RUNTIME_CONFIG, - stringToBytes(endpointRuntimeJson), + RELAY_ISSUER_SECRET, + stringToBytes(payload.relayIssuer ?? payload.relayUrl), ); - } else { - yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); - } - return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId)); + yield* dependencies.secrets.set( + RELAY_ENVIRONMENT_CREDENTIAL_SECRET, + stringToBytes(payload.environmentCredential), + ); + yield* dependencies.secrets.set( + CLOUD_MINT_PUBLIC_KEY, + stringToBytes(payload.cloudMintPublicKey), + ); + yield* dependencies.awarenessRelay.requestCatchUp(); + if (payload.endpointRuntime) { + const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime); + yield* dependencies.secrets.set( + CLOUD_ENDPOINT_RUNTIME_CONFIG, + stringToBytes(endpointRuntimeJson), + ); + } else { + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + } + if (payload.endpointRuntime === null || options?.confirmedOrigin === undefined) { + return { + ok: true, + endpointRuntimeStatus: { status: "disabled" }, + } satisfies EnvironmentCloudRelayConfigResult; + } + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig( + payload.endpointRuntime, + ); + if (endpointRuntimeStatus.status !== "running") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint runtime could not be started.", + endpointRuntimeStatus, + }); + } + const marker = yield* encodeConfirmedOriginJson({ + config: payload.endpointRuntime, + origin: options.confirmedOrigin, + }); + yield* dependencies.secrets.set(CLOUD_ENDPOINT_CONFIRMED_ORIGIN, stringToBytes(marker)); + return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }); + return yield* options?.lockHeld ? apply : dependencies.endpointRuntime.withLinkStateLock(apply); }); const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( function* (dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest) { yield* requireEnvironmentScope(AuthRelayWriteScope); - return yield* applyCloudRelayConfig(dependencies, payload); + const result = yield* applyCloudRelayConfig(dependencies, payload); + if (payload.endpointRuntime?.providerKind === "cloudflare_tunnel") { + const server = yield* HttpServer.HttpServer; + const address = server.address; + if (typeof address === "string" || !("port" in address)) { + return yield* new EnvironmentHttpInternalServerError({ + message: "Could not resolve the local server origin.", + }); + } + const registration = yield* registerManagedCloudTunnelRecovery( + `http://127.0.0.1:${address.port}`, + ).pipe( + Effect.retry({ + times: 2, + while: (error) => + shouldRetryCloudLink(error) && + error._tag !== "EnvironmentCloudEndpointUnavailableError", + }), + ); + if (registration.status === "superseded") { + return yield* new EnvironmentHttpConflictError({ + message: "The managed tunnel configuration changed during registration.", + }); + } + if (registration.status === "recovery_required") { + yield* dependencies.endpointRuntime.requestRecovery(registration.config); + } + if (registration.status !== "ready") { + return yield* new EnvironmentCloudEndpointUnavailableError({ + message: "Managed endpoint origin could not be confirmed.", + endpointRuntimeStatus: { status: "disabled" }, + }); + } + return { + ok: true, + endpointRuntimeStatus: registration.endpointRuntimeStatus, + } satisfies EnvironmentCloudRelayConfigResult; + } + return result; }, Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) => failEnvironmentCloudInternalError(error.message)(error), @@ -559,10 +792,14 @@ const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")( ServerSecretStore.isSecretStoreError, failEnvironmentCloudInternalError("Could not persist environment relay configuration."), ), - Effect.catchTag( - "SchemaError", - failEnvironmentCloudInternalError("Could not persist environment relay configuration."), - ), + Effect.catchTags({ + SchemaError: failEnvironmentCloudInternalError( + "Could not persist environment relay configuration.", + ), + PlatformError: failEnvironmentCloudInternalError( + "Could not register the managed endpoint origin.", + ), + }), ); const relayClientRequest = ( @@ -572,6 +809,7 @@ const relayClientRequest = ( readonly token: string; readonly payload: unknown; readonly schema: Schema.Decoder; + readonly timeout?: Duration.Input; }, ) => HttpClientRequest.post(input.url).pipe( @@ -580,25 +818,20 @@ const relayClientRequest = ( Effect.flatMap(dependencies.httpClient.execute), Effect.flatMap(filterRelayResponse), Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)), + Effect.timeout(input.timeout ?? "10 seconds"), Effect.mapError(relayRequestError), withRelayClientTracing, ); const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")( function* (dependencies: CloudHttpDependencies, localOrigin: string) { - const localUrl = yield* Effect.try({ - try: () => new URL(localOrigin), + const parsedOrigin = yield* Effect.try({ + try: () => parseManagedEndpointLocalOrigin(localOrigin), catch: () => new EnvironmentHttpBadRequestError({ message: "Could not resolve local environment origin.", }), }); - if (localUrl.origin !== localOrigin) { - return yield* new EnvironmentHttpBadRequestError({ - message: "Could not resolve local environment origin.", - }); - } - const localWsOrigin = localOrigin.replace(/^http/u, "ws"); const token = yield* dependencies.cliTokenManager.getExisting.pipe( Effect.flatMap( Option.match({ @@ -631,16 +864,13 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi challenge: challenge.challenge, relayIssuer: relayUrl, endpoint: { - httpBaseUrl: localOrigin, - wsBaseUrl: localWsOrigin, + httpBaseUrl: parsedOrigin.httpBaseUrl, + wsBaseUrl: parsedOrigin.wsBaseUrl, providerKind: managedTunnelsEnabled ? "cloudflare_tunnel" : "manual", }, - origin: { - localHttpHost: localUrl.hostname, - localHttpPort: endpointRequestPort(localUrl), - }, + origin: parsedOrigin.origin, }, - localOrigin, + parsedOrigin.httpBaseUrl, ); const link = yield* relayClientRequest(dependencies, { url: `${relayUrl}/v1/client/environment-links`, @@ -652,16 +882,27 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi managedTunnelsEnabled, }, schema: RelayEnvironmentLinkResponse, + timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT, }); yield* setCliDesiredCloudLink(true, mode); - return yield* applyCloudRelayConfig(dependencies, { - relayUrl, - relayIssuer: link.relayIssuer, - cloudUserId: link.cloudUserId, - environmentCredential: link.environmentCredential, - cloudMintPublicKey: link.cloudMintPublicKey, - endpointRuntime: link.endpointRuntime, - }); + yield* applyCloudRelayConfig( + dependencies, + { + relayUrl, + relayIssuer: link.relayIssuer, + cloudUserId: link.cloudUserId, + environmentCredential: link.environmentCredential, + cloudMintPublicKey: link.cloudMintPublicKey, + endpointRuntime: link.endpointRuntime, + }, + { + lockHeld: true, + confirmedOrigin: parsedOrigin.origin, + }, + ); + // Callers decide on managed tunnel recovery from the mode this link + // actually used, not from a value read before the relay round trip. + return mode; }, Effect.catchIf( ServerSecretStore.isSecretStoreError, @@ -678,7 +919,260 @@ const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesi export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")( function* (localOrigin: string) { - return yield* reconcileDesiredCloudLinkWith(yield* cloudHttpDependencies, localOrigin); + const dependencies = yield* cloudHttpDependencies; + return yield* dependencies.endpointRuntime.withLinkStateLock( + reconcileDesiredCloudLinkWith(dependencies, localOrigin), + ); + }, +); + +export const reconcileDesiredCloudLinkIfStillDesired = Effect.fn( + "environment.cloud.reconcileDesiredLinkIfStillDesired", +)(function* (localOrigin: string) { + const dependencies = yield* cloudHttpDependencies; + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + if (!(yield* readCliDesiredCloudLink)) { + return null; + } + return yield* reconcileDesiredCloudLinkWith(dependencies, localOrigin); + }), + ); +}); + +type ManagedTunnelRecoveryProofInput = { + readonly environmentId: RelayManagedEndpointRecoveryProofPayload["environmentId"]; + readonly cloudUserId: string; + readonly relayUrl: string; +} & ( + | { + readonly action: "register"; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; + } + | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } +); + +const makeManagedTunnelRecoveryProof = Effect.fn( + "environment.cloud.makeManagedTunnelRecoveryProof", +)(function* (dependencies: CloudHttpDependencies, input: ManagedTunnelRecoveryProofInput) { + const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets); + const configuredIssuer = yield* dependencies.secrets.get(RELAY_ISSUER_SECRET); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const claims = { + iss: `t3-env:${input.environmentId}`, + aud: normalizeRelayIssuer( + Option.isSome(configuredIssuer) ? bytesToString(configuredIssuer.value) : input.relayUrl, + ), + sub: input.environmentId, + jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)), + iat: issuedAt, + exp: issuedAt + 60, + environmentId: input.environmentId, + cloudUserId: input.cloudUserId, + }; + const payload = + input.action === "register" + ? { + ...claims, + action: "register" as const, + tunnelId: input.tunnelId, + origin: input.origin, + } + : { ...claims, action: "recover" as const, origin: input.origin }; + + return yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload, + }).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not sign the managed tunnel recovery request.", + }), + ), + ); +}); + +export const registerManagedCloudTunnelRecovery = Effect.fn( + "environment.cloud.registerManagedCloudTunnelRecovery", +)(function* (localOrigin: string, options?: { readonly retryRuntimeFailures?: boolean }) { + const dependencies = yield* cloudHttpDependencies; + const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + if ( + Option.isNone(runtimeConfig) || + Option.isNone(relayUrl) || + Option.isNone(cloudUserId) || + Option.isNone(environmentCredential) + ) { + return { status: "not_linked" as const }; + } + + const config = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); + if (config?.providerKind !== "cloudflare_tunnel") { + return { status: "not_linked" as const }; + } + + const parsedOrigin = yield* Effect.try({ + try: () => parseManagedEndpointLocalOrigin(localOrigin), + catch: () => + new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }), + }); + if (config.tunnelId === undefined) { + return { status: "recovery_required" as const, config }; + } + const origin = parsedOrigin.origin; + const environmentId = yield* dependencies.environment.getEnvironmentId; + const relayUrlValue = bytesToString(relayUrl.value); + const cloudUserIdValue = bytesToString(cloudUserId.value); + const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { + action: "register", + environmentId, + cloudUserId: cloudUserIdValue, + relayUrl: relayUrlValue, + tunnelId: config.tunnelId, + origin, + }); + const registered = yield* relayClientRequest(dependencies, { + url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel/recovery`, + token: bytesToString(environmentCredential.value), + payload: { + cloudUserId: cloudUserIdValue, + tunnelId: config.tunnelId, + origin, + proof, + }, + schema: RelayManagedEndpointRecoveryRegistrationResponse, + }); + if (registered.status === "recovery_required") { + return { status: registered.status, config }; + } + const endpointRuntimeStatus = yield* activateManagedTunnelWithRetry( + dependencies, + { + config, + configJson: bytesToString(runtimeConfig.value), + origin, + }, + options?.retryRuntimeFailures === true, + ); + return endpointRuntimeStatus === null + ? { status: "superseded" as const } + : { status: "ready" as const, endpointRuntimeStatus }; +}); + +export const recoverManagedCloudTunnel = Effect.fn("environment.cloud.recoverManagedCloudTunnel")( + function* ( + localOrigin: string, + expectedConfig?: RelayManagedEndpointRuntimeConfig, + options?: { readonly retryRuntimeFailures?: boolean }, + ) { + const dependencies = yield* cloudHttpDependencies; + const [runtimeConfig, relayUrl, cloudUserId, environmentCredential] = yield* Effect.all([ + dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.get(RELAY_URL_SECRET), + dependencies.secrets.get(CLOUD_LINKED_USER_ID), + dependencies.secrets.get(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + ]); + if ( + Option.isNone(runtimeConfig) || + Option.isNone(relayUrl) || + Option.isNone(cloudUserId) || + Option.isNone(environmentCredential) + ) { + return false; + } + if (expectedConfig !== undefined) { + const current = Option.getOrNull(decodeRuntimeConfig(bytesToString(runtimeConfig.value))); + if ( + current === null || + current.providerKind !== expectedConfig.providerKind || + current.connectorToken !== expectedConfig.connectorToken || + current.tunnelId !== expectedConfig.tunnelId || + current.tunnelName !== expectedConfig.tunnelName + ) { + return false; + } + } + + const parsedOrigin = yield* Effect.try({ + try: () => parseManagedEndpointLocalOrigin(localOrigin), + catch: () => + new EnvironmentHttpBadRequestError({ + message: "Could not resolve local environment origin.", + }), + }); + + const environmentId = yield* dependencies.environment.getEnvironmentId; + const relayUrlValue = bytesToString(relayUrl.value); + const cloudUserIdValue = bytesToString(cloudUserId.value); + const origin = parsedOrigin.origin; + const proof = yield* makeManagedTunnelRecoveryProof(dependencies, { + action: "recover", + environmentId, + cloudUserId: cloudUserIdValue, + relayUrl: relayUrlValue, + origin, + }); + const recovered = yield* relayClientRequest(dependencies, { + url: `${relayUrlValue}/v1/environments/${encodeURIComponent(environmentId)}/tunnel`, + token: bytesToString(environmentCredential.value), + payload: { + cloudUserId: cloudUserIdValue, + origin, + proof, + }, + schema: RelayManagedEndpointRecoveryResponse, + timeout: MANAGED_ENDPOINT_PROVISION_REQUEST_TIMEOUT, + }); + if (recovered.endpointRuntime.providerKind !== "cloudflare_tunnel") { + return yield* new EnvironmentHttpInternalServerError({ + message: "Pylon Connect returned an unsupported managed tunnel configuration.", + }); + } + + const encoded = yield* encodeEndpointRuntimeConfigJson(recovered.endpointRuntime).pipe( + Effect.mapError( + () => + new EnvironmentHttpInternalServerError({ + message: "Could not persist the recovered managed tunnel configuration.", + }), + ), + ); + const stored = yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const currentConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG); + if ( + Option.isNone(currentConfig) || + bytesToString(currentConfig.value) !== bytesToString(runtimeConfig.value) + ) { + return false; + } + yield* dependencies.secrets.set(CLOUD_ENDPOINT_RUNTIME_CONFIG, stringToBytes(encoded)); + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); + return true; + }), + ); + if (!stored) return false; + const status = yield* activateManagedTunnelWithRetry( + dependencies, + { + config: recovered.endpointRuntime, + configJson: encoded, + origin, + }, + options?.retryRuntimeFailures === true, + ); + return status !== null; }, ); @@ -761,11 +1255,10 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( if (Option.isNone(runtimeConfig)) { return false; } - // Only CLI-desired managed links release on shutdown, because the startup - // reconcile that provisions the replacement tunnel only runs for them. A - // link installed by a web/mobile client comes back after a restart by - // reapplying the stored connector token — it has no boot-time re-provision - // path — so its tunnel must survive the restart. (Unlink still deletes it.) + // Only CLI-desired managed links release eagerly because this request uses + // CLI authorization. Web/mobile links register startup recovery with their + // environment credential, and the relay reaper removes them after they are + // down for the configured grace period. Unlink still deletes either kind. if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") { return false; } @@ -822,6 +1315,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value) ) { yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG); + yield* dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN); } return true; }); @@ -868,21 +1362,26 @@ const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")( const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")( function* (dependencies: CloudHttpDependencies) { yield* requireEnvironmentScope(AuthRelayWriteScope); - const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null); - yield* Effect.all( - [ - dependencies.secrets.remove(CLOUD_LINKED_USER_ID), - dependencies.secrets.remove(RELAY_URL_SECRET), - dependencies.secrets.remove(RELAY_ISSUER_SECRET), - dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), - dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), - dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), - dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), - ], - { concurrency: 7 }, + return yield* dependencies.endpointRuntime.withLinkStateLock( + Effect.gen(function* () { + const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null); + yield* Effect.all( + [ + dependencies.secrets.remove(CLOUD_LINKED_USER_ID), + dependencies.secrets.remove(RELAY_URL_SECRET), + dependencies.secrets.remove(RELAY_ISSUER_SECRET), + dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), + dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY), + dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG), + dependencies.secrets.remove(CLOUD_ENDPOINT_CONFIRMED_ORIGIN), + dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET), + ], + { concurrency: 8 }, + ); + yield* setCliDesiredCloudLink(false); + return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; + }), ); - yield* setCliDesiredCloudLink(false); - return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult; }, Effect.catchIf( ServerSecretStore.isSecretStoreError, diff --git a/apps/server/src/cloud/managedTunnelStartup.test.ts b/apps/server/src/cloud/managedTunnelStartup.test.ts new file mode 100644 index 0000000000..1d75f606cd --- /dev/null +++ b/apps/server/src/cloud/managedTunnelStartup.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as TestClock from "effect/testing/TestClock"; + +import { + managedTunnelStartupAction, + retryManagedTunnelRegistration, +} from "./managedTunnelStartup.ts"; + +describe("managedTunnelStartupAction", () => { + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }; + + it("requests tunnel recovery only when the relay proves it is needed", () => { + expect( + managedTunnelStartupAction({ + wantsCliLink: true, + registration: { status: "recovery_required", config }, + }), + ).toEqual({ action: "request_recovery", config }); + }); + + it("creates a desired CLI link only when no local managed link exists", () => { + expect( + managedTunnelStartupAction({ + wantsCliLink: true, + registration: { status: "not_linked" }, + }), + ).toEqual({ action: "reconcile_link" }); + }); + + it.each(["ready", "unavailable"] as const)( + "does not provision after a %s registration result", + (status) => { + expect( + managedTunnelStartupAction({ + wantsCliLink: true, + registration: { status }, + }), + ).toEqual({ action: "none" }); + }, + ); +}); + +describe("retryManagedTunnelRegistration", () => { + it.effect("does not fall back or retry when registration is permanently rejected", () => + Effect.gen(function* () { + let attempts = 0; + let fallbacks = 0; + const error = yield* Effect.flip( + retryManagedTunnelRegistration( + Effect.suspend(() => { + attempts += 1; + return Effect.fail("not authorized"); + }), + () => false, + Effect.sync(() => { + fallbacks += 1; + }), + ), + ); + expect(error).toBe("not authorized"); + expect(attempts).toBe(1); + expect(fallbacks).toBe(0); + }), + ); + + it.effect("stops retrying after the retry window so startup can fall back", () => + Effect.gen(function* () { + let attempts = 0; + const registration = Effect.suspend(() => { + attempts += 1; + return Effect.fail("relay unavailable" as const); + }); + const fiber = yield* Effect.forkChild( + Effect.flip(retryManagedTunnelRegistration(registration, () => true)), + { startImmediately: true }, + ); + yield* TestClock.adjust("15 minutes"); + expect(yield* Fiber.join(fiber)).toBe("relay unavailable"); + // Capped at 30 seconds between attempts, ten minutes allows a bounded run. + expect(attempts).toBeGreaterThan(5); + expect(attempts).toBeLessThan(60); + }), + ); + + it.effect("waits for successful registration before it activates the connector", () => + Effect.gen(function* () { + const firstAttempt = yield* Deferred.make(); + let attempts = 0; + let activations = 0; + let reconciliations = 0; + const registration = Effect.suspend(() => { + attempts += 1; + if (attempts === 1) { + return Deferred.succeed(firstAttempt, undefined).pipe( + Effect.andThen(Effect.fail("relay unavailable" as const)), + ); + } + return Effect.succeed({ status: "ready" as const }); + }); + const startup = retryManagedTunnelRegistration(registration, () => true).pipe( + Effect.tap((result) => + Effect.sync(() => { + const action = managedTunnelStartupAction({ wantsCliLink: true, registration: result }); + if (action.action === "reconcile_link") { + reconciliations += 1; + } + activations += 1; + }), + ), + ); + + const fiber = yield* Effect.forkChild(startup, { startImmediately: true }); + yield* Deferred.await(firstAttempt); + expect(attempts).toBe(1); + expect(activations).toBe(0); + + yield* TestClock.adjust("2 seconds"); + yield* Fiber.join(fiber); + + expect(attempts).toBe(2); + expect(activations).toBe(1); + expect(reconciliations).toBe(0); + }), + ); +}); diff --git a/apps/server/src/cloud/managedTunnelStartup.ts b/apps/server/src/cloud/managedTunnelStartup.ts new file mode 100644 index 0000000000..fe666cd0f2 --- /dev/null +++ b/apps/server/src/cloud/managedTunnelStartup.ts @@ -0,0 +1,77 @@ +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Schedule from "effect/Schedule"; + +export type ManagedTunnelRegistrationResult = + | { readonly status: "not_linked" | "ready" | "unavailable" | "superseded" } + | { + readonly status: "recovery_required"; + readonly config: RelayManagedEndpointRuntimeConfig; + }; + +export type ManagedTunnelStartupAction = + | { readonly action: "none" } + | { readonly action: "reconcile_link" } + | { + readonly action: "request_recovery"; + readonly config: RelayManagedEndpointRuntimeConfig; + }; + +export function managedTunnelStartupAction(input: { + readonly wantsCliLink: boolean; + readonly registration: ManagedTunnelRegistrationResult; +}): ManagedTunnelStartupAction { + if (input.registration.status === "recovery_required") { + return { + action: "request_recovery", + config: input.registration.config, + }; + } + if (input.wantsCliLink && input.registration.status === "not_linked") { + return { action: "reconcile_link" }; + } + return { action: "none" }; +} + +// After this window the host can start its stored connector config while +// registration keeps retrying to reconcile the origin when the relay returns. +const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10); + +export const retryManagedTunnelRegistration = ( + registration: Effect.Effect, + isRetryable: (error: E) => boolean, + onRetryWindowExhausted?: Effect.Effect, +) => { + const schedule = Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.jittered, + ); + return registration.pipe( + Effect.retry({ + while: isRetryable, + schedule: schedule.pipe( + Schedule.upTo({ duration: MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW }), + ), + }), + Effect.catch((error) => + onRetryWindowExhausted !== undefined && isRetryable(error) + ? onRetryWindowExhausted.pipe( + Effect.andThen(registration.pipe(Effect.retry({ while: isRetryable, schedule }))), + ) + : Effect.fail(error), + ), + ); +}; + +// A host asks the relay for a replacement tunnel at most this often. Every +// managed host shares one relay, so a host stuck in a bad loop must not turn +// into a fleet-wide request storm. +export const MANAGED_TUNNEL_RECOVERY_COOLDOWN = Duration.minutes(2); + +// Existing hosts register on their first boot after an upgrade, and desktop +// auto-update delivers that boot to many hosts at once. Spread the first +// registration so the relay and Cloudflare see a ramp instead of a spike. +export const MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER = Duration.seconds(30); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 5fe6a3d025..f41cad44b8 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -1222,6 +1222,9 @@ const buildAppUnderTest = (options?: { CloudManagedEndpointRuntime.CloudManagedEndpointRuntime, CloudManagedEndpointRuntime.CloudManagedEndpointRuntime.of({ applyConfig: () => Effect.succeed({ status: "disabled" }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, ...options?.layers?.cloudManagedEndpointRuntime, }), ), @@ -3339,6 +3342,68 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(loopbackHttpServerTest)), ); + it.effect("rejects a non-Cloudflare managed endpoint runtime without persisting the link", () => + Effect.gen(function* () { + const appliedRuntimeConfigs: Array = []; + yield* buildAppUnderTest({ + layers: { + cloudManagedEndpointRuntime: { + applyConfig: (config) => + Effect.sync(() => { + appliedRuntimeConfigs.push(config); + return config === null + ? ({ status: "disabled" } as const) + : ({ status: "unsupported", providerKind: config.providerKind } as const); + }), + }, + }, + }); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config"); + const relayConfigResponse = yield* fetchEffect(relayConfigUrl, { + method: "POST", + headers: { + cookie: ownerCookie, + "content-type": "application/json", + }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: { + providerKind: "manual", + connectorToken: "manual-token", + }, + }), + }); + const relayConfigBody = yield* responseJsonEffect<{ + readonly _tag?: string; + readonly endpointRuntimeStatus?: { readonly status?: string }; + }>(relayConfigResponse); + const linkStateUrl = yield* getHttpServerUrl("/api/connect/link-state"); + const linkStateResponse = yield* fetchEffect(linkStateUrl, { + headers: { cookie: ownerCookie }, + }); + const linkStateBody = yield* responseJsonEffect<{ readonly linked?: boolean }>( + linkStateResponse, + ); + + assert.equal(relayConfigResponse.status, 503); + assert.equal(relayConfigBody._tag, "EnvironmentCloudEndpointUnavailableError"); + assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "unsupported"); + // The connector is never touched for a rejected runtime. + assert.deepEqual(appliedRuntimeConfigs, []); + assert.equal(linkStateResponse.status, 200); + assert.equal(linkStateBody.linked, false); + }).pipe(Effect.provide(loopbackHttpServerTest)), + ); + it.effect("reports local cloud link state from persisted relay config", () => Effect.gen(function* () { yield* buildAppUnderTest(); @@ -3417,6 +3482,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { it.effect("unlinks local cloud state and disables the managed endpoint runtime", () => Effect.gen(function* () { const appliedRuntimeConfigs: Array = []; + const requestedRecoveryConfigs: Array = []; yield* buildAppUnderTest({ layers: { cloudManagedEndpointRuntime: { @@ -3433,7 +3499,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => { ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}), }); }, + requestRecovery: (config) => + Effect.sync(() => { + requestedRecoveryConfigs.push(config); + }), }, + httpClient: HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ status: "ready" }))), + ), }, }); @@ -3499,6 +3572,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { assert.equal(linkStateBody.relayUrl, null); assert.equal(linkStateBody.relayIssuer, null); assert.deepEqual(appliedRuntimeConfigs, [ + null, { providerKind: "cloudflare_tunnel", connectorToken: "connector-token", @@ -3507,6 +3581,7 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }, null, ]); + assert.deepEqual(requestedRecoveryConfigs, []); }).pipe(Effect.provide(loopbackHttpServerTest)), ); @@ -3893,19 +3968,169 @@ it.layer(NodeServices.layer)("server router seam", (it) => { }).pipe(Effect.provide(loopbackHttpServerTest)), ); + it.effect("keeps a managed connector stopped when relay registration fails", () => + Effect.gen(function* () { + const appliedRuntimeConfigs: Array = []; + const relayRequests: Array = []; + yield* buildAppUnderTest({ + layers: { + cloudManagedEndpointRuntime: { + applyConfig: (config) => + Effect.sync(() => { + appliedRuntimeConfigs.push(config); + return config === null + ? ({ status: "disabled" } as const) + : ({ status: "running", providerKind: "cloudflare_tunnel", pid: 123 } as const); + }), + }, + httpClient: HttpClient.make((request) => + Effect.sync(() => { + relayRequests.push(request); + return HttpClientResponse.fromWeb( + request, + Response.json({ message: "relay unavailable" }, { status: 503 }), + ); + }), + ), + }, + }); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config"); + const relayConfigResponse = yield* fetchEffect(relayConfigUrl, { + method: "POST", + headers: { + cookie: ownerCookie, + "content-type": "application/json", + }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }, + }), + }); + const relayConfigBody = yield* responseJsonEffect<{ readonly _tag?: string }>( + relayConfigResponse, + ); + + assert.equal(relayConfigResponse.status, 500); + assert.equal(relayConfigBody._tag, "EnvironmentHttpInternalServerError"); + assert.equal(relayRequests.length, 3); + assert.deepEqual(appliedRuntimeConfigs, [null]); + }).pipe(Effect.provide(loopbackHttpServerTest)), + ); + + it.effect( + "queues recovery without starting a connector when relay registration requires it", + () => + Effect.gen(function* () { + const appliedRuntimeConfigs: Array = []; + const requestedRecoveryConfigs: Array = []; + const relayRequests: Array = []; + yield* buildAppUnderTest({ + layers: { + cloudManagedEndpointRuntime: { + applyConfig: (config) => + Effect.sync(() => { + appliedRuntimeConfigs.push(config); + return config === null + ? ({ status: "disabled" } as const) + : ({ status: "running", providerKind: "cloudflare_tunnel", pid: 123 } as const); + }), + requestRecovery: (config) => + Effect.sync(() => { + requestedRecoveryConfigs.push(config); + }), + }, + httpClient: HttpClient.make((request) => + Effect.sync(() => { + relayRequests.push(request); + return HttpClientResponse.fromWeb( + request, + Response.json({ status: "recovery_required" }), + ); + }), + ), + }, + }); + + const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const ownerCookie = yield* getAuthenticatedSessionCookieHeader(); + const relayConfigUrl = yield* getHttpServerUrl("/api/connect/relay-config"); + const relayConfigResponse = yield* fetchEffect(relayConfigUrl, { + method: "POST", + headers: { + cookie: ownerCookie, + "content-type": "application/json", + }, + body: jsonRequestBody({ + relayUrl: "https://relay.example.test", + cloudUserId: "user_123", + environmentCredential: "t3env_test_credential", + cloudMintPublicKey: cloudKeyPair.publicKey, + endpointRuntime: { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }, + }), + }); + const relayConfigBody = yield* responseJsonEffect<{ + readonly _tag?: string; + readonly endpointRuntimeStatus?: { readonly status?: string }; + }>(relayConfigResponse); + + assert.equal(relayConfigResponse.status, 503); + assert.equal(relayConfigBody._tag, "EnvironmentCloudEndpointUnavailableError"); + assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "disabled"); + assert.equal(relayRequests.length, 1); + assert.deepEqual(appliedRuntimeConfigs, [null]); + assert.deepEqual(requestedRecoveryConfigs, [ + { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", + }, + ]); + }).pipe(Effect.provide(loopbackHttpServerTest)), + ); + it.effect("fails relay config when the managed endpoint connector cannot start", () => Effect.gen(function* () { + const appliedRuntimeConfigs: Array = []; yield* buildAppUnderTest({ layers: { cloudManagedEndpointRuntime: { - applyConfig: () => - Effect.succeed({ - status: "failed", - providerKind: "cloudflare_tunnel", - reason: "cloudflared missing", - tunnelId: "tunnel-1", + applyConfig: (config) => + Effect.sync(() => { + appliedRuntimeConfigs.push(config); + return config === null + ? ({ status: "disabled" } as const) + : ({ + status: "failed", + providerKind: "cloudflare_tunnel", + failure: "not-installed", + reason: "cloudflared missing", + tunnelId: "tunnel-1", + } as const); }), }, + httpClient: HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ status: "ready" }))), + ), }, }); @@ -3944,33 +4169,14 @@ it.layer(NodeServices.layer)("server router seam", (it) => { assert.equal(relayConfigBody.message, "Managed endpoint runtime could not be started."); assert.equal(relayConfigBody.endpointRuntimeStatus?.status, "failed"); assert.equal(relayConfigBody.endpointRuntimeStatus?.reason, "cloudflared missing"); - - const now = yield* DateTime.now; - const healthRequest = makeCloudEnvironmentHealthRequest({ - privateKey: cloudKeyPair.privateKey, - environmentId: testEnvironmentDescriptor.environmentId, - nonce: "cloud-health-after-failed-runtime", - issuedAt: DateTime.formatIso(now), - expiresAt: DateTime.formatIso(DateTime.add(now, { minutes: 5 })), - }); - const healthUrl = yield* getHttpServerUrl("/api/t3-connect/health"); - const healthResponse = yield* fetchEffect(healthUrl, { - method: "POST", - headers: { - "content-type": "application/json", + assert.deepEqual(appliedRuntimeConfigs, [ + null, + { + providerKind: "cloudflare_tunnel", + connectorToken: "connector-token", + tunnelId: "tunnel-1", }, - body: jsonRequestBody(healthRequest), - }); - const healthBody = yield* responseJsonEffect<{ - _tag?: string; - message?: string; - }>(healthResponse); - assert.equal(healthResponse.status, 500); - assert.equal(healthBody._tag, "EnvironmentHttpInternalServerError"); - assert.equal( - healthBody.message, - "Cloud mint public key is not installed for this environment.", - ); + ]); }).pipe(Effect.provide(loopbackHttpServerTest)), ); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index e1b27a6ab0..b75cad333c 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -6,12 +6,17 @@ import { ProviderDriverKind, type RepositoryIdentity, } from "@t3tools/contracts"; +import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay"; import * as Cause from "effect/Cause"; +import * as Clock from "effect/Clock"; import * as Duration from "effect/Duration"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Random from "effect/Random"; import * as Schedule from "effect/Schedule"; +import * as Semaphore from "effect/Semaphore"; +import * as Stream from "effect/Stream"; import { FetchHttpClient, HttpRouter, HttpServer } from "effect/unstable/http"; import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder"; @@ -126,12 +131,21 @@ import { connectHttpApiLayer, clearDesktopUpdateRestartMarker, pendingServiceUpdateExists, - reconcileDesiredCloudLink, + reconcileDesiredCloudLinkIfStillDesired, + recoverManagedCloudTunnel, + registerManagedCloudTunnelRecovery, + startManagedCloudTunnelIfOriginConfirmed, releaseManagedTunnelOnShutdown, } from "./cloud/http.ts"; import { serverRelayBrokerTracingLayer } from "./cloud/relayTracing.ts"; import { shouldRetryCloudLink } from "./cloud/relayResponse.ts"; import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts"; +import { + MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER, + MANAGED_TUNNEL_RECOVERY_COOLDOWN, + managedTunnelStartupAction, + retryManagedTunnelRegistration, +} from "./cloud/managedTunnelStartup.ts"; import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; import * as ServerSelfUpdate from "./cloud/selfUpdate.ts"; @@ -777,10 +791,6 @@ const makeServerLayer = Layer.unwrap( // A force-killed desktop backend never ran its tunnel finalizer. The // replacement must discard that handoff before its own normal quit. yield* clearDesktopUpdateRestartMarker; - if (!hasCloudPublicConfig) { - yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); - return; - } const releaseManagedTunnel = releaseManagedTunnelOnShutdown().pipe( Effect.timeout("10 seconds"), Effect.tap((released) => @@ -809,33 +819,186 @@ const makeServerLayer = Layer.unwrap( if (!cleanupBeforeActivation) { yield* Effect.addFinalizer(() => releaseManagedTunnel); } - if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) return; + const localOrigin = `http://127.0.0.1:${address.port}`; + const endpointRuntime = yield* CloudManagedEndpointRuntime.CloudManagedEndpointRuntime; + const recoveryLock = yield* Semaphore.make(1); + let lastRecoveryAtMillis = 0; + const recoverManagedTunnel = (config: RelayManagedEndpointRuntimeConfig) => + recoveryLock.withPermits(1)( + Effect.gen(function* () { + const elapsed = (yield* Clock.currentTimeMillis) - lastRecoveryAtMillis; + const wait = Duration.toMillis(MANAGED_TUNNEL_RECOVERY_COOLDOWN) - elapsed; + if (wait > 0) yield* Effect.sleep(Duration.millis(wait)); + lastRecoveryAtMillis = yield* Clock.currentTimeMillis; + }).pipe( + Effect.andThen( + recoverManagedCloudTunnel(localOrigin, config, { + retryRuntimeFailures: true, + }), + ), + Effect.retry({ + while: (error) => + shouldRetryCloudLink(error) && + error._tag !== "EnvironmentCloudEndpointUnavailableError", + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.jittered, + ), + }), + Effect.tap((recovered) => + recovered + ? Effect.logInfo("Pylon Connect managed tunnel recovered") + : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to recover the Pylon Connect managed tunnel", { + cause, + }), + ), + ), + ); + yield* endpointRuntime.recoveryRequests.pipe( + Stream.runForEach(recoverManagedTunnel), + Effect.forkScoped, + ); // No settling delay before the first attempt: routes are already // serving by the time activation opens this gate (the startup // sequence awaits routesReady), and the retry schedule below // covers anything this sleep used to hedge against. Every // millisecond here is dead time on the path to remote // reachability after a restart. - yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe( - Effect.retry({ - while: shouldRetryCloudLink, - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), + const wantsCliLink = hasCloudPublicConfig + ? yield* CloudCliState.readCliDesiredCloudLink.pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to read the desired Pylon Connect link", { + cause, + }).pipe(Effect.as(false)), ), - Schedule.upTo({ duration: "10 minutes" }), - ), - }), - Effect.tap(() => Effect.logInfo("Pylon Connect desired link reconciled on startup")), + ) + : false; + // A failed read must not end this fiber before it registers + // recovery and starts consuming recovery requests. "managed" is + // what a missing value means, so it is the safe fallback. + const desiredCliLinkMode = wantsCliLink + ? yield* CloudCliState.readCliDesiredLinkMode.pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to read the desired Pylon Connect link mode", { + cause, + }).pipe(Effect.as("managed" as const)), + ), + ) + : null; + // A publish-only link must not expose the host, even if a managed + // config from an earlier link is still stored. + const startedConfirmed = + desiredCliLinkMode === "publish_only" + ? false + : yield* startManagedCloudTunnelIfOriginConfirmed(localOrigin).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to start the confirmed Pylon Connect tunnel", { + cause, + }).pipe(Effect.as(false)), + ), + ); + const startStoredManagedTunnel = startManagedCloudTunnelIfOriginConfirmed(localOrigin, { + requireConfirmedOrigin: false, + }).pipe( + Effect.tap((started) => + started + ? Effect.logWarning( + "Pylon Connect started the stored tunnel without relay confirmation", + ) + : Effect.void, + ), Effect.catch((cause) => - Effect.logWarning("Failed to reconcile Pylon Connect desired link on startup", { - message: cause.message, - }), + Effect.logWarning("Failed to start the stored Pylon Connect tunnel", { cause }), ), + Effect.asVoid, ); + const registerManagedTunnel = retryManagedTunnelRegistration( + registerManagedCloudTunnelRecovery(localOrigin, { + retryRuntimeFailures: true, + }), + (error) => + shouldRetryCloudLink(error) && + error._tag !== "EnvironmentCloudEndpointUnavailableError", + startedConfirmed ? Effect.void : startStoredManagedTunnel, + ).pipe( + Effect.tap((result) => + result.status === "ready" + ? Effect.logInfo("Pylon Connect managed tunnel recovery registered") + : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to register Pylon Connect managed tunnel recovery", { + cause, + }).pipe(Effect.as({ status: "unavailable" as const })), + ), + ); + // A host without a confirmed marker is on its first boot after the + // upgrade. Spread those registrations so an auto-update wave does + // not hit the relay all at once. + if (!startedConfirmed && desiredCliLinkMode !== "publish_only") { + const jitter = yield* Random.nextIntBetween( + 0, + Duration.toMillis(MANAGED_TUNNEL_FIRST_REGISTRATION_JITTER), + ); + yield* Effect.sleep(Duration.millis(jitter)); + } + const registration = + desiredCliLinkMode === "publish_only" + ? { status: "not_linked" as const } + : yield* registerManagedTunnel; + // A terminal registration failure also allows the stored config + // to start. Transient outages use the fallback above and keep + // registration retrying in this scoped startup fiber. + if (registration.status === "unavailable" && !startedConfirmed) { + yield* startStoredManagedTunnel; + } + const startupAction = managedTunnelStartupAction({ wantsCliLink, registration }); + if (startupAction.action === "request_recovery") { + yield* endpointRuntime.requestRecovery(startupAction.config); + } + if (startupAction.action === "reconcile_link") { + const reconciledMode = yield* reconcileDesiredCloudLinkIfStillDesired( + localOrigin, + ).pipe( + Effect.retry({ + while: shouldRetryCloudLink, + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), + ), + }), + Effect.tap((mode) => + mode === null + ? Effect.void + : Effect.logInfo("Pylon Connect desired link reconciled on startup"), + ), + Effect.catch((cause) => + Effect.logWarning("Failed to reconcile Pylon Connect desired link on startup", { + cause, + }).pipe(Effect.as(null)), + ), + ); + if (reconciledMode === "managed") { + const afterReconcile = yield* registerManagedTunnel; + if (afterReconcile.status === "recovery_required") { + yield* endpointRuntime.requestRecovery(afterReconcile.config); + } + } + } }), ); yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); diff --git a/docs/internals/t3-connect.md b/docs/internals/t3-connect.md index 4c14936c28..9517647e4e 100644 --- a/docs/internals/t3-connect.md +++ b/docs/internals/t3-connect.md @@ -65,6 +65,47 @@ teardown, because a database failure must leave the active link usable. Failed teardown retains enough state to retry. See the [managed endpoint lifecycle](../../infra/relay/src/environments/ManagedEndpointProvider.ts). +## Idle tunnels are reclaimed and recovered + +Cloudflare bills a tunnel whether or not a connector is attached, so a laptop +that sleeps with a linked environment leaves a paid tunnel behind. The relay's +hourly maintenance job can reclaim those tunnels. `RELAY_TUNNEL_CLEANUP_MODE` +selects `off`, `dry-run`, or `enabled`, with `off` as the default. The mode is +read at deploy time, so changing it means a relay deploy, not a variable flip. +A candidate is a same-stage tunnel that Cloudflare reports down for at least +five minutes, or one that never connected and is at least an hour old. The +longer grace for never-connected tunnels covers a pairing still in progress. + +Cleanup deletes only tunnels whose host has registered recovery. Allocations +without recovery registration belong to hosts that cannot replace a deleted +tunnel and are left alone. Allocations with no recorded tunnel ID, or a +different tunnel ID, are skipped because a provision may own them. A tunnel with +no allocation row at all is counted as `skippedOrphan` and never deleted: there +is no row to lock, so a relink that adopts it by name could race the delete. +Clear those by hand. Each sweep is bounded: at most ten list requests, 100 deletions, a +two-minute deadline, and an early stop on a Cloudflare rate limit. Each sweep +starts one budget further along the candidate list, so a block of deletes that +keep failing cannot starve the tunnels listed after them. See the +[reaper](../../infra/relay/src/environments/ManagedEndpointReaper.ts). + +A host registers recovery at startup by sending its tunnel ID and loopback +origin with a short-lived signature from the environment key. Registration +touches Cloudflare only when the local host or port changed, and once per +existing allocation on the first registration after the upgrade because the +stored origin is empty. First registrations are jittered so an auto-update wave +does not hit the relay at once. The host stores a confirmed-origin marker with +the connector config, and a later boot starts the connector before registration +only when that marker matches the current config and port. If registration +cannot reach the relay for ten minutes, the host starts its stored config anyway +and keeps registering in the background until it can reconcile the origin. +If the connector exits, or `cloudflared` reports repeated tunnel +rejections, the host asks the relay for a replacement, at most once every two +minutes. The relay +provisions under the same allocation, so the hostname and DNS record survive +and clients keep their bindings. Every mutation on an allocation bumps its +`generation`, and deletion locks the row at the generation it claimed, so a +host that reconnects mid-sweep wins. + ## OAuth traps Interactive clients and the headless CLI use the same Clerk application but diff --git a/docs/operations/release.md b/docs/operations/release.md index 2aa477b9cb..d5fadaa220 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -137,6 +137,8 @@ Required `production` environment variables: Optional `production` environment variables: - `RELAY_DOMAIN` when overriding the derived `relay.` domain +- `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use + `off`. Required `production` environment secrets: @@ -160,6 +162,56 @@ Developers deploy personal stages locally rather than through pull-request autom vp run --filter t3code-relay deploy -- --stage "$USER" --env-file .env.local ``` +### Managed tunnel cleanup rollout + +Keep `RELAY_TUNNEL_CLEANUP_MODE=off` for the first production deploy. That deploy applies the +nullable allocation migration and adds the recovery endpoints. Web and mobile clients need no +coordinated release. CLI and desktop server builds must reach users before cleanup is enabled, +because those builds register recovery and replace a deleted tunnel after wake. + +1. Deploy the relay and migration with cleanup `off`. Merging to `pylon` does this; an unset + `RELAY_TUNNEL_CLEANUP_MODE` variable means `off`. +2. Release the server build and confirm current hosts register recovery. Older hosts stay marked + legacy and are never candidates. +3. Set `dry-run`, deploy, and read the sweep counters (`scanned`, `wouldDelete`, `skippedLegacy`, + `skippedOrphan`, `failed`, `truncated`) across several sweeps. +4. Run the disposable-host canary below. +5. Set `enabled` only after the canary recovers without a server restart. + +The job runs hourly, on the relay's existing maintenance schedule, with a five-minute grace period +for tunnels that lost their connector, so a candidate is usually removed within about an hour of +going down. Tunnels that never connected wait at least an hour. One sweep attempts at most 100 deletions, so a backlog takes longer. +`RELAY_TUNNEL_CLEANUP_MODE` is read at deploy time. Changing it, including turning cleanup off during +an incident, needs a relay deploy. + +To roll back, set cleanup to `off` and deploy the relay before downgrading any host. Keep the +recovery endpoints deployed while current server builds are in use. The nullable columns can stay. + +### Disposable-host canary + +This test has not been run against a real Cloudflare account. Run it against a disposable relay +stage, test Cloudflare account, disposable host, and disposable Pylon home (`T3CODE_HOME`). Keep production cleanup at +`off` or `dry-run` until it passes. Do not stop a daily-use Pylon server. + +1. Deploy the disposable stage with cleanup `dry-run`. Link a first disposable environment through + web or mobile settings and confirm its tunnel is healthy and recovery is registered. +2. Stop that host and restart the same Pylon home on a different local port. Confirm the public + hostname reaches the new port and sends nothing to the old one. +3. Link a second disposable environment with a server build that predates recovery registration. + Capture its managed `cloudflared` child PID, confirm it belongs to that host, and pause only that + child with `kill -STOP `. Wait until Cloudflare reports it down for over five minutes. +4. Capture the first environment's `cloudflared` child PID from its server logs, confirm ownership, + and pause it with `kill -STOP `. Wait until Cloudflare reports it down for over five + minutes. +5. Confirm dry-run counts the first tunnel in `wouldDelete` and the second in `skippedLegacy`. +6. Set cleanup `enabled` on the disposable stage and deploy. Confirm in the test Cloudflare account + that the first tunnel is deleted and the legacy tunnel still exists. +7. Resume the first child with `kill -CONT `. Confirm the running server detects the + repeated rejection, requests recovery, and becomes reachable at the same hostname without a + restart. +8. Resume the legacy child with `kill -CONT ` and confirm its tunnel reconnects. +9. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout. + ## Hosted web app release deployment The hosted app is intentionally not deployed by Vercel's Git integration. The diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 78241b42bf..a7ea468958 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -191,6 +191,11 @@ revokes its cloud access, removes any managed tunnel, and frees its host space e is offline or has been wiped. Removing an environment from a device's connection settings only forgets it on that device; it stays registered to your account. +When idle tunnel cleanup is enabled, Pylon Connect removes a linked environment's tunnel after it +stays offline, usually within about an hour. The environment stays linked and keeps the same address. +When the host starts again or wakes, Pylon Connect creates a replacement tunnel on its own. You do not +need to pair again. + On a command-line host, `t3 connect unlink` disables exposure while retaining your login; `t3 connect logout` also clears that login. Background-service [removal](./background-service.md#manage-the-service) is separate. diff --git a/infra/relay/.env.example b/infra/relay/.env.example index f885bff17c..9270ec5ff3 100644 --- a/infra/relay/.env.example +++ b/infra/relay/.env.example @@ -5,6 +5,10 @@ RELAY_API_ZONE_NAME=example.com RELAY_TUNNEL_ZONE_NAME=tunnels.example.com +# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup +# logs, then set enabled. Unset and off both disable cleanup. +# RELAY_TUNNEL_CLEANUP_MODE=off + # Optional: Relay domain override # Set this only when the derived relay hostname should not be used. # RELAY_DOMAIN=relay.example.com diff --git a/infra/relay/README.md b/infra/relay/README.md index af52c2e12a..c6b14cdc5f 100644 --- a/infra/relay/README.md +++ b/infra/relay/README.md @@ -150,6 +150,9 @@ The `production` GitHub environment must define these Actions variables: - `RELAY_API_ZONE_NAME` - `RELAY_TUNNEL_ZONE_NAME` - `RELAY_DOMAIN` if overriding the derived production relay domain +- `RELAY_TUNNEL_CLEANUP_MODE` to reclaim idle managed tunnels: `off` (the default when unset), + `dry-run`, or `enabled`. Follow the [rollout](../../docs/operations/release.md#managed-tunnel-cleanup-rollout) + before changing it. - `CLERK_PUBLISHABLE_KEY` - `CLERK_JWT_AUDIENCE` - `CLERK_JWT_TEMPLATE` diff --git a/infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/migration.sql b/infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/migration.sql new file mode 100644 index 0000000000..f2bbc5bba5 --- /dev/null +++ b/infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/migration.sql @@ -0,0 +1,4 @@ +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_enabled_at" varchar(64);--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "recovery_environment_public_key" text;--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "origin" jsonb;--> statement-breakpoint +ALTER TABLE "relay_managed_endpoint_allocations" ADD COLUMN "generation" integer DEFAULT 0 NOT NULL; diff --git a/infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/snapshot.json b/infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/snapshot.json new file mode 100644 index 0000000000..c066030719 --- /dev/null +++ b/infra/relay/migrations/postgres/20260919015455_managed_endpoint_recovery/snapshot.json @@ -0,0 +1,1568 @@ +{ + "dialect": "postgres", + "id": "3809c51e-3821-4a08-818d-e5d28dd3e9b4", + "prevIds": ["4b6d0d21-8d78-4499-9dde-b42bcf633d05"], + "version": "8", + "ddl": [ + { + "isRlsEnabled": false, + "name": "relay_agent_activity_rows", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_delivery_attempts", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_dpop_proofs", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_credentials", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_environment_links", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_live_activities", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_endpoint_allocations", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_managed_tunnel_limits", + "entityType": "tables", + "schema": "public" + }, + { + "isRlsEnabled": false, + "name": "relay_mobile_devices", + "entityType": "tables", + "schema": "public" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "state_json", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "type": "varchar(36)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(512)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thread_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "kind", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "source_job_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "token_suffix", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_status", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_reason", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "apns_id", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "transport_error", + "entityType": "columns", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "type": "varchar(128)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "thumbprint", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "jti", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "iat", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "credential_hash", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'T3 Environment'", + "generated": null, + "identity": null, + "name": "environment_label", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_http_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_ws_base_url", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(32)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "endpoint_provider_kind", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "notifications_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "true", + "generated": null, + "identity": null, + "name": "live_activities_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "managed_tunnels_enabled", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_by_device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "revoked_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_environment_links" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "activity_push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_start_queued_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "remote_started_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ended_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_aggregate_json", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "last_live_activity_delivery_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_live_activities" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "environment_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "hostname", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "tunnel_name", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "dns_record_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ready_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_enabled_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "recovery_environment_public_key", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "origin", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "generation", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "type": "varchar(191)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "max_tunnels", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_managed_tunnel_limits" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "user_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "device_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'iOS device'", + "generated": null, + "identity": null, + "name": "label", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "platform", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "ios_major_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "android_api_level", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "app_version", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(255)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "bundle_id", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(16)", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "aps_environment", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "push_to_start_token", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "preferences_json", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "type": "varchar(64)", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "updated_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_agent_activity_rows_updated", + "entityType": "indexes", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "thread_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "created_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "source_job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_delivery_attempts_source_job", + "entityType": "indexes", + "schema": "public", + "table": "relay_delivery_attempts" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_dpop_proofs_expires_at", + "entityType": "indexes", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "credential_hash", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_hash", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "environment_public_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_credentials_environment_key", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_credentials" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "environment_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "revoked_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_environment_links_environment", + "entityType": "indexes", + "schema": "public", + "table": "relay_environment_links" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "activity_push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_live_activities_activity_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_live_activities" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "hostname", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_hostname", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "tunnel_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_managed_endpoint_allocations_tunnel_name", + "entityType": "indexes", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "push_to_start_token", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": true, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_relay_mobile_devices_push_to_start_token", + "entityType": "indexes", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["environment_id", "environment_public_key", "thread_id"], + "nameExplicit": false, + "name": "relay_agent_activity_rows_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_agent_activity_rows" + }, + { + "columns": ["thumbprint", "jti"], + "nameExplicit": false, + "name": "relay_dpop_proofs_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_dpop_proofs" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_environment_links_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_environment_links" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_live_activities_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_live_activities" + }, + { + "columns": ["user_id", "environment_id"], + "nameExplicit": false, + "name": "relay_managed_endpoint_allocations_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_managed_endpoint_allocations" + }, + { + "columns": ["user_id", "device_id"], + "nameExplicit": false, + "name": "relay_mobile_devices_pkey", + "entityType": "pks", + "schema": "public", + "table": "relay_mobile_devices" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "relay_delivery_attempts_pkey", + "schema": "public", + "table": "relay_delivery_attempts", + "entityType": "pks" + }, + { + "columns": ["credential_id"], + "nameExplicit": false, + "name": "relay_environment_credentials_pkey", + "schema": "public", + "table": "relay_environment_credentials", + "entityType": "pks" + }, + { + "columns": ["user_id"], + "nameExplicit": false, + "name": "relay_managed_tunnel_limits_pkey", + "schema": "public", + "table": "relay_managed_tunnel_limits", + "entityType": "pks" + } + ], + "renames": [] +} diff --git a/infra/relay/src/Config.test.ts b/infra/relay/src/Config.test.ts new file mode 100644 index 0000000000..e37d605d06 --- /dev/null +++ b/infra/relay/src/Config.test.ts @@ -0,0 +1,35 @@ +import { expect, it } from "@effect/vitest"; +import * as ConfigProvider from "effect/ConfigProvider"; +import * as Effect from "effect/Effect"; + +import { managedEndpointCleanupModeConfig } from "./Config.ts"; + +it.effect.each([ + { name: "missing", env: {}, expected: "off" }, + { name: "empty", env: { RELAY_TUNNEL_CLEANUP_MODE: "" }, expected: "off" }, + { name: "whitespace", env: { RELAY_TUNNEL_CLEANUP_MODE: " \t" }, expected: "off" }, + { name: "off", env: { RELAY_TUNNEL_CLEANUP_MODE: "off" }, expected: "off" }, + { + name: "dry-run", + env: { RELAY_TUNNEL_CLEANUP_MODE: "dry-run" }, + expected: "dry-run", + }, + { name: "enabled", env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled" }, expected: "enabled" }, +] as const)("loads $name cleanup mode as $expected", ({ env, expected }) => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ env }); + expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe(expected); + }), +); + +it.effect("rejects an invalid cleanup mode", () => + Effect.gen(function* () { + const provider = ConfigProvider.fromEnv({ + env: { RELAY_TUNNEL_CLEANUP_MODE: "delete-everything" }, + }); + const error = yield* Effect.flip(managedEndpointCleanupModeConfig.parse(provider)); + + expect(error._tag).toBe("ConfigError"); + expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"'); + }), +); diff --git a/infra/relay/src/Config.ts b/infra/relay/src/Config.ts index 1f9872c08f..e2822b53ca 100644 --- a/infra/relay/src/Config.ts +++ b/infra/relay/src/Config.ts @@ -1,4 +1,6 @@ +import * as Config from "effect/Config"; import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Redacted from "effect/Redacted"; import * as Schema from "effect/Schema"; @@ -6,6 +8,20 @@ import * as Schema from "effect/Schema"; export const ApnsEnvironment = Schema.Literals(["sandbox", "production"]); export type ApnsEnvironment = typeof ApnsEnvironment.Type; +export const ManagedEndpointCleanupMode = Schema.Literals(["off", "dry-run", "enabled"]); +export type ManagedEndpointCleanupMode = typeof ManagedEndpointCleanupMode.Type; +const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpointCleanupMode); + +export const managedEndpointCleanupModeConfig = Config.String("RELAY_TUNNEL_CLEANUP_MODE").pipe( + Config.withDefault("off"), + Config.map((value) => value.trim() || "off"), + Config.mapEffect((value) => + decodeManagedEndpointCleanupMode(value).pipe( + Effect.mapError((error) => new Config.ConfigError(error)), + ), + ), +); + export interface ApnsCredentials { readonly teamId: string; readonly keyId: string; @@ -28,6 +44,7 @@ export class RelayConfiguration extends Context.Service< readonly cloudMintPublicKey: string; readonly managedEndpointBaseDomain: string | undefined; readonly managedEndpointNamespace: string | undefined; + readonly managedEndpointCleanupMode?: ManagedEndpointCleanupMode; } >()("t3code-relay/Config/RelayConfiguration") {} diff --git a/infra/relay/src/dbConfig.test.ts b/infra/relay/src/dbConfig.test.ts deleted file mode 100644 index e97b29a742..0000000000 --- a/infra/relay/src/dbConfig.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; - -import { relayDatabaseMode } from "./dbConfig.ts"; - -describe("relayDatabaseMode", () => { - it("uses the shared database only for production", () => { - expect(relayDatabaseMode("prod")).toBe("shared-database"); - expect(relayDatabaseMode("dev_julius")).toBe("stage-branch"); - expect(relayDatabaseMode("preview")).toBe("stage-branch"); - }); -}); diff --git a/infra/relay/src/deploymentConfig.test.ts b/infra/relay/src/deploymentConfig.test.ts index f090c70ee2..30682d65ba 100644 --- a/infra/relay/src/deploymentConfig.test.ts +++ b/infra/relay/src/deploymentConfig.test.ts @@ -7,6 +7,7 @@ import { managedEndpointHostname, isManagedEndpointHostname, managedEndpointTunnelName, + managedEndpointTunnelNamePrefix, relayOwnsManagedEndpointZone, RelayPublicDomainLabelTooLongError, relayPublicDomainForStage, @@ -84,6 +85,9 @@ describe("managed endpoint names", () => { expect(managedEndpointTunnelName("dev_julius", hash)).toBe( "t3coderelay-managedendpoint-dev-julius-abcdef0123456789", ); + expect(managedEndpointTunnelNamePrefix("dev_julius")).toBe( + "t3coderelay-managedendpoint-dev-julius-", + ); }); it("keeps the DNS label within the provider limit for long stage names", () => { diff --git a/infra/relay/src/deploymentConfig.ts b/infra/relay/src/deploymentConfig.ts index 1a3a332acc..961c6f1b2f 100644 --- a/infra/relay/src/deploymentConfig.ts +++ b/infra/relay/src/deploymentConfig.ts @@ -117,6 +117,10 @@ export function managedEndpointForHostname(hostname: string): RelayManagedEndpoi }; } +export function managedEndpointTunnelNamePrefix(stage: string): string { + return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-`; +} + export function managedEndpointTunnelName(stage: string, hash: string): string { - return `${MANAGED_ENDPOINT_TUNNEL_PREFIX}-${relayStageSlug(stage)}-${stableSuffix(hash)}`; + return `${managedEndpointTunnelNamePrefix(stage)}${stableSuffix(hash)}`; } diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts index 7f536bafb3..624d1241a2 100644 --- a/infra/relay/src/environments/EnvironmentConnector.test.ts +++ b/infra/relay/src/environments/EnvironmentConnector.test.ts @@ -188,7 +188,9 @@ function makeAllocations( tunnelName: "tunnel-name", dnsRecordId: "dns-record-id", readyAt: "2026-05-25T00:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "2026-05-25T00:00:00.000Z", + generation: 1, }, ): ManagedEndpointAllocations.ManagedEndpointAllocations["Service"] { return { @@ -197,7 +199,10 @@ function makeAllocations( recordTunnel: () => Effect.die("unused"), recordDns: () => Effect.die("unused"), markReady: () => Effect.die("unused"), + enableRecovery: () => Effect.die("unused"), + listByTunnelNames: () => Effect.die("unused"), claimRelease: () => Effect.die("unused"), + withClaimedTunnel: () => Effect.die("unused"), claimDeprovision: () => Effect.die("unused"), remove: () => Effect.die("unused"), removeClaimed: () => Effect.die("unused"), @@ -471,7 +476,9 @@ describe("EnvironmentConnector", () => { tunnelName: "tunnel-name", dnsRecordId: "dns-record-id", readyAt: null, + origin: null, updatedAt: "2026-05-25T00:00:00.000Z", + generation: 1, }), }), ), diff --git a/infra/relay/src/environments/EnvironmentLinker.test.ts b/infra/relay/src/environments/EnvironmentLinker.test.ts index c0811e82d9..3de7363c71 100644 --- a/infra/relay/src/environments/EnvironmentLinker.test.ts +++ b/infra/relay/src/environments/EnvironmentLinker.test.ts @@ -136,8 +136,9 @@ function testLayer(input?: { revokeForEnvironmentPublicKey: () => Effect.succeed(false), }), Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, { + reconcileOrigin: () => Effect.succeed("ready"), prepareDeprovision: () => Effect.succeed(null), - deprovision: input?.deprovision ?? (() => Effect.void), + deprovision: input?.deprovision ?? (() => Effect.succeed(true)), release: () => Effect.succeed(true), provision: () => Effect.succeed({ @@ -243,6 +244,7 @@ describe("EnvironmentLinker", () => { deprovision: (input) => Effect.sync(() => { deprovisionedEnvironmentId = input.environmentId; + return true; }), }), ), diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts index ebf51de100..105b10cc6a 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.test.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { PgDialect } from "drizzle-orm/pg-core"; import * as RelayDb from "../db.ts"; import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; @@ -10,17 +12,253 @@ const layerWithDb = (db: RelayDb.RelayDb["Service"]) => ManagedEndpointAllocations.layer.pipe(Layer.provide(Layer.succeed(RelayDb.RelayDb, db))); describe("ManagedEndpointAllocations", () => { + it.effect("clears endpoint readiness and recovery only when the recorded tunnel changes", () => { + let updated: + | { + readonly tunnelId: string; + readonly readyAt: unknown; + readonly recoveryEnabledAt: unknown; + readonly recoveryEnvironmentPublicKey: unknown; + } + | undefined; + const fakeDb = { + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (values: { + readonly tunnelId: string; + readonly readyAt: unknown; + readonly recoveryEnabledAt: unknown; + readonly recoveryEnvironmentPublicKey: unknown; + }) => { + updated = values; + return { + where: () => ({ + returning: () => Effect.succeed([{ generation: 8 }]), + }), + }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.recordTunnel({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "replacement-tunnel", + generation: 7, + }), + ).toBe(8); + expect(updated?.tunnelId).toBe("replacement-tunnel"); + const query = new PgDialect().sqlToQuery(updated?.readyAt as never); + expect(query.sql).toBe( + 'case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."ready_at" else null end', + ); + expect(query.params).toEqual(["replacement-tunnel"]); + for (const [column, value] of [ + ["recovery_enabled_at", updated?.recoveryEnabledAt], + ["recovery_environment_public_key", updated?.recoveryEnvironmentPublicKey], + ] as const) { + const recoveryQuery = new PgDialect().sqlToQuery(value as never); + expect(recoveryQuery.sql).toBe( + `case when "relay_managed_endpoint_allocations"."tunnel_id" = $1 then "relay_managed_endpoint_allocations"."${column}" else null end`, + ); + expect(recoveryQuery.params).toEqual(["replacement-tunnel"]); + } + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("records recovery support and advances the allocation generation", () => { + let updated: + | { + readonly recoveryEnabledAt: string; + readonly recoveryEnvironmentPublicKey: string; + readonly updatedAt: string; + } + | undefined; + let condition: unknown; + const fakeDb = { + update: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + set: (values: { + readonly recoveryEnabledAt: string; + readonly recoveryEnvironmentPublicKey: string; + readonly updatedAt: string; + }) => { + updated = values; + return { + where: (where: unknown) => { + condition = where; + return { + returning: () => Effect.succeed([{ environmentId: "environment-1" }]), + }; + }, + }; + }, + }; + }, + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "tunnel-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toBe(true); + + expect(updated?.recoveryEnabledAt).toBe(updated?.updatedAt); + expect(updated?.recoveryEnabledAt).toBeDefined(); + expect(updated?.recoveryEnvironmentPublicKey).toBe("public-key"); + const query = new PgDialect().sqlToQuery(condition as never); + expect(query.sql).toContain('"relay_managed_endpoint_allocations"."tunnel_id"'); + expect(query.sql).toContain('"relay_environment_links"."environment_public_key"'); + expect(query.sql).toContain('"relay_environment_links"."endpoint_provider_kind"'); + expect(query.sql).toContain('"relay_environment_links"."revoked_at" is null'); + expect(query.sql).toContain("for update"); + expect(query.params).toContain("tunnel-1"); + expect(query.params).toContain("public-key"); + expect(query.params).toContain("cloudflare_tunnel"); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("rejects recovery when the tunnel or active link no longer matches", () => { + const fakeDb = { + update: () => ({ + set: () => ({ + where: () => ({ + returning: () => Effect.succeed([]), + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect( + yield* allocations.enableRecovery({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "missing-tunnel", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toBe(false); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("returns recovery support with tunnel allocation lookups", () => { + const base = { + userId: "user-1", + hostname: "environment.example.test", + tunnelName: "managed-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-08-25T12:00:00.000Z", + updatedAt: "2026-08-25T12:00:00.000Z", + generation: 1, + }; + const fakeDb = { + select: () => ({ + from: (table: unknown) => { + expect(table).toBe(relayManagedEndpointAllocations); + return { + leftJoin: () => ({ + where: () => + Effect.succeed([ + { + ...base, + environmentId: "environment-1", + tunnelId: "tunnel-1", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "current-key", + linkedEnvironmentPublicKey: "current-key", + }, + { + ...base, + environmentId: "environment-2", + tunnelId: "tunnel-2", + recoveryEnabledAt: null, + recoveryEnvironmentPublicKey: null, + linkedEnvironmentPublicKey: "current-key", + }, + { + ...base, + environmentId: "environment-3", + tunnelId: "tunnel-3", + recoveryEnabledAt: "2026-08-25T12:00:00.000Z", + recoveryEnvironmentPublicKey: "old-key", + linkedEnvironmentPublicKey: "new-key", + }, + ]), + }), + }; + }, + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const result = yield* allocations.listByTunnelNames([ + "first-tunnel", + "second-tunnel", + "third-tunnel", + ]); + + expect(result.map((entry) => [entry.tunnelId, entry.recoveryEnabled])).toEqual([ + ["tunnel-1", true], + ["tunnel-2", false], + ["tunnel-3", false], + ]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + + it.effect("skips the database for an empty tunnel lookup", () => + Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + expect(yield* allocations.listByTunnelNames([])).toEqual([]); + }).pipe(Effect.provide(layerWithDb({} as RelayDb.RelayDb["Service"]))), + ); + + it.effect("splits large tunnel lookups into bounded database queries", () => { + const batchSizes: number[] = []; + const fakeDb = { + select: () => ({ + from: () => ({ + leftJoin: () => ({ + where: (condition: unknown) => { + batchSizes.push(new PgDialect().sqlToQuery(condition as never).params.length); + return Effect.succeed([]); + }, + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const names = Array.from({ length: 1_001 }, (_, index) => `tunnel-${index}`); + expect(yield* allocations.listByTunnelNames(names)).toEqual([]); + expect(batchSizes).toEqual([500, 500, 1]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("returns a claim generation only when deprovision wins the allocation CAS", () => { - let claimedAt: string | undefined; const fakeDb = { update: (table: unknown) => { expect(table).toBe(relayManagedEndpointAllocations); return { - set: (values: { readonly updatedAt: string }) => { - claimedAt = values.updatedAt; + set: (_values: { readonly updatedAt: string }) => { return { where: () => ({ - returning: () => Effect.succeed([{ userId: "user-1" }]), + returning: () => Effect.succeed([{ generation: 8 }]), }), }; }, @@ -33,14 +271,58 @@ describe("ManagedEndpointAllocations", () => { const generation = yield* allocations.claimDeprovision({ userId: "user-1", environmentId: "environment-1", - updatedAt: "captured-generation", + generation: 7, }); - expect(generation).toBe(claimedAt); + expect(generation).toBe(8); expect(generation).not.toBeNull(); }).pipe(Effect.provide(layerWithDb(fakeDb))); }); + it.effect("holds the claimed allocation row while deleting its tunnel", () => { + const operations: string[] = []; + const fakeDb = { + $client: { + withTransaction: (effect: Effect.Effect) => + Effect.sync(() => { + operations.push("transaction"); + }).pipe(Effect.andThen(effect)), + }, + select: () => ({ + from: () => ({ + where: () => ({ + limit: () => ({ + for: (strength: string) => + Effect.sync(() => { + operations.push(`lock:${strength}`); + return [{ generation: 7 }]; + }), + }), + }), + }), + }), + } as unknown as RelayDb.RelayDb["Service"]; + + return Effect.gen(function* () { + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const result = yield* allocations.withClaimedTunnel( + { + userId: "user-1", + environmentId: "environment-1", + tunnelId: "tunnel-1", + generation: 7, + }, + Effect.sync(() => { + operations.push("delete"); + return true; + }), + ); + + expect(Option.getOrNull(result)).toBe(true); + expect(operations).toEqual(["transaction", "lock:update", "delete"]); + }).pipe(Effect.provide(layerWithDb(fakeDb))); + }); + it.effect("does not remove an allocation superseded after a deprovision claim", () => { const fakeDb = { delete: (table: unknown) => { @@ -59,7 +341,7 @@ describe("ManagedEndpointAllocations", () => { yield* allocations.removeClaimed({ userId: "user-1", environmentId: "environment-1", - updatedAt: "outdated-claim-generation", + generation: 7, }), ).toBe(false); }).pipe(Effect.provide(layerWithDb(fakeDb))); diff --git a/infra/relay/src/environments/ManagedEndpointAllocations.ts b/infra/relay/src/environments/ManagedEndpointAllocations.ts index a8b5ecde62..4767dd3007 100644 --- a/infra/relay/src/environments/ManagedEndpointAllocations.ts +++ b/infra/relay/src/environments/ManagedEndpointAllocations.ts @@ -1,14 +1,17 @@ -import type { RelayManagedEndpoint } from "@t3tools/contracts/relay"; -import { and, eq } from "drizzle-orm"; +import type { RelayManagedEndpoint, RelayManagedEndpointOrigin } from "@t3tools/contracts/relay"; +import { and, eq, exists, inArray, isNull, sql } from "drizzle-orm"; +import { QueryBuilder } from "drizzle-orm/pg-core"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import * as SqlError from "effect/unstable/sql/SqlError"; import * as RelayDb from "../db.ts"; import { isManagedEndpointHostname, managedEndpointForHostname } from "../deploymentConfig.ts"; -import { relayManagedEndpointAllocations } from "../persistence/schema.ts"; +import { relayEnvironmentLinks, relayManagedEndpointAllocations } from "../persistence/schema.ts"; export interface ManagedEndpointAllocation { readonly userId: string; @@ -18,13 +21,17 @@ export interface ManagedEndpointAllocation { readonly tunnelName: string; readonly dnsRecordId: string | null; readonly readyAt: string | null; - /** - * Doubles as the allocation's generation marker: every mutation rewrites it, - * so `claimRelease` can detect a provision that raced a release. - */ + readonly origin: RelayManagedEndpointOrigin | null; readonly updatedAt: string; + readonly generation: number; } +export interface ManagedEndpointTunnelAllocation extends ManagedEndpointAllocation { + readonly recoveryEnabled: boolean; +} + +export const MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE = 500; + export function resolveReadyManagedEndpoint(input: { readonly allocation: ManagedEndpointAllocation; readonly baseDomain: string | undefined; @@ -50,6 +57,9 @@ export class ManagedEndpointAllocationPersistenceError extends Schema.TaggedErro "record-tunnel", "record-dns", "mark-ready", + "enable-recovery", + "list-tunnels", + "lock-tunnel", "claim-release", "claim-deprovision", "remove", @@ -82,23 +92,38 @@ interface ReserveManagedEndpointAllocationInput extends ManagedEndpointAllocatio interface RecordManagedEndpointTunnelInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; + readonly generation: number; } interface RecordManagedEndpointDnsInput extends ManagedEndpointAllocationKey { readonly dnsRecordId: string; + readonly tunnelId: string; + readonly generation: number; +} + +interface MarkManagedEndpointReadyInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly generation: number; + readonly origin: RelayManagedEndpointOrigin; } interface ClaimManagedEndpointReleaseInput extends ManagedEndpointAllocationKey { readonly tunnelId: string; - readonly updatedAt: string; + readonly generation: number; +} + +interface EnableManagedEndpointRecoveryInput extends ManagedEndpointAllocationKey { + readonly tunnelId: string; + readonly environmentPublicKey: string; + readonly origin: RelayManagedEndpointOrigin; } interface ClaimManagedEndpointDeprovisionInput extends ManagedEndpointAllocationKey { - readonly updatedAt: string; + readonly generation: number; } interface RemoveClaimedManagedEndpointAllocationInput extends ManagedEndpointAllocationKey { - readonly updatedAt: string; + readonly generation: number; } export class ManagedEndpointAllocations extends Context.Service< @@ -112,23 +137,36 @@ export class ManagedEndpointAllocations extends Context.Service< ) => Effect.Effect; readonly recordTunnel: ( input: RecordManagedEndpointTunnelInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly recordDns: ( input: RecordManagedEndpointDnsInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly markReady: ( - input: ManagedEndpointAllocationKey, - ) => Effect.Effect; + input: MarkManagedEndpointReadyInput, + ) => Effect.Effect; + readonly enableRecovery: ( + input: EnableManagedEndpointRecoveryInput, + ) => Effect.Effect; + readonly listByTunnelNames: ( + tunnelNames: ReadonlyArray, + ) => Effect.Effect< + ReadonlyArray, + ManagedEndpointAllocationPersistenceError + >; /** * Atomically claims the right to delete the allocation's tunnel: succeeds * only while the recorded tunnel and generation still match what the - * caller loaded. A concurrent provision rewrites `updatedAt` when it + * caller loaded. A concurrent provision increments `generation` when it * records its tunnel, which makes a stale claim fail and keeps the freshly * issued tunnel alive. */ readonly claimRelease: ( input: ClaimManagedEndpointReleaseInput, - ) => Effect.Effect; + ) => Effect.Effect; + readonly withClaimedTunnel: ( + input: ClaimManagedEndpointReleaseInput, + effect: Effect.Effect, + ) => Effect.Effect, E | ManagedEndpointAllocationPersistenceError, R>; /** * Claims the complete allocation for teardown only if its generation still * matches the snapshot captured by the unlink operation. @@ -138,7 +176,7 @@ export class ManagedEndpointAllocations extends Context.Service< */ readonly claimDeprovision: ( input: ClaimManagedEndpointDeprovisionInput, - ) => Effect.Effect; + ) => Effect.Effect; readonly remove: ( input: ManagedEndpointAllocationKey, ) => Effect.Effect; @@ -156,7 +194,9 @@ const allocationSelection = { tunnelName: relayManagedEndpointAllocations.tunnelName, dnsRecordId: relayManagedEndpointAllocations.dnsRecordId, readyAt: relayManagedEndpointAllocations.readyAt, + origin: relayManagedEndpointAllocations.origin, updatedAt: relayManagedEndpointAllocations.updatedAt, + generation: relayManagedEndpointAllocations.generation, }; const whereAllocation = (input: ManagedEndpointAllocationKey) => @@ -248,14 +288,28 @@ export const make = Effect.gen(function* () { recordTunnel: Effect.fn("relay.managed_endpoint_allocations.record_tunnel")(function* ( input: RecordManagedEndpointTunnelInput, ) { - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ tunnelId: input.tunnelId, + readyAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.readyAt} else null end`, + origin: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.origin} else null end`, + // Recovery registration is per tunnel: a replacement must register + // again before the reaper may treat it as recoverable. + recoveryEnabledAt: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.recoveryEnabledAt} else null end`, + recoveryEnvironmentPublicKey: sql`case when ${relayManagedEndpointAllocations.tunnelId} = ${input.tunnelId} then ${relayManagedEndpointAllocations.recoveryEnvironmentPublicKey} else null end`, updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -270,14 +324,23 @@ export const make = Effect.gen(function* () { recordDns: Effect.fn("relay.managed_endpoint_allocations.record_dns")(function* ( input: RecordManagedEndpointDnsInput, ) { - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ dnsRecordId: input.dnsRecordId, updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -290,17 +353,27 @@ export const make = Effect.gen(function* () { ); }), markReady: Effect.fn("relay.managed_endpoint_allocations.mark_ready")(function* ( - input: ManagedEndpointAllocationKey, + input: MarkManagedEndpointReadyInput, ) { const now = DateTime.formatIso(yield* DateTime.now); - yield* db + return yield* db .update(relayManagedEndpointAllocations) .set({ readyAt: now, + origin: input.origin, updatedAt: now, + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) - .where(whereAllocation(input)) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) .pipe( + Effect.map((rows) => rows.length > 0), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -312,6 +385,124 @@ export const make = Effect.gen(function* () { ), ); }), + enableRecovery: Effect.fn("relay.managed_endpoint_allocations.enable_recovery")(function* ( + input: EnableManagedEndpointRecoveryInput, + ) { + const now = DateTime.formatIso(yield* DateTime.now); + return yield* db + .update(relayManagedEndpointAllocations) + .set({ + recoveryEnabledAt: now, + recoveryEnvironmentPublicKey: input.environmentPublicKey, + updatedAt: now, + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, + }) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.origin, input.origin), + exists( + new QueryBuilder() + .select({ userId: relayEnvironmentLinks.userId }) + .from(relayEnvironmentLinks) + .where( + and( + eq(relayEnvironmentLinks.userId, input.userId), + eq(relayEnvironmentLinks.environmentId, input.environmentId), + eq(relayEnvironmentLinks.environmentPublicKey, input.environmentPublicKey), + eq(relayEnvironmentLinks.endpointProviderKind, "cloudflare_tunnel"), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .for("update"), + ), + ), + ) + .returning({ environmentId: relayManagedEndpointAllocations.environmentId }) + .pipe( + Effect.map((rows) => rows.length > 0), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "enable-recovery", + stage: "database-request", + ...input, + cause, + }), + ), + ); + }), + listByTunnelNames: Effect.fn("relay.managed_endpoint_allocations.list_by_tunnel_names")( + function* (tunnelNames: ReadonlyArray) { + if (tunnelNames.length === 0) { + return []; + } + const batches = Array.from( + { length: Math.ceil(tunnelNames.length / MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE) }, + (_, index) => + tunnelNames.slice( + index * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE, + (index + 1) * MANAGED_ENDPOINT_ALLOCATION_LOOKUP_BATCH_SIZE, + ), + ); + const results = yield* Effect.forEach( + batches, + (batch) => + db + .select({ + ...allocationSelection, + recoveryEnabledAt: relayManagedEndpointAllocations.recoveryEnabledAt, + recoveryEnvironmentPublicKey: + relayManagedEndpointAllocations.recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey: relayEnvironmentLinks.environmentPublicKey, + }) + .from(relayManagedEndpointAllocations) + .leftJoin( + relayEnvironmentLinks, + and( + eq(relayEnvironmentLinks.userId, relayManagedEndpointAllocations.userId), + eq( + relayEnvironmentLinks.environmentId, + relayManagedEndpointAllocations.environmentId, + ), + isNull(relayEnvironmentLinks.revokedAt), + ), + ) + .where(inArray(relayManagedEndpointAllocations.tunnelName, batch)) + .pipe( + Effect.map((rows) => + rows.map( + ({ + recoveryEnabledAt, + recoveryEnvironmentPublicKey, + linkedEnvironmentPublicKey, + ...allocation + }) => ({ + ...allocation, + recoveryEnabled: + recoveryEnabledAt !== null && + recoveryEnvironmentPublicKey !== null && + recoveryEnvironmentPublicKey === linkedEnvironmentPublicKey, + }), + ), + ), + Effect.mapError( + (cause) => + new ManagedEndpointAllocationPersistenceError({ + operation: "list-tunnels", + stage: "database-request", + userId: "*", + environmentId: "*", + cause, + }), + ), + ), + { concurrency: 1 }, + ); + return results.flat(); + }, + ), claimRelease: Effect.fn("relay.managed_endpoint_allocations.claim_release")(function* ( input: ClaimManagedEndpointReleaseInput, ) { @@ -319,17 +510,18 @@ export const make = Effect.gen(function* () { .update(relayManagedEndpointAllocations) .set({ updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, }) .where( and( whereAllocation(input), eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ userId: relayManagedEndpointAllocations.userId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -344,22 +536,66 @@ export const make = Effect.gen(function* () { ); return claimed; }), + withClaimedTunnel: Effect.fn("relay.managed_endpoint_allocations.with_claimed_tunnel")( + function* ( + input: ClaimManagedEndpointReleaseInput, + effect: Effect.Effect, + ): Effect.fn.Return, E | ManagedEndpointAllocationPersistenceError, R> { + const lockError = (cause: unknown) => + new ManagedEndpointAllocationPersistenceError({ + operation: "lock-tunnel", + stage: "database-request", + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + cause, + }); + return yield* db.$client + .withTransaction( + db + .select({ generation: relayManagedEndpointAllocations.generation }) + .from(relayManagedEndpointAllocations) + .where( + and( + whereAllocation(input), + eq(relayManagedEndpointAllocations.tunnelId, input.tunnelId), + eq(relayManagedEndpointAllocations.generation, input.generation), + ), + ) + .limit(1) + .for("update") + .pipe( + Effect.mapError(lockError), + Effect.flatMap((rows) => + rows.length === 0 + ? Effect.succeed(Option.none()) + : effect.pipe(Effect.map(Option.some)), + ), + ), + ) + .pipe( + Effect.mapError((cause) => (SqlError.isSqlError(cause) ? lockError(cause) : cause)), + ); + }, + ), claimDeprovision: Effect.fn("relay.managed_endpoint_allocations.claim_deprovision")(function* ( input: ClaimManagedEndpointDeprovisionInput, ) { - const claimedAt = DateTime.formatIso(yield* DateTime.now); const claimed = yield* db .update(relayManagedEndpointAllocations) - .set({ updatedAt: claimedAt }) + .set({ + updatedAt: DateTime.formatIso(yield* DateTime.now), + generation: sql`${relayManagedEndpointAllocations.generation} + 1`, + }) .where( and( whereAllocation(input), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) - .returning({ userId: relayManagedEndpointAllocations.userId }) + .returning({ generation: relayManagedEndpointAllocations.generation }) .pipe( - Effect.map((rows) => rows.length > 0), + Effect.map((rows) => rows[0]?.generation ?? null), Effect.mapError( (cause) => new ManagedEndpointAllocationPersistenceError({ @@ -371,7 +607,7 @@ export const make = Effect.gen(function* () { }), ), ); - return claimed ? claimedAt : null; + return claimed; }), remove: Effect.fn("relay.managed_endpoint_allocations.remove")(function* ( input: ManagedEndpointAllocationKey, @@ -399,7 +635,7 @@ export const make = Effect.gen(function* () { .where( and( whereAllocation(input), - eq(relayManagedEndpointAllocations.updatedAt, input.updatedAt), + eq(relayManagedEndpointAllocations.generation, input.generation), ), ) .returning({ userId: relayManagedEndpointAllocations.userId }) diff --git a/infra/relay/src/environments/ManagedEndpointProvider.test.ts b/infra/relay/src/environments/ManagedEndpointProvider.test.ts index 4d136658c8..fc02d56b2f 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.test.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.test.ts @@ -6,6 +6,7 @@ import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Redacted from "effect/Redacted"; import * as RelayConfiguration from "../Config.ts"; @@ -33,7 +34,7 @@ const config = RelayConfiguration.RelayConfiguration.of({ }); interface TunnelCall { - readonly operation: "list" | "create" | "putConfiguration" | "getToken" | "delete"; + readonly operation: "get" | "list" | "create" | "putConfiguration" | "getToken" | "delete"; readonly input: unknown; } @@ -62,6 +63,16 @@ function allocationKey(input: { readonly userId: string; readonly environmentId: function makeTunnelClient(calls: TunnelCall[] = []) { return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.sync(() => { + calls.push({ operation: "get", input: tunnelId }); + return { + id: tunnelId, + name: "managed-tunnel", + status: "down", + connsInactiveAt: "2026-06-01T00:00:00.000Z", + }; + }), list: (request) => Effect.sync(() => { calls.push({ operation: "list", input: request }); @@ -91,6 +102,23 @@ function makeTunnelClient(calls: TunnelCall[] = []) { function makePersistentTunnelClient(calls: TunnelCall[] = []) { let tunnel: { readonly id: string; readonly name: string } | null = null; return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.suspend(() => { + calls.push({ operation: "get", input: tunnelId }); + return tunnel === null + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ) + : Effect.succeed({ + ...tunnel, + status: "down", + connsInactiveAt: "2026-06-01T00:00:00.000Z", + }); + }), list: (request) => Effect.sync(() => { calls.push({ operation: "list", input: request }); @@ -159,6 +187,7 @@ function makeDnsClient( function makeAllocations(calls: AllocationCall[] = []) { const allocations = new Map(); + const recoveryEnabled = new Set(); let generation = 0; const mutate = ( key: string, @@ -168,7 +197,11 @@ function makeAllocations(calls: AllocationCall[] = []) { ) => { const allocation = allocations.get(key); if (allocation !== undefined) { - allocations.set(key, { ...change(allocation), updatedAt: `generation-${++generation}` }); + allocations.set(key, { + ...change(allocation), + generation: allocation.generation + 1, + updatedAt: `generation-${++generation}`, + }); } }; return ManagedEndpointAllocations.ManagedEndpointAllocations.of({ @@ -185,7 +218,9 @@ function makeAllocations(calls: AllocationCall[] = []) { tunnelId: null, dnsRecordId: null, readyAt: null, + origin: null, updatedAt: `generation-${++generation}`, + generation: 0, }; allocations.set(allocationKey(input), allocation); return allocation; @@ -193,27 +228,62 @@ function makeAllocations(calls: AllocationCall[] = []) { recordTunnel: (input) => Effect.sync(() => { calls.push({ operation: "recordTunnel", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation) { + return null; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, tunnelId: input.tunnelId, + readyAt: allocation.tunnelId === input.tunnelId ? allocation.readyAt : null, })); + return allocations.get(allocationKey(input))?.generation ?? null; }), recordDns: (input) => Effect.sync(() => { calls.push({ operation: "recordDns", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) { + return null; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, dnsRecordId: input.dnsRecordId, })); + return allocations.get(allocationKey(input))?.generation ?? null; }), markReady: (input) => Effect.sync(() => { calls.push({ operation: "markReady", input }); + const current = allocations.get(allocationKey(input)); + if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) { + return false; + } mutate(allocationKey(input), (allocation) => ({ ...allocation, readyAt: "2026-06-02T00:00:00.000Z", + origin: input.origin, })); + return true; + }), + enableRecovery: (input) => + Effect.sync(() => { + const allocation = allocations.get(allocationKey(input)); + if (allocation?.tunnelId !== input.tunnelId) { + return false; + } + recoveryEnabled.add(allocationKey(input)); + return true; }), + listByTunnelNames: (tunnelNames) => + Effect.sync(() => + [...allocations.values()] + .filter((allocation) => tunnelNames.includes(allocation.tunnelName)) + .map((allocation) => ({ + ...allocation, + recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)), + })), + ), claimRelease: (input) => Effect.sync(() => { calls.push({ operation: "claimRelease", input }); @@ -221,22 +291,29 @@ function makeAllocations(calls: AllocationCall[] = []) { if ( allocation === undefined || allocation.tunnelId !== input.tunnelId || - allocation.updatedAt !== input.updatedAt + allocation.generation !== input.generation ) { - return false; + return null; } mutate(allocationKey(input), (current) => current); - return true; + return allocations.get(allocationKey(input))?.generation ?? null; + }), + withClaimedTunnel: (input, effect) => + Effect.suspend(() => { + const current = allocations.get(allocationKey(input)); + return current?.tunnelId === input.tunnelId && current.generation === input.generation + ? effect.pipe(Effect.map(Option.some)) + : Effect.succeed(Option.none()); }), claimDeprovision: (input) => Effect.sync(() => { calls.push({ operation: "claimDeprovision", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation === undefined || allocation.updatedAt !== input.updatedAt) { + if (allocation === undefined || allocation.generation !== input.generation) { return null; } mutate(allocationKey(input), (current) => current); - return allocations.get(allocationKey(input))?.updatedAt ?? null; + return allocations.get(allocationKey(input))?.generation ?? null; }), remove: (input) => Effect.sync(() => { @@ -247,7 +324,7 @@ function makeAllocations(calls: AllocationCall[] = []) { Effect.sync(() => { calls.push({ operation: "removeClaimed", input }); const allocation = allocations.get(allocationKey(input)); - if (allocation === undefined || allocation.updatedAt !== input.updatedAt) { + if (allocation === undefined || allocation.generation !== input.generation) { return false; } allocations.delete(allocationKey(input)); @@ -306,6 +383,7 @@ function expectedManagedTunnelName(environmentId: string, userId = "user_ABC"): describe("ManagedEndpointProvider", () => { it.effect("does not require the deployment RuntimeContext when building the Worker layer", () => { const tunnelClient = { + get: () => Effect.succeed({ id: "tunnel-id", name: "managed-tunnel" }), list: () => Effect.succeed({ result: [] }), create: (request: { readonly name: string }) => Effect.succeed({ id: "tunnel-id", name: request.name }), @@ -731,7 +809,8 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.andThen(Effect.fail(failure))), deleteRecord: () => Effect.void, }); - const layer = providerLayer(makePersistentTunnelClient(), dnsClient, makeAllocations()); + const allocations = makeAllocations(); + const layer = providerLayer(makePersistentTunnelClient(), dnsClient, allocations); return Effect.gen(function* () { const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; @@ -754,6 +833,10 @@ describe("ManagedEndpointProvider", () => { "createRecord", "updateRecord", ]); + expect(yield* allocations.get(request)).toMatchObject({ + tunnelId: "tunnel-id", + readyAt: "2026-06-02T00:00:00.000Z", + }); }).pipe(Effect.provide(layer)); }); @@ -914,7 +997,7 @@ describe("ManagedEndpointProvider", () => { // longer matches what the release loaded, so the claim fails. const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ ...allocations, - claimRelease: () => Effect.succeed(false), + claimRelease: () => Effect.succeed(null), }); const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated); @@ -939,6 +1022,459 @@ describe("ManagedEndpointProvider", () => { }).pipe(Effect.provide(layer)); }); + it.effect("does not release a tunnel when the requested tunnel id is outdated", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = providerLayer( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(), + makeAllocations(), + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect(yield* provider.release({ ...key, expectedTunnelId: "old-tunnel-id" })).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps a tunnel when a provision replaces an ordinary release generation", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + let replaceAfterClaim = false; + const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + claimRelease: (input) => + allocations.claimRelease(input).pipe( + Effect.tap((claimedGeneration) => { + if (claimedGeneration === null || replaceAfterClaim) return Effect.void; + replaceAfterClaim = true; + return allocations + .recordTunnel({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: "replacement-tunnel", + generation: claimedGeneration, + }) + .pipe(Effect.asVoid); + }), + ), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect(yield* provider.release(key)).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does no Cloudflare work when the registered origin is unchanged", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls)); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const; + const provisioned = yield* provider.provision({ ...key, origin }); + tunnelCalls.length = 0; + + expect( + yield* provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin, + endpoint: provisioned.endpoint, + }), + ).toBe("ready"); + expect(tunnelCalls).toEqual([]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("updates Cloudflare ingress once when the registered port changes", () => { + const tunnelCalls: TunnelCall[] = []; + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls)); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const provisioned = yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + tunnelCalls.length = 0; + + expect( + yield* provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 }, + endpoint: provisioned.endpoint, + }), + ).toBe("ready"); + expect(tunnelCalls).toEqual([ + { + operation: "putConfiguration", + input: { + tunnelId: "tunnel-id", + tunnelConfig: { + ingress: [ + { + hostname: expectedManagedHostname("env_ABC"), + service: "http://127.0.0.1:4884", + }, + { service: "http_status:404" }, + ], + }, + }, + }, + ]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("requires recovery when Cloudflare reports the registered tunnel is missing", () => { + const tunnelCalls: TunnelCall[] = []; + const baseTunnelClient = makePersistentTunnelClient(tunnelCalls); + let tunnelMissing = false; + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + ...baseTunnelClient, + putConfiguration: (tunnelId, tunnelConfig) => + tunnelMissing + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "put-configuration", + tunnelId, + cause: { _tag: "TunnelNotFound" }, + }), + ) + : baseTunnelClient.putConfiguration(tunnelId, tunnelConfig), + }); + const layer = providerLayer(tunnelClient); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const provisioned = yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + tunnelMissing = true; + + expect( + yield* provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 }, + endpoint: provisioned.endpoint, + }), + ).toBe("recovery_required"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("fails origin sync when the allocation generation changes", () => { + const allocations = makeAllocations(); + let loseClaim = false; + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: (input, effect) => + loseClaim ? Effect.succeed(Option.none()) : allocations.withClaimedTunnel(input, effect), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const provisioned = yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + loseClaim = true; + + const error = yield* Effect.flip( + provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 }, + endpoint: provisioned.endpoint, + }), + ); + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "sync-origin", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("rejects an active endpoint that does not match the allocation hostname", () => { + const layer = providerLayer(makePersistentTunnelClient()); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + const provisioned = yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + const error = yield* Effect.flip( + provider.reconcileOrigin({ + ...key, + tunnelId: provisioned.runtime.tunnelId!, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + endpoint: { + ...provisioned.endpoint, + httpBaseUrl: "https://different-host.t3code.test/", + }, + }), + ); + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "verify-endpoint", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect( + "keeps a newly created tunnel available for retry after losing its allocation claim", + () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + let changeGeneration = true; + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + recordTunnel: (input) => + Effect.gen(function* () { + if (changeGeneration) { + changeGeneration = false; + yield* allocations.claimDeprovision(input); + } + return yield* allocations.recordTunnel(input); + }), + }); + const layer = providerLayer( + makePersistentTunnelClient(tunnelCalls), + makeDnsClient(), + changed, + ); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const input = { + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }; + const error = yield* Effect.flip(provider.provision(input)); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-tunnel", + }); + expect(tunnelCalls.map((call) => call.operation)).toEqual(["list", "create"]); + expect((yield* provider.provision(input)).runtime.tunnelId).toBe("tunnel-id"); + expect(tunnelCalls.filter((call) => call.operation === "create")).toHaveLength(1); + expect(tunnelCalls.filter((call) => call.operation === "delete")).toEqual([]); + }).pipe(Effect.provide(layer)); + }, + ); + + it.effect("does not overwrite DNS when tunnel ownership changes during provisioning", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + recordDns: () => Effect.succeed(null), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-dns", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not change tunnel ingress after another provision takes ownership", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: () => Effect.succeed(Option.none()), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "configure-tunnel", + }); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("putConfiguration"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not change DNS after another provision takes ownership", () => { + const dnsCalls: DnsCall[] = []; + const allocations = makeAllocations(); + let lockCount = 0; + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + withClaimedTunnel: (input, effect) => + ++lockCount === 1 + ? allocations.withClaimedTunnel(input, effect) + : Effect.succeed(Option.none()), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(dnsCalls), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "record-dns", + }); + expect(dnsCalls).toEqual([]); + }).pipe(Effect.provide(layer)); + }); + + it.effect("does not mark a superseded tunnel allocation as ready", () => { + const allocations = makeAllocations(); + const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + markReady: () => Effect.succeed(false), + }); + const layer = providerLayer(makePersistentTunnelClient(), makeDnsClient(), changed); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const error = yield* Effect.flip( + provider.provision({ + userId: "user_ABC", + environmentId: "env_ABC", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ + _tag: "ManagedEndpointProvisioningFailed", + stage: "mark-allocation-ready", + }); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps a tunnel that reconnects before scheduled deletion", () => { + const tunnelCalls: TunnelCall[] = []; + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + ...makePersistentTunnelClient(tunnelCalls), + get: (tunnelId) => + Effect.succeed({ + id: tunnelId, + name: expectedManagedTunnelName("env_ABC"), + status: "healthy", + connsInactiveAt: null, + }), + }); + const layer = providerLayer(tunnelClient, makeDnsClient(), makeAllocations()); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: "2026-06-01T00:05:00.000Z", + }), + ).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + + it.effect("keeps a tunnel when a new provision replaces the release generation", () => { + const tunnelCalls: TunnelCall[] = []; + const allocations = makeAllocations(); + const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + ...allocations, + claimRelease: (input) => + allocations.claimRelease(input).pipe( + Effect.tap((claimedGeneration) => + claimedGeneration === null + ? Effect.void + : allocations + .recordTunnel({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: "replacement-tunnel", + generation: claimedGeneration, + }) + .pipe(Effect.asVoid), + ), + ), + }); + const layer = providerLayer(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced); + + return Effect.gen(function* () { + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const key = { userId: "user_ABC", environmentId: "env_ABC" } as const; + yield* provider.provision({ + ...key, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + + expect( + yield* provider.release({ + ...key, + expectedTunnelId: "tunnel-id", + expectedStatus: "down", + expectedInactiveBefore: "2026-06-01T00:05:00.000Z", + }), + ).toBe(false); + expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete"); + }).pipe(Effect.provide(layer)); + }); + it.effect("treats an already deleted tunnel as successfully released", () => { const notFound = { _tag: "NotFound" } as const; const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ diff --git a/infra/relay/src/environments/ManagedEndpointProvider.ts b/infra/relay/src/environments/ManagedEndpointProvider.ts index 977568c274..bc0f4c7457 100644 --- a/infra/relay/src/environments/ManagedEndpointProvider.ts +++ b/infra/relay/src/environments/ManagedEndpointProvider.ts @@ -3,6 +3,7 @@ import * as Cloudflare from "alchemy/Cloudflare"; import * as Arr from "effect/Array"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Encoding from "effect/Encoding"; import * as Layer from "effect/Layer"; @@ -52,6 +53,9 @@ const ManagedEndpointProvisioningStage = Schema.Literals([ "record-dns", "get-tunnel-token", "mark-allocation-ready", + "load-allocation", + "verify-endpoint", + "sync-origin", ]); export class ManagedEndpointProvisioningFailed extends Schema.TaggedError()( @@ -76,6 +80,7 @@ export class ManagedEndpointProvisioningFailed extends Schema.TaggedError Effect.Effect; + readonly reconcileOrigin: (input: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; + readonly endpoint: RelayManagedEndpoint; + }) => Effect.Effect; /** * Captures the allocation generation owned by an unlink before its link * revocation commits. Passing this target to `deprovision` prevents a @@ -150,7 +164,7 @@ export class ManagedEndpointProvider extends Context.Service< readonly userId: string; readonly environmentId: string; readonly target?: ManagedEndpointDeprovisionTarget | null; - }) => Effect.Effect; + }) => Effect.Effect; /** * Deletes the provisioned Cloudflare tunnel while keeping the allocation * (hostname + tunnel name reservation) and DNS record. Cloudflare bills per @@ -166,16 +180,34 @@ export class ManagedEndpointProvider extends Context.Service< readonly release: (input: { readonly userId: string; readonly environmentId: string; + readonly expectedTunnelId?: string; + readonly expectedInactiveBefore?: string; + readonly expectedStatus?: "inactive" | "down"; }) => Effect.Effect; } >()("t3code-relay/environments/ManagedEndpointProvider") {} -interface ManagedEndpointTunnel { +export interface ManagedEndpointTunnel { readonly id?: string | null; readonly name?: string | null; + readonly status?: string | null; + readonly createdAt?: string | null; + readonly connsInactiveAt?: string | null; +} + +export interface ManagedEndpointTunnelListRequest { + readonly isDeleted: false; + readonly name?: string; + readonly includePrefix?: string; + readonly status?: "inactive" | "down"; + readonly existedAt?: string; + readonly wasInactiveAt?: string; + readonly page?: number; + readonly perPage?: number; } const ManagedEndpointTunnelClientOperation = Schema.Literals([ + "get", "list", "create", "put-configuration", @@ -201,11 +233,18 @@ export class ManagedEndpointTunnelClientError extends Schema.TaggedError Effect.Effect< - { readonly result: ReadonlyArray }, + readonly get: ( + tunnelId: string, + ) => Effect.Effect; + readonly list: (request: ManagedEndpointTunnelListRequest) => Effect.Effect< + { + readonly result: ReadonlyArray; + readonly resultInfo?: { + readonly page?: number | null; + readonly perPage?: number | null; + readonly totalCount?: number | null; + } | null; + }, ManagedEndpointTunnelClientError >; readonly create: (request: { @@ -333,17 +372,17 @@ function isLoopbackOrigin(origin: RelayManagedEndpointOrigin): boolean { ); } -function isNotFoundCause(cause: unknown): boolean { +export function isManagedEndpointNotFound(cause: unknown): boolean { if (typeof cause !== "object" || cause === null) { return false; } - if ("_tag" in cause && cause._tag === "NotFound") { + if ("_tag" in cause && (cause._tag === "NotFound" || cause._tag === "TunnelNotFound")) { return true; } if ("status" in cause && cause.status === 404) { return true; } - return "cause" in cause && isNotFoundCause(cause.cause); + return "cause" in cause && isManagedEndpointNotFound(cause.cause); } type ManagedEndpointClientError = ManagedEndpointTunnelClientError | ManagedEndpointDnsClientError; @@ -355,9 +394,9 @@ const ignoreNotFound = ( Effect.asVoid, Effect.catchTags({ ManagedEndpointTunnelClientError: (error) => - isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error), ManagedEndpointDnsClientError: (error) => - isNotFoundCause(error.cause) ? Effect.void : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.void : Effect.fail(error), }), ); @@ -399,7 +438,7 @@ export const make = Effect.gen(function* () { Effect.as(true), Effect.catchTags({ ManagedEndpointDnsClientError: (error) => - isNotFoundCause(error.cause) ? Effect.succeed(false) : Effect.fail(error), + isManagedEndpointNotFound(error.cause) ? Effect.succeed(false) : Effect.fail(error), }), ); if (checkpointedRecordUpdated) { @@ -455,8 +494,129 @@ export const make = Effect.gen(function* () { }, ); + const reconcileOrigin = Effect.fn("relay.managed_endpoint_provider.reconcile_origin")( + function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; + readonly endpoint: RelayManagedEndpoint; + }) { + if (!isLoopbackOrigin(input.origin)) { + return yield* new ManagedEndpointOriginNotAllowed({ + userId: input.userId, + environmentId: input.environmentId, + host: input.origin.localHttpHost, + port: input.origin.localHttpPort, + }); + } + const allocation = yield* allocations.get(input).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + ...input, + stage: "load-allocation", + cause, + }), + ), + ); + if ( + allocation === null || + allocation.tunnelId !== input.tunnelId || + allocation.dnsRecordId === null || + allocation.readyAt === null + ) { + return "recovery_required"; + } + const cf = yield* requireCloudflareSettings(config, input); + const recordedEndpoint = ManagedEndpointAllocations.resolveReadyManagedEndpoint({ + allocation, + baseDomain: cf.baseDomain, + }); + if ( + recordedEndpoint === null || + recordedEndpoint.httpBaseUrl !== input.endpoint.httpBaseUrl || + recordedEndpoint.wsBaseUrl !== input.endpoint.wsBaseUrl || + recordedEndpoint.providerKind !== input.endpoint.providerKind + ) { + return yield* new ManagedEndpointProvisioningFailed({ + ...input, + stage: "verify-endpoint", + hostname: allocation.hostname, + }); + } + if ( + allocation.origin?.localHttpHost === input.origin.localHttpHost && + allocation.origin.localHttpPort === input.origin.localHttpPort + ) { + return "ready"; + } + + const updated = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + generation: allocation.generation, + }, + tunnels + .putConfiguration(input.tunnelId, { + ingress: [ + { + hostname: allocation.hostname, + service: formatOriginService(input.origin), + }, + { service: "http_status:404" }, + ], + }) + .pipe( + Effect.as("configured" as const), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (error) => + isManagedEndpointNotFound(error.cause) + ? Effect.succeed("missing" as const) + : Effect.fail(error), + }), + Effect.flatMap((result) => + result === "missing" + ? Effect.succeed(result) + : allocations + .markReady({ + ...input, + generation: allocation.generation, + }) + .pipe( + Effect.map((updated) => + updated ? ("configured" as const) : ("stale" as const), + ), + ), + ), + ), + ) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + ...input, + stage: "sync-origin", + cause, + }), + ), + ); + if (Option.isNone(updated) || updated.value === "stale") { + return yield* new ManagedEndpointProvisioningFailed({ + ...input, + stage: "sync-origin", + }); + } + return updated.value === "configured" ? "ready" : "recovery_required"; + }, + ); + return ManagedEndpointProvider.of({ prepareDeprovision, + reconcileOrigin, deprovision: Effect.fn("relay.managed_endpoint_provider.deprovision")(function* (input) { yield* Effect.annotateCurrentSpan({ "relay.user_id": input.userId, @@ -465,13 +625,13 @@ export const make = Effect.gen(function* () { const allocation = input.target === undefined ? yield* prepareDeprovision(input) : input.target; if (allocation === null) { - return; + return true; } - const claimedAt = yield* allocations + const claimedGeneration = yield* allocations .claimDeprovision({ userId: input.userId, environmentId: input.environmentId, - updatedAt: allocation.updatedAt, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -485,55 +645,86 @@ export const make = Effect.gen(function* () { }), ), ); - if (claimedAt === null) { - return; - } - const dnsRecordId = allocation.dnsRecordId; - if (dnsRecordId !== null) { - yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "delete-dns-record", - dnsRecordId, - cause, - }), - ), - ); + if (claimedGeneration === null) { + return false; } const tunnelId = allocation.tunnelId; - if (tunnelId !== null) { - yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "delete-tunnel", - tunnelId, - cause, - }), - ), - ); + const deprovision = Effect.gen(function* () { + const dnsRecordId = allocation.dnsRecordId; + if (dnsRecordId !== null) { + yield* ignoreNotFound(dns.deleteRecord(dnsRecordId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-dns-record", + dnsRecordId, + cause, + }), + ), + ); + } + if (tunnelId !== null) { + yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "delete-tunnel", + tunnelId, + cause, + }), + ), + ); + } + return yield* allocations + .removeClaimed({ + userId: input.userId, + environmentId: input.environmentId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "remove-allocation", + ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }), + ...(allocation.dnsRecordId === null + ? {} + : { dnsRecordId: allocation.dnsRecordId }), + cause, + }), + ), + ); + }); + if (tunnelId === null) { + return yield* deprovision; } - yield* allocations - .removeClaimed({ - userId: input.userId, - environmentId: input.environmentId, - updatedAt: claimedAt, - }) + const removed = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + deprovision, + ) .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointDeprovisioningFailed({ - ...input, - stage: "remove-allocation", - ...(allocation.tunnelId === null ? {} : { tunnelId: allocation.tunnelId }), - ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }), - cause, - }), - ), + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-deprovision", + tunnelId, + cause, + }), + ), + }), ); + return Option.getOrElse(removed, () => false); }), release: Effect.fn("relay.managed_endpoint_provider.release")(function* (input) { yield* Effect.annotateCurrentSpan({ @@ -554,19 +745,22 @@ export const make = Effect.gen(function* () { if (allocation === null || tunnelId === null) { return true; } + if (input.expectedTunnelId !== undefined && input.expectedTunnelId !== tunnelId) { + return false; + } // Claim the release against the allocation's current generation before // touching Cloudflare. A provision racing this release (fast environment - // restart) rewrites updatedAt when it records its tunnel, so a stale + // restart) increments the generation when it records its tunnel, so a stale // claim means the recorded tunnel may already back a fresh connector and // must be left alive. A provision that starts after the claim instead // fails loudly on the deleted tunnel and the client-side retry // provisions a replacement. - const claimed = yield* allocations + const claimedGeneration = yield* allocations .claimRelease({ userId: input.userId, environmentId: input.environmentId, tunnelId, - updatedAt: allocation.updatedAt, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -579,10 +773,10 @@ export const make = Effect.gen(function* () { }), ), ); - if (!claimed) { + if (claimedGeneration === null) { return false; } - yield* ignoreNotFound(tunnels.delete(tunnelId)).pipe( + const deleteTunnel = ignoreNotFound(tunnels.delete(tunnelId)).pipe( Effect.mapError( (cause) => new ManagedEndpointDeprovisioningFailed({ @@ -593,13 +787,103 @@ export const make = Effect.gen(function* () { }), ), ); + if (input.expectedInactiveBefore !== undefined && input.expectedStatus !== undefined) { + const expectedStatus = input.expectedStatus; + const inactiveBefore = input.expectedInactiveBefore; + const currentTunnel = yield* tunnels.get(tunnelId).pipe( + Effect.map(Option.some), + Effect.catchTags({ + ManagedEndpointTunnelClientError: (cause) => + isManagedEndpointNotFound(cause.cause) + ? Effect.succeed(Option.none()) + : Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "load-tunnel", + tunnelId, + cause, + }), + ), + }), + ); + if (Option.isNone(currentTunnel)) { + return true; + } + const inactiveAt = + expectedStatus === "down" + ? currentTunnel.value.connsInactiveAt + : currentTunnel.value.createdAt; + if ( + currentTunnel.value.id !== tunnelId || + currentTunnel.value.status !== expectedStatus || + typeof inactiveAt !== "string" + ) { + return false; + } + const inactiveTime = DateTime.make(inactiveAt); + const cutoff = DateTime.make(inactiveBefore); + if ( + Option.isNone(inactiveTime) || + Option.isNone(cutoff) || + inactiveTime.value.epochMilliseconds > cutoff.value.epochMilliseconds + ) { + return false; + } + } + const released = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }, + Effect.gen(function* () { + const finalGeneration = yield* allocations + .claimRelease({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId, + generation: claimedGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + ); + if (finalGeneration === null) { + return false; + } + yield* deleteTunnel; + return true; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointDeprovisioningFailed({ + ...input, + stage: "claim-release", + tunnelId, + cause, + }), + ), + }), + ); // The recorded tunnelId is now stale, but the allocation row is left // untouched deliberately: connect/status authorization requires a fully // recorded allocation, and an offline environment must keep reporting // "offline" (health probe fails) rather than "not authorized". The next // provision lists tunnels by name, finds none, creates a replacement and // re-records the fresh id. - return true; + return Option.getOrElse(released, () => false); }), provision: Effect.fn("relay.managed_endpoint_provider.provision")(function* (input) { yield* Effect.annotateCurrentSpan({ @@ -717,11 +1001,12 @@ export const make = Effect.gen(function* () { }); } const tunnel = { id: tunnelResponse.id, name: tunnelResponse.name }; - yield* allocations + const tunnelGeneration = yield* allocations .recordTunnel({ userId: input.userId, environmentId: input.environmentId, tunnelId: tunnel.id, + generation: allocation.generation, }) .pipe( Effect.mapError( @@ -737,31 +1022,78 @@ export const make = Effect.gen(function* () { }), ), ); + if (tunnelGeneration === null) { + // A newer provision can adopt this tunnel by name at any point after + // our claim fails. Leave it available for that provision or a retry. + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + }); + } - yield* tunnels - .putConfiguration(tunnel.id, { - ingress: [ - { - hostname, - service: formatOriginService(input.origin), - }, - { service: "http_status:404" }, - ], - }) + const configured = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }, + tunnels + .putConfiguration(tunnel.id, { + ingress: [ + { + hostname, + service: formatOriginService(input.origin), + }, + { service: "http_status:404" }, + ], + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + ), + ) .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ - userId: input.userId, - environmentId: input.environmentId, - stage: "configure-tunnel", - hostname, - tunnelName, - tunnelId: tunnel.id, - cause, - }), - ), + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + }), ); + if (Option.isNone(configured)) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "configure-tunnel", + hostname, + tunnelName, + tunnelId: tunnel.id, + }); + } const dnsRecord = { type: "CNAME", @@ -771,31 +1103,61 @@ export const make = Effect.gen(function* () { proxied: true, } as const; - const dnsRecordId = yield* ensureDnsRecord(hostname, allocation.dnsRecordId, dnsRecord).pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ - userId: input.userId, - environmentId: input.environmentId, - stage: "ensure-dns-record", + const recordedDns = yield* allocations + .withClaimedTunnel( + { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }, + Effect.gen(function* () { + const dnsRecordId = yield* ensureDnsRecord( hostname, - tunnelName, - tunnelId: tunnel.id, - ...(allocation.dnsRecordId === null ? {} : { dnsRecordId: allocation.dnsRecordId }), - cause, - }), - ), - ); - yield* allocations - .recordDns({ - userId: input.userId, - environmentId: input.environmentId, - dnsRecordId, - }) - .pipe( - Effect.mapError( - (cause) => - new ManagedEndpointProvisioningFailed({ + allocation.dnsRecordId, + dnsRecord, + ).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "ensure-dns-record", + hostname, + tunnelName, + tunnelId: tunnel.id, + ...(allocation.dnsRecordId === null + ? {} + : { dnsRecordId: allocation.dnsRecordId }), + cause, + }), + ), + ); + const dnsGeneration = yield* allocations + .recordDns({ + userId: input.userId, + environmentId: input.environmentId, + dnsRecordId, + tunnelId: tunnel.id, + generation: tunnelGeneration, + }) + .pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + cause, + }), + ), + ); + if (dnsGeneration === null) { + return yield* new ManagedEndpointProvisioningFailed({ userId: input.userId, environmentId: input.environmentId, stage: "record-dns", @@ -803,10 +1165,38 @@ export const make = Effect.gen(function* () { tunnelName, tunnelId: tunnel.id, dnsRecordId, - cause, - }), - ), + }); + } + return { dnsRecordId, dnsGeneration }; + }), + ) + .pipe( + Effect.catchTags({ + ManagedEndpointAllocationPersistenceError: (cause) => + Effect.fail( + new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + cause, + }), + ), + }), ); + if (Option.isNone(recordedDns)) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "record-dns", + hostname, + tunnelName, + tunnelId: tunnel.id, + }); + } + const { dnsRecordId, dnsGeneration } = recordedDns.value; const connectorToken = yield* tunnels.getToken(tunnel.id).pipe( Effect.mapError( @@ -823,10 +1213,13 @@ export const make = Effect.gen(function* () { }), ), ); - yield* allocations + const ready = yield* allocations .markReady({ userId: input.userId, environmentId: input.environmentId, + tunnelId: tunnel.id, + generation: dnsGeneration, + origin: input.origin, }) .pipe( Effect.mapError( @@ -843,6 +1236,17 @@ export const make = Effect.gen(function* () { }), ), ); + if (!ready) { + return yield* new ManagedEndpointProvisioningFailed({ + userId: input.userId, + environmentId: input.environmentId, + stage: "mark-allocation-ready", + hostname, + tunnelName, + tunnelId: tunnel.id, + dnsRecordId, + }); + } return { endpoint: managedEndpointForHostname(hostname), @@ -865,16 +1269,30 @@ export const layerCloudflareBindings = ( alchemyRuntimeContext: Alchemy.BaseRuntimeContext, ) => layer.pipe( - Layer.provide( + Layer.provideMerge( Layer.mergeAll( layerTunnelClient({ + get: (tunnelId) => + tunnelClient.get(tunnelId).pipe( + Effect.timeout("8 seconds"), + Effect.mapError( + (cause) => + new ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause, + }), + ), + Effect.provideService(Alchemy.RuntimeContext, alchemyRuntimeContext), + ), list: (request) => tunnelClient.list(request).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ operation: "list", - tunnelName: request.name, + ...(request.name === undefined ? {} : { tunnelName: request.name }), cause, }), ), @@ -882,6 +1300,7 @@ export const layerCloudflareBindings = ( ), create: (request) => tunnelClient.create(request).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -894,6 +1313,7 @@ export const layerCloudflareBindings = ( ), putConfiguration: (tunnelId, config) => tunnelClient.putConfiguration(tunnelId, config).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -906,6 +1326,7 @@ export const layerCloudflareBindings = ( ), getToken: (tunnelId) => tunnelClient.getToken(tunnelId).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -918,6 +1339,7 @@ export const layerCloudflareBindings = ( ), delete: (tunnelId) => tunnelClient.delete(tunnelId).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointTunnelClientError({ @@ -932,6 +1354,7 @@ export const layerCloudflareBindings = ( layerDnsClient({ listRecords: (hostname) => dnsClient.listDnsRecords({ search: hostname }).pipe( + Effect.timeout("8 seconds"), Effect.map((response) => response.result.filter( (record): record is typeof record & { readonly id: string } => @@ -951,6 +1374,7 @@ export const layerCloudflareBindings = ( ), createRecord: (request) => dnsClient.createDnsRecord(request).pipe( + Effect.timeout("8 seconds"), Effect.map((response) => ({ id: response.id })), Effect.mapError( (cause) => @@ -964,6 +1388,7 @@ export const layerCloudflareBindings = ( ), updateRecord: (dnsRecordId, request) => dnsClient.updateDnsRecord(dnsRecordId, request).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointDnsClientError({ @@ -977,6 +1402,7 @@ export const layerCloudflareBindings = ( ), deleteRecord: (dnsRecordId) => dnsClient.deleteDnsRecord(dnsRecordId).pipe( + Effect.timeout("8 seconds"), Effect.mapError( (cause) => new ManagedEndpointDnsClientError({ diff --git a/infra/relay/src/environments/ManagedEndpointReaper.test.ts b/infra/relay/src/environments/ManagedEndpointReaper.test.ts new file mode 100644 index 0000000000..b1d1b22f60 --- /dev/null +++ b/infra/relay/src/environments/ManagedEndpointReaper.test.ts @@ -0,0 +1,812 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Redacted from "effect/Redacted"; +import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; + +import * as RelayConfiguration from "../Config.ts"; +import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; +import * as ManagedEndpointReaper from "./ManagedEndpointReaper.ts"; + +const NOW = "2026-08-25T12:00:00.000Z"; +const NOW_MILLIS = DateTime.makeUnsafe(NOW).epochMilliseconds; +const SWEEP_INTERVAL_MILLIS = + ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_INTERVAL_MINUTES * 60 * 1_000; +const PREFIX = "t3coderelay-managedendpoint-prod-"; + +function tunnel(input: { + readonly id: string; + readonly suffix: string; + readonly status: "down" | "inactive" | "healthy" | "degraded"; + readonly timestamp?: string | null; + readonly prefix?: string; +}): ManagedEndpointProvider.ManagedEndpointTunnel { + return { + id: input.id, + name: `${input.prefix ?? PREFIX}${input.suffix}`, + status: input.status, + ...(input.timestamp === undefined + ? {} + : input.status === "inactive" + ? { createdAt: input.timestamp } + : { connsInactiveAt: input.timestamp }), + }; +} + +function recoverableOwners( + tunnels: ReadonlyArray, +): ReadonlyArray { + return tunnels.map((entry) => allocation({ tunnelId: entry.id!, recoveryEnabled: true })); +} + +function allocation(input: { + readonly tunnelId: string | null; + readonly recoveryEnabled: boolean; +}): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation { + return { + userId: "user-1", + environmentId: `environment-${input.tunnelId ?? "pending"}`, + hostname: `${input.tunnelId ?? "pending"}.example.test`, + tunnelId: input.tunnelId, + tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`, + dnsRecordId: "dns-1", + readyAt: "2026-08-25T11:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + updatedAt: "2026-08-25T11:00:00.000Z", + generation: 1, + recoveryEnabled: input.recoveryEnabled, + }; +} + +function harness(input?: { + readonly tunnels?: ReadonlyArray; + readonly allocations?: ReadonlyArray; + readonly namespace?: string; + readonly failTunnelId?: string; + readonly rateLimitedTunnelId?: string; + readonly failAllDeletes?: boolean; + readonly failDeleteWhen?: (tunnelId: string) => boolean; + readonly missingOnDeleteTunnelId?: string; + readonly missingOnGetTunnelId?: string; + readonly reserveOnGetTunnelId?: string; + readonly refreshedTunnels?: ReadonlyMap; + readonly skipTunnelId?: string; + readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode; +}) { + const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = []; + const deleted: string[] = []; + const releases: Array< + Parameters[0] + > = []; + const remaining = [...(input?.tunnels ?? [])]; + const recorded = (input?.allocations ?? []).map((entry) => { + const matching = remaining.find((candidate) => candidate.id === entry.tunnelId); + return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry; + }); + const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({ + get: (tunnelId) => + Effect.suspend(() => { + if (tunnelId === input?.missingOnGetTunnelId) { + return Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ); + } + const found = + input?.refreshedTunnels?.get(tunnelId) ?? + remaining.find((candidate) => candidate.id === tunnelId); + if (found === undefined) { + return Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "get", + tunnelId, + cause: { _tag: "NotFound" }, + }), + ); + } + if (tunnelId === input?.reserveOnGetTunnelId && typeof found.name === "string") { + recorded.push({ + ...allocation({ tunnelId, recoveryEnabled: false }), + tunnelName: found.name, + }); + } + return Effect.succeed(found); + }), + list: (request) => + Effect.sync(() => { + listRequests.push(request); + const matching = remaining.filter((entry) => entry.status === request.status); + const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100); + return { + result: matching.slice(start, start + (request.perPage ?? 100)), + resultInfo: { + page: request.page ?? 1, + perPage: request.perPage ?? 100, + totalCount: matching.length, + }, + }; + }), + create: () => Effect.die("unused"), + putConfiguration: () => Effect.die("unused"), + getToken: () => Effect.die("unused"), + delete: (tunnelId) => + input?.failAllDeletes === true || + input?.failDeleteWhen?.(tunnelId) === true || + tunnelId === input?.failTunnelId || + tunnelId === input?.rateLimitedTunnelId || + tunnelId === input?.missingOnDeleteTunnelId + ? Effect.fail( + new ManagedEndpointProvider.ManagedEndpointTunnelClientError({ + operation: "delete", + tunnelId, + cause: + tunnelId === input?.missingOnDeleteTunnelId + ? { _tag: "NotFound" } + : tunnelId === input?.rateLimitedTunnelId + ? { + cause: { + _tag: "TooManyRequests", + message: "Cloudflare rate limit exceeded", + retryAfter: 60, + }, + } + : "Cloudflare refused the deletion", + }), + ) + : Effect.sync(() => { + deleted.push(tunnelId); + const index = remaining.findIndex((candidate) => candidate.id === tunnelId); + if (index !== -1) { + remaining.splice(index, 1); + } + }), + }); + const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({ + get: () => Effect.die("unused"), + reserve: () => Effect.die("unused"), + recordTunnel: () => Effect.die("unused"), + recordDns: () => Effect.die("unused"), + markReady: () => Effect.die("unused"), + enableRecovery: () => Effect.die("unused"), + listByTunnelNames: (tunnelNames) => + Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))), + claimRelease: () => Effect.die("unused"), + withClaimedTunnel: () => Effect.die("unused"), + claimDeprovision: () => Effect.die("unused"), + remove: () => Effect.die("unused"), + removeClaimed: () => Effect.die("unused"), + }); + const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({ + provision: () => Effect.die("unused"), + reconcileOrigin: () => Effect.die("unused"), + prepareDeprovision: () => Effect.die("unused"), + deprovision: () => Effect.die("unused"), + release: (request) => + Effect.gen(function* () { + releases.push(request); + if (request.expectedTunnelId === input?.skipTunnelId) { + return false; + } + if (request.expectedTunnelId !== undefined) { + // Surface Cloudflare failures the same way the real release does. + yield* tunnelClient.delete(request.expectedTunnelId).pipe( + Effect.mapError( + (cause) => + new ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed({ + stage: "delete-tunnel", + userId: request.userId, + environmentId: request.environmentId, + tunnelId: request.expectedTunnelId!, + cause, + }), + ), + ); + } + return true; + }), + }); + const config = RelayConfiguration.RelayConfiguration.of({ + relayIssuer: "https://relay.example.test", + apns: { + environment: "sandbox", + teamId: "team-id", + keyId: "key-id", + privateKey: Redacted.make("private-key"), + bundleId: "com.t3tools.t3code.dev", + }, + apnsDeliveryJobSigningSecret: Redacted.make("job-secret"), + clerkSecretKey: Redacted.make("clerk-secret"), + clerkPublishableKey: "pk_test_test", + clerkJwtAudience: "t3-code-relay", + cloudMintPrivateKey: Redacted.make("cloud-private-key"), + cloudMintPublicKey: "cloud-public-key", + managedEndpointBaseDomain: "example.test", + managedEndpointNamespace: input?.namespace ?? "prod", + managedEndpointCleanupMode: input?.cleanupMode ?? "enabled", + }); + + return { + listRequests, + deleted, + releases, + layer: ManagedEndpointReaper.layer.pipe( + Layer.provide( + Layer.mergeAll( + RelayConfiguration.layer(config), + ManagedEndpointProvider.layerTunnelClient(tunnelClient), + Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, provider), + Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocationService), + ), + ), + ), + }; +} + +describe("ManagedEndpointReaper", () => { + it.effect("removes expired down and inactive tunnels from recoverable environments", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "down-1", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:55:00.000Z", + }), + tunnel({ + id: "inactive-1", + suffix: "bbbbbbbbbbbbbbbb", + status: "inactive", + timestamp: "2026-08-25T10:59:00.000Z", + }), + ], + allocations: [ + allocation({ tunnelId: "down-1", recoveryEnabled: true }), + allocation({ tunnelId: "inactive-1", recoveryEnabled: true }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + scanned: 2, + deleted: 2, + skippedLegacy: 0, + failed: 0, + }); + expect(state.deleted).toEqual(["down-1", "inactive-1"]); + expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([ + "down-1", + "inactive-1", + ]); + expect(state.listRequests).toEqual([ + { + isDeleted: false, + includePrefix: PREFIX, + status: "down", + existedAt: "2026-08-25T11:55:00.000Z", + wasInactiveAt: "2026-08-25T11:55:00.000Z", + page: 1, + perPage: 100, + }, + { + isDeleted: false, + includePrefix: PREFIX, + status: "inactive", + existedAt: "2026-08-25T11:00:00.000Z", + page: 1, + perPage: 100, + }, + ]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps a tunnel that never connected until it is an hour old", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "pairing", + suffix: "cccccccccccccccc", + status: "inactive", + timestamp: "2026-08-25T11:30:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "pairing", recoveryEnabled: true })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps recent tunnels, other stages, and tunnels without valid timestamps", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "recent", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:55:01.000Z", + }), + tunnel({ + id: "other-stage", + prefix: `${PREFIX}julius-`, + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "missing-time", + suffix: "cccccccccccccccc", + status: "inactive", + timestamp: null, + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + scanned: 0, + deleted: 0, + skippedLegacy: 0, + failed: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("keeps tunnels owned by environments that cannot recover yet", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "legacy", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "legacy", recoveryEnabled: false })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + scanned: 1, + deleted: 0, + skippedLegacy: 1, + failed: 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("counts an expired tunnel with no allocation instead of deleting it", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "orphan", + suffix: "cccccccccccccccc", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + scanned: 1, + wouldDelete: 0, + deleted: 0, + skippedOrphan: 1, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + it.effect("keeps an expired tunnel while its allocation is incomplete", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "unrecorded", + suffix: "aaaaaaaaaaaaaaaa", + status: "inactive", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: null, recoveryEnabled: false })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("does not count a tunnel that was replaced before its release", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "replaced", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "replaced", recoveryEnabled: true })], + skipTunnelId: "replaced", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(0); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("stops the sweep after a structured Cloudflare rate limit error", () => { + const state = harness({ + tunnels: [ + tunnel({ + id: "limited", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "next", + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [ + allocation({ tunnelId: "limited", recoveryEnabled: true }), + allocation({ tunnelId: "next", recoveryEnabled: true }), + ], + rateLimitedTunnelId: "limited", + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + attempted: 1, + deleted: 0, + failed: 1, + truncated: true, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("continues past a page of older hosts to find recoverable tunnels", () => { + const entries = Array.from({ length: 101 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: entries.map((entry, index) => + allocation({ tunnelId: entry.id!, recoveryEnabled: index === 100 }), + ), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + scanned: 101, + deleted: 1, + skippedLegacy: 100, + failed: 0, + }); + expect(state.deleted).toEqual(["tunnel-100"]); + expect( + state.listRequests + .filter((request) => request.status === "down") + .map((request) => request.page), + ).toEqual([1, 2]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("collects every page before deletions shift Cloudflare pagination", () => { + const entries = Array.from({ length: 120 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: entries.map((entry, index) => + allocation({ tunnelId: entry.id!, recoveryEnabled: index < 50 || index >= 100 }), + ), + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + scanned: 120, + deleted: 70, + skippedLegacy: 50, + failed: 0, + }); + expect(state.deleted).toContain("tunnel-119"); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("limits each cleanup run to 100 tunnel deletions", () => { + const entries = Array.from({ length: 105 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect((yield* reaper.sweep).deleted).toBe(100); + expect(state.deleted).toHaveLength(100); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("records dry-run counters on the sweep span", () => { + const spans: Array = []; + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options); + spans.push(span); + return span; + }, + }); + const expired = [ + tunnel({ + id: "recoverable", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + tunnel({ + id: "legacy", + suffix: "bbbbbbbbbbbbbbbb", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ]; + const state = harness({ + cleanupMode: "dry-run", + tunnels: expired, + allocations: [ + allocation({ tunnelId: "recoverable", recoveryEnabled: true }), + allocation({ tunnelId: "legacy", recoveryEnabled: false }), + ], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + yield* reaper.sweep; + const sweepSpan = spans.find((span) => span.name === "relay.managed_endpoint_reaper.sweep"); + expect(Object.fromEntries(sweepSpan?.attributes ?? [])).toMatchObject({ + "relay.managed_endpoint_reaper.mode": "dry-run", + "relay.managed_endpoint_reaper.scanned": 2, + "relay.managed_endpoint_reaper.wouldDelete": 1, + "relay.managed_endpoint_reaper.skippedLegacy": 1, + "relay.managed_endpoint_reaper.deleted": 0, + }); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer), Effect.withTracer(tracer)); + }); + + it.effect("does no Cloudflare work while cleanup is off", () => { + const state = harness({ + cleanupMode: "off", + tunnels: [ + tunnel({ + id: "expired", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toEqual({ + mode: "off", + listRequests: 0, + scanned: 0, + attempted: 0, + deleted: 0, + wouldDelete: 0, + skippedLegacy: 0, + skippedOrphan: 0, + failed: 0, + truncated: false, + }); + expect(state.listRequests).toEqual([]); + expect(state.deleted).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("reports candidates without mutating them in dry-run mode", () => { + const state = harness({ + cleanupMode: "dry-run", + tunnels: [ + tunnel({ + id: "expired", + suffix: "aaaaaaaaaaaaaaaa", + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ], + allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })], + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + mode: "dry-run", + scanned: 1, + attempted: 0, + deleted: 0, + wouldDelete: 1, + }); + expect(state.deleted).toEqual([]); + expect(state.releases).toEqual([]); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("caps failed deletion attempts and Cloudflare list pages", () => { + const entries = Array.from({ length: 250 }, (_, index) => + tunnel({ + id: `failed-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + tunnels: entries, + allocations: recoverableOwners(entries), + failAllDeletes: true, + }); + + return Effect.gen(function* () { + yield* TestClock.setTime(NOW_MILLIS); + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + expect(yield* reaper.sweep).toMatchObject({ + attempted: 100, + deleted: 0, + failed: 100, + truncated: true, + }); + expect(state.listRequests.length).toBeLessThanOrEqual( + ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT, + ); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("rotates the deletion order so persistent failures do not starve later tunnels", () => { + const entries = Array.from({ length: 200 }, (_, index) => + tunnel({ + id: `tunnel-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + // The first 100 candidates always fail to delete. + const state = harness({ + tunnels: entries, + allocations: recoverableOwners(entries), + failDeleteWhen: (id) => Number(id.split("-")[1]) < 100, + }); + + return Effect.gen(function* () { + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + yield* TestClock.setTime(NOW_MILLIS); + yield* reaper.sweep; + yield* TestClock.setTime(NOW_MILLIS + SWEEP_INTERVAL_MILLIS); + yield* reaper.sweep; + const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= 100); + expect(later.length).toBeGreaterThan(0); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("rotates bounded pages across a large legacy prefix", () => { + const entries = Array.from({ length: 1_000 }, (_, index) => + tunnel({ + id: `legacy-${index}`, + suffix: index.toString(16).padStart(16, "0"), + status: "down", + timestamp: "2026-08-25T11:00:00.000Z", + }), + ); + const state = harness({ + cleanupMode: "dry-run", + tunnels: entries, + allocations: entries.map((entry) => + allocation({ tunnelId: entry.id!, recoveryEnabled: false }), + ), + }); + + return Effect.gen(function* () { + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + yield* TestClock.setTime(NOW_MILLIS); + yield* reaper.sweep; + const firstPages = state.listRequests + .filter((request) => request.status === "down") + .map((request) => request.page); + state.listRequests.length = 0; + yield* TestClock.setTime(NOW_MILLIS + SWEEP_INTERVAL_MILLIS); + yield* reaper.sweep; + const secondPages = state.listRequests + .filter((request) => request.status === "down") + .map((request) => request.page); + expect(firstPages).not.toEqual(secondPages); + expect(firstPages).toHaveLength(5); + expect(secondPages).toHaveLength(5); + }).pipe(Effect.provide(state.layer)); + }); + + it.effect("rotates the attempt budget across both statuses over consecutive sweeps", () => { + const entries = (["down", "inactive"] as const).flatMap((status) => + Array.from({ length: 100 }, (_, index) => + tunnel({ + id: `${status}-${index}`, + suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`, + status, + timestamp: "2026-08-25T10:00:00.000Z", + }), + ), + ); + const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) }); + + return Effect.gen(function* () { + const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper; + yield* TestClock.setTime(NOW_MILLIS); + yield* reaper.sweep; + const firstStatus = state.deleted[0]!.split("-")[0]; + expect(state.deleted).toHaveLength(100); + expect(state.deleted.every((id) => id.startsWith(`${firstStatus}-`))).toBe(true); + + state.deleted.length = 0; + yield* TestClock.setTime(NOW_MILLIS + SWEEP_INTERVAL_MILLIS); + yield* reaper.sweep; + expect(state.deleted).toHaveLength(100); + expect(state.deleted.every((id) => !id.startsWith(`${firstStatus}-`))).toBe(true); + }).pipe(Effect.provide(state.layer)); + }); +}); diff --git a/infra/relay/src/environments/ManagedEndpointReaper.ts b/infra/relay/src/environments/ManagedEndpointReaper.ts new file mode 100644 index 0000000000..875f21d46f --- /dev/null +++ b/infra/relay/src/environments/ManagedEndpointReaper.ts @@ -0,0 +1,312 @@ +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; + +import type { ManagedEndpointCleanupMode } from "../Config.ts"; +import * as RelayConfiguration from "../Config.ts"; +import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts"; +import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts"; +import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts"; + +export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5; +// A tunnel that never connected is usually a link still being set up: a slow +// cloudflared download or a user who walked away mid-pairing. Give it an hour. +export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60; +// Matches the worker's hourly cron. Consecutive sweeps must land in +// consecutive rotation slots, or the page and attempt rotation below can +// return to the same starting point every run. +export const MANAGED_ENDPOINT_SWEEP_INTERVAL_MINUTES = 60; +export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100; +export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100; +export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10; + +export interface ManagedEndpointSweepResult { + readonly mode: ManagedEndpointCleanupMode; + readonly listRequests: number; + readonly scanned: number; + readonly attempted: number; + readonly deleted: number; + readonly wouldDelete: number; + readonly skippedLegacy: number; + readonly skippedOrphan: number; + readonly failed: number; + readonly truncated: boolean; +} + +export class ManagedEndpointReaper extends Context.Service< + ManagedEndpointReaper, + { + readonly sweep: Effect.Effect< + ManagedEndpointSweepResult, + | ManagedEndpointProvider.ManagedEndpointTunnelClientError + | ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError + >; + } +>()("t3code-relay/environments/ManagedEndpointReaper") {} + +function isExpiredManagedTunnel(input: { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel; + readonly status: "down" | "inactive"; + readonly prefix: string; + readonly cutoff: DateTime.Utc; +}): input is typeof input & { + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; +} { + const { tunnel, status, prefix, cutoff } = input; + if ( + typeof tunnel.id !== "string" || + typeof tunnel.name !== "string" || + tunnel.status !== status || + !tunnel.name.startsWith(prefix) || + !/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length)) + ) { + return false; + } + const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt; + if (typeof inactiveAt !== "string") { + return false; + } + const timestamp = DateTime.make(inactiveAt); + return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds; +} + +function isRateLimited(cause: unknown): boolean { + if (typeof cause !== "object" || cause === null) { + return false; + } + if ("_tag" in cause && cause._tag === "TooManyRequests") { + return true; + } + if ("status" in cause && cause.status === 429) { + return true; + } + return "cause" in cause && isRateLimited(cause.cause); +} + +function rotatedPages(input: { + readonly totalCount: number | undefined; + readonly slot: number; + readonly limit: number; +}): ReadonlyArray { + if (input.limit <= 0) return []; + if (input.totalCount === undefined) { + return Array.from({ length: input.limit }, (_, index) => index + 2); + } + const laterPageCount = Math.max( + 0, + Math.ceil(input.totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) - 1, + ); + if (laterPageCount === 0) return []; + const count = Math.min(input.limit, laterPageCount); + const start = input.slot % laterPageCount; + return Array.from({ length: count }, (_, index) => 2 + ((start + index) % laterPageCount)); +} + +const emptyResult = (mode: ManagedEndpointCleanupMode): ManagedEndpointSweepResult => ({ + mode, + listRequests: 0, + scanned: 0, + attempted: 0, + deleted: 0, + wouldDelete: 0, + skippedLegacy: 0, + skippedOrphan: 0, + failed: 0, + truncated: false, +}); + +export const make = Effect.gen(function* () { + const config = yield* RelayConfiguration.RelayConfiguration; + const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + + const sweep = Effect.gen(function* () { + const mode = config.managedEndpointCleanupMode ?? "off"; + const namespace = config.managedEndpointNamespace; + if (mode === "off" || !namespace) return emptyResult(mode); + + const now = yield* DateTime.now; + const cutoffFor = (status: "down" | "inactive") => + DateTime.subtract(now, { + minutes: + status === "down" + ? MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES + : MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES, + }); + const prefix = managedEndpointTunnelNamePrefix(namespace); + const slot = Math.floor( + now.epochMilliseconds / (MANAGED_ENDPOINT_SWEEP_INTERVAL_MINUTES * 60 * 1_000), + ); + let listRequests = 0; + let truncated = false; + const expired: Array<{ + readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & { + readonly id: string; + readonly name: string; + }; + readonly status: "down" | "inactive"; + readonly cutoff: DateTime.Utc; + }> = []; + + for (const status of ["down", "inactive"] as const) { + const cutoff = cutoffFor(status); + const cutoffIso = DateTime.formatIso(cutoff); + const listPage = (page: number) => { + listRequests += 1; + return tunnels.list({ + isDeleted: false, + includePrefix: prefix, + status, + existedAt: cutoffIso, + ...(status === "down" ? { wasInactiveAt: cutoffIso } : {}), + page, + perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE, + }); + }; + const first = yield* listPage(1); + const totalCount = + typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined; + const pages = rotatedPages({ + totalCount, + slot, + limit: Math.floor(MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT / 2) - 1, + }); + const responses = [first, ...(yield* Effect.forEach(pages, listPage, { concurrency: 1 }))]; + if ( + totalCount !== undefined && + Math.ceil(totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) > responses.length + ) { + truncated = true; + } else if ( + totalCount === undefined && + responses.at(-1)?.result.length === MANAGED_ENDPOINT_SWEEP_PAGE_SIZE + ) { + truncated = true; + } + for (const response of responses) { + expired.push( + ...response.result + .map((tunnel) => ({ tunnel, status, prefix, cutoff })) + .filter(isExpiredManagedTunnel) + .map(({ tunnel }) => ({ tunnel, status, cutoff })), + ); + } + } + + const collected = [...new Map(expired.map((entry) => [entry.tunnel.id, entry])).values()]; + // Start each sweep one attempt budget further along so a run of + // candidates whose deletes keep failing cannot hold the budget forever + // and starve everything listed after them. + const offset = + collected.length === 0 ? 0 : (slot * MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) % collected.length; + const uniqueExpired = [...collected.slice(offset), ...collected.slice(0, offset)]; + const recorded = yield* allocations.listByTunnelNames( + uniqueExpired.map(({ tunnel }) => tunnel.name), + ); + const recordedByTunnelName = new Map( + recorded.map((allocation) => [allocation.tunnelName, allocation]), + ); + let attempted = 0; + let deleted = 0; + let wouldDelete = 0; + let skippedLegacy = 0; + let skippedOrphan = 0; + let failed = 0; + + for (const { tunnel, status, cutoff } of uniqueExpired) { + const cutoffIso = DateTime.formatIso(cutoff); + const allocation = recordedByTunnelName.get(tunnel.name); + if ( + allocation !== undefined && + allocation.tunnelId !== null && + allocation.tunnelId !== tunnel.id + ) { + continue; + } + const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined; + if (owner !== undefined && !owner.recoveryEnabled) { + skippedLegacy += 1; + continue; + } + if (allocation !== undefined && owner === undefined) continue; + // A tunnel with no allocation row cannot be claimed, so a relink that + // adopts it by name races any delete here. Count it and leave it for a + // manual sweep instead. + if (owner === undefined) { + skippedOrphan += 1; + continue; + } + wouldDelete += 1; + if (mode === "dry-run") continue; + if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) { + truncated = true; + break; + } + attempted += 1; + const result = yield* provider + .release({ + userId: owner.userId, + environmentId: owner.environmentId, + expectedTunnelId: tunnel.id, + expectedInactiveBefore: cutoffIso, + expectedStatus: status, + }) + .pipe(Effect.result); + if (result._tag === "Failure") { + failed += 1; + yield* Effect.logWarning("Failed to delete an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + cause: result.failure, + }); + if (isRateLimited(result.failure)) { + truncated = true; + break; + } + } else if (result.success) { + deleted += 1; + yield* Effect.logInfo("Deleted an inactive managed tunnel", { + tunnelId: tunnel.id, + tunnelName: tunnel.name, + status, + }); + } + } + + return { + mode, + listRequests, + scanned: uniqueExpired.length, + attempted, + deleted, + wouldDelete, + skippedLegacy, + skippedOrphan, + failed, + truncated, + }; + }).pipe( + // Dry-run rollout reads these counters from the exported span. + Effect.tap((result) => + Effect.annotateCurrentSpan( + Object.fromEntries( + Object.entries(result).map(([key, value]) => [ + `relay.managed_endpoint_reaper.${key}`, + value, + ]), + ), + ), + ), + Effect.withSpan("relay.managed_endpoint_reaper.sweep"), + ); + + return ManagedEndpointReaper.of({ sweep }); +}); + +export const layer = Layer.effect(ManagedEndpointReaper, make); diff --git a/infra/relay/src/http/Api.test.ts b/infra/relay/src/http/Api.test.ts index bf7bd1f5a4..f6e07966d3 100644 --- a/infra/relay/src/http/Api.test.ts +++ b/infra/relay/src/http/Api.test.ts @@ -7,13 +7,15 @@ import { import * as EnvironmentLinker from "../environments/EnvironmentLinker.ts"; import * as RelayTokens from "../auth/RelayTokens.ts"; import * as Devices from "../agentActivity/Devices.ts"; +import * as NodeCrypto from "node:crypto"; import { createClerkClient, verifyToken } from "@clerk/backend"; import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import { vi } from "vite-plus/test"; import * as Context from "effect/Context"; -import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeCryptoLayer from "@effect/platform-node/NodeCrypto"; +import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -35,6 +37,7 @@ import { RelayEnvironmentPrincipal, RelayApi, } from "@t3tools/contracts/relay"; +import { RELAY_MANAGED_TUNNEL_RECOVERY_TYP, signRelayJwt } from "@t3tools/shared/relayJwt"; import { RELAY_HTTP_ROUTER_CONFIG, @@ -44,18 +47,22 @@ import { relayDocsRedirectRoute, relayEnvironmentAuthLayer, relayNotFoundRoute, + recoverEnvironmentTunnelRecord, + registerEnvironmentTunnelRecovery, relayDpopFailureReason, revokeEnvironmentLinkRecord, serverApi, traceRelayHttpRequestWith, unlinkEnvironmentRecord, verifyRelayClientBearerToken, + verifyEnvironmentTunnelRecoveryProof, withoutCapturedParentSpan, } from "./Api.ts"; import * as RelayConfiguration from "../Config.ts"; import * as RelayDb from "../db.ts"; import * as EnvironmentCredentials from "../environments/EnvironmentCredentials.ts"; import * as EnvironmentLinks from "../environments/EnvironmentLinks.ts"; +import * as ManagedEndpointAllocations from "../environments/ManagedEndpointAllocations.ts"; import * as ManagedEndpointProvider from "../environments/ManagedEndpointProvider.ts"; import * as AgentActivityPublisher from "../agentActivity/AgentActivityPublisher.ts"; import * as EnvironmentPublishSignatures from "../environments/EnvironmentPublishSignatures.ts"; @@ -122,7 +129,7 @@ describe("device listing compatibility", () => { Layer.provide( Layer.mergeAll( Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings), - NodeCrypto.layer, + NodeCryptoLayer.layer, Layer.mock(RelayTokens.RelayTokens, { resolveDpopAccessTokenScopes: () => null }), Layer.mock(EnvironmentLinker.EnvironmentLinker, {}), Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), @@ -307,6 +314,9 @@ function relayUnlinkTestLayer(input?: { readonly revokeCredential?: EnvironmentCredentials.EnvironmentCredentials["Service"]["revokeForEnvironmentPublicKey"]; readonly prepareDeprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["prepareDeprovision"]; readonly deprovision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["deprovision"]; + readonly provision?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["provision"]; + readonly reconcileOrigin?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["reconcileOrigin"]; + readonly release?: ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]; }) { return Layer.mergeAll( Layer.succeed( @@ -338,10 +348,11 @@ function relayUnlinkTestLayer(input?: { Layer.succeed( ManagedEndpointProvider.ManagedEndpointProvider, ManagedEndpointProvider.ManagedEndpointProvider.of({ - provision: () => Effect.die("unused provision"), + provision: input?.provision ?? (() => Effect.die("unused provision")), + reconcileOrigin: input?.reconcileOrigin ?? (() => Effect.succeed("ready")), prepareDeprovision: input?.prepareDeprovision ?? (() => Effect.succeed(null)), - deprovision: input?.deprovision ?? (() => Effect.void), - release: () => Effect.die("unused release"), + deprovision: input?.deprovision ?? (() => Effect.succeed(true)), + release: input?.release ?? (() => Effect.die("unused release")), }), ), ); @@ -359,6 +370,502 @@ const linkedEnvironmentRecord = { linkedAt: "2026-07-28T00:00:00.000Z", } as const; +describe("relay managed tunnel recovery", () => { + it.effect("binds recovery requests to the host, cloud user, and T3 service origin", () => + Effect.gen(function* () { + const keyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const proof = yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload: { + iss: "t3-env:environment-1", + aud: "https://relay.example.test", + sub: "environment-1", + jti: "recovery-proof", + iat: issuedAt, + exp: issuedAt + 60, + action: "recover", + environmentId: "environment-1", + cloudUserId: "user-1", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }, + }); + const request = { + action: "recover" as const, + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }; + + yield* verifyEnvironmentTunnelRecoveryProof(request); + + const wrongOwner = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ ...request, userId: "user-2" }), + ); + expect(wrongOwner).toMatchObject({ _tag: "Unauthorized" }); + + const wrongOrigin = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + ...request, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 }, + }), + ); + expect(wrongOrigin).toMatchObject({ _tag: "Unauthorized" }); + + const wrongAction = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(wrongAction).toMatchObject({ _tag: "Unauthorized" }); + }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)), + ); + + it.effect("rejects a signed registration proof for a different origin", () => + Effect.gen(function* () { + const keyPair = NodeCrypto.generateKeyPairSync("ed25519", { + privateKeyEncoding: { format: "pem", type: "pkcs8" }, + publicKeyEncoding: { format: "pem", type: "spki" }, + }); + const now = yield* DateTime.now; + const issuedAt = Math.floor(now.epochMilliseconds / 1_000); + const proof = yield* signRelayJwt({ + privateKey: keyPair.privateKey, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + payload: { + iss: "t3-env:environment-1", + aud: "https://relay.example.test", + sub: "environment-1", + jti: "registration-origin-proof", + iat: issuedAt, + exp: issuedAt + 60, + action: "register", + environmentId: "environment-1", + cloudUserId: "user-1", + tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }, + }); + + const error = yield* Effect.flip( + verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof, + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: keyPair.publicKey, + tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 5432 }, + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe(Effect.provideService(RelayConfiguration.RelayConfiguration, relaySettings)), + ); + + it.effect("registers recovery for an existing tunnel without provisioning it", () => { + let recoveryEnabledFor: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly environmentPublicKey: string; + readonly origin: { readonly localHttpHost: string; readonly localHttpPort: number }; + } | null = null; + + return Effect.gen(function* () { + expect( + yield* registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toEqual({ status: "ready" }); + expect(recoveryEnabledFor).toEqual({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "existing-tunnel", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => Effect.die("registration must not provision a tunnel"), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabledFor = input; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("requests recovery without enabling a stale tunnel", () => { + let recoveryEnabled = false; + + return Effect.gen(function* () { + expect( + yield* registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "deleted-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toEqual({ status: "recovery_required" }); + expect(recoveryEnabled).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + reconcileOrigin: () => Effect.succeed("recovery_required"), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects recovery registration for a different environment key", () => { + let recoveryEnabled = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "different-public-key", + tunnelId: "existing-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(recoveryEnabled).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects recovery registration when the recorded tunnel changed", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + registerEnvironmentTunnelRecovery({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + tunnelId: "stale-tunnel", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.succeed(false), + }), + ), + ), + ), + ); + + it.effect("recovers a linked environment and marks its tunnel as recoverable", () => { + let recoveryEnabledFor: { + readonly userId: string; + readonly environmentId: string; + readonly tunnelId: string; + readonly environmentPublicKey: string; + } | null = null; + const runtime = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "replacement-token", + tunnelId: "replacement-tunnel", + }; + + return Effect.gen(function* () { + expect( + yield* recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ).toEqual({ + endpoint: linkedEnvironmentRecord.endpoint, + endpointRuntime: runtime, + }); + expect(recoveryEnabledFor).toEqual({ + userId: "user-1", + environmentId: "environment-1", + tunnelId: "replacement-tunnel", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.succeed({ + endpoint: linkedEnvironmentRecord.endpoint, + runtime, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: (input) => + Effect.sync(() => { + recoveryEnabledFor = input; + return true; + }), + }), + ), + ), + ); + }); + + it.effect("rejects a credential from a different environment owner", () => { + let provisioned = false; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "different-public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(provisioned).toBe(false); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.sync(() => { + provisioned = true; + return { + endpoint: linkedEnvironmentRecord.endpoint, + runtime: { providerKind: "cloudflare_tunnel", connectorToken: "token" }, + }; + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.die("unused"), + }), + ), + ), + ); + }); + + it.effect("does not recover a publish-only environment", () => + Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => + Effect.succeed({ + ...linkedEnvironmentRecord, + endpoint: { + ...linkedEnvironmentRecord.endpoint, + providerKind: "manual" as const, + }, + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.die("unused"), + }), + ), + ), + ), + ); + + it.effect("rejects a recovered tunnel that changes the linked endpoint", () => { + let recoveryEnabled = false; + const cleaned: Array = []; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(recoveryEnabled).toBe(false); + expect(cleaned).toEqual(["replacement-tunnel"]); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => Effect.succeed(linkedEnvironmentRecord), + provision: () => + Effect.succeed({ + endpoint: { + httpBaseUrl: "https://different.example.test/", + wsBaseUrl: "wss://different.example.test/ws", + providerKind: "cloudflare_tunnel", + }, + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken: "token", + tunnelId: "replacement-tunnel", + }, + }), + prepareDeprovision: () => Effect.die("must keep the active allocation"), + deprovision: () => Effect.die("must keep the active link DNS"), + release: ({ expectedTunnelId }) => + Effect.sync(() => { + if (expectedTunnelId) { + cleaned.push(expectedTunnelId); + } + return true; + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => + Effect.sync(() => { + recoveryEnabled = true; + return true; + }), + }), + ), + ), + ); + }); + + it.effect.each([ + { state: "removed", currentLink: null }, + { + state: "publish-only", + currentLink: { + ...linkedEnvironmentRecord, + endpoint: { + ...linkedEnvironmentRecord.endpoint, + providerKind: "manual" as const, + }, + }, + }, + ])("removes a recovered tunnel when its link becomes $state", ({ currentLink }) => { + let lookups = 0; + const cleaned: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "environment-1.example.test", + tunnelId: "replacement-tunnel", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + updatedAt: "replacement-generation", + generation: 3, + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; + + return Effect.gen(function* () { + const error = yield* Effect.flip( + recoverEnvironmentTunnelRecord({ + userId: "user-1", + environmentId: "environment-1", + environmentPublicKey: "public-key", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + }), + ); + expect(error).toMatchObject({ _tag: "Unauthorized" }); + expect(cleaned).toEqual(["replacement-tunnel"]); + }).pipe( + Effect.provide( + Layer.merge( + relayUnlinkTestLayer({ + getForUser: () => + Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : currentLink)), + provision: () => + Effect.succeed({ + endpoint: linkedEnvironmentRecord.endpoint, + runtime: { + providerKind: "cloudflare_tunnel", + connectorToken: "replacement-token", + tunnelId: "replacement-tunnel", + }, + }), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + if (captured?.tunnelId) { + cleaned.push(captured.tunnelId); + } + return true; + }), + }), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations)({ + enableRecovery: () => Effect.succeed(false), + }), + ), + ), + ); + }); +}); + describe("relay environment unlink", () => { it.effect("revokes the link and its credentials in one database transaction", () => { const calls: Array = []; @@ -403,7 +910,9 @@ describe("relay environment unlink", () => { tunnelName: "environment-1-tunnel", dnsRecordId: "dns-1", readyAt: "2026-07-28T00:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, updatedAt: "generation-before-unlink", + generation: 1, } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; return Effect.gen(function* () { @@ -452,6 +961,7 @@ describe("relay environment unlink", () => { Effect.sync(() => { expect(request.target).toBe(deprovisionTarget); calls.push("deprovision"); + return true; }), }), ), @@ -500,6 +1010,7 @@ describe("relay environment unlink", () => { deprovision: () => Effect.sync(() => { calls.push("deprovision"); + return true; }), }), ), @@ -527,6 +1038,47 @@ describe("relay environment unlink", () => { deprovision: () => Effect.sync(() => { calls.push("deprovision"); + return true; + }), + }), + ), + ); + }); + + it.effect("retries unlink cleanup when a concurrent tunnel release wins the first claim", () => { + let lookups = 0; + const targets: Array = []; + const target = { + userId: "user-1", + environmentId: "environment-1", + hostname: "environment-1.example.test", + tunnelId: "tunnel-1", + tunnelName: "environment-1-tunnel", + dnsRecordId: "dns-1", + readyAt: "2026-07-28T00:00:00.000Z", + origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 }, + updatedAt: "original-generation", + generation: 1, + } satisfies ManagedEndpointProvider.ManagedEndpointDeprovisionTarget; + + return Effect.gen(function* () { + expect( + yield* unlinkEnvironmentRecord({ + userId: "user-1", + environmentId: "environment-1", + }), + ).toBe(true); + expect(targets).toEqual([target, target]); + }).pipe( + Effect.provide( + relayUnlinkTestLayer({ + getForUser: () => Effect.sync(() => (++lookups === 1 ? linkedEnvironmentRecord : null)), + revokeForUser: () => Effect.succeed(true), + prepareDeprovision: () => Effect.succeed(target), + deprovision: ({ target: captured }) => + Effect.sync(() => { + targets.push(captured ?? undefined); + return targets.length > 1; }), }), ), @@ -654,7 +1206,19 @@ describe("relay routing fallback", () => { const routes = HttpApiBuilder.layer( HttpApi.make("RelayApi").add(RelayApi.groups.server), ).pipe( - Layer.provide(serverApi.pipe(Layer.provide([publisher, signatures]))), + Layer.provide( + serverApi.pipe( + HttpRouter.provideRequest( + Layer.mergeAll( + Layer.succeed(RelayConfiguration.RelayConfiguration, relaySettings), + Layer.mock(EnvironmentLinks.EnvironmentLinks, {}), + Layer.mock(ManagedEndpointAllocations.ManagedEndpointAllocations, {}), + Layer.mock(ManagedEndpointProvider.ManagedEndpointProvider, {}), + ), + ), + Layer.provide([publisher, signatures]), + ), + ), Layer.provide(auth), Layer.provide([NodeServices.layer, NodeHttpPlatform.layer, Etag.layerWeak]), ); diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts index 5a4d0c9ca4..d6384add16 100644 --- a/infra/relay/src/http/Api.ts +++ b/infra/relay/src/http/Api.ts @@ -47,10 +47,16 @@ import { RelayEnvironmentLinkLimitExceededError, RelayEnvironmentPrincipal, type RelayEnvironmentConnectRequest, + type RelayManagedEndpointOrigin, + RelayManagedEndpointRecoveryProofPayload, type RelayDpopAccessTokenScope, RelayInternalError, } from "@t3tools/contracts/relay"; -import { normalizeRelayIssuer } from "@t3tools/shared/relayJwt"; +import { + normalizeRelayIssuer, + RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + verifyRelayJwt, +} from "@t3tools/shared/relayJwt"; import * as DeliveryAttempts from "../agentActivity/DeliveryAttempts.ts"; import * as AgentActivityRows from "../agentActivity/AgentActivityRows.ts"; @@ -99,6 +105,10 @@ const relayCorsPreflightHeaders = { "access-control-max-age": "86400", } as const; +const decodeManagedTunnelRecoveryProof = Schema.decodeUnknownEffect( + RelayManagedEndpointRecoveryProofPayload, +); + const appendRelayCredentialResponseHeaders = HttpEffect.appendPreResponseHandler( (_request, response) => Effect.succeed( @@ -462,15 +472,207 @@ export const unlinkEnvironmentRecord = Effect.fn("relay.api.client.unlinkEnviron // revocation commits so a database failure leaves a fully usable active // link. Still run teardown when the link is already revoked, allowing a // retry to finish cleanup after an earlier Cloudflare failure. - yield* managedEndpointProvider.deprovision({ + const deprovisioned = yield* managedEndpointProvider.deprovision({ userId: input.userId, environmentId: input.environmentId, target: deprovisionTarget, }); + if (!deprovisioned) { + const retryTarget = yield* managedEndpointProvider.prepareDeprovision(input); + if (retryTarget !== null && (yield* links.getForUser(input)) === null) { + yield* managedEndpointProvider.deprovision({ ...input, target: retryTarget }); + } + } return unlinked; }, ); +type EnvironmentTunnelRecoveryProofInput = { + readonly proof: string; + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; +} & ( + | { + readonly action: "register"; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; + } + | { readonly action: "recover"; readonly origin: RelayManagedEndpointOrigin } +); + +export const verifyEnvironmentTunnelRecoveryProof = Effect.fn( + "relay.api.server.verifyEnvironmentTunnelRecoveryProof", +)(function* (input: EnvironmentTunnelRecoveryProofInput) { + const config = yield* RelayConfiguration.RelayConfiguration; + const now = yield* DateTime.now; + const verified = yield* verifyRelayJwt({ + publicKey: input.environmentPublicKey, + token: input.proof, + typ: RELAY_MANAGED_TUNNEL_RECOVERY_TYP, + issuer: `t3-env:${input.environmentId}`, + audience: normalizeRelayIssuer(config.relayIssuer), + nowEpochSeconds: Math.floor(now.epochMilliseconds / 1_000), + }).pipe( + Effect.flatMap(decodeManagedTunnelRecoveryProof), + Effect.mapError(() => new HttpApiError.Unauthorized({})), + ); + + if ( + verified.environmentId !== input.environmentId || + verified.sub !== input.environmentId || + verified.cloudUserId !== input.userId || + verified.action !== input.action + ) { + return yield* new HttpApiError.Unauthorized({}); + } + if (input.action === "register") { + if ( + verified.action !== "register" || + verified.tunnelId !== input.tunnelId || + verified.origin.localHttpHost !== input.origin.localHttpHost || + verified.origin.localHttpPort !== input.origin.localHttpPort + ) { + return yield* new HttpApiError.Unauthorized({}); + } + return; + } + if ( + verified.action !== "recover" || + verified.origin.localHttpHost !== input.origin.localHttpHost || + verified.origin.localHttpPort !== input.origin.localHttpPort + ) { + return yield* new HttpApiError.Unauthorized({}); + } +}); + +export const registerEnvironmentTunnelRecovery = Effect.fn( + "relay.api.server.registerEnvironmentTunnelRecovery", +)(function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly tunnelId: string; + readonly origin: RelayManagedEndpointOrigin; +}) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const link = yield* links.getForUser({ + userId: input.userId, + environmentId: input.environmentId, + }); + if ( + link === null || + link.environmentPublicKey !== input.environmentPublicKey || + link.endpoint.providerKind !== "cloudflare_tunnel" + ) { + return yield* new HttpApiError.Unauthorized({}); + } + const status = yield* managedEndpointProvider.reconcileOrigin({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: input.tunnelId, + origin: input.origin, + endpoint: link.endpoint, + }); + if (status === "recovery_required") { + return { status }; + } + if (!(yield* allocations.enableRecovery(input))) { + return yield* new HttpApiError.Unauthorized({}); + } + return { status }; +}); + +export const recoverEnvironmentTunnelRecord = Effect.fn( + "relay.api.server.recoverEnvironmentTunnelRecord", +)(function* (input: { + readonly userId: string; + readonly environmentId: string; + readonly environmentPublicKey: string; + readonly origin: RelayManagedEndpointOrigin; +}) { + const links = yield* EnvironmentLinks.EnvironmentLinks; + const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations; + const managedEndpointProvider = yield* ManagedEndpointProvider.ManagedEndpointProvider; + const link = yield* links.getForUser({ + userId: input.userId, + environmentId: input.environmentId, + }); + if ( + link === null || + link.environmentPublicKey !== input.environmentPublicKey || + link.endpoint.providerKind !== "cloudflare_tunnel" + ) { + return yield* new HttpApiError.Unauthorized({}); + } + + const recovered = yield* managedEndpointProvider.provision({ + userId: input.userId, + environmentId: input.environmentId, + origin: input.origin, + }); + const recoveredTunnelId = recovered.runtime.tunnelId; + if ( + recoveredTunnelId === undefined || + recovered.endpoint.httpBaseUrl !== link.endpoint.httpBaseUrl || + recovered.endpoint.wsBaseUrl !== link.endpoint.wsBaseUrl + ) { + if (recoveredTunnelId !== undefined) { + yield* managedEndpointProvider + .release({ + userId: input.userId, + environmentId: input.environmentId, + expectedTunnelId: recoveredTunnelId, + }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to clean up a tunnel with a mismatched endpoint", { + userId: input.userId, + environmentId: input.environmentId, + tunnelId: recoveredTunnelId, + cause, + }), + ), + ); + } + return yield* new HttpApiError.Unauthorized({}); + } + + const enabled = yield* allocations.enableRecovery({ + userId: input.userId, + environmentId: input.environmentId, + tunnelId: recoveredTunnelId, + environmentPublicKey: input.environmentPublicKey, + origin: input.origin, + }); + if (!enabled) { + const owner = { userId: input.userId, environmentId: input.environmentId }; + const target = yield* managedEndpointProvider.prepareDeprovision(owner); + const currentLink = target === null ? null : yield* links.getForUser(input); + if ( + target !== null && + (currentLink === null || currentLink.endpoint.providerKind !== "cloudflare_tunnel") + ) { + yield* managedEndpointProvider.deprovision({ ...owner, target }).pipe( + Effect.catch((cause) => + Effect.logWarning("Failed to clean up a tunnel after its managed link was removed", { + userId: input.userId, + environmentId: input.environmentId, + cause, + }), + ), + ); + } + return yield* new HttpApiError.Unauthorized({}); + } + return { + endpoint: recovered.endpoint, + endpointRuntime: recovered.runtime, + }; +}); + export const mobileApi = HttpApiBuilder.group( RelayApi, "mobile", @@ -879,7 +1081,7 @@ export const serverApi = HttpApiBuilder.group( Effect.fnUntraced(function* (handlers) { const publisher = yield* AgentActivityPublisher.AgentActivityPublisher; const publishSignatures = yield* EnvironmentPublishSignatures.EnvironmentPublishSignatures; - return handlers.handle( + const activityHandlers = handlers.handle( "publishAgentActivity", Effect.fn("relay.api.server.publishAgentActivity")( function* (args) { @@ -1013,6 +1215,82 @@ export const serverApi = HttpApiBuilder.group( mapRelayCommonApiErrors("not_authorized"), ), ); + + return activityHandlers + .handle( + "registerManagedEndpointRecovery", + Effect.fn("relay.api.server.registerManagedEndpointRecovery")( + function* ({ params, payload }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "register", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + tunnelId: payload.tunnelId, + origin: payload.origin, + }); + yield* appendRelayCredentialResponseHeaders; + return yield* registerEnvironmentTunnelRecovery({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + tunnelId: payload.tunnelId, + origin: payload.origin, + }); + }, + Effect.catchTags({ + ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), + ManagedEndpointProvisioningNotConfigured: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointProvisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), + }), + mapRelayCommonApiErrors("not_authorized"), + ), + ) + .handle( + "recoverManagedEndpoint", + Effect.fn("relay.api.server.recoverManagedEndpoint")( + function* ({ params, payload }) { + const principal = yield* RelayEnvironmentPrincipal; + if (principal.environmentId !== params.environmentId) { + return yield* new HttpApiError.Unauthorized({}); + } + yield* verifyEnvironmentTunnelRecoveryProof({ + action: "recover", + proof: payload.proof, + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); + yield* appendRelayCredentialResponseHeaders; + return yield* recoverEnvironmentTunnelRecord({ + userId: payload.cloudUserId, + environmentId: params.environmentId, + environmentPublicKey: principal.environmentPublicKey, + origin: payload.origin, + }); + }, + Effect.catchTags({ + ManagedEndpointOriginNotAllowed: () => Effect.fail(new HttpApiError.Unauthorized({})), + ManagedEndpointProvisioningNotConfigured: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointProvisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedEndpointDeprovisioningFailed: () => + relayInternalErrorResponse("upstream_unavailable"), + ManagedTunnelLimitExceeded: () => relayInternalErrorResponse("upstream_unavailable"), + }), + mapRelayCommonApiErrors("not_authorized"), + ), + ); }), ); diff --git a/infra/relay/src/persistence/schema.ts b/infra/relay/src/persistence/schema.ts index fd766c7589..1f5d8c4d94 100644 --- a/infra/relay/src/persistence/schema.ts +++ b/infra/relay/src/persistence/schema.ts @@ -2,6 +2,7 @@ import type { RelayAgentActivityAggregateState, RelayAgentActivityState, RelayAgentAwarenessPreferences, + RelayManagedEndpointOrigin, } from "@t3tools/contracts/relay"; import { boolean, @@ -94,6 +95,10 @@ export const relayManagedEndpointAllocations = pgTable( tunnelName: text("tunnel_name").notNull(), dnsRecordId: varchar("dns_record_id", { length: 191 }), readyAt: varchar("ready_at", { length: 64 }), + recoveryEnabledAt: varchar("recovery_enabled_at", { length: 64 }), + recoveryEnvironmentPublicKey: text("recovery_environment_public_key"), + origin: jsonb("origin").$type(), + generation: integer("generation").notNull().default(0), createdAt: varchar("created_at", { length: 64 }).notNull(), updatedAt: varchar("updated_at", { length: 64 }).notNull(), }, diff --git a/infra/relay/src/worker.ts b/infra/relay/src/worker.ts index f125bf7fb4..d3d1b399cc 100644 --- a/infra/relay/src/worker.ts +++ b/infra/relay/src/worker.ts @@ -2,6 +2,7 @@ import * as Alchemy from "alchemy"; import * as Cloudflare from "alchemy/Cloudflare"; import * as Drizzle from "alchemy/Drizzle/Postgres"; import * as Config from "effect/Config"; +import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; @@ -69,6 +70,7 @@ import * as EnvironmentConnector from "./environments/EnvironmentConnector.ts"; import * as EnvironmentLinker from "./environments/EnvironmentLinker.ts"; import * as EnvironmentPublishSignatures from "./environments/EnvironmentPublishSignatures.ts"; import * as ManagedEndpointProvider from "./environments/ManagedEndpointProvider.ts"; +import * as ManagedEndpointReaper from "./environments/ManagedEndpointReaper.ts"; import * as ManagedTunnelLimits from "./environments/ManagedTunnelLimits.ts"; import * as MobileRegistrations from "./agentActivity/MobileRegistrations.ts"; @@ -180,6 +182,7 @@ export const ApiLive = Api.make( yield* yield* relayApiZone.zoneId; const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone); const managedEndpointZoneName = yield* managedEndpointZone.name; + const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig; // // 3. Runtime layers and app construction @@ -199,6 +202,7 @@ export const ApiLive = Api.make( cloudMintPublicKey: yield* cloudMintPublicKey, managedEndpointBaseDomain: yield* managedEndpointZoneName, managedEndpointNamespace: stage, + managedEndpointCleanupMode, }); }); @@ -215,7 +219,9 @@ export const ApiLive = Api.make( Layer.provideMerge(AgentActivityPublisher.layer), Layer.provideMerge(EnvironmentConnector.layer), Layer.provideMerge(EnvironmentLinker.layer), - Layer.provideMerge(EnvironmentPublishSignatures.layer), + Layer.provideMerge( + Layer.merge(EnvironmentPublishSignatures.layer, ManagedEndpointReaper.layer), + ), Layer.provideMerge( ManagedEndpointProvider.layerCloudflareBindings( managedEndpointTunnelBinding, @@ -322,22 +328,49 @@ export const ApiLive = Api.make( // reads these rows on a schedule — DPoP replay is an insert conflict and // terminal Live Activity rows stop rendering after // TERMINAL_AGENT_ACTIVITY_DISPLAY_TTL_MS — so the sweep only reclaims space. + // The tunnel reaper rides the same schedule. It touches Cloudflare only, + // and the database only when a sweep finds candidates, so it does not + // change that budget; a down tunnel is reclaimed within about an hour. yield* Cloudflare.Workers.cron("0 * * * *", () => - DpopProofs.DpopProofReplay.pipe( - Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired), - // Terminal thread rows are kept briefly so finished agents show as - // Done/Failed in the Live Activity; sweep them once they age out. - Effect.andThen( - Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe( - Effect.flatMap(([activityRows, now]) => - activityRows.pruneTerminal({ - updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })), - }), + Effect.all( + [ + DpopProofs.DpopProofReplay.pipe( + Effect.flatMap((dpopProofs) => dpopProofs.pruneExpired), + // Keep completed thread rows long enough to show their final state. + Effect.andThen( + Effect.all([AgentActivityRows.AgentActivityRows, DateTime.now]).pipe( + Effect.flatMap(([activityRows, now]) => + activityRows.pruneTerminal({ + updatedBefore: DateTime.formatIso(DateTime.subtract(now, { minutes: 30 })), + }), + ), + ), + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to prune expired relay state", { cause }), ), ), - ), + ManagedEndpointReaper.ManagedEndpointReaper.pipe( + Effect.flatMap((reaper) => reaper.sweep.pipe(Effect.timeout("2 minutes"))), + Effect.tap((result) => + result.scanned > 0 + ? Effect.logInfo("Finished managed tunnel cleanup", result) + : Effect.void, + ), + Effect.catchCause((cause) => + Cause.hasInterrupts(cause) + ? Effect.interrupt + : Effect.logWarning("Failed to clean up inactive managed tunnels", { cause }), + ), + ), + ], + { concurrency: 2, discard: true }, + ).pipe( Effect.withSpan("relay.cron.prune_expired_state"), - Effect.provide(runtimeLayer), + // Export cron spans to Axiom like HTTP spans; the scope flushes them before the run ends. + Effect.provide(Layer.merge(runtimeLayer, relayTraceLayer)), ), ); diff --git a/packages/contracts/src/relay.ts b/packages/contracts/src/relay.ts index 194970b53d..4582a95f59 100644 --- a/packages/contracts/src/relay.ts +++ b/packages/contracts/src/relay.ts @@ -186,6 +186,34 @@ export const RelayManagedEndpointRuntimeConfig = Schema.Struct({ }); export type RelayManagedEndpointRuntimeConfig = typeof RelayManagedEndpointRuntimeConfig.Type; +export const RelayManagedEndpointRecoveryRequest = Schema.Struct({ + cloudUserId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, + proof: TrimmedNonEmptyString, +}); +export type RelayManagedEndpointRecoveryRequest = typeof RelayManagedEndpointRecoveryRequest.Type; + +export const RelayManagedEndpointRecoveryRegistrationRequest = Schema.Struct({ + cloudUserId: TrimmedNonEmptyString, + tunnelId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, + proof: TrimmedNonEmptyString, +}); +export type RelayManagedEndpointRecoveryRegistrationRequest = + typeof RelayManagedEndpointRecoveryRegistrationRequest.Type; + +export const RelayManagedEndpointRecoveryRegistrationResponse = Schema.Struct({ + status: Schema.Literals(["ready", "recovery_required"]), +}); +export type RelayManagedEndpointRecoveryRegistrationResponse = + typeof RelayManagedEndpointRecoveryRegistrationResponse.Type; + +export const RelayManagedEndpointRecoveryResponse = Schema.Struct({ + endpoint: RelayManagedEndpoint, + endpointRuntime: RelayManagedEndpointRuntimeConfig, +}); +export type RelayManagedEndpointRecoveryResponse = typeof RelayManagedEndpointRecoveryResponse.Type; + export const RelayLinkProofRequest = Schema.Struct({ challenge: Schema.String, relayIssuer: Schema.String, @@ -213,6 +241,26 @@ const RelaySignedJwtRegisteredClaims = { exp: Schema.Int, } as const; +export const RelayManagedEndpointRecoveryProofPayload = Schema.Union([ + Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + action: Schema.Literal("register"), + environmentId: EnvironmentId, + cloudUserId: TrimmedNonEmptyString, + tunnelId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, + }), + Schema.Struct({ + ...RelaySignedJwtRegisteredClaims, + action: Schema.Literal("recover"), + environmentId: EnvironmentId, + cloudUserId: TrimmedNonEmptyString, + origin: RelayManagedEndpointOrigin, + }), +]); +export type RelayManagedEndpointRecoveryProofPayload = + typeof RelayManagedEndpointRecoveryProofPayload.Type; + export const RelayAgentActivityPublishProofPayload = Schema.Struct({ ...RelaySignedJwtRegisteredClaims, environmentId: EnvironmentId, @@ -1090,6 +1138,26 @@ const RelayDpopClientGroup = HttpApiGroup.make("dpopClient") const RelayServerGroup = HttpApiGroup.make("server") .add( + HttpApiEndpoint.post( + "registerManagedEndpointRecovery", + "/v1/environments/:environmentId/tunnel/recovery", + { + params: Schema.Struct({ + environmentId: EnvironmentId, + }), + payload: RelayManagedEndpointRecoveryRegistrationRequest, + success: RelayManagedEndpointRecoveryRegistrationResponse, + error: RelayAuthAndInternalErrors, + }, + ).annotate(OpenApi.Summary, "Register managed tunnel recovery without provisioning"), + HttpApiEndpoint.post("recoverManagedEndpoint", "/v1/environments/:environmentId/tunnel", { + params: Schema.Struct({ + environmentId: EnvironmentId, + }), + payload: RelayManagedEndpointRecoveryRequest, + success: RelayManagedEndpointRecoveryResponse, + error: RelayAuthAndInternalErrors, + }).annotate(OpenApi.Summary, "Recover an environment's managed tunnel"), HttpApiEndpoint.post( "publishAgentActivity", "/v1/environments/:environmentId/threads/:threadId/agent-activity", diff --git a/packages/shared/src/relayJwt.ts b/packages/shared/src/relayJwt.ts index f63f458a22..cabe340d70 100644 --- a/packages/shared/src/relayJwt.ts +++ b/packages/shared/src/relayJwt.ts @@ -10,6 +10,7 @@ export const RELAY_HEALTH_REQUEST_TYP = "t3-cloud-health+jwt"; export const RELAY_MINT_RESPONSE_TYP = "t3-env-mint+jwt"; export const RELAY_HEALTH_RESPONSE_TYP = "t3-env-health+jwt"; export const RELAY_ACTIVITY_PUBLISH_TYP = "t3-env-activity+jwt"; +export const RELAY_MANAGED_TUNNEL_RECOVERY_TYP = "t3-env-managed-tunnel-recovery+jwt"; export class RelayJwtError extends Schema.TaggedError()("RelayJwtError", { operation: Schema.Literals(["sign", "verify"]),