diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 42c5d9b20e1d..a7aceba20158 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -190,6 +190,30 @@ describe("CloudManagedEndpointRuntime", () => { '2026-06-17T02:00:00Z ERR Register tunnel error from server side error="connection timed out" connIndex=0', ), ).toBe(false); + // The edge's reason for a tunnel the relay reaper deleted. + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-10-06T06:44:22Z ERR Register tunnel error from server side error="Unauthorized: Tunnel not found" connIndex=0 event=0 ip=198.41.200.43', + ), + ).toBe(true); + // Over QUIC, cloudflared hides the edge's reason behind an opaque control + // stream failure. Only the supervisor's per-attempt line counts; the + // connection-level line repeats the same error for the same attempt. + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-10-06T06:43:35Z ERR Serve tunnel error error="control stream encountered a failure while serving" connIndex=0 event=0 ip=198.41.200.43', + ), + ).toBe(true); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-10-06T06:43:35Z ERR failed to serve tunnel connection error="control stream encountered a failure while serving" connIndex=0 event=0 ip=198.41.200.43', + ), + ).toBe(false); + expect( + ManagedEndpointRuntime.isRejectedRelayClientTunnelOutput( + '2026-10-06T06:43:35Z ERR Serve tunnel error error="failed to accept QUIC stream: timeout: no recent network activity" connIndex=0 event=0 ip=198.41.200.43', + ), + ).toBe(false); }); it.effect("keeps recovery requests sent before the server starts consuming them", () => @@ -318,6 +342,59 @@ describe("CloudManagedEndpointRuntime", () => { }), ); + it.effect("recovers a tunnel rejected over QUIC, where cloudflared hides the edge's reason", () => + Effect.gen(function* () { + const output = yield* Queue.unbounded(); + const checkpointObserved = yield* Deferred.make(); + const recoveryRequested = yield* Deferred.make(); + const encoder = new TextEncoder(); + const connectorOutput = Stream.fromQueue(output).pipe( + Stream.tap((chunk) => + new TextDecoder().decode(chunk) === "checkpoint\n" + ? Deferred.succeed(checkpointObserved, undefined).pipe(Effect.asVoid) + : Effect.void, + ), + ); + const spawner = ChildProcessSpawner.make(() => + Effect.gen(function* () { + const handle = makeHandle({ pid: 610, onKill: () => {}, output: connectorOutput }); + yield* Effect.addFinalizer(() => handle.kill().pipe(Effect.ignore)); + return handle; + }), + ); + const runtime = yield* buildCloudManagedEndpointRuntime(spawner); + const config = { + providerKind: "cloudflare_tunnel" as const, + connectorToken: "token", + tunnelId: "reaped-tunnel", + }; + // One failed registration attempt as cloudflared 2026.5.2 logs it over + // QUIC at `--loglevel info`: no server-side reason, two lines carrying + // the same opaque error, then the retry announcement. + const failedAttempt = + '2026-10-06T06:43:35Z ERR failed to serve tunnel connection error="control stream encountered a failure while serving" connIndex=0 event=0 ip=198.41.200.43\n' + + '2026-10-06T06:43:35Z ERR Serve tunnel error error="control stream encountered a failure while serving" connIndex=0 event=0 ip=198.41.200.43\n' + + "2026-10-06T06:43:35Z INF Retrying connection in up to 2s connIndex=0 event=0 ip=198.41.200.43\n"; + + yield* runtime.recoveryRequests.pipe( + Stream.runForEach((requested) => + Deferred.succeed(recoveryRequested, requested).pipe(Effect.asVoid), + ), + Effect.forkChild, + ); + yield* runtime.applyConfig(config); + + yield* Queue.offer(output, encoder.encode(failedAttempt.repeat(3))); + yield* Queue.offer(output, encoder.encode("checkpoint\n")); + yield* Deferred.await(checkpointObserved); + expect(yield* Deferred.isDone(recoveryRequested)).toBe(false); + + yield* Queue.offer(output, encoder.encode(failedAttempt)); + + expect(yield* Deferred.await(recoveryRequested)).toEqual(config); + }), + ); + it.effect("starts, deduplicates, rotates, and stops the Cloudflare connector", () => Effect.gen(function* () { const spawned: Array = []; diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 6b54b7ee3bdf..456b72c940f2 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -84,16 +84,33 @@ export function classifyRelayClientOutput(line: string): "connected" | "warning" /** * Cloudflare's edge rejects a connector whose tunnel was deleted or whose - * token no longer matches. Current edge output is - * `error="Failed to get tunnel"` with no prefix; older edges prefixed the - * same messages with `Unauthorized:`. Match both so recovery fires on either. + * token no longer matches. Over HTTP/2 the supervisor logs the edge's reason: + * `error="Unauthorized: Tunnel not found"` for a tunnel the relay reaper + * deleted, `error="Failed to get tunnel"` for an unknown tunnel ID (older + * edges prefixed every reason with `Unauthorized:`), and + * `error="Unauthorized: Invalid tunnel secret"` for a stale token. + * + * Over QUIC, cloudflared's default and auto-selected transport, the pinned + * supervisor never sees that reason: the QUIC connection collapses every + * control-stream failure into an opaque `ControlStreamError`, so the same + * rejection is logged only as `Serve tunnel error error="control stream + * encountered a failure while serving"`. The control stream is the first + * thing to fail only while a connection is still registering (a connection + * lost after registration fails its stream listener or datagram handler + * first), so repeated failures without a registered connection in between are + * treated as a rejection as well; the threshold above absorbs transient ones. + * `failed to serve tunnel connection` carries the same error for the same + * attempt and is deliberately not matched, so one failed attempt counts once. */ export function isRejectedRelayClientTunnelOutput(line: string): boolean { - return ( - /\bRegister tunnel error from server side\b/iu.test(line) && - /error="(?:Unauthorized:\s*)?(?:Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test( + if (/\bRegister tunnel error from server side\b/iu.test(line)) { + return /error="(?:Unauthorized:\s*)?(?:Tunnel not found|Failed to get tunnel|Record for tunnel not found|Invalid tunnel secret)"/iu.test( line, - ) + ); + } + return ( + /\bServe tunnel error\b/iu.test(line) && + /error="control stream encountered a failure while serving"/iu.test(line) ); }