From b492a9e306effc96b87f4ebe14b45db09cd4871a Mon Sep 17 00:00:00 2001 From: Lakshmi Tanmay Date: Sat, 31 Oct 2026 16:42:02 +0530 Subject: [PATCH] fix(server): remember T3 Connect HTTP/2 fallback across restarts --- .../src/cloud/ManagedEndpointRuntime.test.ts | 127 +++++++++++++++++- .../src/cloud/ManagedEndpointRuntime.ts | 32 ++++- 2 files changed, 154 insertions(+), 5 deletions(-) diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts index 409ed3ce2fb61..b6d455f3a8837 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.test.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.test.ts @@ -1,4 +1,6 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; +import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -19,6 +21,7 @@ import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay import * as RelayClient from "@t3tools/shared/relayClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerConfig from "../config.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; const layerRelayClientAvailable = Layer.succeed( @@ -38,29 +41,34 @@ const layerRelayClientAvailable = Layer.succeed( const unusedHttpClient = HttpClient.make(() => Effect.die("unused")); +const emptySecretStoreLayer = Layer.mock(ServerSecretStore.ServerSecretStore)({ + get: () => Effect.succeedNone, + set: () => Effect.void, +}); + const runtimeDependencies = ( spawner: ReturnType, relayClientLayer = layerRelayClientAvailable, httpClient = unusedHttpClient, + secretStoreLayer = emptySecretStoreLayer, ) => Layer.mergeAll( Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), Layer.succeed(HttpClient.HttpClient, httpClient), relayClientLayer, - Layer.mock(ServerSecretStore.ServerSecretStore)({ - get: () => Effect.succeedNone, - }), + secretStoreLayer, ); const buildCloudManagedEndpointRuntime = ( spawner: ReturnType, relayClientLayer = layerRelayClientAvailable, httpClient = unusedHttpClient, + secretStoreLayer = emptySecretStoreLayer, ) => Effect.gen(function* () { const context = yield* Layer.build( ManagedEndpointRuntime.layer.pipe( - Layer.provide(runtimeDependencies(spawner, relayClientLayer, httpClient)), + Layer.provide(runtimeDependencies(spawner, relayClientLayer, httpClient, secretStoreLayer)), ), ); return yield* Effect.service(ManagedEndpointRuntime.CloudManagedEndpointRuntime).pipe( @@ -403,6 +411,117 @@ describe("CloudManagedEndpointRuntime", () => { }), ); + it.effect( + "retains automatic HTTP/2 fallback across restarts until expiry and respects overrides", + () => + Effect.gen(function* () { + const context = yield* Layer.build( + ServerSecretStore.layer.pipe( + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-transport-test-" })), + Layer.provide(NodeServices.layer), + ), + ); + const secretStoreLayer = Layer.succeed( + ServerSecretStore.ServerSecretStore, + context.pipe(Context.get(ServerSecretStore.ServerSecretStore)), + ); + const fixture = yield* makeTransportSpawner(); + const start = (spawner = fixture.spawner) => + buildCloudManagedEndpointRuntime( + spawner, + layerRelayClientAvailable, + unusedHttpClient, + secretStoreLayer, + ); + yield* Effect.gen(function* () { + const runtime = yield* start(); + yield* runtime.applyConfig(tunnelConfig); + expect(yield* Queue.take(fixture.started)).toBe(0); + expect(fixture.spawned[0]!.args).not.toContain("--protocol"); + yield* fixture.emit(0, quicDialWarning.repeat(30)); + expect(yield* Queue.take(fixture.started)).toBe(1); + yield* fixture.observe(runtime, 1); + expect(fixture.spawned[1]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]); + }).pipe(Effect.scoped); + + yield* TestClock.adjust(Duration.hours(1)); + for (const transport of [undefined, "quic", "http2", "auto", undefined]) { + yield* Effect.gen(function* () { + const next = yield* makeTransportSpawner(transport ? { transport } : {}); + const restarted = yield* start(next.spawner); + yield* restarted.applyConfig(tunnelConfig); + if (transport === undefined) { + expect(next.spawned[0]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]); + } else { + expect(next.spawned[0]!.args).not.toContain("--protocol"); + expect(next.spawned[0]!.options.env?.TUNNEL_TRANSPORT_PROTOCOL).toBe(transport); + yield* next.observe(restarted, 0, quicDialWarning.repeat(30)); + expect(next.spawned).toHaveLength(1); + } + }).pipe(Effect.scoped); + } + + // Reusing the hint one hour later must not extend its original lifetime. + yield* TestClock.adjust(Duration.hours(23)); + yield* Effect.gen(function* () { + const next = yield* makeTransportSpawner(); + const restarted = yield* start(next.spawner); + yield* restarted.applyConfig(tunnelConfig); + expect(next.spawned[0]!.args).not.toContain("--protocol"); + expect(yield* Queue.take(next.started)).toBe(0); + yield* next.emit(0, quicDatagramWarning.repeat(30)); + expect(yield* Queue.take(next.started)).toBe(1); + expect(next.spawned[1]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]); + }).pipe(Effect.scoped); + }), + ); + + it.effect( + "ignores invalid or unreadable fallback state and still falls back if persistence fails", + () => + Effect.gen(function* () { + for (const cached of [ + "not json", + '{"fellBackAt":"invalid"}', + '{"fellBackAt":9007199254740991}', + null, + ]) { + yield* Effect.gen(function* () { + const fixture = yield* makeTransportSpawner(); + const runtime = yield* buildCloudManagedEndpointRuntime( + fixture.spawner, + layerRelayClientAvailable, + unusedHttpClient, + Layer.mock(ServerSecretStore.ServerSecretStore)({ + get: () => + cached === null + ? Effect.fail( + new ServerSecretStore.SecretStoreReadError({ + resource: "test", + cause: "unreadable", + }), + ) + : Effect.succeedSome(new TextEncoder().encode(cached)), + set: () => + Effect.fail( + new ServerSecretStore.SecretStorePersistError({ + resource: "test", + cause: "read-only", + }), + ), + }), + ); + yield* runtime.applyConfig(tunnelConfig); + expect(yield* Queue.take(fixture.started)).toBe(0); + expect(fixture.spawned[0]!.args).not.toContain("--protocol"); + yield* fixture.emit(0, quicDialWarning.repeat(30)); + expect(yield* Queue.take(fixture.started)).toBe(1); + expect(fixture.spawned[1]!.args.slice(-3)).toEqual(["--protocol", "http2", "run"]); + }).pipe(Effect.scoped); + } + }), + ); + it.effect("requests recovery after a failed HTTP/2 spawn and retries with HTTP/2", () => Effect.gen(function* () { const fixture = yield* makeTransportSpawner({ failSpawn: 1 }); diff --git a/apps/server/src/cloud/ManagedEndpointRuntime.ts b/apps/server/src/cloud/ManagedEndpointRuntime.ts index 2528a08630a1a..de4ad782cefcb 100644 --- a/apps/server/src/cloud/ManagedEndpointRuntime.ts +++ b/apps/server/src/cloud/ManagedEndpointRuntime.ts @@ -19,6 +19,8 @@ import { HttpClient, HttpClientResponse } from "effect/http"; import * as ChildProcess from "effect/process/ChildProcess"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; + export type CloudManagedEndpointRuntimeStatus = | { readonly status: "disabled"; @@ -109,6 +111,12 @@ const ConnectorConfigBody = Schema.Struct({ // immediately. Keep that evidence across registrations and connector restarts. const QUIC_STALLS_BEFORE_HTTP2 = 30; const QUIC_STALL_WINDOW_MS = 5 * 60_000; +// Retry QUIC on a later startup after transient failures or network changes; reuse never renews this hint. +const HTTP2_FALLBACK_TTL_MS = 24 * 60 * 60_000; +const HTTP2_FALLBACK_STATE = "cloud-endpoint-http2-fallback"; +const Http2Fallback = Schema.fromJsonString(Schema.Struct({ fellBackAt: Schema.Number })); +const decodeHttp2Fallback = Schema.decodeUnknownOption(Http2Fallback); +const encodeHttp2Fallback = Schema.encodeSync(Http2Fallback); // A failed background install of the pinned relay client is retried no sooner // than this, so a crash-looping connector or an offline host does not redownload // on every reconcile. @@ -182,6 +190,7 @@ const stopConnector = (connector: ActiveConnector | null) => export const make = Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const relayClient = yield* RelayClient.RelayClient; + const secrets = yield* ServerSecretStore.ServerSecretStore; const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient); const activeRef = yield* Ref.make(null); const desiredConfigRef = yield* Ref.make(null); @@ -199,7 +208,18 @@ export const make = Effect.gen(function* () { const notReadyChecksBeforeRestartRef = yield* Ref.make(CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART); // An explicit cloudflared transport in the environment is the user's choice. const transportChosenByUser = process.env.TUNNEL_TRANSPORT_PROTOCOL !== undefined; - const http2Ref = yield* Ref.make(false); + const previousFallback = transportChosenByUser + ? Option.none() + : yield* secrets.get(HTTP2_FALLBACK_STATE).pipe( + Effect.map(Option.flatMap((bytes) => decodeHttp2Fallback(new TextDecoder().decode(bytes)))), + Effect.orElseSucceed(() => Option.none()), + ); + const now = yield* Clock.currentTimeMillis; + const http2Ref = yield* Ref.make( + Option.isSome(previousFallback) && + now >= previousFallback.value.fellBackAt && + now - previousFallback.value.fellBackAt < HTTP2_FALLBACK_TTL_MS, + ); const quicStallsRef = yield* Ref.make>([]); let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"]; @@ -452,6 +472,16 @@ export const make = Effect.gen(function* () { const desiredConfig = yield* Ref.get(desiredConfigRef); if ((yield* Ref.get(http2Ref)) || active !== connector || !desiredConfig) return; yield* Ref.set(http2Ref, true); + yield* secrets + .set( + HTTP2_FALLBACK_STATE, + new TextEncoder().encode( + encodeHttp2Fallback({ fellBackAt: yield* Clock.currentTimeMillis }), + ), + ) + .pipe( + Effect.catch(() => Effect.logWarning("Could not persist the relay client HTTP/2 fallback")), + ); yield* Effect.logWarning("Relay client keeps losing QUIC connections; switching to HTTP/2", { pid: Number(connector.child.pid), tunnelId: connector.config.tunnelId,