Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
174 commits
Select commit Hold shift + click to select a range
fdf6fb2
fix(relay): stop dead accept work, spread control rotations, fail dir…
Jinwoo-H Sep 4, 2026
e5ccd4a
fix(relay): check client abandonment between each accept lookup
Jinwoo-H Sep 4, 2026
aa3a7ef
fix(relay): make the control lease jitter symmetric
Jinwoo-H Sep 4, 2026
d338b34
docs: relay reconnect investigation findings and roll status
Jinwoo-H Sep 4, 2026
8ebff89
docs(cloud): move relay reconnect findings under cloud/docs
Jinwoo-H Sep 4, 2026
16c6aef
docs(cloud): record retries-bar decision and PR #18580 basis
Jinwoo-H Sep 4, 2026
7a8d1fa
docs(cloud): record monitor dry-run dispatch inputs
Jinwoo-H Sep 4, 2026
d85fe9e
docs(cloud): record canary blast radius and dry-run #4
Jinwoo-H Sep 4, 2026
bad866a
docs(cloud): record dry-run #4 freeze (c27 crash) and dry-run #5
Jinwoo-H Sep 4, 2026
262a188
docs(cloud): record dry-run #5 freeze on c27 autoheal recreate
Jinwoo-H Sep 4, 2026
5a96670
docs(cloud): record Asia-cell autoheal recreate amplifier
Jinwoo-H Sep 4, 2026
4d80d24
docs(cloud): record dry-run #6 dispatch and PR states
Jinwoo-H Sep 4, 2026
87047a2
docs(cloud): record dry-run #6 pass and c7 canary dispatch
Jinwoo-H Sep 4, 2026
903da34
docs(cloud): record c7 drain effect on the director
Jinwoo-H Sep 4, 2026
169d6ff
docs(cloud): record c7 drain recovery numbers
Jinwoo-H Sep 4, 2026
f61a879
docs(cloud): record c27/c29 crash storm during c7 apply
Jinwoo-H Sep 4, 2026
60f60ed
docs(cloud): attribute the 06:20Z 503 burst
Jinwoo-H Sep 4, 2026
f8dd8bd
docs(cloud): c7 up on target image
Jinwoo-H Sep 4, 2026
c4af2a2
docs(cloud): c7 canary succeeded
Jinwoo-H Sep 4, 2026
d0ff6f6
docs(cloud): c7 post-restore observations
Jinwoo-H Sep 4, 2026
6fffb33
docs(cloud): record the agreed lock-first plan
Jinwoo-H Sep 4, 2026
21f69de
docs(cloud): lock-removal PR status
Jinwoo-H Sep 4, 2026
600b959
docs(cloud): faster same-cap rollout design from measured limits
Jinwoo-H Sep 4, 2026
9f162c1
docs(cloud): correct the rollout-speed plan after reading the cell job
Jinwoo-H Sep 4, 2026
ffb833a
docs(cloud): lock-removal PR review outcome
Jinwoo-H Sep 4, 2026
efaefdc
docs(cloud): PR #18606 opened; c7 two-hour evidence
Jinwoo-H Sep 4, 2026
5de21d4
docs(cloud): lock PR re-verification outcome
Jinwoo-H Sep 4, 2026
cc36ced
docs(cloud): post-merge dispatch plan for the lock fix
Jinwoo-H Sep 4, 2026
e4b54d6
docs(cloud): #18606 merged, image publish dispatched
Jinwoo-H Sep 4, 2026
817b0a8
docs(cloud): image 519f4914 published; director deploy dispatched
Jinwoo-H Sep 4, 2026
5a33e5a
docs(cloud): director deploy dispatched; record required predecessor …
Jinwoo-H Sep 4, 2026
63ceeef
docs(cloud): director on 519f4914; Finding 8 ten-cell cascade
Jinwoo-H Sep 4, 2026
d907776
docs(cloud): director rollback command
Jinwoo-H Sep 4, 2026
017b3c0
docs(cloud): Finding 9, director lock retries down ~10x on 519f4914
Jinwoo-H Sep 4, 2026
ad043fd
docs(cloud): fourth cascade 09:00Z, five cells
Jinwoo-H Sep 4, 2026
d52ae4b
docs(cloud): director 500 shape; verify + dry-run #7 dispatched
Jinwoo-H Sep 4, 2026
b869093
docs(cloud): verify passed; dry-run #7 running; selector gen 112
Jinwoo-H Sep 4, 2026
48286f8
docs(cloud): dry-run #7 froze on aged director 500s
Jinwoo-H Sep 4, 2026
8724fb6
docs(cloud): dry-run #8 dispatched with auto-canary on success
Jinwoo-H Sep 4, 2026
2c9399f
docs(cloud): dry-run #8 froze on the monitor's own 500; #9 armed
Jinwoo-H Sep 4, 2026
580630c
docs(cloud): dry-run #9 dispatched; 09:31 cascade
Jinwoo-H Sep 4, 2026
97898f5
docs(cloud): 09:34 cascade
Jinwoo-H Sep 4, 2026
c8b4cc7
docs(cloud): dry-run #9 froze; #10 armed; gate decision put to owner
Jinwoo-H Sep 4, 2026
5632975
docs(cloud): 09:39 cascade; per-instance crash ranking
Jinwoo-H Sep 4, 2026
65c2fe9
docs(cloud): deploy-vs-crash-rate check
Jinwoo-H Sep 4, 2026
1533409
docs(cloud): dry-run #10 froze on a 4 s Postgres connect stall
Jinwoo-H Sep 4, 2026
c623f12
docs(cloud): checkpoint-phase check negative
Jinwoo-H Sep 4, 2026
e37089e
docs(cloud): stall census
Jinwoo-H Sep 4, 2026
f18962e
docs(cloud): cascade gap math; dry-run #11 armed
Jinwoo-H Sep 4, 2026
7605acd
docs(cloud): dry-run #11 dispatched into a crash
Jinwoo-H Sep 4, 2026
0fdbfe6
docs(cloud): dry-run #11 froze; #12 armed with longer window
Jinwoo-H Sep 4, 2026
4acab49
docs(cloud): dry-run #12 froze on c16 probe; #13 armed
Jinwoo-H Sep 4, 2026
5f22ce4
docs(cloud): dry-run #12 froze on a probe miss during a c16 stall
Jinwoo-H Sep 4, 2026
f896864
docs(cloud): #13 waiter held by 10:24 stall
Jinwoo-H Sep 4, 2026
aa73dbc
docs(cloud): waiter re-armed with monitor-matched window
Jinwoo-H Sep 4, 2026
2993e8f
docs(cloud): dry-run #13 dispatched by hand; waiter grep bug
Jinwoo-H Sep 4, 2026
41a3214
docs(cloud): dry-run #13 froze at minute 14; #14 armed
Jinwoo-H Sep 4, 2026
1389827
docs(cloud): c28 crash loop and recreate
Jinwoo-H Sep 4, 2026
bb2a690
docs(cloud): 11:17 cascade
Jinwoo-H Sep 4, 2026
3243e46
docs(cloud): clean-window census since 08:40
Jinwoo-H Sep 4, 2026
8dac99f
docs(cloud): 11:31 cascade during dry-run #14
Jinwoo-H Sep 4, 2026
929e9aa
docs(cloud): dry-run #14 froze; #15 armed
Jinwoo-H Sep 4, 2026
ad0bea5
docs(cloud): 11:34 cascade
Jinwoo-H Sep 4, 2026
ca7e1c2
docs(cloud): c27 crash loop; per-hour die counts
Jinwoo-H Sep 4, 2026
f076e87
docs(cloud): Asia cell crash loops 11:38-11:45
Jinwoo-H Sep 4, 2026
a35490a
docs(cloud): 11:59 and 12:02 cascades
Jinwoo-H Sep 4, 2026
f425a3f
docs(cloud): no 15-min window in the last hour
Jinwoo-H Sep 4, 2026
93568ca
docs(cloud): 12:10 eight-cell cascade
Jinwoo-H Sep 4, 2026
3550b4d
docs(cloud): 12:20 cascade; stop per-cascade logging
Jinwoo-H Sep 4, 2026
169eb7d
docs(cloud): dry-run #15 froze on a fleet-wide 5 s SQL stall
Jinwoo-H Sep 4, 2026
e37a7ab
docs(cloud): 12:30 stall shape
Jinwoo-H Sep 4, 2026
6334491
docs(cloud): Finding 10, Cloud SQL disk write saturation since 11:58Z
Jinwoo-H Sep 4, 2026
66a0cc2
docs(cloud): Finding 10 corrections; disk owner is the other Terrafor…
Jinwoo-H Sep 4, 2026
ff4340d
docs(cloud): checkpoint loop mechanism and max_wal_size lever
Jinwoo-H Sep 4, 2026
668ee48
docs(cloud): owner summary: disk first, then roll, then gate
Jinwoo-H Sep 4, 2026
5c797be
docs(cloud): exact Terraform owner and change for the Cloud SQL disk
Jinwoo-H Sep 4, 2026
4f00db1
docs(cloud): dry-run #16 froze on probe latency; loop stopped pending…
Jinwoo-H Sep 4, 2026
59f676c
docs(cloud): checkpoint loop broke at 12:39; #17 armed with sync guard
Jinwoo-H Sep 4, 2026
712dd39
docs(cloud): dry-run #17 froze; #18 armed
Jinwoo-H Sep 4, 2026
8cf66dd
docs(cloud): Finding 11, Cloud SQL proxy dial timeouts as a second st…
Jinwoo-H Sep 4, 2026
436c0fe
docs(cloud): dial-timeout burst shape and path attribution
Jinwoo-H Sep 4, 2026
51fa0e8
docs(cloud): Finding 11 is Cloud NAT port exhaustion; exact Terraform…
Jinwoo-H Sep 4, 2026
df042b2
docs(cloud): NAT drop proof; owner summary reordered
Jinwoo-H Sep 4, 2026
68f5b45
docs(cloud): Finding 12, auth service 429 storm, fleet emptied
Jinwoo-H Sep 4, 2026
6b529ab
docs(cloud): auth incident state at 13:14Z
Jinwoo-H Sep 4, 2026
db61101
docs(cloud): dry-run #18 mis-dispatch on expired gcloud auth; loops s…
Jinwoo-H Sep 4, 2026
d74c1bf
docs(relay): record auth outage mitigation and refresh_tokens findings
Jinwoo-H Sep 4, 2026
4ccdbbe
docs(relay): record auth outage resolution and sign-out residual
Jinwoo-H Sep 4, 2026
fff7db7
docs(relay): record affected-user count for the auth outage
Jinwoo-H Sep 4, 2026
1dcecfc
docs(relay): record stale-Connected desktop UX finding after revocation
Jinwoo-H Sep 4, 2026
04b81e2
docs(relay): record alerting/NAT PRs and the phone signed-out copy de…
Jinwoo-H Sep 4, 2026
a71782f
docs(relay): record desktop revoked-session PR #18694
Jinwoo-H Sep 4, 2026
72df3a7
docs(relay): record refresh_tokens pruning PR #476
Jinwoo-H Sep 4, 2026
cbe90f8
docs(relay): pruning job operating rules and enablement gap
Jinwoo-H Sep 4, 2026
453557d
docs(relay): record phone signed-out reason PR #18698
Jinwoo-H Sep 4, 2026
f161c40
docs(relay): record the 2026-09-04 landing and production applies
Jinwoo-H Sep 4, 2026
b544511
docs(relay): auth deploy 00031 live; both NATs on dynamic ports
Jinwoo-H Sep 4, 2026
fcb278f
docs(relay): post-outage improvement roadmap
Jinwoo-H Sep 4, 2026
911cdef
docs(relay): implementation checklist with lanes and disruption per item
Jinwoo-H Sep 4, 2026
d3c6c10
docs(relay): defer the relay/auth database split; private IP proceeds…
Jinwoo-H Sep 4, 2026
15a5a81
docs(relay): authoritative cell image census before Roll 1
Jinwoo-H Sep 4, 2026
3c7756e
docs(relay): wave 1 status and the cos_system MESSAGE-field correction
Jinwoo-H Sep 4, 2026
64df90a
docs(relay): monitor freeze was a Cloud Monitoring publish gap
Jinwoo-H Sep 4, 2026
4f15eb5
docs(relay): record desktop refresh/jitter PR #18719 and the coordina…
Jinwoo-H Sep 4, 2026
45f4bd2
docs(relay): private IP restarts the instance and is one-way; directo…
Jinwoo-H Sep 4, 2026
9992cc4
docs(relay): dry-run #2 froze on a thrown probe fetch, not auth; roll…
Jinwoo-H Sep 4, 2026
54c1822
docs(relay): record relay lock/timeout PR #18722 and the placement-lo…
Jinwoo-H Sep 4, 2026
39141c0
docs(relay): grace window #478 merged; TTL env-strip follow-up
Jinwoo-H Sep 4, 2026
b403690
docs(relay): monitor probe fix PR #18723
Jinwoo-H Sep 4, 2026
32f400f
docs(relay): runtime-metric label drift blocks the dashboard apply
Jinwoo-H Sep 4, 2026
e88d52a
docs(relay): wave 1 closed; what is merged vs deployed vs held
Jinwoo-H Sep 4, 2026
ba6647e
docs(relay): checklist status block; tick items merged on 2026-09-04
Jinwoo-H Sep 4, 2026
65cb3ee
docs(relay): owner go; dry-run #19 green, c8 canary provenance mismat…
Jinwoo-H Sep 4, 2026
4241557
docs(relay): checklist ticks for auth deploy, label drift, pruner merge
Jinwoo-H Sep 4, 2026
4308f36
docs(relay): auth deploy live; dry-runs #20-#23; provenance blocker a…
Jinwoo-H Sep 5, 2026
2877978
docs(relay): dry-run #24 director instance recycle
Jinwoo-H Sep 5, 2026
d92bffe
docs(relay): provenance fix merged; dry-run #27 dispatched
Jinwoo-H Sep 5, 2026
40fd235
docs(relay): follow-ups surfaced by the MIG-retry and provenance reviews
Jinwoo-H Sep 5, 2026
b8b25e2
docs(relay): c8 canary #2 verified then tripped on a single 503; reco…
Jinwoo-H Sep 5, 2026
a8b8735
docs(relay): checklist follow-up for the verify-step curl
Jinwoo-H Sep 5, 2026
8686406
docs(relay): follow-up for unregistered cloud dev-script tests
Jinwoo-H Sep 5, 2026
75ea8e6
docs(relay): c8 restored to general on 519f4914 at gen 114; #18769 me…
Jinwoo-H Sep 5, 2026
21eb02f
docs(relay): pruner enabled; first run summary
Jinwoo-H Sep 5, 2026
a78ba73
docs(relay): tick pruner apply and first run
Jinwoo-H Sep 5, 2026
984b666
docs(relay): c9 canary succeeded; batch authority sealed at gen 116
Jinwoo-H Sep 5, 2026
af01e93
docs(relay): gate #30 and batch 1 plan
Jinwoo-H Sep 5, 2026
afbb2e8
docs(relay): batch 1 dispatched
Jinwoo-H Sep 5, 2026
e2c52aa
docs(relay): batch 1 attempt 1 failed on wave-0 preflight staleness; …
Jinwoo-H Sep 5, 2026
335d01e
docs(relay): capacity-job preflight follow-up
Jinwoo-H Sep 5, 2026
e67aa35
docs(relay): gate #31 green, batch 1 attempt 2 dispatched, #18778 merged
Jinwoo-H Sep 5, 2026
50c1904
docs(relay): batch 1 c10 rolled
Jinwoo-H Sep 5, 2026
7fe7944
docs(relay): batch 1 c13 rolled
Jinwoo-H Sep 5, 2026
c3f0569
docs(relay): batch 1 c14 rolled
Jinwoo-H Sep 5, 2026
b7b9132
docs(relay): batch 1 complete at gen 124; boot-race follow-up
Jinwoo-H Sep 5, 2026
c144d21
docs(relay): gate #32 green, c16 canary dispatched
Jinwoo-H Sep 5, 2026
ef00e0e
docs(relay): c16 canary succeeded
Jinwoo-H Sep 5, 2026
bbdab91
docs(relay): gate #33 continuity deadline after one stale sample
Jinwoo-H Sep 5, 2026
af1f21c
docs(relay): freshness recalibration PR
Jinwoo-H Sep 5, 2026
764473f
docs(relay): tolerated-gap rendering follow-up
Jinwoo-H Sep 5, 2026
f898a3d
docs(relay): gate #34 green, batch 2 dispatched, #18798 merged
Jinwoo-H Sep 5, 2026
cecb2a7
docs(relay): batch 2 c19 rolled
Jinwoo-H Sep 5, 2026
fb6da0a
docs(relay): batch 2 c20 rolled
Jinwoo-H Sep 5, 2026
5be4fed
docs(relay): record Roll 1 batch 2 completion (gen 134, 15/23 on new …
Jinwoo-H Sep 5, 2026
196b509
docs(relay): gate #35 green, c23 canary dispatched
Jinwoo-H Sep 5, 2026
2d7ab76
docs(relay): c23 canary succeeded (gen 136), gate #36 dispatched
Jinwoo-H Sep 5, 2026
696293b
docs(relay): gate #36 green, batch 3 dispatched
Jinwoo-H Sep 5, 2026
0bcb867
docs(relay): batch 3 complete (gen 142, all US cells on new images), …
Jinwoo-H Sep 5, 2026
af18058
docs(relay): gate #37 green, c27 Asia canary dispatched
Jinwoo-H Sep 5, 2026
d98901b
docs(relay): c27 canary failed closed on operator protocol input; gat…
Jinwoo-H Sep 5, 2026
ba1b550
docs(relay): gate #38 green, c27 canary redispatched with protocol 0
Jinwoo-H Sep 5, 2026
ee2ddd7
docs(relay): c27 canary #2 failed closed on US-only allowlist in isol…
Jinwoo-H Sep 5, 2026
9dca742
docs(relay): link #18811 and rehome-trust probe follow-up
Jinwoo-H Sep 5, 2026
524e53b
docs(relay): #18811 merged, gate #40 dispatched for c27
Jinwoo-H Sep 5, 2026
621118f
docs(relay): gate #39 green, selector unchanged
Jinwoo-H Sep 5, 2026
c9441a0
docs(relay): gate #40 green, c27 canary attempt 3 dispatched
Jinwoo-H Sep 5, 2026
040262c
docs(relay): c27 canary #3 failed at plan validation after isolate; r…
Jinwoo-H Sep 5, 2026
30b3da6
docs(relay): c27 live state after canary #3
Jinwoo-H Sep 5, 2026
39e4145
docs(relay): link #18818
Jinwoo-H Sep 5, 2026
cf674b1
docs(relay): record #18818 merge and gate #41 for c27 recovery
Jinwoo-H Sep 5, 2026
dab47f4
docs(relay): record c27 recovery #1 stale-authority miss and gate #42
Jinwoo-H Sep 5, 2026
c78dfae
docs(relay): record c27 recovery #2 miss (laptop sleep) and gate #43
Jinwoo-H Sep 5, 2026
a0a3e54
docs(relay): record c27 recovery #3 preflight probe miss and gate #44
Jinwoo-H Sep 5, 2026
0e26da7
docs(relay): checklist follow-up for opaque preflight failures and As…
Jinwoo-H Sep 5, 2026
c74843b
docs(relay): record gate #44 c27 probe freeze and gate #45
Jinwoo-H Sep 5, 2026
cb38451
docs(relay): gate #45 freeze, c27 /ready tail analysis, gate #46
Jinwoo-H Sep 5, 2026
abcc36c
docs(relay): gate #46 freeze, Asia /ready tail, stop retrying pending…
Jinwoo-H Sep 5, 2026
0cf59f5
docs(relay): link #18877 and note probe-retry bar residual
Jinwoo-H Sep 5, 2026
b529128
docs(relay): record #18877 merge and gate #47
Jinwoo-H Sep 5, 2026
5dfb4af
docs(relay): c27 recovered onto target image (gen 144), gate #48 for c28
Jinwoo-H Sep 5, 2026
5a60767
docs(relay): record gate #48 green and c28 canary success (gen 146)
Jinwoo-H Sep 5, 2026
afdf65b
docs(relay): record gate #49 dispatch for c29 canary
Jinwoo-H Sep 5, 2026
1e77dae
docs(relay): record gate #49 continuity freeze, gate #50 green, c29 c…
Jinwoo-H Sep 5, 2026
e69452b
docs(relay): record c29 canary success and Roll 1 completion census (…
Jinwoo-H Sep 5, 2026
6564834
docs(relay): Roll 2 and close-out implementation plan
Jinwoo-H Sep 5, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
345 changes: 345 additions & 0 deletions cloud/apps/relay/src/host-session-client-accept.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,345 @@
import { EventEmitter } from 'node:events'
import { RELAY_CLOSE_CODE } from '@orca-cloud/relay-contract'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type WebSocket from 'ws'
import type { RelayAssignmentStore } from './assignment-store.js'
import type { RelayConfig } from './config.js'
import type { CredentialReservation, RelayCredentialStore } from './credential-store.js'
import {
CONTROL_LEASE_JITTER_MS,
CONTROL_LEASE_MS,
HostSessionRegistry
} from './host-session-registry.js'
import type { RelayRuntimeObserver } from './relay-observability.js'
import type { RelayTokenClaims } from './relay-token-verifier.js'
import { ProcessQueuedByteBudget } from './splice-forwarder.js'

// Incident 2026-09-04 ~01:05Z: the phone's dial bound ran out while the cell was
// still inside acceptClient's serialized Postgres phase (cell-inventory lock
// contention). The cell then finished the work for a socket nobody held, leaked
// a 90s activity lease, and logged `host_data_reservation_already_bound`.

class FakeSocket extends EventEmitter {
readonly OPEN = 1
readonly CLOSING = 2
readonly CLOSED = 3
readyState = this.OPEN
readonly send = vi.fn()
readonly close = vi.fn((code?: number, reason?: string) => {
this.readyState = this.CLOSED
this.emit('close', code, Buffer.from(reason ?? ''))
})
readonly terminate = vi.fn(() => {
this.readyState = this.CLOSED
this.emit('close')
})
}

const config = {
port: 8080,
publicUrl: 'https://relay-c3.example.com',
cellUrl: 'https://relay-c3.example.com',
authIssuer: 'https://auth.example.com',
authAudience: 'orca-relay',
jwksUrl: 'https://auth.example.com/jwks',
assignmentSigningKey: new Uint8Array(32),
role: 'cell',
cellId: 'production-gce-c3',
cells: [{ id: 'production-gce-c3', url: 'https://relay-c3.example.com', capacityRequests: 4_000 }],
adminAudience: 'https://relay-c3.example.com/v1/admin/drain',
deployServiceAccount: 'deploy@example.com',
runtimeServiceAccount: 'runtime@example.com',
adminJwksUrl: 'https://auth.example.com/admin-jwks',
databasePoolMax: 10,
publicAssignmentsEnabled: true,
publicAssignmentConcurrency: 2,
publicAssignmentQueueMax: 128,
publicAssignmentWaitMs: 4_000,
publicResolveConcurrency: 1,
publicResolveWaitMs: 5_000,
publicAssignmentRetryAfterSeconds: 5,
dataDir: './test-data'
} satisfies RelayConfig

const identity = {
sub: 'user-1',
prof: 'profile-1',
relayHostId: 'abcdefghijklmnop',
purpose: 'host-control',
exp: 4_102_444_800
} satisfies RelayTokenClaims

function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void } {
let resolve!: (value: T) => void
const promise = new Promise<T>((next) => (resolve = next))
return { promise, resolve }
}

const reservation: CredentialReservation = {
userId: identity.sub,
relayHostId: identity.relayHostId,
credentialKind: 'resume',
relayDeviceId: 'device-1',
tokenHash: 'hash',
reservationId: 'reservation-1',
leaseExpiresAt: Date.now() + 60_000,
acceptedCredentialVersion: 2,
acceptedAs: 'current'
}

function harness(options: { random?: () => number; now?: () => number } = {}) {
const acquireActivity = vi.fn().mockResolvedValue(undefined)
const releaseActivity = vi.fn().mockResolvedValue(true)
const assignments = {
activateControl: vi.fn().mockResolvedValue('control:production-gce-c3:1'),
markMigrationTargetRegistered: vi.fn().mockResolvedValue(undefined),
resolve: vi.fn().mockResolvedValue({ cellId: config.cellId }),
acquireActivity,
renewControlActivity: vi.fn().mockResolvedValue(undefined),
releaseActivity
} as unknown as RelayAssignmentStore
const store = {
resolveResume: vi.fn().mockResolvedValue({ userId: identity.sub }),
reserveCredential: vi.fn().mockResolvedValue(reservation),
failReservation: vi.fn().mockResolvedValue(undefined)
}
const observer = {
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql: vi.fn(),
recordClientAcceptAbandoned: vi.fn()
} satisfies RelayRuntimeObserver
const registry = new HostSessionRegistry(
config,
vi.fn(),
store as unknown as RelayCredentialStore,
assignments,
new ProcessQueuedByteBudget(),
observer,
options.now,
options.random
)
const activate = (
registry as unknown as {
activate: (
socket: WebSocket,
identity: RelayTokenClaims,
existing: null,
generation: number,
rebind: boolean,
assignmentEpoch: number,
appVersion: string
) => Promise<void>
}
).activate.bind(registry)
return { registry, store, assignments, acquireActivity, releaseActivity, observer, activate }
}

async function activeHost(h: ReturnType<typeof harness>): Promise<FakeSocket> {
const control = new FakeSocket()
await h.activate(control as unknown as WebSocket, identity, null, 1, false, 1, '1.4.197')
return control
}

describe('client accept abandoned mid-DB-phase', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})

it('stops after a slow activity acquire when the phone already hung up', async () => {
const h = harness()
const control = await activeHost(h)
const slowAcquire = deferred<void>()
h.acquireActivity.mockReturnValueOnce(slowAcquire.promise)
const capacity = { bind: vi.fn(), release: vi.fn() }
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
await vi.advanceTimersByTimeAsync(0)
expect(h.acquireActivity).toHaveBeenCalledOnce()
// The phone's 12s bound fires while the cell still waits on Postgres.
client.close(1000, 'client bound')
capacity.release()
slowAcquire.resolve()
await accepting

// No conn-open reached the desktop; nothing pending; the lease it just took is
// released instead of leaking to expiry cleanup; bind never throws.
expect(control.send).not.toHaveBeenCalledWith(expect.stringContaining('conn-open'))
expect(capacity.bind).not.toHaveBeenCalled()
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })
expect(session?.pendingConns.size).toBe(0)
expect(h.store.failReservation).toHaveBeenCalledWith(reservation)
expect(h.releaseActivity).toHaveBeenCalledWith(
{ userId: identity.sub, relayHostId: identity.relayHostId },
expect.stringMatching(/^confirmation:/)
)
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'activity',
expect.any(Number)
)
const line = warn.mock.calls.map((call) => String(call[0])).find((entry) =>
entry.includes('orca_relay_client_accept_abandoned')
)
expect(line).toBeDefined()
expect(JSON.parse(line!)).toMatchObject({ stage: 'activity' })
expect(line).not.toContain(identity.relayHostId)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})

it('stops after a slow credential reservation without acquiring an activity lease', async () => {
const h = harness()
await activeHost(h)
const slowReserve = deferred<CredentialReservation>()
h.store.reserveCredential.mockReturnValueOnce(slowReserve.promise)
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
await vi.advanceTimersByTimeAsync(0)
client.close(1000, 'client bound')
slowReserve.resolve(reservation)
await accepting

expect(h.acquireActivity).not.toHaveBeenCalled()
expect(h.store.failReservation).toHaveBeenCalledWith(reservation)
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'credential',
expect.any(Number)
)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})

it('stops after a slow resume lookup before starting the invite and assignment lookups', async () => {
const h = harness()
await activeHost(h)
const store = h.store as typeof h.store & { resolveInviteForMove: ReturnType<typeof vi.fn> }
store.resolveInviteForMove = vi.fn().mockResolvedValue(null)
const slowResume = deferred<null>()
h.store.resolveResume.mockReturnValueOnce(slowResume.promise)
const resolveAssignment = (h.assignments as unknown as { resolve: ReturnType<typeof vi.fn> })
.resolve
resolveAssignment.mockClear()
const client = new FakeSocket()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const accepting = h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential'
)
await vi.advanceTimersByTimeAsync(0)
client.close(1000, 'client bound')
slowResume.resolve(null)
await accepting

expect(store.resolveInviteForMove).not.toHaveBeenCalled()
expect(resolveAssignment).not.toHaveBeenCalled()
expect(h.store.reserveCredential).not.toHaveBeenCalled()
expect(h.observer.recordClientAcceptAbandoned).toHaveBeenCalledWith(
'assignment',
expect.any(Number)
)
} finally {
warn.mockRestore()
h.registry.drain(0)
vi.advanceTimersByTime(0)
}
})

it('still opens the connection when the phone is holding on', async () => {
const h = harness()
const control = await activeHost(h)
const capacity = { bind: vi.fn(), release: vi.fn() }
const client = new FakeSocket()
await h.registry.acceptClient(
client as unknown as WebSocket,
identity.relayHostId,
'credential',
capacity
)
expect(control.send).toHaveBeenCalledWith(expect.stringContaining('"type":"conn-open"'))
expect(capacity.bind).toHaveBeenCalledOnce()
expect(h.observer.recordClientAcceptAbandoned).not.toHaveBeenCalled()
expect(client.close).not.toHaveBeenCalled()
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
})

describe('control lease jitter', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => {
vi.clearAllTimers()
vi.useRealTimers()
})

it('grants a lease uniformly around 55min so cohorts drift apart at the same mean rate', async () => {
const now = 1_700_000_000_000
const helloAck = (socket: FakeSocket) =>
JSON.parse(
String(socket.send.mock.calls.find((call) => String(call[0]).includes('host-hello-ack'))![0])
) as { leaseExpiresAt: number }

const shortest = harness({ now: () => now, random: () => 0 })
const shortestAck = helloAck(await activeHost(shortest))
const centered = harness({ now: () => now, random: () => 0.5 })
const centeredAck = helloAck(await activeHost(centered))
const longest = harness({ now: () => now, random: () => 0.999999 })
const longestAck = helloAck(await activeHost(longest))

expect(shortestAck.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS - CONTROL_LEASE_JITTER_MS)
expect(centeredAck.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS)
expect(longestAck.leaseExpiresAt).toBeLessThan(now + CONTROL_LEASE_MS + CONTROL_LEASE_JITTER_MS)
// The mean grant stays at 55 min, so steady-state rebind load is unchanged;
// hosts that connected together now differ by minutes, not zero.
expect(longestAck.leaseExpiresAt - shortestAck.leaseExpiresAt).toBeGreaterThan(9 * 60 * 1000)
shortest.registry.drain(0)
centered.registry.drain(0)
longest.registry.drain(0)
vi.advanceTimersByTime(0)
})

it('rebinds re-roll the jitter instead of pinning the cohort phase', async () => {
const now = 1_700_000_000_000
let roll = 0
const h = harness({ now: () => now, random: () => roll })
const first = await activeHost(h)
const session = h.registry.get({ userId: identity.sub, relayHostId: identity.relayHostId })!
const firstLease = session.leaseExpiresAt
roll = 0.75
const rebind = new FakeSocket()
await (
h.registry as unknown as {
activate: (...args: unknown[]) => Promise<void>
}
).activate(rebind as unknown as WebSocket, identity, session, 1, true, 1, '1.4.197')
expect(session.leaseExpiresAt).toBe(now + CONTROL_LEASE_MS + CONTROL_LEASE_JITTER_MS / 2)
expect(session.leaseExpiresAt).not.toBe(firstLease)
expect(first.close).toHaveBeenCalledWith(RELAY_CLOSE_CODE.PEER_DROPPED, 'control rebound')
h.registry.drain(0)
vi.advanceTimersByTime(0)
})
})
Loading