Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 61 additions & 19 deletions cloud/apps/relay/src/assignment-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -645,9 +645,17 @@ export class RelayAssignmentStore {
): Promise<RelayAssignment | null> {
const now = this.now()
return await this.database.transaction(async (transaction) => {
const lockedCells = inventoryFirst
? await this.lockCellInventory(transaction, lockMode)
// Why: the retry exists to take a cell row before the assignment row, the
// order placement uses. It only ever needs the one cell this host is
// pinned to, so read the pin unlocked and lock that row alone; taking all
// 23 queued every sticky refresh in the fleet behind every other one.
const pinnedCellId = inventoryFirst
? await this.pinnedCellId(transaction, identity)
: undefined
const lockedCells =
pinnedCellId === undefined
? undefined
: await this.lockCellRows(transaction, [pinnedCellId], lockMode)
const existing = await this.assignmentRow(transaction, identity, inventoryFirst)
if (!existing) return null
const activityLeases = await this.lockAssignmentActivities(transaction, identity, true)
Expand All @@ -661,6 +669,11 @@ export class RelayAssignmentStore {
}

const currentCellId = text(existing, 'cell_id')
// The pin moved between the unlocked read and the assignment lock, so the
// row held is the wrong one. Same recovery as losing the lock: retry.
if (pinnedCellId !== undefined && pinnedCellId !== currentCellId) {
throw new Error('database_lock_unavailable')
}
const hadControl = holdsControlLease(
activityLeases,
currentCellId,
Expand Down Expand Up @@ -701,14 +714,9 @@ export class RelayAssignmentStore {
if (hadControl) {
await this.touchAssignment(transaction, identity, leaseExpiresAt, now)
} else {
const nextReservation = integer(currentRow, 'reserved_requests') + 1
if (nextReservation > integer(currentRow, 'capacity_requests')) {
throw new Error('relay_capacity_exhausted')
}
await transaction.query(
`UPDATE relay_cells SET reserved_requests = ?, updated_at = ? WHERE cell_id = ?`,
[nextReservation, now, currentCellId]
)
// Delta, not the value read from the snapshot: an absolute write here
// would clobber any concurrent movement of the same counter.
await this.adjustCellReservationAtomically(transaction, currentCellId, 1)
await this.adjustActivityCount(transaction, identity, 'control', 1, leaseExpiresAt, now)
await this.insertPendingControlLease(
transaction,
Expand Down Expand Up @@ -6864,13 +6872,24 @@ export class RelayAssignmentStore {
targetCellId
]
)
const cells = await this.lockCellInventory(transaction, 'pool-default')
// Only the two cells this repairs need holding. The id set below is an
// existence check against a table that only reconcileCells writes, so it

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The justification is slightly off: configureCell (assignment-store.ts:2687) and insertCell in cell-admission-migration-registration.ts:199 also write relay_cells. The safety-relevant fact is that no production path deletes a cell row (all three are inserts/upserts), which is what makes the unlocked read safe — worth wording it that way so a future editor doesn't assume the table has a single writer.

// reads unlocked instead of dragging the other 21 rows into the section.
const cellIds = new Set(
(await transaction.query(`SELECT cell_id FROM relay_cells`)).map((row) =>
text(row, 'cell_id')
)
)
const cells = await this.lockCellRows(
transaction,
[sourceCellId, targetCellId],
'pool-default'
)
const assignmentKeys = new Set(
assignments.map((row) =>
assignmentKey(text(row, 'user_id'), text(row, 'relay_host_id'))
)
)
const cellIds = new Set(cells.map((row) => text(row, 'cell_id')))
const assignmentCounts = new Map<
string,
{ counts: Record<AssignmentActivityKind, number>; leaseExpiresAt: number }
Expand Down Expand Up @@ -6924,9 +6943,7 @@ export class RelayAssignmentStore {
)
}

for (const row of cells.filter((cell) =>
[sourceCellId, targetCellId].includes(text(cell, 'cell_id'))
)) {
for (const row of cells) {
const cellId = text(row, 'cell_id')
const expected = cellUnits.get(cellId) ?? 0
if (expected > integer(row, 'capacity_requests')) {
Expand Down Expand Up @@ -6958,16 +6975,40 @@ export class RelayAssignmentStore {
// Per-connection paths touch one or two cells. Locking exactly those rows,
// in the same ascending order the inventory lock uses (ORDER BY fixes the
// row-lock order), keeps them off the fleet-wide lock without a cycle.
private async lockCellRows(database: RelayDatabase, cellIds: string[]): Promise<SqlRow[]> {
// The wait policy follows the caller for the same reason the inventory lock's
// does: a sweep must not fail terminally on ordinary contention. Hold time is
// deliberately not sampled here — the metric tracks the fleet-wide lock these
// rows replace, and mixing in short single-row holds would flatter it.
private async lockCellRows(
database: RelayDatabase,
cellIds: string[],
mode: CellInventoryLockMode = 'request'
): Promise<SqlRow[]> {
const distinct = [...new Set(cellIds)]
const { measureHoldMs: _sampled, ...wait } = cellInventoryLockOptions(mode)
return await database.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id IN (${distinct.map(() => '?').join(', ')})
ORDER BY cell_id ASC`,
distinct,
{ lockTimeoutMs: CELL_INVENTORY_LOCK_TIMEOUT_MS }
wait
)
}

// Unlocked on purpose: this only names the row to lock next, and the caller
// re-checks the pin once the assignment row is held.
private async pinnedCellId(
database: RelayDatabase,
identity: AssignmentIdentity
): Promise<string | undefined> {
const row = (
await database.query(
`SELECT cell_id FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]
return row ? text(row, 'cell_id') : undefined
}

private async lockGeneralCellInventory(
database: RelayDatabase,
mode: CellInventoryLockMode
Expand All @@ -6986,10 +7027,11 @@ export class RelayAssignmentStore {

private async leastLoadedCell(
database: RelayDatabase,
lockedCells: SqlRow[] | undefined,
// Required: the one caller has already locked the inventory it selects from,
// and an optional parameter left a second fleet-wide lock reachable here.
rows: SqlRow[],
preferredRegion: RelayRegion
): Promise<CellRow | null> {
const rows = lockedCells ?? (await this.lockCellInventory(database, 'pool-default'))
const regions = new Map(
(await database.query(`SELECT cell_id, region FROM relay_cell_regions`)).map((row) => [
text(row, 'cell_id'),
Expand Down
9 changes: 6 additions & 3 deletions cloud/apps/relay/src/cell-inventory-lock-census.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ type CensusEntry = { method: string; mode: CensusMode; reach: Reachability }
// assignment-store.ts, in source order. A new site fails this test until it is
// classified here, which is the point.
const CENSUS: CensusEntry[] = [
{ method: 'assignStickyOnce', mode: 'caller', reach: 'both' },
// assignStickyOnce is gone from this list: its retry now locks only the row
// the host is pinned to (lockCellRows), which is what a sticky refresh
// touches. Placement below is the one genuinely fleet-wide decision left.
{ method: 'assignOnce', mode: 'caller', reach: 'both' },
{ method: 'assignOnce', mode: 'caller', reach: 'both' },
{ method: 'assignOnce', mode: 'nowait', reach: 'both' },
Expand Down Expand Up @@ -49,8 +51,9 @@ const CENSUS: CensusEntry[] = [
{ method: 'abortExpiredEvacuations', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivityLeases', mode: 'nowait', reach: 'sweep' },
{ method: 'releaseExpiredActivity', mode: 'nowait', reach: 'sweep' },
{ method: 'reconcileReservationAccounting', mode: 'pool-default', reach: 'both' },
{ method: 'leastLoadedCell', mode: 'pool-default', reach: 'both' }
// reconcileReservationAccounting and leastLoadedCell are gone too: the first
// repairs exactly two cells' counters and now holds only those rows, and the
// second selects from the inventory its single caller has already locked.
]

// Every inline `FROM relay_cells ... FOR UPDATE` outside the named lock helpers,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'

const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip

// Sorted ascending, and the host is pinned to the LAST id on purpose: the
// fleet-wide lock is one ordered scan, so it holds every earlier row while it
// waits on the pinned one. Pinning to the first id would make the two locking
// models indistinguishable.
const cells = ['a', 'b', 'c'].map((suffix) => ({
id: `percell-postgres-${suffix}`,
url: `https://percell-postgres-${suffix}.example.com`,
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}))
const [cellA, cellB, cellC] = cells as [(typeof cells)[0], (typeof cells)[0], (typeof cells)[0]]
const identity = { userId: 'percell-postgres-user', relayHostId: 'percellhost00001' }

function heartbeat(cell: (typeof cells)[number]) {
return {
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
}

describePostgres('PostgreSQL per-cell inventory locking', () => {
const databases: RelayDatabase[] = []

beforeAll(async () => {
for (let index = 0; index < 3; index++) {
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
}
})

async function removeTestRows(database: RelayDatabase): Promise<void> {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id LIKE 'percell-postgres-%'`
)
for (const table of [
'relay_assignment_activity_leases',
'relay_post_drain_migration_pins',
'relay_assignment_migration_incarnations',
'relay_assignment_migrations',
'relay_assignment_region_preferences',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id LIKE 'percell-postgres-%'`)
}
for (const cell of cells) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
}
}
}

afterAll(async () => {
if (databases[0]) await removeTestRows(databases[0])
for (const connection of databases) await connection.close()
})

async function pinHostToLastCell(store: RelayAssignmentStore): Promise<void> {
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)
const assignment = await store.assign(identity)
expect(assignment.cellId).toBe(cellC.id)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
}

async function lockWaiterAppeared(database: RelayDatabase): Promise<boolean> {
const deadline = Date.now() + 4_000
while (Date.now() < deadline) {
const rows = await database.query(
`SELECT count(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'`
)
if (Number(rows[0]!.waiting) > 0) return true
await new Promise((resolve) => setTimeout(resolve, 10))
}
return false
}

// Why: a sticky refresh whose first NOWAIT probe loses retries by taking a
// cell row before the assignment row. That retry used to take the whole
// inventory, so one busy cell stalled every other cell's reconnects.
it('waits only on the pinned cell row while refreshing a sticky assignment', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await pinHostToLastCell(store)
// A host whose control lease was already reaped still holds its pin; that
// is the shape that reaches the cell-row probe instead of touchAssignment.
await databases[0]!.query(
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
[identity.userId]
)

let releaseRow!: () => void
const rowReleased = new Promise<void>((resolve) => {
releaseRow = resolve
})
let rowHeld!: () => void
const rowHeldPromise = new Promise<void>((resolve) => {
rowHeld = resolve
})
const holder = databases[1]!.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellC.id])
rowHeld()
await rowReleased
})
await rowHeldPromise

const refresh = store.assign(identity)
expect(await lockWaiterAppeared(databases[2]!)).toBe(true)
// The refresh is blocked on cell C. Every earlier row must still be free:
// the ordered fleet-wide scan would be holding both of them by now.
const heldWhileRefreshWaits: string[] = []
await databases[2]!.transaction(async (transaction) => {
for (const cell of [cellA, cellB]) {
try {
await transaction.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id = ?`,
[cell.id],
{ failIfUnavailable: true }
)
} catch {
heldWhileRefreshWaits.push(cell.id)
}
}
})
releaseRow()
await holder

expect(heldWhileRefreshWaits).toEqual([])
expect((await refresh).cellId).toBe(cellC.id)
}, 15_000)

// Why: the counter moves by a delta now instead of an absolute value read
// from a snapshot, so concurrent movement on the same cell must still sum.
it('keeps a cell reservation exact under concurrent same-cell activity', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)

const hosts = Array.from({ length: 6 }, (_, index) => ({
userId: `percell-postgres-user-${index}`,
relayHostId: `percellhost0000${index}`
}))
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
await Promise.all(hosts.map((host, index) => stores[index % stores.length]!.assign(host)))

// One splice each (2 units) on the same cell, from three connections at once.
await Promise.all(
hosts.map((host, index) =>
stores[index % stores.length]!.acquireActivity(host, {
activityId: `splice:percell-${index}`,
kind: 'splice',
cellId: cellC.id
})
)
)
const afterAcquire = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellC.id]
)
// 6 pending control grants + 6 splices at 2 units each.
expect(Number(afterAcquire[0]!.reserved_requests)).toBe(6 + 12)

await Promise.all(
hosts.map((host, index) =>
stores[index % stores.length]!.releaseActivity(host, `splice:percell-${index}`)
)
)
const afterRelease = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellC.id]
)
expect(Number(afterRelease[0]!.reserved_requests)).toBe(6)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
}, 15_000)
})
Loading
Loading