diff --git a/cloud/apps/relay/src/database-move-catalog.ts b/cloud/apps/relay/src/database-move-catalog.ts new file mode 100644 index 000000000000..06c7aa58b7cd --- /dev/null +++ b/cloud/apps/relay/src/database-move-catalog.ts @@ -0,0 +1,244 @@ +import pg from 'pg' +import { relayPostgresSchemaStatements } from './database.js' +import { MoveRefusal } from './database-move-events.js' + +export type MoveColumn = { name: string; type: string; notNull: boolean; generated: boolean } +export type MoveForeignKey = { columns: string[]; parent: string; parentColumns: string[] } +export type MoveTable = { + name: string + columns: MoveColumn[] + primaryKey: string[] + // Non-primary unique constraints and plain unique indexes (no predicate, no expression). + uniqueKeys: string[][] + foreignKeys: MoveForeignKey[] +} + +export const RELAY_DATABASE_AUTHORITY_TABLE = 'relay_database_authority' +const MOVE_TABLE_PREFIX = 'relay_move_' +// No longer created by the schema; left empty on databases that have them. +const RETIRED_RELAY_TABLES = new Set([ + 'relay_confirmable_splices', + 'relay_cell_drain_attempts', + 'relay_migration_leases' +]) + +const CATALOG_SQL = ` +SELECT c.relname AS name, + (SELECT json_agg(json_build_object('name', a.attname, 'type', format_type(a.atttypid, a.atttypmod), + 'notNull', a.attnotnull, 'generated', a.attgenerated <> '') ORDER BY a.attnum) + FROM pg_attribute a WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped) AS columns, + (SELECT json_agg(a.attname ORDER BY k.ord) + FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum + WHERE i.indrelid = c.oid AND i.indisprimary) AS primary_key, + (SELECT coalesce(json_agg(u.cols), '[]'::json) FROM ( + SELECT (SELECT json_agg(a.attname ORDER BY k.ord) + FROM unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum) AS cols + FROM pg_index i + WHERE i.indrelid = c.oid AND i.indisunique AND NOT i.indisprimary AND i.indpred IS NULL + AND i.indexprs IS NULL) u) AS unique_keys, + (SELECT coalesce(json_agg(json_build_object( + 'parent', p.relname, + 'columns', (SELECT json_agg(a.attname ORDER BY k.ord) FROM unnest(f.conkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_attribute a ON a.attrelid = f.conrelid AND a.attnum = k.attnum), + 'parentColumns', (SELECT json_agg(a.attname ORDER BY k.ord) FROM unnest(f.confkey) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_attribute a ON a.attrelid = f.confrelid AND a.attnum = k.attnum))), '[]'::json) + FROM pg_constraint f JOIN pg_class p ON p.oid = f.confrelid + WHERE f.conrelid = c.oid AND f.contype = 'f') AS foreign_keys +FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname = 'public' AND c.relkind = 'r' +ORDER BY c.relname` + +export function quoteIdent(name: string): string { + return pg.escapeIdentifier(name) +} + +export function quoteLiteral(value: string): string { + return pg.escapeLiteral(value) +} + +function isMoveCandidate(name: string): boolean { + return ( + name !== RELAY_DATABASE_AUTHORITY_TABLE && + !name.startsWith(MOVE_TABLE_PREFIX) && + !RETIRED_RELAY_TABLES.has(name) + ) +} + +// The tables the schema creates, read from the statements the relay itself applies. +export function relaySchemaTableNames(statements = relayPostgresSchemaStatements()): string[] { + const names = new Set() + for (const statement of statements) { + for (const match of statement.matchAll( + /CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS\s+([a-z_][a-z0-9_]*)/gi + )) { + if (match[1] && isMoveCandidate(match[1])) names.add(match[1]) + } + } + return [...names].sort() +} + +// Every relay table that moves, in parent-before-child order. Refuses when the database and the +// schema disagree about which tables exist, or a table cannot be keyed. +export async function readMoveTables( + client: pg.Client, + schemaTables = relaySchemaTableNames() +): Promise { + const result = await client.query(CATALOG_SQL) + const tables: MoveTable[] = [] + for (const row of result.rows) { + const name = String(row.name) + if (!isMoveCandidate(name)) continue + tables.push({ + name, + columns: row.columns ?? [], + primaryKey: row.primary_key ?? [], + uniqueKeys: row.unique_keys ?? [], + foreignKeys: row.foreign_keys ?? [] + }) + } + const present = tables.map((table) => table.name) + const missing = schemaTables.filter((name) => !present.includes(name)) + const unexpected = present.filter((name) => !schemaTables.includes(name)) + if (missing.length > 0 || unexpected.length > 0) { + throw new MoveRefusal('table_list_differs_from_schema', { missing, unexpected }) + } + const unkeyed = tables.filter((table) => table.primaryKey.length === 0).map((table) => table.name) + const generated = tables.filter((table) => table.columns.some((column) => column.generated)) + if (unkeyed.length > 0 || generated.length > 0) { + throw new MoveRefusal('table_not_movable', { + withoutPrimaryKey: unkeyed, + withGeneratedColumns: generated.map((table) => table.name) + }) + } + return orderParentsFirst(tables) +} + +function orderParentsFirst(tables: MoveTable[]): MoveTable[] { + const byName = new Map(tables.map((table) => [table.name, table])) + const ordered: MoveTable[] = [] + const visiting = new Set() + const visit = (table: MoveTable): void => { + if (ordered.includes(table)) return + if (visiting.has(table.name)) throw new MoveRefusal('foreign_key_cycle', { table: table.name }) + visiting.add(table.name) + for (const key of table.foreignKeys) { + const parent = byName.get(key.parent) + if (!parent) throw new MoveRefusal('foreign_key_parent_not_moved', { table: table.name }) + if (parent !== table) visit(parent) + } + visiting.delete(table.name) + ordered.push(table) + } + for (const table of tables) visit(table) + return ordered +} + +// What a row's copy depends on, independent of column position. +export function moveTableShape(table: MoveTable): string { + return JSON.stringify({ + columns: [...table.columns] + .sort((left, right) => left.name.localeCompare(right.name)) + .map((column) => [column.name, column.type, column.notNull]), + primaryKey: table.primaryKey, + uniqueKeys: table.uniqueKeys.map((key) => key.join(',')).sort(), + foreignKeys: table.foreignKeys + .map((key) => `${key.columns.join(',')}>${key.parent}(${key.parentColumns.join(',')})`) + .sort() + }) +} + +// Names, types, nullability, keys and the database collation (which orders text keys): what the +// copy, the sync and the range hashes depend on being the same on both sides. +export async function compareMoveSchemas( + primary: { client: pg.Client; tables: MoveTable[] }, + dedicated: { client: pg.Client; tables: MoveTable[] } +): Promise { + const shape = moveTableShape + const dedicatedShapes = new Map(dedicated.tables.map((table) => [table.name, shape(table)])) + const drifted = primary.tables + .filter((table) => dedicatedShapes.get(table.name) !== shape(table)) + .map((table) => table.name) + const collation = async (client: pg.Client): Promise => + String( + ( + await client.query( + `SELECT datcollate || '/' || datctype AS collation FROM pg_database WHERE datname = current_database()` + ) + ).rows[0]?.collation + ) + const [primaryCollation, dedicatedCollation] = await Promise.all([ + collation(primary.client), + collation(dedicated.client) + ]) + if (drifted.length > 0 || primaryCollation !== dedicatedCollation) { + throw new MoveRefusal('schema_drift', { + tables: drifted, + ...(primaryCollation === dedicatedCollation ? {} : { collation: 'differs' }) + }) + } +} + +// SQL fragments keyed to one table. `alias` is a row source in the statement. +export function pkTuple(table: MoveTable, alias: string): string { + return `(${table.primaryKey.map((column) => `${alias}.${quoteIdent(column)}`).join(', ')})` +} + +// Keys in (after, through] for rows aliased mv_r; null = open end. Parameters start at $1. +export function pkRange( + table: MoveTable, + after: string | null, + through: string | null +): { clauses: string[]; params: string[] } { + const params: string[] = [] + const clauses: string[] = [] + const columns = table.primaryKey.map((column) => `mv_k.${quoteIdent(column)}`).join(', ') + for (const [op, value] of [ + ['>', after], + ['<=', through] + ] as const) { + if (value === null) continue + params.push(value) + clauses.push( + `${pkTuple(table, 'mv_r')} ${op} (SELECT ${columns} FROM ${jsonRecord(table, params.length)} mv_k)` + ) + } + return { clauses, params } +} + +export function columnTuple(columns: string[], alias: string): string { + return `(${columns.map((column) => `${alias}.${quoteIdent(column)}`).join(', ')})` +} + +// jsonb text is canonical, so the same key always prints the same way on both sides. +export function pkJson(table: MoveTable, alias: string): string { + const pairs = table.primaryKey.map( + (column) => `${quoteLiteral(column)}, ${alias}.${quoteIdent(column)}` + ) + return `jsonb_build_object(${pairs.join(', ')})` +} + +// Rows of the table's own type from a JSON array parameter: values are parsed by Postgres, +// never by JavaScript, so BIGINTs keep every digit. +export function jsonRecords(table: MoveTable, parameter: number): string { + return `jsonb_populate_recordset(null::public.${quoteIdent(table.name)}, $${parameter}::jsonb)` +} + +export function jsonRecord(table: MoveTable, parameter: number): string { + return `jsonb_populate_record(null::public.${quoteIdent(table.name)}, $${parameter}::jsonb)` +} + +export function tableRef(table: MoveTable): string { + return `public.${quoteIdent(table.name)}` +} + +// Hash input: every column by name, in name order, so column position never matters. +export function hashedRowText(table: MoveTable, alias: string): string { + const names = table.columns.map((column) => column.name).sort() + return `ROW(${names.map((name) => `${alias}.${quoteIdent(name)}`).join(', ')})::text` +} + +export function jsonArray(items: string[]): string { + return `[${items.join(',')}]` +} diff --git a/cloud/apps/relay/src/database-move-cli.test.ts b/cloud/apps/relay/src/database-move-cli.test.ts new file mode 100644 index 000000000000..067864f17db2 --- /dev/null +++ b/cloud/apps/relay/src/database-move-cli.test.ts @@ -0,0 +1,30 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { runMoveCli } from './database-move-cli.js' + +describe('database move CLI', () => { + afterEach(() => vi.restoreAllMocks()) + + it('refuses bad flags and unknown commands before touching anything', async () => { + const lines: string[] = [] + vi.spyOn(console, 'log').mockImplementation((line: string) => lines.push(line)) + vi.spyOn(console, 'error').mockImplementation(() => undefined) + expect(await runMoveCli(['copy', '--from', 'elsewhere'], {})).toBe(2) + expect(await runMoveCli(['sync'], {})).toBe(2) + expect(await runMoveCli(['bogus'], {})).toBe(2) + expect(lines.map((line) => JSON.parse(line).reason)).toEqual([ + 'flag_needs_primary_or_dedicated', + 'flag_needs_primary_or_dedicated' + ]) + }) + + it('names a missing credential without opening a session', async () => { + const lines: string[] = [] + vi.spyOn(console, 'log').mockImplementation((line: string) => lines.push(line)) + expect(await runMoveCli(['install-tracking', '--side', 'primary'], {})).toBe(2) + expect(JSON.parse(lines[0] ?? '{}')).toMatchObject({ + reason: 'credential_not_configured', + side: 'primary', + role: 'move' + }) + }) +}) diff --git a/cloud/apps/relay/src/database-move-cli.ts b/cloud/apps/relay/src/database-move-cli.ts new file mode 100644 index 000000000000..3a59b3badf53 --- /dev/null +++ b/cloud/apps/relay/src/database-move-cli.ts @@ -0,0 +1,167 @@ +import { pathToFileURL } from 'node:url' +import { parseArgs } from 'node:util' +import { + MoveDatabases, + type MoveCredentialSource, + type MoveCredentialSources, + type MoveRole, + type MoveSide +} from './database-move-connections.js' +import { + copyCommand, + installTrackingCommand, + prepareTargetCommand, + syncCommand, + uninstallTrackingCommand, + verifyCommand, + type MoveCommandContext +} from './database-move-commands.js' +import { describeMoveError, logMoveEvent, MoveRefusal } from './database-move-events.js' +import { gcsDatabaseTargetPointerStore } from './database-move-pointer-store.js' +import { googleMetadataAccess } from './google-metadata-access-token.js' + +// node apps/relay/dist/database-move-cli.js [flags]; runs as a Cloud Run job on the +// relay image. Credentials are Secret Manager version names (flags or env); a *_URL env var is +// accepted instead for local runs only. Exit 0 = done, 2 = refused (nothing unsafe happened), +// 1 = failed, 3 = copy stopped by a signal (run it again to resume). +const USAGE = `commands: + install-tracking --side primary|dedicated uninstall-tracking --side primary|dedicated + prepare-target copy --from primary|dedicated [--restart] + sync --from primary|dedicated [--until-empty] verify [--tables a,b]` + +// Both sides together; a 13 M-row database verifies in about 11 minutes. +const DEFAULT_VERIFY_ROWS_PER_SECOND = 40_000 + +const OPTIONS = { + side: { type: 'string' }, + from: { type: 'string' }, + restart: { type: 'boolean' }, + 'until-empty': { type: 'boolean' }, + tables: { type: 'string' }, + 'batch-rows': { type: 'string' }, + 'batch-keys': { type: 'string' }, + 'max-bytes-per-second': { type: 'string' }, + 'chunk-rows': { type: 'string' }, + 'max-rows-per-second': { type: 'string' }, + 'primary-app-secret': { type: 'string' }, + 'primary-move-secret': { type: 'string' }, + 'dedicated-app-secret': { type: 'string' }, + 'dedicated-move-secret': { type: 'string' }, + 'pointer-bucket': { type: 'string' } +} as const + +type ParsedFlags = ReturnType< + typeof parseArgs<{ options: typeof OPTIONS; allowPositionals: true }> +>['values'] + +function side(value: string | undefined, flag: string): MoveSide { + if (value === 'primary' || value === 'dedicated') return value + throw new MoveRefusal('flag_needs_primary_or_dedicated', { flag }) +} + +function count(value: string | undefined, flag: string): number | undefined { + if (value === undefined) return undefined + const parsed = Number(value) + if (!Number.isSafeInteger(parsed) || parsed < 0) + throw new MoveRefusal('flag_needs_a_count', { flag }) + return parsed +} + +function credentialSources(flags: ParsedFlags, env: NodeJS.ProcessEnv): MoveCredentialSources { + const sources: MoveCredentialSources = {} + for (const moveSide of ['primary', 'dedicated'] as const) { + for (const role of ['app', 'move'] as const satisfies readonly MoveRole[]) { + const flag = flags[`${moveSide}-${role}-secret`] + const prefix = `ORCA_RELAY_MOVE_${moveSide.toUpperCase()}_${role.toUpperCase()}` + const secret = flag ?? env[`${prefix}_SECRET`] + const url = env[`${prefix}_URL`] + const source: MoveCredentialSource | null = secret ? { secret } : url ? { url } : null + if (source) sources[moveSide] = { ...sources[moveSide], [role]: source } + } + } + return sources +} + +export async function runMoveCli( + argv: string[], + env: NodeJS.ProcessEnv = process.env +): Promise { + let databases: MoveDatabases | null = null + try { + const { values: flags, positionals } = parseArgs({ + args: argv, + options: OPTIONS, + allowPositionals: true, + strict: true + }) + const command = positionals[0] + const google = googleMetadataAccess() + databases = new MoveDatabases(credentialSources(flags, env), google) + const ctx: MoveCommandContext = { + databases, + google, + pointerStore: async () => + gcsDatabaseTargetPointerStore({ + bucket: flags['pointer-bucket'] ?? `${await google.projectId()}-relay-control`, + google + }) + } + // Only the long-running commands stop cleanly; the rest keep Node's default. + const stop = { requested: false } + if (command === 'sync' || command === 'copy') { + // `on`, not `once`: a repeated signal must not kill a stop that is still cleaning up. + process.on('SIGTERM', () => (stop.requested = true)) + process.on('SIGINT', () => (stop.requested = true)) + } + switch (command) { + case 'install-tracking': + await installTrackingCommand(ctx, side(flags.side, 'side')) + return 0 + case 'uninstall-tracking': + await uninstallTrackingCommand(ctx, side(flags.side, 'side')) + return 0 + case 'prepare-target': + await prepareTargetCommand(ctx) + return 0 + case 'copy': { + const { stopped } = await copyCommand(ctx, side(flags.from, 'from'), { + restart: flags.restart ?? false, + batchRows: count(flags['batch-rows'], 'batch-rows'), + maxBytesPerSecond: count(flags['max-bytes-per-second'], 'max-bytes-per-second'), + maxRowsPerSecond: count(flags['max-rows-per-second'], 'max-rows-per-second'), + shouldStop: () => stop.requested + }) + return stopped ? 3 : 0 + } + case 'sync': + await syncCommand(ctx, side(flags.from, 'from'), { + batchKeys: count(flags['batch-keys'], 'batch-keys'), + ...(flags['until-empty'] ? { stopAtBacklog: 0 } : {}), + shouldStop: () => stop.requested + }) + return 0 + case 'verify': + return (await verifyCommand(ctx, { + tables: flags.tables?.split(','), + chunkRows: count(flags['chunk-rows'], 'chunk-rows'), + maxRowsPerSecond: + count(flags['max-rows-per-second'], 'max-rows-per-second') ?? + DEFAULT_VERIFY_ROWS_PER_SECOND + })) + ? 0 + : 2 + default: + console.error(USAGE) + return 2 + } + } catch (error) { + logMoveEvent('command_failed', describeMoveError(error)) + return error instanceof MoveRefusal ? 2 : 1 + } finally { + databases?.close() + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exitCode = await runMoveCli(process.argv.slice(2)) +} diff --git a/cloud/apps/relay/src/database-move-commands.ts b/cloud/apps/relay/src/database-move-commands.ts new file mode 100644 index 000000000000..4423d6c0a453 --- /dev/null +++ b/cloud/apps/relay/src/database-move-commands.ts @@ -0,0 +1,233 @@ +import type pg from 'pg' +import { applyRelayPostgresSchema, openRelayPostgresPool } from './database.js' +import { compareMoveSchemas, readMoveTables, type MoveTable } from './database-move-catalog.js' +import { + endQuietly, + moveApplicationName, + otherMoveSide, + type MoveDatabases, + type MoveSide +} from './database-move-connections.js' +import { runCopy, type MoveCopyOptions } from './database-move-copy.js' +import { logMoveEvent, MoveRefusal } from './database-move-events.js' +import { + replayClaims, + requireCurrentCopy, + requireTracking, + runSyncLoop, + takeSyncLock, + type MoveSyncOptions, + type MoveSyncSessions +} from './database-move-sync.js' +import { + createMoveBookkeeping, + grantMoveRole, + installTracking, + readAuthorityGeneration, + uninstallTracking +} from './database-move-tracking.js' +import { verifyMoveTables, verdictsEqual, type MoveVerifyOptions } from './database-move-verify.js' +import type { DatabaseTargetPointerStore } from './database-move-pointer-store.js' +import type { GoogleMetadataAccess } from './google-metadata-access-token.js' + +export type MoveCommandContext = { + databases: MoveDatabases + google: GoogleMetadataAccess | null + pointerStore: () => Promise +} + +export async function withClients( + open: Array>, + run: (clients: pg.Client[]) => Promise +): Promise { + const settled = await Promise.allSettled(open) + const clients = settled.flatMap((result) => (result.status === 'fulfilled' ? [result.value] : [])) + try { + const failed = settled.find((result) => result.status === 'rejected') + if (failed?.status === 'rejected') throw failed.reason + return await run(clients) + } finally { + await Promise.all(clients.map((client) => endQuietly(client))) + } +} + +function pair(clients: pg.Client[]): [pg.Client, pg.Client] { + const [first, second] = clients + if (!first || !second) throw new MoveRefusal('session_missing') + return [first, second] +} + +// Processes serve the side with the higher authority generation (a tie is primary's); copy and +// sync must never write into it. +async function refuseWritingTheActiveSide( + sessions: MoveSyncSessions, + from: MoveSide +): Promise { + const [source, target] = await Promise.all([ + readAuthorityGeneration(sessions.source), + readAuthorityGeneration(sessions.target) + ]) + const dedicatedActive = from === 'dedicated' ? source > target : target > source + if ((from === 'primary') === dedicatedActive) { + throw new MoveRefusal('target_is_the_active_side', { from }) + } +} + +// Both sides as the tool role, after proving their tables agree. +async function withMoveSessions( + ctx: MoveCommandContext, + from: MoveSide, + purpose: string, + run: (sessions: MoveSyncSessions, tables: MoveTable[]) => Promise +): Promise { + const to = otherMoveSide(from) + return await withClients( + [ctx.databases.connect(from, 'move', purpose), ctx.databases.connect(to, 'move', purpose)], + async (clients) => { + const [source, target] = pair(clients) + const tables = await readMoveTables(source) + const targetTables = await readMoveTables(target) + const primary = + from === 'primary' ? { client: source, tables } : { client: target, tables: targetTables } + const dedicated = + from === 'primary' ? { client: target, tables: targetTables } : { client: source, tables } + await compareMoveSchemas(primary, dedicated) + return await run({ source, target }, tables) + } + ) +} + +export async function installTrackingCommand( + ctx: MoveCommandContext, + side: MoveSide +): Promise { + const toolUser = (await ctx.databases.login(side, 'move')).user + await withClients([ctx.databases.connect(side, 'app', 'install-tracking')], async ([app]) => { + if (!app) throw new MoveRefusal('session_missing') + const tables = await readMoveTables(app) + const { sentinel } = await installTracking(app, { toolUser, tables }) + logMoveEvent('install_tracking_done', { side, tables: tables.length, sentinel }) + if (sentinel === 'missing') logMoveEvent('sentinel_missing', { side, run: 'copy --restart' }) + }) +} + +export async function uninstallTrackingCommand( + ctx: MoveCommandContext, + side: MoveSide +): Promise { + // A side a cutover left is fenced: the relay role cannot log in there any more, and + // restore-password refuses it. Its triggers go with the database. + const connecting = ctx.databases + .connect(side, 'app', 'uninstall-tracking') + .catch((error: unknown) => { + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === '28P01' + ) { + throw new MoveRefusal('side_is_fenced', { + side, + run: 'once the move is finished, drop the old database through the Cloud SQL API (gcloud sql databases delete), which needs no login' + }) + } + throw error + }) + await withClients([connecting], async ([app]) => { + if (!app) throw new MoveRefusal('session_missing') + const tables = await readMoveTables(app) + await uninstallTracking(app, tables) + logMoveEvent('uninstall_tracking_done', { side, tables: tables.length }) + }) +} + +// The dedicated side gets the relay schema exactly as a director applies it; both sides get the +// authority and progress tables and the tool role's grants; then the two schemas must agree. +// Creating the authority table on primary starts the period in which a process serves only after +// reading the pointer, so the pointer must already be there and readable (`finish` ends it). +export async function prepareTargetCommand(ctx: MoveCommandContext): Promise { + const stored = await (await ctx.pointerStore()).read() + if (!stored) throw new MoveRefusal('pointer_absent', { run: 'write database-target.json first' }) + const connection = await ctx.databases.connection('dedicated', 'app') + const applicationName = moveApplicationName('prepare-target') + const { database } = openRelayPostgresPool(connection, { applicationName, poolMax: 1 }) + try { + await applyRelayPostgresSchema(connection, applicationName, database) + } finally { + await database.close().catch(() => undefined) + } + logMoveEvent('schema_applied', { side: 'dedicated' }) + const sides = ['primary', 'dedicated'] as const + const toolUsers = await Promise.all( + sides.map(async (side) => (await ctx.databases.login(side, 'move')).user) + ) + await withClients( + sides.map((side) => ctx.databases.connect(side, 'app', 'prepare-target')), + async (clients) => { + const [primary, dedicated] = pair(clients) + const tables: MoveTable[][] = [] + for (const [index, client] of [primary, dedicated].entries()) { + const sideTables = await readMoveTables(client) + await createMoveBookkeeping(client) + await grantMoveRole(client, toolUsers[index] ?? '', sideTables) + tables.push(sideTables) + } + await compareMoveSchemas( + { client: primary, tables: tables[0] ?? [] }, + { client: dedicated, tables: tables[1] ?? [] } + ) + logMoveEvent('prepare_target_done', { tables: tables[0]?.length ?? 0 }) + } + ) +} + +export async function copyCommand( + ctx: MoveCommandContext, + from: MoveSide, + options: MoveCopyOptions +): Promise<{ stopped: boolean }> { + return await withMoveSessions(ctx, from, 'copy', async (sessions, tables) => { + // Lock first: a cutover holding it may still move which side is active. + await takeSyncLock(sessions.source) + await refuseWritingTheActiveSide(sessions, from) + return await runCopy(sessions, tables, options) + }) +} + +export async function syncCommand( + ctx: MoveCommandContext, + from: MoveSide, + options: MoveSyncOptions +): Promise { + await withMoveSessions(ctx, from, 'sync', async (sessions, tables) => { + await takeSyncLock(sessions.source) + await refuseWritingTheActiveSide(sessions, from) + await requireTracking(sessions.source, tables) + await requireCurrentCopy(sessions, tables, 'apply') + const replayed = await replayClaims(sessions, tables) + logMoveEvent('sync_started', { from, replayed }) + const result = await runSyncLoop(sessions, tables, { ...options, label: { from } }) + logMoveEvent('sync_stopped', { from, ...result }) + }) +} + +export async function verifyCommand( + ctx: MoveCommandContext, + options: MoveVerifyOptions & { tables?: string[] } +): Promise { + return await withMoveSessions(ctx, 'primary', 'verify', async (sessions, tables) => { + const selected = options.tables + ? tables.filter((table) => options.tables?.includes(table.name)) + : tables + const verdicts = await verifyMoveTables(sessions.source, sessions.target, selected, options) + const equal = verdictsEqual(verdicts) + logMoveEvent('verify_done', { + equal, + tables: selected.length, + differing: verdicts + .filter((verdict) => verdict.mismatches.length > 0) + .map((verdict) => verdict.table) + }) + return equal + }) +} diff --git a/cloud/apps/relay/src/database-move-connections.test.ts b/cloud/apps/relay/src/database-move-connections.test.ts new file mode 100644 index 000000000000..1e442821520e --- /dev/null +++ b/cloud/apps/relay/src/database-move-connections.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from 'vitest' +import { parseMoveLogin } from './database-move-connections.js' +import { MoveRefusal } from './database-move-events.js' + +const PASSWORD = 'pa:ss/w@rd' + +describe('database move login parsing', () => { + it('takes the instance from a Cloud SQL socket URL and refuses without echoing', () => { + const url = `postgresql://orca_relay_move:${encodeURIComponent(PASSWORD)}@/orca_relay?host=/cloudsql/p:us-central1:relay-db` + expect(parseMoveLogin(url)).toEqual({ + user: 'orca_relay_move', + password: PASSWORD, + database: 'orca_relay', + instanceConnectionName: 'p:us-central1:relay-db' + }) + const error = (() => { + try { + return parseMoveLogin(`postgresql://u@/db?host=/x ${PASSWORD}`) + } catch (caught) { + return caught + } + })() + expect(error).toBeInstanceOf(MoveRefusal) + expect(String(error instanceof Error ? error.message : '')).not.toContain(PASSWORD) + }) +}) diff --git a/cloud/apps/relay/src/database-move-connections.ts b/cloud/apps/relay/src/database-move-connections.ts new file mode 100644 index 000000000000..37974fbe97db --- /dev/null +++ b/cloud/apps/relay/src/database-move-connections.ts @@ -0,0 +1,134 @@ +import { randomBytes } from 'node:crypto' +import { AuthTypes, Connector, IpAddressTypes } from '@google-cloud/cloud-sql-connector' +import pg from 'pg' +import { parse as parseConnectionString } from 'pg-connection-string' +import { markRelayDatabaseMoveCapability, type RelayPostgresConnection } from './database.js' +import { MoveRefusal } from './database-move-events.js' +import type { GoogleMetadataAccess } from './google-metadata-access-token.js' +import { readSecretManagerText } from './relay-dedicated-database-secret.js' + +export type MoveSide = 'primary' | 'dedicated' +// app = the relay's own role (owns the tables); move = the tool's role, which the triggers ignore. +export type MoveRole = 'app' | 'move' +// A Secret Manager version name in production; a URL only for local runs and tests. +export type MoveCredentialSource = { secret: string } | { url: string } +export type MoveCredentialSources = Partial< + Record>> +> + +export type MoveLogin = { + user: string + password: string + database: string + // From a `host=/cloudsql/` URL; null for a plain host (local runs). + instanceConnectionName: string | null +} + +const CLOUD_SQL_SOCKET_PREFIX = '/cloudsql/' +const TOOL_STATEMENT_TIMEOUT_MS = 60_000 + +// One id per tool run, in the relay's own mark format; the cutover's gate leaves the tool's +// sessions out by their prefix. +const TOOL_PROCESS_ID = randomBytes(4).toString('hex') + +export function moveApplicationName(purpose: string): string { + return markRelayDatabaseMoveCapability(`orca-relay-move/${purpose}`, TOOL_PROCESS_ID) +} + +export function otherMoveSide(side: MoveSide): MoveSide { + return side === 'primary' ? 'dedicated' : 'primary' +} + +export function parseMoveLogin(databaseUrl: string): MoveLogin { + let parsed: ReturnType + try { + parsed = parseConnectionString(databaseUrl) + } catch { + throw new MoveRefusal('database_url_unparseable') + } + const { user, password, database, host } = parsed + if (!user || !password || !database) throw new MoveRefusal('database_url_incomplete') + const instanceConnectionName = host?.startsWith(CLOUD_SQL_SOCKET_PREFIX) + ? host.slice(CLOUD_SQL_SOCKET_PREFIX.length) + : null + return { user, password, database, instanceConnectionName } +} + +// Resolves each credential once, on first use, and opens one session per call. Cloud SQL URLs go +// through the in-process connector (cloudsql.client), so the job needs no proxy sidecar. +export class MoveDatabases { + private readonly urls = new Map>() + private connector: Connector | null = null + + constructor( + private readonly sources: MoveCredentialSources, + private readonly google: GoogleMetadataAccess | null + ) {} + + has(side: MoveSide, role: MoveRole): boolean { + return this.sources[side]?.[role] !== undefined + } + + async login(side: MoveSide, role: MoveRole): Promise { + return parseMoveLogin(await this.url(side, role)) + } + + // Where a session for this side and role goes, in the shape the relay's own pools take. + async connection(side: MoveSide, role: MoveRole): Promise { + const url = await this.url(side, role) + const login = parseMoveLogin(url) + if (!login.instanceConnectionName) return { connectionString: url } + this.connector ??= new Connector() + const { stream } = await this.connector.getOptions({ + instanceConnectionName: login.instanceConnectionName, + authType: AuthTypes.PASSWORD, + ipType: IpAddressTypes.PUBLIC + }) + return { stream, user: login.user, password: login.password, database: login.database } + } + + // application_name carries the capability mark, so the tool's own app-role sessions never count + // as an unmarked relay client. + async connect(side: MoveSide, role: MoveRole, purpose: string): Promise { + const client = new pg.Client({ + ...(await this.connection(side, role)), + application_name: moveApplicationName(purpose), + statement_timeout: TOOL_STATEMENT_TIMEOUT_MS + }) + // A terminated session errors on the idle client; the next query reports it instead. + client.on('error', () => undefined) + await client.connect() + return client + } + + close(): void { + this.connector?.close() + this.connector = null + } + + private url(side: MoveSide, role: MoveRole): Promise { + const key = `${side}/${role}` + let url = this.urls.get(key) + if (!url) { + url = this.resolve(side, role) + this.urls.set(key, url) + } + return url + } + + private async resolve(side: MoveSide, role: MoveRole): Promise { + const source = this.sources[side]?.[role] + if (!source) throw new MoveRefusal('credential_not_configured', { side, role }) + if ('url' in source) return source.url + if (!this.google) throw new MoveRefusal('google_access_unavailable') + try { + return await readSecretManagerText(source.secret, this.google) + } catch { + throw new MoveRefusal('credential_secret_unreadable', { side, role }) + } + } +} + +export async function endQuietly(client: pg.Client | null | undefined): Promise { + await client?.end().catch(() => undefined) +} diff --git a/cloud/apps/relay/src/database-move-copy.ts b/cloud/apps/relay/src/database-move-copy.ts new file mode 100644 index 000000000000..e33b2906a1bd --- /dev/null +++ b/cloud/apps/relay/src/database-move-copy.ts @@ -0,0 +1,168 @@ +import type pg from 'pg' +import { tableRef, type MoveTable } from './database-move-catalog.js' +import { logMoveEvent, sleep } from './database-move-events.js' +import { + addReferencedParents, + addTargetStrays, + applyMoveChanges, + readSourcePage, + type MoveChanges +} from './database-move-row-apply.js' +import { + MOVE_SCHEMA_CHECK_EVERY_MS, + requireCurrentCopy, + requireTracking, + requireUnchangedMoveState, + type MoveSyncSessions +} from './database-move-sync.js' +import { + markSourceDirty, + MOVE_COPY_PROGRESS_TABLE, + readTrackingArm, + sentinelInsertSql +} from './database-move-tracking.js' + +// ~1 GB/h of source reads: well under the 7-20 GB/h the shared instance already reads, so the +// copy adds little to its cache churn. +export const DEFAULT_COPY_BYTES_PER_SECOND = 300_000 +export const DEFAULT_COPY_ROWS_PER_SECOND = 2_000 + +// What the source read from disk for one table: heap, indexes and TOAST, from the cumulative +// statistics. A bloated table costs its dead pages too, which the copied JSON never shows. +// Other sessions' reads of the table count as well, which only slows the copy. +async function tableBlocksRead(source: pg.Client, table: MoveTable): Promise { + const result = await source.query( + `SELECT (coalesce(heap_blks_read, 0) + coalesce(idx_blks_read, 0) + coalesce(toast_blks_read, 0) + + coalesce(tidx_blks_read, 0))::text AS n + FROM pg_statio_user_tables WHERE relid = $1::regclass`, + [tableRef(table)] + ) + return Number(result.rows[0]?.n ?? 0) +} + +// From 15 a backend's own counters reach the shared statistics only when it next goes idle. +async function flushOwnStatistics(source: pg.Client, supported: boolean): Promise { + if (supported) await source.query('SELECT pg_stat_force_next_flush()') +} + +export type MoveCopyOptions = { + batchRows?: number + // Source disk reads per second (block reads x block size, or the JSON size if larger). + maxBytesPerSecond?: number + maxRowsPerSecond?: number + // Forget the saved position and start every table again; also re-arms an emptied dirty set. + restart?: boolean + // Checked between pages; the saved position makes the next run resume. + shouldStop?: () => boolean +} + +// Every table in parent-first order, in primary-key pages; each page also deletes target rows in +// its key range that the source no longer has. The position is saved on the target in +// the same transaction as the rows, so a stopped copy resumes exactly where it committed. Rows +// that change behind the copy are on the dirty set, which the sync replays. The caller holds the +// sync lock: a sync applying a key between this page's read and its write would be overwritten +// with the older row, and the key would already be off the dirty set. +export async function runCopy( + sessions: MoveSyncSessions, + tables: MoveTable[], + options: MoveCopyOptions = {} +): Promise<{ rows: number; bytes: number; readBytes: number; stopped: boolean }> { + const batchRows = options.batchRows ?? 500 + const maxBytesPerSecond = options.maxBytesPerSecond ?? DEFAULT_COPY_BYTES_PER_SECOND + const maxRowsPerSecond = options.maxRowsPerSecond ?? DEFAULT_COPY_ROWS_PER_SECOND + if (options.restart) { + // Armed before the first page: every change from here on is either copied or dirty. + await sessions.source.query(sentinelInsertSql()) + await sessions.target.query(`DELETE FROM ${MOVE_COPY_PROGRESS_TABLE}`) + logMoveEvent('copy_restarted') + } + await requireTracking(sessions.source, tables) + await requireCurrentCopy(sessions, tables, 'copy') + const arm = await readTrackingArm(sessions.source) + const settings = await sessions.source.query( + `SELECT current_setting('block_size')::int AS block_size, + current_setting('server_version_num')::int >= 150000 AS flushes` + ) + const blockSize = Number(settings.rows[0]?.block_size ?? 8192) + const flushes = Boolean(settings.rows[0]?.flushes) + const startedAt = Date.now() + let bytes = 0 + let readBytes = 0 + let rows = 0 + let schemaCheckedAt = 0 + for (const table of tables) { + const saved = await sessions.target.query( + `SELECT after_pk::text AS after_pk, done, rows_copied::text AS rows_copied + FROM ${MOVE_COPY_PROGRESS_TABLE} WHERE table_name = $1`, + [table.name] + ) + if (saved.rows[0]?.done) continue + await requireUnchangedMoveState(sessions, tables) + schemaCheckedAt = Date.now() + await flushOwnStatistics(sessions.source, flushes) + let blocksBefore = await tableBlocksRead(sessions.source, table) + let afterPk: string | null = saved.rows[0]?.after_pk ?? null + let tableRows = Number(saved.rows[0]?.rows_copied ?? 0) + logMoveEvent('copy_table_started', { table: table.name, resumed: afterPk !== null }) + for (;;) { + if (options.shouldStop?.()) { + logMoveEvent('copy_stopped', { table: table.name, rows }) + return { rows, bytes, readBytes, stopped: true } + } + if (Date.now() - schemaCheckedAt >= MOVE_SCHEMA_CHECK_EVERY_MS) { + await requireUnchangedMoveState(sessions, tables) + schemaCheckedAt = Date.now() + } + const changes: MoveChanges = new Map() + await sessions.source.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY') + let page: Awaited> + try { + page = await readSourcePage(sessions.source, table, afterPk, batchRows, changes) + bytes += page.bytes + (await addReferencedParents(sessions.source, tables, changes)) + await sessions.source.query('COMMIT') + } catch (error) { + await sessions.source.query('ROLLBACK').catch(() => undefined) + throw error + } + const done = page.rows < batchRows + const range = { after: afterPk, through: done ? null : page.lastPk, pagePks: page.pks } + afterPk = page.lastPk ?? afterPk + tableRows += page.rows + await sessions.target.query('BEGIN') + try { + await addTargetStrays(sessions.target, table, range, changes) + await applyMoveChanges(sessions.target, tables, changes, (displaced) => + markSourceDirty(sessions.source, displaced) + ) + await sessions.target.query( + `INSERT INTO ${MOVE_COPY_PROGRESS_TABLE} + (table_name, after_pk, done, rows_copied, updated_at, tracking_arm) + VALUES ($1, $2::jsonb, $3, $4, $5, $6) + ON CONFLICT (table_name) DO UPDATE SET after_pk = EXCLUDED.after_pk, done = EXCLUDED.done, + rows_copied = EXCLUDED.rows_copied, updated_at = EXCLUDED.updated_at, + tracking_arm = EXCLUDED.tracking_arm`, + [table.name, afterPk, done, tableRows, Date.now(), arm] + ) + await sessions.target.query('COMMIT') + } catch (error) { + await sessions.target.query('ROLLBACK').catch(() => undefined) + throw error + } + rows += page.rows + await flushOwnStatistics(sessions.source, flushes) + const blocksNow = await tableBlocksRead(sessions.source, table) + readBytes += Math.max(0, blocksNow - blocksBefore) * blockSize + blocksBefore = blocksNow + // Sleep until both running averages are back under their limits. + const budgetMs = Math.max( + (Math.max(readBytes, bytes) * 1000) / maxBytesPerSecond, + (rows * 1000) / maxRowsPerSecond + ) + await sleep(budgetMs - (Date.now() - startedAt)) + if (done) break + } + logMoveEvent('copy_table_done', { table: table.name, rows: tableRows }) + } + logMoveEvent('copy_done', { rows, bytes, readBytes, ms: Date.now() - startedAt }) + return { rows, bytes, readBytes, stopped: false } +} diff --git a/cloud/apps/relay/src/database-move-events.ts b/cloud/apps/relay/src/database-move-events.ts new file mode 100644 index 000000000000..113dc6a7fcaa --- /dev/null +++ b/cloud/apps/relay/src/database-move-events.ts @@ -0,0 +1,43 @@ +// One JSON line per phase of the database move tool. Fields are counts, names and timestamps: +// no URL, password, or row content is ever passed here. +export function logMoveEvent(event: string, fields: Record = {}): void { + console.log( + JSON.stringify({ event: `orca_relay_move_${event}`, at: new Date().toISOString(), ...fields }) + ) +} + +// A precondition or check the tool refuses on. The message is the tool's own text, safe to print. +export class MoveRefusal extends Error { + constructor( + readonly reason: string, + readonly details: Record = {} + ) { + super(reason) + } +} + +// Driver and network errors can quote connection settings, so only a refusal's own text, a +// SQLSTATE and a server message (which never echoes credentials) are reported. +export function describeMoveError(error: unknown): Record { + if (error instanceof MoveRefusal) return { reason: error.reason, ...error.details } + if (typeof error !== 'object' || error === null) return { error: 'unknown' } + const code = 'code' in error && typeof error.code === 'string' ? error.code : undefined + const serverMessage = + 'severity' in error && 'message' in error && typeof error.message === 'string' + ? error.message + : undefined + const name = error instanceof Error ? error.name : 'unknown' + // Our own error tokens (snake_case words) carry no values, so they are safe to print. + const token = + error instanceof Error && /^[a-z_]+$/.test(error.message) ? error.message : undefined + return { + error: name, + ...(code ? { code } : {}), + ...(serverMessage ? { serverMessage } : {}), + ...(token ? { token } : {}) + } +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms))) +} diff --git a/cloud/apps/relay/src/database-move-pointer-store.ts b/cloud/apps/relay/src/database-move-pointer-store.ts new file mode 100644 index 000000000000..0664553eb0c1 --- /dev/null +++ b/cloud/apps/relay/src/database-move-pointer-store.ts @@ -0,0 +1,99 @@ +import { MoveRefusal } from './database-move-events.js' +import type { GoogleMetadataAccess } from './google-metadata-access-token.js' +import { + DATABASE_TARGET_POINTER_OBJECT, + parseDatabaseTargetPointer, + type DatabaseTargetPointer +} from './relay-database-target-pointer.js' + +const STORAGE_ROOT = 'https://storage.googleapis.com' +const REQUEST_TIMEOUT_MS = 5_000 + +export type StoredDatabaseTargetPointer = { pointer: DatabaseTargetPointer; generation: string } + +// Every write is a compare-and-swap on the object generation the tool last read, so a pointer +// changed by anyone else in between fails the write instead of being overwritten. +export type DatabaseTargetPointerStore = { + read(): Promise + write(pointer: DatabaseTargetPointer, ifGenerationMatch: string): Promise + // Ends the move: processes read the pointer as absent and serve DATABASE_URL alone. + delete(ifGenerationMatch: string): Promise +} + +export class PointerGenerationConflict extends MoveRefusal { + constructor() { + super('pointer_changed_by_someone_else') + } +} + +// Needs storage.objects.get and .create/.delete on database-target.json in the control bucket. +export function gcsDatabaseTargetPointerStore(input: { + bucket: string + google: GoogleMetadataAccess + fetchImpl?: typeof fetch +}): DatabaseTargetPointerStore { + const fetchImpl = input.fetchImpl ?? fetch + const object = encodeURIComponent(DATABASE_TARGET_POINTER_OBJECT) + const bucket = encodeURIComponent(input.bucket) + const request = async (url: string, init: RequestInit = {}): Promise => { + const response = await fetchImpl(url, { + ...init, + headers: { + authorization: `Bearer ${await input.google.accessToken()}`, + ...(init.body ? { 'content-type': 'application/json' } : {}) + }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }).catch(() => { + throw new MoveRefusal('pointer_store_unreachable') + }) + if (response.status === 401) input.google.forgetAccessToken() + return response + } + return { + read: async () => { + const response = await request(`${STORAGE_ROOT}/storage/v1/b/${bucket}/o/${object}?alt=media`) + if (response.status === 404) { + void response.body?.cancel().catch(() => undefined) + return null + } + const generation = response.headers.get('x-goog-generation') + if (!response.ok || !generation) { + void response.body?.cancel().catch(() => undefined) + throw new MoveRefusal('pointer_read_failed', { status: response.status }) + } + const parsed = parseDatabaseTargetPointer(await response.json().catch(() => null)) + if (!parsed?.flags) throw new MoveRefusal('pointer_malformed') + return { pointer: parsed.flags, generation } + }, + delete: async (ifGenerationMatch) => { + const url = new URL(`${STORAGE_ROOT}/storage/v1/b/${bucket}/o/${object}`) + url.searchParams.set('ifGenerationMatch', ifGenerationMatch) + const response = await request(url.toString(), { method: 'DELETE' }) + void response.body?.cancel().catch(() => undefined) + if (response.status === 412) throw new PointerGenerationConflict() + if (!response.ok) throw new MoveRefusal('pointer_delete_failed', { status: response.status }) + }, + write: async (pointer, ifGenerationMatch) => { + const url = new URL(`${STORAGE_ROOT}/upload/storage/v1/b/${bucket}/o`) + url.searchParams.set('uploadType', 'media') + url.searchParams.set('name', DATABASE_TARGET_POINTER_OBJECT) + url.searchParams.set('ifGenerationMatch', ifGenerationMatch) + const response = await request(url.toString(), { + method: 'POST', + body: JSON.stringify(pointer) + }) + if (response.status === 412) { + void response.body?.cancel().catch(() => undefined) + throw new PointerGenerationConflict() + } + const body: unknown = await response.json().catch(() => null) + const generation = + typeof body === 'object' && body !== null && 'generation' in body + ? String(body.generation) + : '' + if (!response.ok || !generation) + throw new MoveRefusal('pointer_write_failed', { status: response.status }) + return generation + } + } +} diff --git a/cloud/apps/relay/src/database-move-row-apply.ts b/cloud/apps/relay/src/database-move-row-apply.ts new file mode 100644 index 000000000000..464f2bb583bf --- /dev/null +++ b/cloud/apps/relay/src/database-move-row-apply.ts @@ -0,0 +1,226 @@ +import type pg from 'pg' +import { + jsonArray, + jsonRecord, + jsonRecords, + pkJson, + pkRange, + pkTuple, + quoteIdent, + tableRef, + type MoveTable +} from './database-move-catalog.js' + +// Rows travel as Postgres-made JSON text (row_to_json) and are rebuilt by Postgres +// (jsonb_populate_recordset), so no value is ever parsed in JavaScript. +export type MoveTableChanges = { + // pk JSON text -> row JSON text, both as Postgres printed them. + upserts: Map + deletes: Set +} +export type MoveChanges = Map +export type MoveKey = { table: string; pk: string } + +function list(columns: string[], alias: string): string { + return columns.map((column) => `${alias}.${quoteIdent(column)}`).join(', ') +} + +function tuple(columns: string[], alias: string): string { + return `(${list(columns, alias)})` +} + +export function changesFor(changes: MoveChanges, table: string): MoveTableChanges { + let entry = changes.get(table) + if (!entry) { + entry = { upserts: new Map(), deletes: new Set() } + changes.set(table, entry) + } + return entry +} + +function rowsToChanges(rows: Array>, target: Map): number { + let bytes = 0 + for (const row of rows) { + const pk = String(row.pk) + const json = String(row.row) + bytes += json.length + if (!target.has(pk)) target.set(pk, json) + } + return bytes +} + +// Current source rows for these keys; a key with no row is a delete. +export async function readSourceRows( + source: pg.Client, + table: MoveTable, + pks: string[], + changes: MoveChanges +): Promise { + const entry = changesFor(changes, table.name) + const result = await source.query( + `SELECT ${pkJson(table, 'mv_r')}::text AS pk, row_to_json(mv_r)::text AS row + FROM ${tableRef(table)} mv_r + WHERE ${pkTuple(table, 'mv_r')} IN (SELECT ${list(table.primaryKey, 'mv_k')} FROM ${jsonRecords(table, 1)} mv_k)`, + [jsonArray(pks)] + ) + const bytes = rowsToChanges(result.rows, entry.upserts) + for (const pk of pks) if (!entry.upserts.has(pk)) entry.deletes.add(pk) + return bytes +} + +// The next keyset page of a table, after `afterPk` (pk JSON text) or from the start. +export async function readSourcePage( + source: pg.Client, + table: MoveTable, + afterPk: string | null, + limit: number, + changes: MoveChanges +): Promise<{ rows: number; bytes: number; lastPk: string | null; pks: string[] }> { + const after = + afterPk === null + ? '' + : `WHERE ${pkTuple(table, 'mv_r')} > (SELECT ${list(table.primaryKey, 'mv_k')} FROM ${jsonRecord(table, 2)} mv_k)` + const result = await source.query( + `SELECT ${pkJson(table, 'mv_r')}::text AS pk, row_to_json(mv_r)::text AS row + FROM ${tableRef(table)} mv_r ${after} + ORDER BY ${list(table.primaryKey, 'mv_r')} LIMIT $1`, + afterPk === null ? [limit] : [limit, afterPk] + ) + const bytes = rowsToChanges(result.rows, changesFor(changes, table.name).upserts) + const pks = result.rows.map((row) => String(row.pk)) + return { rows: pks.length, bytes, lastPk: pks.at(-1) ?? null, pks } +} + +// Target rows in a copied page's key range that the source page lacks: deleted on the source +// while their keys were not tracked (before tracking, or after a lost dirty set). +export async function addTargetStrays( + target: pg.Client, + table: MoveTable, + range: { after: string | null; through: string | null; pagePks: string[] }, + changes: MoveChanges +): Promise { + const { clauses, params } = pkRange(table, range.after, range.through) + const page = params.length + 1 + const result = await target.query( + `SELECT ${pkJson(table, 'mv_r')}::text AS pk FROM ${tableRef(table)} mv_r + WHERE ${[ + ...clauses, + `${pkTuple(table, 'mv_r')} NOT IN (SELECT ${list(table.primaryKey, 'mv_k')} FROM ${jsonRecords(table, page)} mv_k)` + ].join(' AND ')}`, + [...params, jsonArray(range.pagePks)] + ) + const entry = changesFor(changes, table.name) + for (const row of result.rows) entry.deletes.add(String(row.pk)) +} + +// A child row can only land after its parent does, so every upserted child brings the parent row +// it references, read in the same source snapshot. Children are visited first so a parent that is +// itself a child brings its own parent too. +export async function addReferencedParents( + source: pg.Client, + tables: MoveTable[], + changes: MoveChanges +): Promise { + const byName = new Map(tables.map((table) => [table.name, table])) + let bytes = 0 + for (const table of tables.toReversed()) { + const childRows = [...(changes.get(table.name)?.upserts.values() ?? [])] + if (childRows.length === 0) continue + for (const key of table.foreignKeys) { + const parent = byName.get(key.parent) + if (!parent) continue + const result = await source.query( + `SELECT ${pkJson(parent, 'mv_p')}::text AS pk, row_to_json(mv_p)::text AS row + FROM ${tableRef(parent)} mv_p + WHERE ${tuple(key.parentColumns, 'mv_p')} IN (SELECT ${list(key.columns, 'mv_c')} FROM ${jsonRecords(table, 1)} mv_c)`, + [jsonArray(childRows)] + ) + const entry = changesFor(changes, parent.name) + bytes += rowsToChanges(result.rows, entry.upserts) + for (const row of result.rows) entry.deletes.delete(String(row.pk)) + } + } + return bytes +} + +// Applies one batch inside the caller's open target transaction: deletes children first, then +// upserts parents first. Rows the target holds that block a change (a stale child of a deleted +// parent, or a stale row holding a unique value the source now gives another key) are deleted and +// handed to `markDirty`, so the next sync pass copies whatever the source still has for them. +export async function applyMoveChanges( + target: pg.Client, + tables: MoveTable[], + changes: MoveChanges, + markDirty: (keys: MoveKey[]) => Promise +): Promise<{ upserted: number; deleted: number; displaced: number }> { + const displaced: MoveKey[] = [] + let deleted = 0 + let upserted = 0 + + const deleteKeys = async (table: MoveTable, pks: string[], displace: boolean): Promise => { + if (pks.length === 0) return + for (const child of tables) { + for (const key of child.foreignKeys.filter((foreign) => foreign.parent === table.name)) { + const children = await target.query( + `SELECT ${pkJson(child, 'mv_c')}::text AS pk FROM ${tableRef(child)} mv_c + WHERE ${tuple(key.columns, 'mv_c')} IN (SELECT ${list(key.parentColumns, 'mv_k')} FROM ${jsonRecords(table, 1)} mv_k)`, + [jsonArray(pks)] + ) + await deleteKeys( + child, + children.rows.map((row) => String(row.pk)), + true + ) + } + } + const result = await target.query( + `DELETE FROM ${tableRef(table)} mv_t + WHERE ${pkTuple(table, 'mv_t')} IN (SELECT ${list(table.primaryKey, 'mv_k')} FROM ${jsonRecords(table, 1)} mv_k)`, + [jsonArray(pks)] + ) + deleted += result.rowCount ?? 0 + if (displace) displaced.push(...pks.map((pk) => ({ table: table.name, pk }))) + } + + for (const table of tables.toReversed()) { + await deleteKeys(table, [...(changes.get(table.name)?.deletes ?? [])], false) + } + for (const table of tables) { + const rows = [...(changes.get(table.name)?.upserts.values() ?? [])] + if (rows.length === 0) continue + const incoming = jsonArray(rows) + // A target row holding an incoming unique value is stale unless it is that same incoming row + // with that same value; this also clears values two incoming rows swap. + for (const unique of table.uniqueKeys) { + const sameRow = [...table.primaryKey, ...unique] + .map( + (column) => `mv_s.${quoteIdent(column)} IS NOT DISTINCT FROM mv_t.${quoteIdent(column)}` + ) + .join(' AND ') + const colliding = await target.query( + `SELECT ${pkJson(table, 'mv_t')}::text AS pk FROM ${tableRef(table)} mv_t + WHERE ${tuple(unique, 'mv_t')} IN (SELECT ${list(unique, 'mv_r')} FROM ${jsonRecords(table, 1)} mv_r) + AND NOT EXISTS (SELECT 1 FROM ${jsonRecords(table, 1)} mv_s WHERE ${sameRow})`, + [incoming] + ) + await deleteKeys( + table, + colliding.rows.map((row) => String(row.pk)), + true + ) + } + const updatable = table.columns.filter((column) => !table.primaryKey.includes(column.name)) + const onConflict = + updatable.length === 0 + ? 'DO NOTHING' + : `DO UPDATE SET ${updatable.map((column) => `${quoteIdent(column.name)} = EXCLUDED.${quoteIdent(column.name)}`).join(', ')}` + const result = await target.query( + `INSERT INTO ${tableRef(table)} SELECT * FROM ${jsonRecords(table, 1)} + ON CONFLICT (${table.primaryKey.map(quoteIdent).join(', ')}) ${onConflict}`, + [incoming] + ) + upserted += result.rowCount ?? 0 + } + if (displaced.length > 0) await markDirty(displaced) + return { upserted, deleted, displaced: displaced.length } +} diff --git a/cloud/apps/relay/src/database-move-sync-postgres.test.ts b/cloud/apps/relay/src/database-move-sync-postgres.test.ts new file mode 100644 index 000000000000..8cfdffed3eae --- /dev/null +++ b/cloud/apps/relay/src/database-move-sync-postgres.test.ts @@ -0,0 +1,535 @@ +import pg from 'pg' +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' +import { readMoveTables, type MoveTable } from './database-move-catalog.js' +import { prepareTargetCommand, syncCommand } from './database-move-commands.js' +import type { MoveDatabases } from './database-move-connections.js' +import { runCopy } from './database-move-copy.js' +import { runSyncLoop, type MoveSyncSessions } from './database-move-sync.js' +import { + installTracking, + readDirtyBacklog, + uninstallTracking, + type TrackingConflictAction +} from './database-move-tracking.js' +import { verifyMoveTables, verdictsEqual } from './database-move-verify.js' +import { + createMoveTestSides, + MemoryPointerStore, + testPointerStore, + withPgClient, + type MoveTestSides +} from './test-fixtures/database-move-sides.js' + +const adminUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL +const describePostgres = adminUrl ? describe : describe.skip + +async function connect(url: string): Promise { + const client = new pg.Client({ connectionString: url }) + client.on('error', () => undefined) + await client.connect() + return client +} + +describePostgres('database move: tracking, copy, sync and verify', () => { + let sides: MoveTestSides + let databases: MoveDatabases + let sessions: MoveSyncSessions + let app: pg.Client + let tables: MoveTable[] + + const table = (name: string): MoveTable => { + const found = tables.find((candidate) => candidate.name === name) + if (!found) throw new Error(`no table ${name}`) + return found + } + const track = async (conflictAction: TrackingConflictAction = 'update'): Promise => { + await installTracking(app, { toolUser: sides.primary.move.user, tables, conflictAction }) + } + const drain = async (): Promise => { + await runSyncLoop(sessions, tables, { stopAtBacklog: 0, idleMs: 1, reportEveryMs: 60_000 }) + } + const equal = async (only?: string[]): Promise => + verdictsEqual( + await verifyMoveTables( + sessions.source, + sessions.target, + only ? tables.filter((candidate) => only.includes(candidate.name)) : tables, + { chunkRows: 7 } + ) + ) + const clearRelayRows = async (): Promise => { + for (const url of [sides.primary.adminUrl, sides.dedicated.adminUrl]) { + await withPgClient(url, async (admin) => { + await admin.query(`TRUNCATE ${tables.map((candidate) => candidate.name).join(', ')}`) + await admin.query(`DELETE FROM relay_move_copy_progress`) + }) + } + } + + beforeAll(async () => { + vi.spyOn(console, 'log').mockImplementation(() => undefined) + sides = await createMoveTestSides(adminUrl ?? '') + databases = sides.databases() + await prepareTargetCommand({ + databases, + google: null, + pointerStore: async () => testPointerStore() + }) + app = await connect(sides.primary.app.url) + tables = await readMoveTables(app) + sessions = { + source: await connect(sides.primary.move.url), + target: await connect(sides.dedicated.move.url) + } + }, 60_000) + + afterEach(async () => { + await uninstallTracking(app, tables) + await clearRelayRows() + }) + + afterAll(async () => { + for (const client of [app, sessions?.source, sessions?.target]) + await client?.end().catch(() => undefined) + databases?.close() + await sides?.drop() + vi.restoreAllMocks() + }) + + it('refuses to prepare the target before the pointer exists', async () => { + await expect( + prepareTargetCommand({ + databases, + google: null, + pointerStore: async () => new MemoryPointerStore(null) + }) + ).rejects.toThrow('pointer_absent') + }) + + it('covers every schema table except the authority, move and retired tables', () => { + expect(tables.length).toBeGreaterThan(35) + const names = tables.map((candidate) => candidate.name) + expect(names).not.toContain('relay_database_authority') + expect(names.filter((name) => name.startsWith('relay_move_'))).toEqual([]) + // Parents first, for the three foreign keys. + expect(names.indexOf('relay_cell_fence_attempts')).toBeLessThan( + names.indexOf('relay_cell_fence_plan_bindings') + ) + expect(names.indexOf('relay_cell_drain_attempt_states')).toBeLessThan( + names.indexOf('relay_cell_drain_recovery_attempts') + ) + }) + + it('ignores the tool role and marks every relay-role change, old and new key', async () => { + await track() + await sessions.source.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) VALUES ('tool-1', 1, 't', '{}')` + ) + expect(await readDirtyBacklog(sessions.source)).toBe(0) + await app.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) VALUES ('app-1', 1, 't', '{}')` + ) + await app.query(`UPDATE relay_audit_events SET id = 'app-2' WHERE id = 'app-1'`) + const keys = await sessions.source.query( + `SELECT pk::text AS pk FROM relay_move_dirty WHERE table_name = 'relay_audit_events' ORDER BY 1` + ) + expect(keys.rows.map((row) => row.pk)).toEqual(['{"id": "app-1"}', '{"id": "app-2"}']) + }) + + it('stops the sync when the dirty set was emptied, until a restarted copy re-arms it', async () => { + await track() + await app.query(`TRUNCATE relay_audit_events`) + await expect(drain()).rejects.toThrow('dirty_set_was_emptied_recopy_needed') + await runCopy(sessions, tables, { restart: true, maxBytesPerSecond: 1e9 }) + await drain() + }) + + it('lets another role that writes relay rows keep writing once tracking is on', async () => { + const writer = `${sides.primary.app.user}_writer` + await withPgClient(sides.primary.adminUrl, async (admin) => { + await admin.query(`CREATE ROLE ${writer} LOGIN PASSWORD 'writer'`) + }) + try { + await app.query(`GRANT INSERT ON relay_audit_events TO ${writer}`) + await track() + const url = new URL(sides.primary.adminUrl) + url.username = writer + url.password = 'writer' + await withPgClient(url.toString(), (client) => + client.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) VALUES ('x', 1, 't', '{}')` + ) + ) + expect(await readDirtyBacklog(sessions.source)).toBe(1) + } finally { + await withPgClient(sides.primary.adminUrl, async (admin) => { + await admin.query(`DROP OWNED BY ${writer}`) + await admin.query(`DROP ROLE ${writer}`) + }) + } + }) + + it('stops copy and sync when a table loses its triggers', async () => { + await track() + await app.query(`DROP TRIGGER relay_move_track_upd ON relay_cells`) + await expect(runCopy(sessions, tables, { maxBytesPerSecond: 1e9 })).rejects.toThrow( + 'tables_without_tracking' + ) + await expect( + runSyncLoop(sessions, tables, { stopAtBacklog: 0, schemaCheckEveryMs: 0 }) + ).rejects.toThrow('tables_without_tracking') + }) + + it("stops on a column added mid-run and resets that table's copy position", async () => { + await track() + await runCopy(sessions, tables, { maxBytesPerSecond: 1e9 }) + try { + await app.query(`ALTER TABLE relay_audit_events ADD COLUMN mid_run TEXT`) + const refusal = await runSyncLoop(sessions, tables, { + stopAtBacklog: 0, + schemaCheckEveryMs: 0 + }).catch((error: unknown) => error) + expect(refusal).toMatchObject({ + reason: 'schema_changed_during_run', + details: { tables: ['relay_audit_events'], copyProgressReset: true } + }) + const progress = await sessions.target.query( + `SELECT table_name FROM relay_move_copy_progress WHERE table_name = 'relay_audit_events'` + ) + expect(progress.rows).toEqual([]) + await expect(runCopy(sessions, tables, { maxBytesPerSecond: 1e9 })).rejects.toThrow( + 'schema_changed_during_run' + ) + } finally { + await app.query(`ALTER TABLE relay_audit_events DROP COLUMN IF EXISTS mid_run`) + } + }) + + it('paces the copy on what the source reads from disk, dead pages included', async (context) => { + // Evicting the table from shared buffers needs pg_buffercache_evict (Postgres 17+). + const evictable = await withPgClient(sides.primary.adminUrl, async (admin) => { + await admin.query('CREATE EXTENSION IF NOT EXISTS pg_buffercache') + const found = await admin.query(`SELECT to_regproc('pg_buffercache_evict') IS NOT NULL AS ok`) + return Boolean(found.rows[0]?.ok) + }) + if (!evictable) context.skip() + await track() + // 20,000 rows written and 19,900 deleted: the live JSON is tiny, the pages are not. + await app.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) + SELECT 'b' || lpad(n::text, 6, '0'), n, 't', repeat('x', 300) FROM generate_series(1, 20000) n` + ) + await app.query(`DELETE FROM relay_audit_events WHERE at > 100`) + const evicted = await withPgClient(sides.primary.adminUrl, async (admin) => { + const result = await admin.query( + `SELECT count(*) FILTER (WHERE pg_buffercache_evict(bufferid))::int AS n FROM pg_buffercache + WHERE relfilenode IN (SELECT pg_relation_filenode(oid) FROM pg_class + WHERE relname LIKE 'relay_audit_events%')` + ) + return Number(result.rows[0]?.n) + }) + expect(evicted).toBeGreaterThan(100) + const budget = 2_000_000 + const startedAt = Date.now() + const result = await runCopy(sessions, tables, { maxBytesPerSecond: budget, batchRows: 50 }) + const elapsedMs = Date.now() - startedAt + console.error(JSON.stringify({ pacing: result, elapsedMs })) + expect(result.readBytes).toBeGreaterThan(result.bytes * 10) + expect(elapsedMs).toBeGreaterThanOrEqual((result.readBytes * 1000) / budget - 50) + }) + + it('caps the copy at the row rate', async () => { + await track() + await app.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) + SELECT 'r' || n, n, 't', '{}' FROM generate_series(1, 300) n` + ) + const startedAt = Date.now() + await runCopy(sessions, tables, { maxBytesPerSecond: 1e9, maxRowsPerSecond: 500 }) + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(550) + }) + + it('refuses a copy made before tracking was installed again, until it is redone', async () => { + await track() + await app.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) + SELECT 'u' || n, n, 't', '{}' FROM generate_series(1, 200) n` + ) + await runCopy(sessions, tables, { maxBytesPerSecond: 1e9 }) + await drain() + // Tracking off, relay writes, tracking on: nothing recorded those writes. + await uninstallTracking(app, tables) + await app.query(`UPDATE relay_audit_events SET at = -at WHERE at <= 50`) + await app.query(`DELETE FROM relay_audit_events WHERE at > 150`) + await track() + await expect(runCopy(sessions, tables, { maxBytesPerSecond: 1e9 })).rejects.toThrow( + 'copy_predates_tracking' + ) + await expect( + syncCommand( + { databases, google: null, pointerStore: async () => testPointerStore() }, + 'primary', + { stopAtBacklog: 0 } + ) + ).rejects.toThrow('copy_predates_tracking') + expect(await equal(['relay_audit_events'])).toBe(false) + const redone = await runCopy(sessions, tables, { restart: true, maxBytesPerSecond: 1e9 }) + expect(redone.rows).toBe(150) + await drain() + expect(await equal()).toBe(true) + }) + + it('starts a new arming when a table was left without triggers, voiding the copy', async () => { + await track() + await runCopy(sessions, tables, { maxBytesPerSecond: 1e9 }) + // An untrack that stopped after one table: that table's writes go unrecorded. + for (const suffix of ['ins', 'upd', 'del', 'trunc']) { + await app.query(`DROP TRIGGER relay_move_track_${suffix} ON relay_audit_events`) + } + await app.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) VALUES ('gap', 1, 't', '{}')` + ) + await track() + await expect(runCopy(sessions, tables, { maxBytesPerSecond: 1e9 })).rejects.toThrow( + 'copy_predates_tracking' + ) + await runCopy(sessions, tables, { restart: true, maxBytesPerSecond: 1e9 }) + expect(await equal(['relay_audit_events'])).toBe(true) + }) + + it('refuses to sync into a target with no finished copy under the current arming', async () => { + await track() + await expect( + syncCommand( + { databases, google: null, pointerStore: async () => testPointerStore() }, + 'primary', + { stopAtBacklog: 0 } + ) + ).rejects.toThrow('target_not_current') + }) + + it('keeps BIGINTs past 2^53 exact through copy and sync', async () => { + await track() + const big = '9223372036854775807' + const odd = '9007199254740993' + await app.query( + `INSERT INTO relay_rate_windows (scope_key, window_kind, window_started_at, count) VALUES ('s', 'k', $1, $2)`, + [odd, big] + ) + await runCopy(sessions, tables, { maxBytesPerSecond: 1e9 }) + await app.query( + `UPDATE relay_rate_windows SET count = count - 1 WHERE window_started_at = $1`, + [odd] + ) + await drain() + const row = await sessions.target.query( + `SELECT window_started_at::text AS started, count::text AS count FROM relay_rate_windows` + ) + expect(row.rows).toEqual([{ started: odd, count: '9223372036854775806' }]) + expect(await equal(['relay_rate_windows'])).toBe(true) + }) + + it('lands children after parents and deletes them first, one key per batch', async () => { + await track() + for (let n = 0; n < 6; n++) { + await app.query('BEGIN') + await app.query( + `INSERT INTO relay_cell_fence_attempts (attempt_id, environment, cell_id, cell_incarnation, mig_name, + instance_group, generation_identity, fence_commit, plan_sha256, created_at, expires_at) + VALUES ($1, 'e', 'c', 'i', 'm', 'g', 'gi', 'fc', 'sha', 1, 2)`, + [`a${n}`] + ) + await app.query( + `INSERT INTO relay_cell_fence_apply_invocations (invocation_id, attempt_id, request_reason, started_at) + VALUES ($1, $2, $3, 1)`, + [`i${n}`, `a${n}`, `reason-${n}`] + ) + await app.query('COMMIT') + } + await runSyncLoop(sessions, tables, { stopAtBacklog: 0, batchKeys: 1, idleMs: 1 }) + expect(await equal(['relay_cell_fence_attempts', 'relay_cell_fence_apply_invocations'])).toBe( + true + ) + // A unique value moved to a new key, and parents deleted with their children. + await app.query(`DELETE FROM relay_cell_fence_apply_invocations WHERE invocation_id = 'i0'`) + await app.query( + `INSERT INTO relay_cell_fence_apply_invocations (invocation_id, attempt_id, request_reason, started_at) + VALUES ('i0-new', 'a1', 'reason-0', 2)` + ) + await app.query( + `DELETE FROM relay_cell_fence_apply_invocations WHERE attempt_id IN ('a2', 'a3')` + ) + await app.query(`DELETE FROM relay_cell_fence_attempts WHERE attempt_id IN ('a2', 'a3')`) + await runSyncLoop(sessions, tables, { stopAtBacklog: 0, batchKeys: 1, idleMs: 1 }) + expect(await equal(['relay_cell_fence_attempts', 'relay_cell_fence_apply_invocations'])).toBe( + true + ) + }) + + it('settles unique values two keys swap within one batch', async () => { + await track() + const cell = (id: string, url: string): Promise => + app.query( + `INSERT INTO relay_cells (cell_id, cell_url, enabled, capacity_requests, reserved_requests, + observed_requests, last_heartbeat_at, updated_at) VALUES ($1, $2, 1, 1, 0, 0, 0, 0)`, + [id, url] + ) + await cell('a', 'https://one') + await cell('b', 'https://two') + await drain() + await app.query('BEGIN') + await app.query(`UPDATE relay_cells SET cell_url = 'https://swap' WHERE cell_id = 'a'`) + await app.query(`UPDATE relay_cells SET cell_url = 'https://one' WHERE cell_id = 'b'`) + await app.query(`UPDATE relay_cells SET cell_url = 'https://two' WHERE cell_id = 'a'`) + await app.query('COMMIT') + await drain() + expect(await equal(['relay_cells'])).toBe(true) + }) + + it('removes target rows the source deleted while untracked when the copy runs again', async () => { + await track() + await app.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) + SELECT 's' || lpad(n::text, 3, '0'), n, 't', '{}' FROM generate_series(1, 120) n` + ) + await runCopy(sessions, tables, { batchRows: 25, maxBytesPerSecond: 1e9 }) + await drain() + // Deleted as the tool role, so no key is tracked: what a lost dirty set leaves behind. + await sessions.source.query( + `DELETE FROM relay_audit_events WHERE id IN ('s001', 's050', 's051', 's120')` + ) + expect(await equal(['relay_audit_events'])).toBe(false) + await runCopy(sessions, tables, { batchRows: 25, restart: true, maxBytesPerSecond: 1e9 }) + expect(await equal(['relay_audit_events'])).toBe(true) + }) + + it('resumes a copy from the last committed page', async () => { + await track() + await app.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) + SELECT 'e' || lpad(n::text, 5, '0'), n, 't', '{}' FROM generate_series(1, 1200) n` + ) + // A stalled copy: the target session dies after a few pages. + const target = await connect(sides.dedicated.move.url) + const pid = (await target.query('SELECT pg_backend_pid() AS pid')).rows[0]?.pid + // Killed once the first pages of the table have committed. + const killer = setInterval(() => { + void withPgClient(sides.dedicated.adminUrl, async (admin) => { + const progress = await admin.query( + `SELECT rows_copied FROM relay_move_copy_progress WHERE table_name = 'relay_audit_events'` + ) + if (Number(progress.rows[0]?.rows_copied ?? 0) > 0) { + await admin.query('SELECT pg_terminate_backend($1)', [pid]) + } + }) + }, 20) + await expect( + runCopy({ source: sessions.source, target }, tables, { + batchRows: 50, + maxBytesPerSecond: 20_000 + }) + ).rejects.toThrow() + clearInterval(killer) + await target.end().catch(() => undefined) + const partial = await sessions.target.query( + `SELECT rows_copied::int AS rows, done FROM relay_move_copy_progress WHERE table_name = 'relay_audit_events'` + ) + const first = Number(partial.rows[0]?.rows ?? 0) + expect(first).toBeGreaterThan(0) + expect(first).toBeLessThan(1200) + const resumed = await runCopy(sessions, tables, { batchRows: 50, maxBytesPerSecond: 1e9 }) + expect(resumed.rows).toBe(1200 - first) + expect(await equal(['relay_audit_events'])).toBe(true) + }) + + it('names the keys of a range that differs', async () => { + await track() + await app.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) + SELECT 'v' || n, n, 't', '{}' FROM generate_series(1, 30) n` + ) + await runCopy(sessions, tables, { maxBytesPerSecond: 1e9 }) + await withPgClient(sides.dedicated.adminUrl, async (admin) => { + await admin.query(`UPDATE relay_audit_events SET at = -1 WHERE id = 'v17'`) + await admin.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) VALUES ('zz', 1, 't', '{}')` + ) + }) + const verdicts = await verifyMoveTables( + sessions.source, + sessions.target, + [table('relay_audit_events')], + { + chunkRows: 7 + } + ) + const keys = verdicts[0]?.mismatches.flatMap((mismatch) => mismatch.keys) + expect(keys).toEqual(['{"id": "v17"}', '{"id": "zz"}']) + }) + + // Writers hold their transaction open after the trigger marked a key, while the sync claims + // and reads as fast as it can. With DO UPDATE the claim must pass over a held key, so nothing + // committed after the read is lost; with DO NOTHING it is not, and the copies drift. + async function raceRound(): Promise<{ equal: boolean; operations: number }> { + const writers = await Promise.all( + Array.from({ length: 6 }, () => connect(sides.primary.app.url)) + ) + let stop = false + let operations = 0 + const write = async (client: pg.Client): Promise => { + while (!stop) { + const key = `k${Math.floor(Math.random() * 12)}` + const roll = Math.random() + await client.query('BEGIN') + if (roll < 0.15) { + await client.query(`DELETE FROM relay_audit_events WHERE id = $1`, [key]) + } else { + await client.query( + `INSERT INTO relay_audit_events (id, at, type, detail_json) VALUES ($1, $2, 't', '{}') + ON CONFLICT (id) DO UPDATE SET at = EXCLUDED.at`, + [key, Math.floor(Math.random() * 1e9)] + ) + } + await client.query(`SELECT pg_sleep($1)`, [Math.random() * 0.004]) + await client.query('COMMIT') + operations += 1 + } + } + const syncing = runSyncLoop(sessions, tables, { + shouldStop: () => stop, + idleMs: 0, + batchKeys: 4 + }) + const running = writers.map(write) + await new Promise((resolve) => setTimeout(resolve, 1_500)) + stop = true + await Promise.all(running) + await syncing + await drain() + await Promise.all(writers.map((client) => client.end())) + return { equal: await equal(['relay_audit_events']), operations } + } + + it('loses no change under concurrent writers and sync claims (DO UPDATE)', async () => { + await track('update') + let operations = 0 + for (let round = 0; round < 6; round++) { + const result = await raceRound() + operations += result.operations + expect(result.equal).toBe(true) + } + console.error(JSON.stringify({ race: 'do-update', rounds: 6, operations })) + }, 60_000) + + it('the same race loses changes when the trigger uses DO NOTHING', async () => { + await track('nothing') + let diverged = 0 + let rounds = 0 + while (rounds < 12 && diverged === 0) { + rounds += 1 + if (!(await raceRound()).equal) diverged += 1 + } + console.error(JSON.stringify({ race: 'do-nothing', rounds, diverged })) + expect(diverged).toBeGreaterThan(0) + }, 60_000) +}) diff --git a/cloud/apps/relay/src/database-move-sync.ts b/cloud/apps/relay/src/database-move-sync.ts new file mode 100644 index 000000000000..bb04d56e674c --- /dev/null +++ b/cloud/apps/relay/src/database-move-sync.ts @@ -0,0 +1,285 @@ +import { randomUUID } from 'node:crypto' +import type pg from 'pg' +import { moveTableShape, readMoveTables, type MoveTable } from './database-move-catalog.js' +import { logMoveEvent, MoveRefusal, sleep } from './database-move-events.js' +import { + addReferencedParents, + applyMoveChanges, + readSourceRows, + type MoveChanges, + type MoveKey +} from './database-move-row-apply.js' +import { + markSourceDirty, + MOVE_CLAIMED_TABLE, + MOVE_DIRTY_TABLE, + MOVE_COPY_PROGRESS_TABLE, + MOVE_SENTINEL_TABLE_NAME, + readDirtyBacklog, + readAuthorityTrackingArm, + readCurrentCopiedTables, + readStaleCopyProgress, + readTrackingArm, + readTrackingState, + sentinelPresent +} from './database-move-tracking.js' + +// A director boot can add a column mid-run; rows copied without it would silently drop it. +export const MOVE_SCHEMA_CHECK_EVERY_MS = 30_000 + +// One syncer per source: the claimed journal is replayed on start, which is only safe when no +// other syncer is mid-batch. The cutover takes the same lock. +const SYNC_LOCK_KEYS = [0x6f72_6361, 0x6d6f_7665] as const + +export type MoveSyncSessions = { + // Both as the tool role, so neither the claim nor the apply marks anything dirty. + source: pg.Client + target: pg.Client +} + +export type MoveBatchResult = { keys: number; upserted: number; deleted: number; displaced: number } + +export async function takeSyncLock(source: pg.Client): Promise { + const result = await source.query(`SELECT pg_try_advisory_lock($1, $2) AS taken`, [ + ...SYNC_LOCK_KEYS + ]) + if (!result.rows[0]?.taken) throw new MoveRefusal('another_sync_is_running') +} + +export async function requireSentinel(source: pg.Client): Promise { + if (!(await sentinelPresent(source))) { + throw new MoveRefusal('dirty_set_was_emptied_recopy_needed') + } +} + +// Every source table must still carry its triggers: a key changed on one without them is lost. +export async function requireTracking(source: pg.Client, tables: MoveTable[]): Promise { + const state = await readTrackingState(source, tables) + if (!state.sentinel) throw new MoveRefusal('dirty_set_was_emptied_recopy_needed') + if (state.untracked.length > 0) { + throw new MoveRefusal('tables_without_tracking', { tables: state.untracked }) + } +} + +// Tracking that was removed and installed again missed every change in between, so a copy +// made under the earlier arming no longer counts. Applying changes (sync, cutover) also needs the +// target to be level with the source as a whole: a finished copy under this arming, or this +// arming unchanged since the source itself became the target (a move back after a cutover). +export async function requireCurrentCopy( + sessions: MoveSyncSessions, + tables: MoveTable[], + purpose: 'copy' | 'apply' +): Promise { + const arm = await readTrackingArm(sessions.source) + const stale = await readStaleCopyProgress(sessions.target, arm) + if (stale.length > 0) { + throw new MoveRefusal('copy_predates_tracking', { tables: stale, run: 'copy --restart' }) + } + if (purpose === 'copy') return + const copied = await readCurrentCopiedTables(sessions.target, arm) + if (tables.every((table) => copied.has(table.name))) return + if (arm !== null && (await readAuthorityTrackingArm(sessions.source)) === arm) return + throw new MoveRefusal('target_not_current', { + uncopied: tables.filter((table) => !copied.has(table.name)).map((table) => table.name), + run: 'copy --restart from this source' + }) +} + +// Both sides still have the tables the run started with. A drifted table's copy position is +// reset, because rows already copied may lack a column. +export async function requireUnchangedMoveState( + sessions: MoveSyncSessions, + tables: MoveTable[] +): Promise { + const [source, target] = await Promise.all([ + readMoveTables(sessions.source), + readMoveTables(sessions.target) + ]) + const started = new Map(tables.map((table) => [table.name, moveTableShape(table)])) + const targetShapes = new Map(target.map((table) => [table.name, moveTableShape(table)])) + const drifted = source + .filter((table) => { + const shape = moveTableShape(table) + return started.get(table.name) !== shape || targetShapes.get(table.name) !== shape + }) + .map((table) => table.name) + if (drifted.length > 0) { + await sessions.target.query( + `DELETE FROM ${MOVE_COPY_PROGRESS_TABLE} WHERE table_name = ANY($1)`, + [drifted] + ) + throw new MoveRefusal('schema_changed_during_run', { + tables: drifted, + copyProgressReset: true, + run: 'prepare-target, then copy and sync again' + }) + } + await requireTracking(sessions.source, tables) +} + +// Reads every claimed key's current source row in one snapshot, then applies the lot in one +// target transaction. The claim is dropped only after the target commits. +async function applyKeys( + sessions: MoveSyncSessions, + tables: MoveTable[], + keys: MoveKey[], + claimId: string +): Promise { + const changes: MoveChanges = new Map() + const known = new Map(tables.map((table) => [table.name, table])) + await sessions.source.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY') + try { + for (const table of tables) { + const pks = keys.filter((key) => key.table === table.name).map((key) => key.pk) + if (pks.length > 0) await readSourceRows(sessions.source, table, pks, changes) + } + await addReferencedParents(sessions.source, tables, changes) + await sessions.source.query('COMMIT') + } catch (error) { + await sessions.source.query('ROLLBACK').catch(() => undefined) + throw error + } + const unknown = keys.filter((key) => !known.has(key.table)).length + if (unknown > 0) logMoveEvent('sync_unknown_table_keys', { keys: unknown }) + await sessions.target.query('BEGIN') + let applied: Omit + try { + applied = await applyMoveChanges(sessions.target, tables, changes, (displaced) => + markSourceDirty(sessions.source, displaced) + ) + await sessions.target.query('COMMIT') + } catch (error) { + await sessions.target.query('ROLLBACK').catch(() => undefined) + throw error + } + await sessions.source.query(`DELETE FROM ${MOVE_CLAIMED_TABLE} WHERE claim_id = $1`, [claimId]) + return { keys: keys.length, ...applied } +} + +function claimedKeys(rows: Array>): MoveKey[] { + return rows.map((row) => ({ table: String(row.table_name), pk: String(row.pk) })) +} + +// Claims run as their own committed statement, so the claim's row locks are held for one +// statement only: writers marking the same key wait at most that long. +export async function syncBatch( + sessions: MoveSyncSessions, + tables: MoveTable[], + batchKeys: number +): Promise { + const claimId = randomUUID() + const claimed = await sessions.source.query( + `WITH taken AS ( + DELETE FROM ${MOVE_DIRTY_TABLE} + WHERE (table_name, pk) IN ( + SELECT table_name, pk FROM ${MOVE_DIRTY_TABLE} WHERE table_name <> $2 + LIMIT $3 FOR UPDATE SKIP LOCKED) + RETURNING table_name, pk) + INSERT INTO ${MOVE_CLAIMED_TABLE} (claim_id, table_name, pk) + SELECT $1, table_name, pk FROM taken + RETURNING table_name, pk::text AS pk`, + [claimId, MOVE_SENTINEL_TABLE_NAME, batchKeys] + ) + if (claimed.rows.length === 0) return { keys: 0, upserted: 0, deleted: 0, displaced: 0 } + return await applyKeys(sessions, tables, claimedKeys(claimed.rows), claimId) +} + +// Claims left by a syncer that died between its claim and its apply. +export async function replayClaims( + sessions: MoveSyncSessions, + tables: MoveTable[] +): Promise { + let replayed = 0 + for (;;) { + const next = await sessions.source.query(`SELECT claim_id FROM ${MOVE_CLAIMED_TABLE} LIMIT 1`) + const claimId = next.rows[0]?.claim_id + if (claimId === undefined) return replayed + const keys = await sessions.source.query( + `SELECT table_name, pk::text AS pk FROM ${MOVE_CLAIMED_TABLE} WHERE claim_id = $1`, + [claimId] + ) + replayed += (await applyKeys(sessions, tables, claimedKeys(keys.rows), String(claimId))).keys + } +} + +export type MoveSyncOptions = { + batchKeys?: number + idleMs?: number + reportEveryMs?: number + // Stop once the backlog is at most this (0 = empty, nothing held); otherwise run until stopped. + stopAtBacklog?: number + // The cutover drains with a hard deadline (epoch ms). + deadline?: number + shouldStop?: () => boolean + label?: Record + schemaCheckEveryMs?: number +} + +// Keys a writer holds are skipped (SKIP LOCKED) and stay in the backlog, so a drain to 0 also +// waits out every open writer. +export async function runSyncLoop( + sessions: MoveSyncSessions, + tables: MoveTable[], + options: MoveSyncOptions = {} +): Promise<{ keys: number; batches: number }> { + const batchKeys = options.batchKeys ?? 500 + const idleMs = options.idleMs ?? 200 + const reportEveryMs = options.reportEveryMs ?? 10_000 + let keys = 0 + let batches = 0 + let lastEmptyAt = Date.now() + let lastReportAt = 0 + let lastLoopMs = 0 + let windowKeys = 0 + let schemaCheckedAt = Date.now() + for (;;) { + if (options.shouldStop?.()) return { keys, batches } + if (options.deadline !== undefined && Date.now() > options.deadline) { + throw new MoveRefusal('sync_deadline_passed', { keys }) + } + if ( + Date.now() - schemaCheckedAt >= + (options.schemaCheckEveryMs ?? MOVE_SCHEMA_CHECK_EVERY_MS) + ) { + await requireUnchangedMoveState(sessions, tables) + schemaCheckedAt = Date.now() + } + await requireSentinel(sessions.source) + const startedAt = Date.now() + const result = await syncBatch(sessions, tables, batchKeys) + lastLoopMs = Date.now() - startedAt + keys += result.keys + windowKeys += result.keys + if (result.keys > 0) batches += 1 + const now = Date.now() + const report = now - lastReportAt >= reportEveryMs + // Between reports, count only far enough to decide, so a big backlog stays a cheap read. + const backlog = report + ? await readDirtyBacklog(sessions.source) + : result.keys === 0 || options.stopAtBacklog !== undefined + ? await readDirtyBacklog(sessions.source, (options.stopAtBacklog ?? 0) + 1) + : null + if (backlog === 0) lastEmptyAt = now + if (report) { + logMoveEvent('sync_progress', { + ...options.label, + backlog, + keysPerSecond: Math.round( + (windowKeys * 1000) / Math.max(1, now - (lastReportAt || startedAt)) + ), + loopMs: lastLoopMs, + behindMs: backlog === 0 ? 0 : now - lastEmptyAt + }) + lastReportAt = now + windowKeys = 0 + } + if ( + options.stopAtBacklog !== undefined && + backlog !== null && + backlog <= options.stopAtBacklog + ) { + return { keys, batches } + } + if (result.keys === 0) await sleep(backlog === 0 ? idleMs : Math.min(idleMs, 20)) + } +} diff --git a/cloud/apps/relay/src/database-move-tracking.ts b/cloud/apps/relay/src/database-move-tracking.ts new file mode 100644 index 000000000000..d844d940072e --- /dev/null +++ b/cloud/apps/relay/src/database-move-tracking.ts @@ -0,0 +1,415 @@ +import type pg from 'pg' +import { + pkJson, + quoteIdent, + quoteLiteral, + RELAY_DATABASE_AUTHORITY_TABLE, + tableRef, + type MoveTable +} from './database-move-catalog.js' +import { logMoveEvent, MoveRefusal, sleep } from './database-move-events.js' + +// Unlogged: no WAL, nothing in the shared instance's archive. A crash or failover empties it, +// sentinel included, and the sentinel's absence is how the sync knows keys were lost. +export const MOVE_DIRTY_TABLE = 'relay_move_dirty' +// Keys a sync has taken off the dirty set but not yet applied; replayed after a crash. +export const MOVE_CLAIMED_TABLE = 'relay_move_claimed' +export const MOVE_COPY_PROGRESS_TABLE = 'relay_move_copy_progress' +export const MOVE_SENTINEL_TABLE_NAME = 'relay_move_sentinel' + +const TRIGGER_PREFIX = 'relay_move_track_' +const TRUNCATE_FUNCTION = 'relay_move_truncated' +const TRIGGERS = ['ins', 'upd', 'del', 'trunc'] as const +const POSTGRES_NAME_MAX_BYTES = 63 +const LOCK_NOT_AVAILABLE = '55P03' + +// Missing table or row = 0, as the relay reads it. +export async function readAuthorityGeneration(client: pg.Client): Promise { + try { + const result = await client.query( + `SELECT generation::text AS generation FROM ${RELAY_DATABASE_AUTHORITY_TABLE}` + ) + return Number(result.rows[0]?.generation ?? 0) + } catch (error) { + if (typeof error === 'object' && error !== null && 'code' in error && error.code === '42P01') + return 0 + throw error + } +} + +export type TrackingConflictAction = 'update' | 'nothing' + +export const AUTHORITY_TABLE_SQL = `CREATE TABLE IF NOT EXISTS ${RELAY_DATABASE_AUTHORITY_TABLE} ( + singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton), + generation BIGINT NOT NULL, + written_at BIGINT NOT NULL +)` + +// The move tool's own column, which the relay never reads: this side's tracking arming when it +// became the target, so a move back can tell whether the copy kept in step since. +const AUTHORITY_MOVE_COLUMNS_SQL = [ + `ALTER TABLE ${RELAY_DATABASE_AUTHORITY_TABLE} ADD COLUMN IF NOT EXISTS tracking_arm TEXT` +] + +const MOVE_TABLES_SQL = [ + `CREATE UNLOGGED TABLE IF NOT EXISTS ${MOVE_CLAIMED_TABLE} ( + claim_id TEXT NOT NULL, table_name TEXT NOT NULL, pk JSONB NOT NULL, + PRIMARY KEY (claim_id, table_name, pk))`, + `CREATE TABLE IF NOT EXISTS ${MOVE_COPY_PROGRESS_TABLE} ( + table_name TEXT PRIMARY KEY, after_pk JSONB, done BOOLEAN NOT NULL DEFAULT FALSE, + rows_copied BIGINT NOT NULL DEFAULT 0, updated_at BIGINT NOT NULL)`, + // Which arming of the source's tracking the copy ran under; see readTrackingArm. + `ALTER TABLE ${MOVE_COPY_PROGRESS_TABLE} ADD COLUMN IF NOT EXISTS tracking_arm TEXT` +] + +function trackingFunctionName(table: MoveTable): string { + const name = `${TRIGGER_PREFIX}${table.name}` + if (Buffer.byteLength(name) > POSTGRES_NAME_MAX_BYTES) { + throw new MoveRefusal('table_name_too_long_for_tracking', { table: table.name }) + } + return name +} + +// DO UPDATE, never DO NOTHING: the row lock it takes on an existing dirty key makes the sync's +// SKIP LOCKED take pass that key until this writer commits, so a change committed after the +// sync read the row is never lost with the key. +export function trackingFunctionSql( + table: MoveTable, + toolUser: string, + conflictAction: TrackingConflictAction = 'update' +): string { + const mark = (source: string): string => ` + INSERT INTO ${MOVE_DIRTY_TABLE} (table_name, pk, touched) + SELECT ${quoteLiteral(table.name)}, ${pkJson(table, 'k')}, 1 FROM (${source}) k + ON CONFLICT (table_name, pk) DO ${ + conflictAction === 'update' + ? `UPDATE SET touched = ${MOVE_DIRTY_TABLE}.touched + 1` + : 'NOTHING' + };` + const columns = table.primaryKey.map((column) => quoteIdent(column)).join(', ') + return `CREATE OR REPLACE FUNCTION public.${quoteIdent(trackingFunctionName(table))}() +RETURNS trigger LANGUAGE plpgsql AS $relay_move$ +BEGIN + IF current_user = ${quoteLiteral(toolUser)} THEN RETURN NULL; END IF; + IF TG_OP = 'INSERT' THEN${mark(`SELECT ${columns} FROM relay_move_new`)} + ELSIF TG_OP = 'UPDATE' THEN${mark( + `SELECT ${columns} FROM relay_move_new UNION SELECT ${columns} FROM relay_move_old` + )} + ELSE${mark(`SELECT ${columns} FROM relay_move_old`)} + END IF; + RETURN NULL; +END $relay_move$` +} + +function triggerStatements(table: MoveTable): string[] { + const fn = `public.${quoteIdent(trackingFunctionName(table))}()` + const on = tableRef(table) + return [ + ...TRIGGERS.map((suffix) => `DROP TRIGGER IF EXISTS ${TRIGGER_PREFIX}${suffix} ON ${on}`), + `CREATE TRIGGER ${TRIGGER_PREFIX}ins AFTER INSERT ON ${on} + REFERENCING NEW TABLE AS relay_move_new FOR EACH STATEMENT EXECUTE FUNCTION ${fn}`, + `CREATE TRIGGER ${TRIGGER_PREFIX}upd AFTER UPDATE ON ${on} + REFERENCING OLD TABLE AS relay_move_old NEW TABLE AS relay_move_new + FOR EACH STATEMENT EXECUTE FUNCTION ${fn}`, + `CREATE TRIGGER ${TRIGGER_PREFIX}del AFTER DELETE ON ${on} + REFERENCING OLD TABLE AS relay_move_old FOR EACH STATEMENT EXECUTE FUNCTION ${fn}`, + // A truncate cannot be listed key by key, so it voids the dirty set as a failover would. + `CREATE TRIGGER ${TRIGGER_PREFIX}trunc AFTER TRUNCATE ON ${on} + FOR EACH STATEMENT EXECUTE FUNCTION public.${TRUNCATE_FUNCTION}()` + ] +} + +// Each table's DDL is one short transaction under a 200 ms lock wait, so a writer queued behind +// it waits at most that long; a timeout backs off and retries. +async function withShortLocks( + client: pg.Client, + label: string, + statements: string[], + attempts: number +): Promise { + for (let attempt = 1; ; attempt++) { + try { + await client.query('BEGIN') + await client.query(`SET LOCAL lock_timeout = '200ms'`) + for (const statement of statements) await client.query(statement) + await client.query('COMMIT') + return + } catch (error) { + await client.query('ROLLBACK').catch(() => undefined) + const code = + typeof error === 'object' && error !== null && 'code' in error ? error.code : null + if (code !== LOCK_NOT_AVAILABLE || attempt >= attempts) throw error + logMoveEvent('lock_retry', { object: label, attempt }) + await sleep(Math.min(250 * 2 ** (attempt - 1), 2_000) * (0.5 + Math.random() / 2)) + } + } +} + +// The tool role reads and writes relay rows and the move's own tables; it owns nothing. +export async function grantMoveRole( + client: pg.Client, + toolUser: string, + tables: MoveTable[], + attempts = 25 +): Promise { + const role = quoteIdent(toolUser) + const statements = [ + ...tables.map( + (table) => `GRANT SELECT, INSERT, UPDATE, DELETE ON ${tableRef(table)} TO ${role}` + ), + `GRANT SELECT, INSERT, UPDATE ON ${RELAY_DATABASE_AUTHORITY_TABLE} TO ${role}`, + `GRANT SELECT, INSERT, UPDATE, DELETE ON ${MOVE_CLAIMED_TABLE}, ${MOVE_COPY_PROGRESS_TABLE} TO ${role}` + ] + if (await dirtyTableExists(client)) { + // The trigger writes the dirty set as whichever role changed the row, so every role that + // may write relay rows needs it too, or tracking would fail that role's writes. + const writers = await client.query( + `SELECT DISTINCT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE a.grantee::regrole::text END AS role + FROM pg_class c CROSS JOIN LATERAL aclexplode(c.relacl) a + WHERE c.relnamespace = 'public'::regnamespace AND c.relname = ANY($1) + AND a.privilege_type IN ('INSERT', 'UPDATE', 'DELETE', 'TRUNCATE') AND a.grantee <> c.relowner`, + [tables.map((table) => table.name)] + ) + const grantees = new Set([role, ...writers.rows.map((row) => String(row.role))]) + if (grantees.size > 1) logMoveEvent('tracking_writer_roles', { roles: [...grantees] }) + statements.push( + `GRANT SELECT, INSERT, UPDATE, DELETE ON ${MOVE_DIRTY_TABLE} TO ${[...grantees].join(', ')}` + ) + } + for (const statement of statements) await withShortLocks(client, 'grant', [statement], attempts) +} + +// The move's own tables that exist on both sides whether or not tracking is on. +export async function createMoveBookkeeping(client: pg.Client, attempts = 25): Promise { + for (const statement of [ + AUTHORITY_TABLE_SQL, + ...AUTHORITY_MOVE_COLUMNS_SQL, + ...MOVE_TABLES_SQL + ]) { + await withShortLocks(client, 'bookkeeping', [statement], attempts) + } +} + +export async function installTracking( + client: pg.Client, + input: { + toolUser: string + tables: MoveTable[] + attempts?: number + conflictAction?: TrackingConflictAction + } +): Promise<{ sentinel: 'armed' | 'present' | 'missing' }> { + const attempts = input.attempts ?? 25 + await createMoveBookkeeping(client, attempts) + const existed = await dirtyTableExists(client) + // A table left without triggers (an interrupted untrack) missed changes: start a new arming, + // which voids every copy made under the old one. + const partial = existed && (await readTrackingState(client, input.tables)).untracked.length > 0 + if (partial && (await sentinelPresent(client))) { + await client.query(`DELETE FROM ${MOVE_DIRTY_TABLE} WHERE table_name = $1`, [ + MOVE_SENTINEL_TABLE_NAME + ]) + await client.query(sentinelInsertSql()) + logMoveEvent('tracking_rearmed', { reason: 'tables_without_triggers' }) + } + // The sentinel is armed only with a new table: re-arming an emptied one would hide lost keys. + await withShortLocks( + client, + MOVE_DIRTY_TABLE, + [ + `CREATE UNLOGGED TABLE IF NOT EXISTS ${MOVE_DIRTY_TABLE} ( + table_name TEXT NOT NULL, pk JSONB NOT NULL, touched BIGINT NOT NULL, + PRIMARY KEY (table_name, pk))`, + ...(existed ? [] : [sentinelInsertSql()]), + `CREATE OR REPLACE FUNCTION public.${TRUNCATE_FUNCTION}() RETURNS trigger LANGUAGE plpgsql AS + $relay_move$ BEGIN + DELETE FROM ${MOVE_DIRTY_TABLE} WHERE table_name = ${quoteLiteral(MOVE_SENTINEL_TABLE_NAME)}; + RETURN NULL; + END $relay_move$` + ], + attempts + ) + await grantMoveRole(client, input.toolUser, input.tables, attempts) + for (const table of input.tables) { + await withShortLocks( + client, + table.name, + [ + trackingFunctionSql(table, input.toolUser, input.conflictAction), + ...triggerStatements(table) + ], + attempts + ) + logMoveEvent('tracking_installed', { table: table.name }) + } + const sentinel = existed ? ((await sentinelPresent(client)) ? 'present' : 'missing') : 'armed' + return { sentinel } +} + +export async function uninstallTracking( + client: pg.Client, + tables: MoveTable[], + attempts = 25 +): Promise { + // First and committed on its own: an untrack that stops partway leaves no arming behind for a + // later install to keep. + if (await dirtyTableExists(client)) { + await client.query(`DELETE FROM ${MOVE_DIRTY_TABLE} WHERE table_name = $1`, [ + MOVE_SENTINEL_TABLE_NAME + ]) + } + for (const table of tables) { + await withShortLocks( + client, + table.name, + [ + ...TRIGGERS.map( + (suffix) => `DROP TRIGGER IF EXISTS ${TRIGGER_PREFIX}${suffix} ON ${tableRef(table)}` + ), + `DROP FUNCTION IF EXISTS public.${quoteIdent(trackingFunctionName(table))}()` + ], + attempts + ) + logMoveEvent('tracking_removed', { table: table.name }) + } + await withShortLocks( + client, + MOVE_DIRTY_TABLE, + [ + `DROP FUNCTION IF EXISTS public.${TRUNCATE_FUNCTION}()`, + `DROP TABLE IF EXISTS ${MOVE_DIRTY_TABLE}, ${MOVE_CLAIMED_TABLE}` + ], + attempts + ) +} + +// Each arming gets a fresh id in the sentinel's key. Rows copied under an earlier arming may +// have missed changes made while tracking was off, so a copy is current only under this id. +export function sentinelInsertSql(): string { + const sentinel = quoteLiteral(MOVE_SENTINEL_TABLE_NAME) + return `INSERT INTO ${MOVE_DIRTY_TABLE} (table_name, pk, touched) + SELECT ${sentinel}, jsonb_build_object('arm', gen_random_uuid()::text), 0 + WHERE NOT EXISTS (SELECT 1 FROM ${MOVE_DIRTY_TABLE} WHERE table_name = ${sentinel})` +} + +// The current arming's id; '' for a sentinel armed before ids existed, null with none. +export async function readTrackingArm(client: pg.Client): Promise { + if (!(await dirtyTableExists(client))) return null + const result = await client.query( + `SELECT coalesce(pk->>'arm', '') AS arm FROM ${MOVE_DIRTY_TABLE} WHERE table_name = $1 LIMIT 1`, + [MOVE_SENTINEL_TABLE_NAME] + ) + const arm = result.rows[0]?.arm + return arm === undefined ? null : String(arm) +} + +// The arming recorded when this side last became the target; null if it never was. +export async function readAuthorityTrackingArm(client: pg.Client): Promise { + const result = await client + .query(`SELECT tracking_arm FROM ${RELAY_DATABASE_AUTHORITY_TABLE}`) + .catch((error: unknown) => { + // 42P01/42703: never prepared, or prepared before the column existed. + const code = + typeof error === 'object' && error !== null && 'code' in error ? error.code : null + if (code === '42P01' || code === '42703') return null + throw error + }) + const arm = result?.rows[0]?.tracking_arm + return typeof arm === 'string' ? arm : null +} + +// Tables with a finished copy position under this arming. +export async function readCurrentCopiedTables( + target: pg.Client, + sourceArm: string | null +): Promise> { + const result = await target.query( + `SELECT table_name FROM ${MOVE_COPY_PROGRESS_TABLE} WHERE done AND tracking_arm = $1`, + [sourceArm] + ) + return new Set(result.rows.map((row) => String(row.table_name))) +} + +// Copy positions on the target written under another arming of the source's tracking. +export async function readStaleCopyProgress( + target: pg.Client, + sourceArm: string | null +): Promise { + const result = await target.query( + `SELECT table_name FROM ${MOVE_COPY_PROGRESS_TABLE} + WHERE tracking_arm IS DISTINCT FROM $1 ORDER BY table_name`, + [sourceArm] + ) + return result.rows.map((row) => String(row.table_name)) +} + +// False with no dirty table too: tracking was never installed, or was removed. +export async function sentinelPresent(client: pg.Client): Promise { + if (!(await dirtyTableExists(client))) return false + const result = await client.query( + `SELECT EXISTS (SELECT 1 FROM ${MOVE_DIRTY_TABLE} WHERE table_name = $1) AS present`, + [MOVE_SENTINEL_TABLE_NAME] + ) + return Boolean(result.rows[0]?.present) +} + +async function dirtyTableExists(client: pg.Client): Promise { + const result = await client.query(`SELECT to_regclass($1) IS NOT NULL AS present`, [ + MOVE_DIRTY_TABLE + ]) + return Boolean(result.rows[0]?.present) +} + +export type TrackingState = { + dirtyTable: boolean + sentinel: boolean + // Tables lacking any of the four triggers. + untracked: string[] +} + +export async function readTrackingState( + client: pg.Client, + tables: MoveTable[] +): Promise { + const dirtyTable = await dirtyTableExists(client) + const triggers = await client.query( + `SELECT c.relname AS name, count(*)::int AS triggers FROM pg_trigger t + JOIN pg_class c ON c.oid = t.tgrelid JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND t.tgname LIKE 'relay\\_move\\_track\\_%' AND NOT t.tgisinternal + AND t.tgenabled <> 'D' + GROUP BY c.relname` + ) + const counts = new Map(triggers.rows.map((row) => [String(row.name), Number(row.triggers)])) + return { + dirtyTable, + sentinel: await sentinelPresent(client), + untracked: tables + .filter((table) => (counts.get(table.name) ?? 0) < TRIGGERS.length) + .map((table) => table.name) + } +} + +// Keys waiting to be synced, sentinel excluded; capped so a huge backlog stays a cheap read. +export async function readDirtyBacklog(client: pg.Client, cap = 1_000_000): Promise { + const result = await client.query( + `SELECT count(*)::bigint AS n FROM ( + SELECT 1 FROM ${MOVE_DIRTY_TABLE} WHERE table_name <> $1 LIMIT $2) d`, + [MOVE_SENTINEL_TABLE_NAME, cap] + ) + return Number(result.rows[0]?.n ?? 0) +} + +// Puts keys back on the source's dirty set (as the tool role; the dirty table has no trigger). +export async function markSourceDirty( + source: pg.Client, + keys: Array<{ table: string; pk: string }> +): Promise { + if (keys.length === 0) return + const items = keys.map((key) => `{"table":${JSON.stringify(key.table)},"pk":${key.pk}}`) + await source.query( + `INSERT INTO ${MOVE_DIRTY_TABLE} (table_name, pk, touched) + SELECT k->>'table', k->'pk', 1 FROM jsonb_array_elements($1::jsonb) k + ON CONFLICT (table_name, pk) DO UPDATE SET touched = ${MOVE_DIRTY_TABLE}.touched + 1`, + [`[${items.join(',')}]`] + ) +} diff --git a/cloud/apps/relay/src/database-move-verify.ts b/cloud/apps/relay/src/database-move-verify.ts new file mode 100644 index 000000000000..1267c21055a3 --- /dev/null +++ b/cloud/apps/relay/src/database-move-verify.ts @@ -0,0 +1,156 @@ +import type pg from 'pg' +import { + hashedRowText, + pkJson, + pkRange, + quoteIdent, + tableRef, + type MoveTable +} from './database-move-catalog.js' +import { logMoveEvent, sleep } from './database-move-events.js' + +export type MoveRangeMismatch = { + // Exclusive lower and inclusive upper pk (JSON text); null = open end. + after: string | null + through: string | null + referenceRows: number + otherRows: number + keys: string[] +} + +export type MoveTableVerdict = { + table: string + referenceRows: number + otherRows: number + mismatches: MoveRangeMismatch[] +} + +export type MoveVerifyOptions = { + chunkRows?: number + // Rows read on both sides per second; 0 = unpaced. + maxRowsPerSecond?: number + maxReportedKeys?: number +} + +function rangeWhere( + table: MoveTable, + after: string | null, + through: string | null +): { sql: string; params: string[] } { + const range = pkRange(table, after, through) + return { + sql: range.clauses.length > 0 ? `WHERE ${range.clauses.join(' AND ')}` : '', + params: range.params + } +} + +function orderBy(table: MoveTable): string { + return table.primaryKey.map((column) => `mv_r.${quoteIdent(column)}`).join(', ') +} + +async function rangeHash( + client: pg.Client, + table: MoveTable, + after: string | null, + through: string | null +): Promise<{ rows: number; hash: string }> { + const where = rangeWhere(table, after, through) + const result = await client.query( + `SELECT count(*)::text AS n, + coalesce(md5(string_agg(md5(${hashedRowText(table, 'mv_r')}), '' ORDER BY ${orderBy(table)})), '') AS h + FROM ${tableRef(table)} mv_r ${where.sql}`, + where.params + ) + return { rows: Number(result.rows[0]?.n ?? 0), hash: String(result.rows[0]?.h ?? '') } +} + +async function rangeRowHashes( + client: pg.Client, + table: MoveTable, + after: string | null, + through: string | null +): Promise> { + const where = rangeWhere(table, after, through) + const result = await client.query( + `SELECT ${pkJson(table, 'mv_r')}::text AS pk, md5(${hashedRowText(table, 'mv_r')}) AS h + FROM ${tableRef(table)} mv_r ${where.sql}`, + where.params + ) + return new Map(result.rows.map((row) => [String(row.pk), String(row.h)])) +} + +// Row counts and md5 over primary-key ranges cut on the reference side. Every column is hashed by +// name in a fixed order. A range that differs is listed key by key. +export async function verifyMoveTables( + reference: pg.Client, + other: pg.Client, + tables: MoveTable[], + options: MoveVerifyOptions = {} +): Promise { + const chunkRows = options.chunkRows ?? 5_000 + const maxRowsPerSecond = options.maxRowsPerSecond ?? 0 + const maxReportedKeys = options.maxReportedKeys ?? 20 + const startedAt = Date.now() + let scanned = 0 + const verdicts: MoveTableVerdict[] = [] + for (const table of tables) { + const verdict: MoveTableVerdict = { + table: table.name, + referenceRows: 0, + otherRows: 0, + mismatches: [] + } + let after: string | null = null + for (;;) { + const where = rangeWhere(table, after, null) + const boundary = await reference.query( + `SELECT ${pkJson(table, 'mv_r')}::text AS pk FROM ${tableRef(table)} mv_r ${where.sql} + ORDER BY ${orderBy(table)} OFFSET ${chunkRows - 1} LIMIT 1`, + where.params + ) + // The last range is open, so rows only the other side has past the end are counted too. + const through: string | null = boundary.rows[0] ? String(boundary.rows[0].pk) : null + const [left, right] = await Promise.all([ + rangeHash(reference, table, after, through), + rangeHash(other, table, after, through) + ]) + verdict.referenceRows += left.rows + verdict.otherRows += right.rows + scanned += left.rows + right.rows + if (left.hash !== right.hash || left.rows !== right.rows) { + const [leftRows, rightRows] = await Promise.all([ + rangeRowHashes(reference, table, after, through), + rangeRowHashes(other, table, after, through) + ]) + const keys = new Set() + for (const [pk, hash] of leftRows) if (rightRows.get(pk) !== hash) keys.add(pk) + for (const pk of rightRows.keys()) if (!leftRows.has(pk)) keys.add(pk) + verdict.mismatches.push({ + after, + through, + referenceRows: left.rows, + otherRows: right.rows, + keys: [...keys].slice(0, maxReportedKeys) + }) + } + if (maxRowsPerSecond > 0) { + await sleep((scanned * 1000) / maxRowsPerSecond - (Date.now() - startedAt)) + } + if (through === null) break + after = through + } + logMoveEvent('verify_table', { + table: table.name, + referenceRows: verdict.referenceRows, + otherRows: verdict.otherRows, + mismatchedRanges: verdict.mismatches.length, + ...(verdict.mismatches.length > 0 ? { mismatches: verdict.mismatches } : {}) + }) + verdicts.push(verdict) + } + return verdicts +} + +export function verdictsEqual(verdicts: MoveTableVerdict[]): boolean { + return verdicts.every((verdict) => verdict.mismatches.length === 0) +} diff --git a/cloud/apps/relay/src/relay-dedicated-database-secret.ts b/cloud/apps/relay/src/relay-dedicated-database-secret.ts index ab9b781c8017..5a9b4dded2f4 100644 --- a/cloud/apps/relay/src/relay-dedicated-database-secret.ts +++ b/cloud/apps/relay/src/relay-dedicated-database-secret.ts @@ -21,6 +21,14 @@ export async function readDedicatedDatabaseCredentials( google: GoogleMetadataAccess, fetchImpl: typeof fetch = fetch ): Promise { + return parseDedicatedDatabaseUrl(await readSecretManagerText(secretVersion, google, fetchImpl)) +} + +export async function readSecretManagerText( + secretVersion: string, + google: GoogleMetadataAccess, + fetchImpl: typeof fetch = fetch +): Promise { const response = await fetchImpl(`${SECRET_MANAGER_ROOT}/${secretVersion}:access`, { headers: { authorization: `Bearer ${await google.accessToken()}` }, signal: AbortSignal.timeout(SECRET_READ_TIMEOUT_MS) @@ -46,7 +54,7 @@ export async function readDedicatedDatabaseCredentials( if (encoded === null || encoded.length > MAX_SECRET_BYTES * 2) { throw new DedicatedDatabaseSecretError('payload') } - return parseDedicatedDatabaseUrl(Buffer.from(encoded, 'base64').toString('utf8').trim()) + return Buffer.from(encoded, 'base64').toString('utf8').trim() } export function parseDedicatedDatabaseUrl(databaseUrl: string): DedicatedDatabaseCredentials { diff --git a/cloud/apps/relay/src/test-fixtures/database-move-sides.ts b/cloud/apps/relay/src/test-fixtures/database-move-sides.ts new file mode 100644 index 000000000000..0cb6d2afdde4 --- /dev/null +++ b/cloud/apps/relay/src/test-fixtures/database-move-sides.ts @@ -0,0 +1,151 @@ +import { randomBytes } from 'node:crypto' +import pg from 'pg' +import { applyRelayPostgresSchema, openRelayPostgresPool } from '../database.js' +import { MoveDatabases, type MoveSide } from '../database-move-connections.js' +import { + PointerGenerationConflict, + type DatabaseTargetPointerStore, + type StoredDatabaseTargetPointer +} from '../database-move-pointer-store.js' +import type { DatabaseTargetPointer } from '../relay-database-target-pointer.js' + +// Two databases on one test server stand in for the shared and the dedicated instance. Each side +// has its own app and tool role, as the two Cloud SQL instances would; roles are cluster-wide +// here, so distinct names keep a fence on one side from touching the other. +export type MoveTestSide = { + database: string + app: { user: string; password: string; url: string } + move: { user: string; password: string; url: string } + // Superuser on this side's database: reads what the relay role can no longer log in to see. + adminUrl: string +} + +export type MoveTestSides = Record & { + databases: () => MoveDatabases + drop: () => Promise +} + +function urlFor(adminUrl: string, user: string, password: string, database: string): string { + const url = new URL(adminUrl) + url.username = user + url.password = password + url.pathname = `/${database}` + return url.toString() +} + +export async function withPgClient( + url: string, + run: (client: pg.Client) => Promise +): Promise { + const client = new pg.Client({ connectionString: url }) + client.on('error', () => undefined) + await client.connect() + try { + return await run(client) + } finally { + await client.end().catch(() => undefined) + } +} + +export async function createMoveTestSides( + adminUrl: string, + options: { schemaOn?: MoveSide[] } = {} +): Promise { + const suffix = randomBytes(4).toString('hex') + const make = (side: MoveSide): MoveTestSide => { + const tag = side === 'primary' ? 'p' : 'd' + const database = `mv_${tag}_${suffix}` + const app = { user: `mv_${tag}_app_${suffix}`, password: randomBytes(12).toString('hex') } + const move = { user: `mv_${tag}_tool_${suffix}`, password: randomBytes(12).toString('hex') } + return { + database, + app: { ...app, url: urlFor(adminUrl, app.user, app.password, database) }, + move: { ...move, url: urlFor(adminUrl, move.user, move.password, database) }, + adminUrl: (() => { + const url = new URL(adminUrl) + url.pathname = `/${database}` + return url.toString() + })() + } + } + const sides = { primary: make('primary'), dedicated: make('dedicated') } + await withPgClient(adminUrl, async (admin) => { + for (const side of Object.values(sides)) { + // No CREATEROLE: the fence must work for a role that can only change itself. + await admin.query( + `CREATE ROLE ${side.app.user} LOGIN NOCREATEROLE PASSWORD '${side.app.password}'` + ) + await admin.query( + `CREATE ROLE ${side.move.user} LOGIN NOCREATEROLE PASSWORD '${side.move.password}'` + ) + await admin.query(`CREATE DATABASE ${side.database} OWNER ${side.app.user}`) + } + }) + for (const name of options.schemaOn ?? ['primary']) { + const connection = { connectionString: sides[name].app.url } + const { database } = openRelayPostgresPool(connection, { poolMax: 1 }) + try { + await applyRelayPostgresSchema(connection, undefined, database) + } finally { + await database.close() + } + } + return { + ...sides, + databases: () => + new MoveDatabases( + { + primary: { app: { url: sides.primary.app.url }, move: { url: sides.primary.move.url } }, + dedicated: { + app: { url: sides.dedicated.app.url }, + move: { url: sides.dedicated.move.url } + } + }, + null + ), + drop: async () => { + await withPgClient(adminUrl, async (admin) => { + for (const side of Object.values(sides)) { + await admin.query(`DROP DATABASE IF EXISTS ${side.database} WITH (FORCE)`) + await admin.query(`DROP ROLE IF EXISTS ${side.app.user}`) + await admin.query(`DROP ROLE IF EXISTS ${side.move.user}`) + } + }) + } + } +} + +// The pointer object in memory, with the same generation preconditions as Cloud Storage. +export class MemoryPointerStore implements DatabaseTargetPointerStore { + private generation = 1 + constructor(private stored: StoredDatabaseTargetPointer | null) {} + current(): DatabaseTargetPointer | null { + return this.stored ? structuredClone(this.stored.pointer) : null + } + async read(): Promise { + return this.stored ? structuredClone(this.stored) : null + } + async write(pointer: DatabaseTargetPointer, ifGenerationMatch: string): Promise { + if (this.stored?.generation !== ifGenerationMatch) throw new PointerGenerationConflict() + this.generation += 1 + this.stored = { pointer: structuredClone(pointer), generation: String(this.generation) } + return this.stored.generation + } + async delete(ifGenerationMatch: string): Promise { + if (this.stored?.generation !== ifGenerationMatch) throw new PointerGenerationConflict() + this.stored = null + } +} + +export function testPointerStore(): MemoryPointerStore { + return new MemoryPointerStore({ + generation: '1', + pointer: { + version: 1, + dedicated: { + instanceConnectionName: 'test-project:us-central1:relay-db', + databaseUrlSecret: 'projects/test-project/secrets/relay-db-url/versions/latest' + } + } + }) +}