Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions cloud/dev/contracts/production-cloud-sql-app-consumers.json
Original file line number Diff line number Diff line change
@@ -1,15 +1,19 @@
{
"comment": "Production Cloud SQL consumers owned by the private orca-cloud application tree (auth and API services). The relay ships without them, so the values the connection budget needs are published here; the private repository binds every field back to its source in its own CI.",
"authInstances": 2,
"comment": "Production Cloud SQL consumers owned by the private orca-cloud application tree (auth and API services). The relay ships without them, so the values the connection budget needs are published here; the private repository binds every field back to its source in its own CI. The mobile push gateway is deliberately absent: it runs on its own orca-cloud-push-db instance and consumes none of this ceiling.",
"authInstances": 20,
"authPoolMax": 10,
"authCandidateMinInstances": 1,
"apiInstances": 10,
"apiPoolMax": 5,
"apiCandidateMinInstances": 1,
"maxConnections": 500,
"sources": {
"authInstances": "private apps tfvars: auth service max instances",
"authPoolMax": "private auth service: pg.Pool max",
"apiInstances": "private apps tfvars: API service max instances",
"authInstances": "private apps tfvars: auth_max_instances. Raised 2 -> 20 in orca-cloud PR #474 on 2026-09-04 after a cap of 2 starved desktop token refresh for hours. Production has used it: a 14-day Monitoring pull shows instance maxima of 21 (09-04), 16 (09-13) and 18 (09-17)",
"authPoolMax": "private auth service: pg.Pool max on the request pool. Per-database num_backends on those three days was 205, 165 and 114, i.e. 9.2-9.8 connections per instance, so the pool does fill",
"authCandidateMinInstances": "the auth deploy script builds its candidate with --tag and --no-traffic and never passes --min, so the candidate inherits auth_min_instances = 1 until the atomic --to-tags=<tag>=100 flip; observed 2026-09-17, four revisions with live instances totalled 165 backends",
"apiInstances": "private apps tfvars: max_instances for the artifact API service",
"apiPoolMax": "private API service: pg.Pool max",
"apiCandidateMinInstances": "the API deploy script also uses --tag and --no-traffic with no --min, and the service min_instances is 0; the candidate is driven above zero only by the serial smoke run against its tag origin before promotion, so one instance is the honest pre-flip figure",
"maxConnections": "measured SHOW max_connections = 500 on the live instance 2026-09-16; no max_connections flag is set, so this is the tier default and the previous 400 was an unverified assumption about it"
}
}
17 changes: 14 additions & 3 deletions cloud/dev/scripts/production-cloud-sql-rollout-lock.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ test('census: no workflow rolls out against the shared instance outside the leas
// The API and auth deploy scripts share this contract but stay in the private repository.
const serviceCapScripts = ['dev/scripts/deploy-relay-blue-green.mjs']

test('budgets tagged Cloud Run candidates outside the service-wide instance cap', () => {
test('budgets tagged Cloud Run candidates at their pre-flip instance floor', () => {
for (const file of serviceCapScripts) {
const script = readFileSync(new URL(`../../${file}`, import.meta.url), 'utf8')
assert.match(script, /'--no-traffic'/, file)
Expand All @@ -196,15 +196,26 @@ test('budgets tagged Cloud Run candidates outside the service-wide instance cap'
new URL('../../dev/scripts/relay-cloud-sql-connection-budget.mjs', import.meta.url),
'utf8'
)
assert.match(budget, /directly addressable tagged revisions outside service-level caps/)
assert.match(budget, /the candidate revision at its pre-flip instance floor/)
// Auth and API tag a --no-traffic candidate without --min, so each costs its service floor.
assert.match(
budget,
/apiCandidate: retainedDirectorRollback \+ inputs\.apiInstances \* inputs\.apiPoolMax/
/apiCandidate: retainedDirectorRollback \+ inputs\.apiCandidateMinInstances \* inputs\.apiPoolMax/
)
assert.match(
budget,
/authCandidate: retainedDirectorRollback \+ inputs\.authCandidateMinInstances \* inputs\.authPoolMax/
)
// The director candidate is the exception: it inherits the Terraform floor and dual-serves.
assert.match(budget, /relayDirectorCandidate: retainedDirectorRollback \* 2/)
const director = readWorkflow(relayWorkflowFile('deploy-relay-production-director.yml'))
const capacity = readWorkflow(relayWorkflowFile('deploy-relay-production-capacity-job.yml'))
const asia = readWorkflow(relayWorkflowFile('operate-relay-asia-admission.yml'))
assert.match(director, /--max-instances "\$\{DIRECTOR_MAX_INSTANCES\}"/)
assert.match(capacity, /--max-instances 5/)
assert.match(asia, /--max-instances "\$\{DIRECTOR_MAX_INSTANCES\}"/)
// No --min-instances on the blue/green call is what makes the director candidate inherit 5.
const blueGreenCall = director.match(/node dev\/scripts\/deploy-relay-blue-green\.mjs[\s\S]*?\n\n/)
assert.ok(blueGreenCall)
assert.doesNotMatch(blueGreenCall[0], /--min-instances/)
})
22 changes: 18 additions & 4 deletions cloud/dev/scripts/relay-cloud-sql-connection-budget.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,15 @@ const APP_CONSUMERS_CONTRACT = new URL(
import.meta.url
)

const APP_CONSUMER_FIELDS = ['authInstances', 'authPoolMax', 'apiInstances', 'apiPoolMax', 'maxConnections']
const APP_CONSUMER_FIELDS = [
'authInstances',
'authPoolMax',
'authCandidateMinInstances',
'apiInstances',
'apiPoolMax',
'apiCandidateMinInstances',
'maxConnections'
]

export function readProductionCloudSqlAppConsumers(contract) {
const parsed = contract ?? JSON.parse(readFileSync(APP_CONSUMERS_CONTRACT, 'utf8'))
Expand Down Expand Up @@ -60,10 +68,14 @@ export function calculateRelayCloudSqlConnectionBudget(inputs) {
}
const configuredMaximum = Object.values(consumers).reduce((total, value) => total + value, 0)
const retainedDirectorRollback = inputs.directorInstances * inputs.directorPoolMax
// A candidate revision costs what it actually runs before the traffic flip, not its ceiling.
// Auth and API tag a --no-traffic candidate and never pass --min, so each holds its service
// floor. The director candidate inherits the Terraform-owned floor of directorInstances and
// genuinely dual-serves, so it keeps the full term.
const candidateOverlap = {
relayDirectorCandidate: retainedDirectorRollback * 2,
apiCandidate: retainedDirectorRollback + inputs.apiInstances * inputs.apiPoolMax,
authCandidate: retainedDirectorRollback + inputs.authInstances * inputs.authPoolMax,
apiCandidate: retainedDirectorRollback + inputs.apiCandidateMinInstances * inputs.apiPoolMax,
authCandidate: retainedDirectorRollback + inputs.authCandidateMinInstances * inputs.authPoolMax,
relayCells: retainedDirectorRollback
}
const rolloutOverlap = Math.max(...Object.values(candidateOverlap))
Expand All @@ -79,7 +91,7 @@ export function calculateRelayCloudSqlConnectionBudget(inputs) {
...candidateOverlap,
retainedDirectorRollback,
maximum: rolloutOverlap,
reason: 'serialized rollouts include directly addressable tagged revisions outside service-level caps'
reason: 'a serialized rollout adds the retained director rollback plus the candidate revision at its pre-flip instance floor'
},
maintenanceAdminAllowance: inputs.maintenanceAdminAllowance,
maintenanceAdminAllowanceReason: 'covers bounded work outside configured services',
Expand Down Expand Up @@ -133,8 +145,10 @@ export function readRelayCloudSqlConnectionBudget({
),
authInstances: apps.authInstances,
authPoolMax: apps.authPoolMax,
authCandidateMinInstances: apps.authCandidateMinInstances,
apiInstances: apps.apiInstances,
apiPoolMax: apps.apiPoolMax,
apiCandidateMinInstances: apps.apiCandidateMinInstances,
maxConnections: maxConnections ?? apps.maxConnections,
maintenanceAdminAllowance,
explicitReserve
Expand Down
73 changes: 60 additions & 13 deletions cloud/dev/scripts/relay-cloud-sql-connection-budget.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,27 +6,54 @@ import {
readRelayCloudSqlConnectionBudget
} from './relay-cloud-sql-connection-budget.mjs'

test('production shared consumers keep allowance and reserve below the ceiling', () => {
const arithmetic = (report) =>
[
`cells ${report.consumers.cells}`,
`directors ${report.consumers.directors}`,
`auth ${report.consumers.auth}`,
`api ${report.consumers.api}`,
`= ${report.configuredMaximum} configured`,
`+ ${report.rolloutOverlap.maximum} rollout overlap`,
`+ ${report.maintenanceAdminAllowance} admin allowance`,
`= ${report.operatingMaximum} operating`,
`against ${report.maxConnections} max_connections less a ${report.explicitReserve} reserve`
].join(', ')

test('the production budget reads the committed pools and the measured ceiling', () => {
// cells: 20 pools at 10 (200) + the three asia-east2 pools at 16 (48).
const report = readRelayCloudSqlConnectionBudget()

assert.deepEqual(report.consumers, { cells: 248, directors: 15, auth: 20, api: 50 })
assert.equal(report.maxConnections, 500)
assert.deepEqual(report.consumers, { cells: 248, directors: 15, auth: 200, api: 50 })
assert.deepEqual(report.asia, { cells: 3, poolMax: 16 })
assert.equal(report.configuredMaximum, 333)
assert.equal(report.configuredMaximum, 513)
// The director candidate inherits a floor of 5 and dual-serves; auth and API candidates hold
// one instance each until their traffic flip, so the director roll is now the widest overlap.
assert.equal(report.rolloutOverlap.relayDirectorCandidate, 30)
assert.equal(report.rolloutOverlap.apiCandidate, 65)
assert.equal(report.rolloutOverlap.authCandidate, 35)
assert.equal(report.rolloutOverlap.apiCandidate, 20)
assert.equal(report.rolloutOverlap.authCandidate, 25)
assert.equal(report.rolloutOverlap.relayCells, 15)
assert.equal(report.rolloutOverlap.retainedDirectorRollback, 15)
assert.equal(report.rolloutOverlap.maximum, 65)
assert.equal(report.rolloutOverlap.maximum, 30)
assert.equal(report.maintenanceAdminAllowance, 5)
assert.equal(report.explicitReserve, 10)
assert.equal(report.usableCeiling, 490)
assert.equal(report.operatingMaximum, 403)
assert.equal(report.remainingWithinUsableCeiling, 87)
assert.equal(report.budgetedTotal, 413)
assert.equal(report.unallocated, 87)
assert.equal(report.withinBudget, true)
assert.equal(report.operatingMaximum, 548)
})

test('configured pools fit under the ceiling less the stated reserve', () => {
const report = readRelayCloudSqlConnectionBudget()

assert.ok(
report.configuredMaximum <= report.usableCeiling,
`configured pools exceed the usable ceiling: ${arithmetic(report)}`
)
})

test('a serialized rollout still fits under the ceiling less the stated reserve', () => {
const report = readRelayCloudSqlConnectionBudget()

assert.ok(report.withinBudget, `the operating maximum exceeds the usable ceiling: ${arithmetic(report)}`)
})

test('fails closed when pool growth consumes the explicit reserve', () => {
Expand All @@ -38,8 +65,10 @@ test('fails closed when pool growth consumes the explicit reserve', () => {
directorPoolMax: 3,
authInstances: 2,
authPoolMax: 10,
authCandidateMinInstances: 2,
apiInstances: 20,
apiPoolMax: 5,
apiCandidateMinInstances: 20,
maxConnections: 400,
maintenanceAdminAllowance: 5,
explicitReserve: 10
Expand All @@ -52,7 +81,15 @@ test('fails closed when pool growth consumes the explicit reserve', () => {
test('excludes fenced cell pools and reads per-cell pool overrides', () => {
const report = readRelayCloudSqlConnectionBudget({
proposedAsiaCellCount: 1,
appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 },
appConsumers: {
authInstances: 1,
authPoolMax: 10,
authCandidateMinInstances: 1,
apiInstances: 1,
apiPoolMax: 5,
apiCandidateMinInstances: 1,
maxConnections: 100
},
sources: {
productionTfvars: `
relay_max_instances = 1
Expand Down Expand Up @@ -84,7 +121,15 @@ test('excludes fenced cell pools and reads per-cell pool overrides', () => {
test('dedicated push scaling does not consume shared capacity', () => {
const report = readRelayCloudSqlConnectionBudget({
proposedAsiaCellCount: 1,
appConsumers: { authInstances: 1, authPoolMax: 10, apiInstances: 1, apiPoolMax: 5, maxConnections: 100 },
appConsumers: {
authInstances: 1,
authPoolMax: 10,
authCandidateMinInstances: 1,
apiInstances: 1,
apiPoolMax: 5,
apiCandidateMinInstances: 1,
maxConnections: 100
},
sources: {
productionTfvars: `
relay_max_instances = 1
Expand Down Expand Up @@ -121,8 +166,10 @@ test('requires strict headroom below the physical ceiling', () => {
directorPoolMax: 3,
authInstances: 1,
authPoolMax: 10,
authCandidateMinInstances: 1,
apiInstances: 1,
apiPoolMax: 5,
apiCandidateMinInstances: 1,
maxConnections: 50,
maintenanceAdminAllowance: 9,
explicitReserve: 3
Expand Down
Loading