Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ jobs:
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
# cell_1..cell_10 in the calling wave; the chain is static, so this range is too.
[[ "${WAVE_INDEX}" =~ ^[0-9]$ ]]
# The caller validated this too; re-check here so a malformed override
# can never reach a mutation through this reusable workflow.
if test -n "${GATE_OVERRIDE_REASON}${GATE_OVERRIDE_CONFIRMATION}"; then
Expand Down
146 changes: 145 additions & 1 deletion .github/workflows/cloud-deploy-relay-production-same-cap.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ on:
type: choice
options: [verify, canary-apply, batch-apply, rollback]
cell-ids:
description: Ordered comma-separated serving cells; one canary or two to four batch cells
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
required: true
type: string
target-image-digest:
Expand Down Expand Up @@ -313,6 +313,144 @@ jobs:
wave-index: '3'
secrets: inherit

cell_5:
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
needs: [gate, cell_4]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '4'
secrets: inherit

cell_6:
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
needs: [gate, cell_5]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '5'
secrets: inherit

cell_7:
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
needs: [gate, cell_6]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '6'
secrets: inherit

cell_8:
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
needs: [gate, cell_7]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '7'
secrets: inherit

cell_9:
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
needs: [gate, cell_8]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '8'
secrets: inherit

cell_10:
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
needs: [gate, cell_9]
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
with:
mode: ${{ needs.gate.outputs.job-mode }}
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
target-image-digest: ${{ inputs.target-image-digest }}
rollback-image-digest: ${{ inputs.rollback-image-digest }}
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
expected-selector-generation: ${{ inputs.expected-selector-generation }}
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
expected-general-cells: ${{ inputs.expected-general-cells }}
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
monitor-run-id: ${{ inputs.monitor-run-id }}
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
gate-override-reason: ${{ inputs.gate-override-reason }}
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
wave-index: '9'
secrets: inherit

seal_canary:
if: ${{ inputs.mode == 'canary-apply' }}
needs: [gate, cell_1]
Expand Down Expand Up @@ -360,6 +498,12 @@ jobs:
- cell_2
- cell_3
- cell_4
- cell_5
- cell_6
- cell_7
- cell_8
- cell_9
- cell_10
- seal_canary
runs-on: blacksmith-2vcpu-ubuntu-2204
timeout-minutes: 10
Expand Down
11 changes: 9 additions & 2 deletions cloud/apps/relay-ops/src/incident-live-preflight-cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -242,9 +242,16 @@ describe('relay incident live preflight', () => {
await expect(runIncidentLivePreflight(
['--state-file', agedState(235 * 60_000 + 1), '--wave-index', '3'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
// The wave index is a strict single-use 0-3 argument.
// The last cell of a ten-cell same-cap batch: 10min + 9 * 75min exactly.
await expect(runIncidentLivePreflight(
['--state-file', stateFile(), '--wave-index', '4'], deps
['--state-file', agedState(685 * 60_000), '--wave-index', '9'], deps
)).resolves.toBeUndefined()
await expect(runIncidentLivePreflight(
['--state-file', agedState(685 * 60_000 + 1), '--wave-index', '9'], deps
)).rejects.toThrow('monitor evidence is incomplete or stale')
// The wave index is a strict single-use 0-9 argument.
await expect(runIncidentLivePreflight(
['--state-file', stateFile(), '--wave-index', '10'], deps
)).rejects.toThrow('usage:')
await expect(runIncidentLivePreflight(
['--state-file', stateFile(), '--wave-index', ''], deps
Expand Down
4 changes: 3 additions & 1 deletion cloud/apps/relay-ops/src/incident-live-preflight-cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ const FRESHNESS_RETRY_INTERVAL_MS = 15_000
const MONITOR_EVIDENCE_MAX_AGE_MS = 10 * 60_000
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
const WAVE_INDEX_PATTERN = /^[0-3]$/
// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow
// pins its own narrower range.
const WAVE_INDEX_PATTERN = /^[0-9]$/
// 2 for a general cell's isolate-and-restore wave, 0 for a migration-only cell's no-op pair.
const SELECTOR_WAVE_DELTA_PATTERN = /^[02]$/

Expand Down
4 changes: 3 additions & 1 deletion cloud/dev/scripts/relay-monitor-evidence.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ const JWT = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/
const EVIDENCE_MAX_AGE_MS = 5 * 60_000
// Matches the same-cap cell job timeout-minutes; bounds each predecessor wave.
const WAVE_PREDECESSOR_TIMEOUT_MS = 75 * 60_000
const WAVE_INDEX = /^[0-3]$/
// Widest any wave chain declares (same-cap's cell_1..cell_10); each job workflow
// pins its own narrower range.
const WAVE_INDEX = /^[0-9]$/
const EVIDENCE_SAMPLE_INTERVAL_MS = 60_000
const EVIDENCE_MAX_LINEAGE_MS = 25 * 60_000
const MIGRATION_POLICIES = new Set([
Expand Down
7 changes: 6 additions & 1 deletion cloud/dev/scripts/relay-monitor-evidence.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ test('later same-cap waves accept evidence aged by predecessor cell rolls', asyn
await ageState(17 * 60_000)
await assert.rejects(authorityAt('0'), /authority is incomplete or stale/)
await assert.doesNotReject(authorityAt('1'))
await assert.rejects(authorityAt('4'), /wave index is invalid/)
await assert.rejects(authorityAt('10'), /wave index is invalid/)
await assert.rejects(authorityAt('x'), /wave index is invalid/)
// Both edges of one predecessor job timeout: 5min + 75min exactly.
await ageState(80 * 60_000)
Expand All @@ -259,6 +259,11 @@ test('later same-cap waves accept evidence aged by predecessor cell rolls', asyn
await assert.doesNotReject(authorityAt('3'))
await ageState(230 * 60_000 + 1)
await assert.rejects(authorityAt('3'), /authority is incomplete or stale/)
// The last cell of a ten-cell same-cap batch: 5min + 9 * 75min exactly.
await ageState(680 * 60_000)
await assert.doesNotReject(authorityAt('9'))
await ageState(680 * 60_000 + 1)
await assert.rejects(authorityAt('9'), /authority is incomplete or stale/)
} finally {
await rm(directory, { recursive: true, force: true })
}
Expand Down
7 changes: 4 additions & 3 deletions cloud/dev/scripts/relay-production-same-cap-wave.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ function cells(value) {
const parsed = value.split(',').map((cell) => cell.trim()).filter(Boolean)
if (
parsed.length < 1 ||
parsed.length > 4 ||
parsed.length > 10 ||
new Set(parsed).size !== parsed.length ||
parsed.some((cell) => !SAME_CAP_CELLS.includes(cell))
) throw new Error('same-cap wave cells are invalid')
Expand Down Expand Up @@ -76,8 +76,9 @@ export function validateSameCapWave(input) {
if (input.mode === 'canary-apply' && selected.length !== 1) {
throw new Error('canary mode requires exactly one cell')
}
if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 4)) {
throw new Error('batch mode requires two to four cells')
// Ten is the wave workflow's statically declared serial cell-job chain, cell_1..cell_10.
if (input.mode === 'batch-apply' && (selected.length < 2 || selected.length > 10)) {
throw new Error('batch mode requires two to ten cells')
}
// Later waves expect the selector to advance by exactly 2 per predecessor,
// which a resumed rollback cell (isolate skipped, +1) violates.
Expand Down
42 changes: 42 additions & 0 deletions cloud/dev/scripts/relay-production-same-cap-wave.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,48 @@ test('requires one canary or a bounded reviewed batch', () => {
}), /cells/)
})

// The bound is the wave workflow's static cell_1..cell_10 chain: a batch longer than the
// chain would silently drop its tail cells, so it is refused before any mutation.
test('a batch fills the serial cell chain and never overflows it', () => {
const general = SAME_CAP_CELLS.filter((cell) => entryAdmission(cell) === 'general')
const batch = (count) => {
const cellIds = general.slice(0, count).join(',')
return validateSameCapWave({
mode: 'batch-apply',
cellIds,
targetDigest,
rollbackDigest,
confirmation: `ROLL_RELAY_SAME_CAP ${targetDigest} ${cellIds}`,
canaryRunId: '42'
})
}
assert.equal(batch(10).cells.length, 10)
assert.throws(() => batch(11), /same-cap wave cells are invalid/)
assert.throws(() => batch(1), /batch mode requires two to ten cells/)
})

// The validator's ten-cell bound is only true if the workflow really declares ten strictly
// serial cell jobs and frees the lease after all of them.
test('the wave workflow chains exactly ten serial cell jobs', () => {
const dispatch = readRelayWorkflow('deploy-relay-production-same-cap.yml')
for (let index = 0; index < 10; index += 1) {
const job = index + 1
assert.match(dispatch, new RegExp(`\n cell_${job}:\n`), `cell_${job} is missing`)
assert.match(dispatch, new RegExp(`fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\]`))
assert.match(dispatch, new RegExp(`wave-index: '${index}'`))
if (index > 0) {
assert.match(dispatch, new RegExp(`needs: \\[gate, cell_${index}\\]`))
assert.match(
dispatch,
new RegExp(`if: \\$\\{\\{ needs\\.cell_${index}\\.result == 'success' && ` +
`fromJSON\\(needs\\.gate\\.outputs\\.cells\\)\\[${index}\\] != null \\}\\}`)
)
}
assert.match(dispatch, new RegExp(`\n - cell_${job}\n`), `release_lease must need cell_${job}`)
}
assert.doesNotMatch(dispatch, /\n cell_11:/)
})

test('rolls the migration-only cells but never mixes the two classes in one wave', () => {
for (const cellId of SAME_CAP_MIGRATION_ONLY_CELLS) {
assert.equal(SAME_CAP_CELLS.includes(cellId), true, cellId)
Expand Down
7 changes: 4 additions & 3 deletions cloud/dev/scripts/relay-regional-rehome-workflow.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,10 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
// The wrapper validates the override before anything runs, passes it to every
// cell, seals it into the canary artifact, and prints it in the run summary.
assert.match(wrapper, /--gate-override-reason "\$\{GATE_OVERRIDE_REASON\}" \\\n {12}--gate-override-confirmation "\$\{GATE_OVERRIDE_CONFIRMATION\}"\)/)
// One per cell job in the serial cell_1..cell_10 chain.
assert.equal(
wrapper.match(/gate-override-confirmation: \$\{\{ inputs\.gate-override-confirmation \}\}/g).length,
4
10
)
assert.match(wrapper, /Aggregate monitor gate overridden \(break-glass\)/)
assert.match(wrapper, /ACTOR: \$\{\{ github\.actor \}\}/)
Expand All @@ -163,12 +164,12 @@ test('same-cap wrapper is reusable, canary-bound, and sequential', () => {
]) {
const body = readFileSync(fileURLToPath(new URL(source, import.meta.url)), 'utf8')
assert.match(body, /WAVE_PREDECESSOR_TIMEOUT_MS = 75 \* 60_000/)
assert.match(body, /\^\[0-3\]\$/)
assert.match(body, /\^\[0-9\]\$/)
}
// Aged-evidence replay via job re-runs is fenced: mutations are
// single-dispatch, so a failed cell needs a fresh gate and monitor run.
assert.match(job, /test "\$\{GITHUB_RUN_ATTEMPT\}" = 1/)
for (const index of [0, 1, 2, 3]) {
for (const index of [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) {
assert.match(wrapper, new RegExp(`wave-index: '${index}'`))
}
assert.doesNotMatch(job, /EFFECTIVE_SELECTOR_GENERATION \+ 1\)/)
Expand Down
9 changes: 7 additions & 2 deletions cloud/dev/scripts/validate-relay-asia-topology-plan.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@ import { readFileSync } from 'node:fs'
import { fileURLToPath } from 'node:url'

const REGION = 'asia-east2'
// Mirrors local.relay_gce_topology in infra/terraform/relay-gce-cells.tf, which Terraform
// cannot export to JS; the census test below the validator equates the two by reading the
// .tf source, so this pair and the topology `check` assert cannot drift apart.
export const RELAY_CELL_BACKEND_TIMEOUT_SECONDS = 86_400
export const RELAY_CELL_CONNECTION_DRAIN_SECONDS = 60
const CELL_SHAPES = {
production: {
domain: 'relay.onorca.dev',
Expand Down Expand Up @@ -117,8 +122,8 @@ function requireCellBackend(change, config, cellId) {
const hostname = cellId.split('-').at(-1)
const name = `${relayGceName(config.environment)}-${hostname}`
if (
after?.timeout_sec !== 86_400 ||
after?.connection_draining_timeout_sec !== 300 ||
after?.timeout_sec !== RELAY_CELL_BACKEND_TIMEOUT_SECONDS ||
after?.connection_draining_timeout_sec !== RELAY_CELL_CONNECTION_DRAIN_SECONDS ||
after?.load_balancing_scheme !== 'EXTERNAL_MANAGED' ||
after?.protocol !== 'HTTP' ||
after?.port_name !== 'relay' ||
Expand Down
Loading
Loading