chore(cloud): add the relay fence broker, ops console, Terraform root, scripts, and 24 cloud-* workflows

.github/CODEOWNERS

@@ -3,3 +3,8 @@
 /config/scripts/*localization*.mjs @brennanb2025
 /config/scripts/*locale*.mjs @brennanb2025
 /config/i18next.config.ts @brennanb2025
+
+# The relay's deploy and operate surface: workspace, workflows, and the rollout lease action.
+/cloud/ @Jinwoo-H
+/.github/workflows/cloud-*.yml @Jinwoo-H
+/.github/actions/cloud-sql-rollout-lease/ @Jinwoo-H

.github/actions/cloud-sql-rollout-lease/README.md

@@ -0,0 +1,152 @@
+# Cloud SQL rollout lease
+
+A compare-and-swap lease on one Cloud Storage object, used to serialize Cloud SQL
+**connection-budget** rollouts across two repositories.
+
+`concurrency.group: production-cloud-sql-rollout` only serializes runs inside a single repository.
+Once the relay workflows live in `stablyai/orca` and the app workflows stay in
+`stablyai/orca-cloud`, there are two independent queues pointed at one shared Cloud SQL instance.
+`relay-cloud-sql-connection-budget.mjs` computes `rolloutOverlap` as a `Math.max` over the relay
+director, api, auth and relay-cell candidates, which is only sound when exactly one rollout is in
+flight. This lease is what keeps that assumption true. Keep the per-repo concurrency groups **and**
+the lease; they solve different halves of the problem.
+
+## What it protects
+
+No workflow runs a Cloud SQL schema migration. Every locked workflow either deploys a Cloud Run
+revision or applies a GCE instance template against the shared instance, so the lease must cover
+**all rollouts**, not just migrations.
+
+## Usage
+
+The lease step must run **after** `google-github-actions/setup-gcloud`, and after
+`actions/checkout` — `uses: ./.github/actions/...` resolves against the checked-out workspace.
+It belongs in the first job of the workflow that holds a GCP credential, which is not always the
+gate job: `deploy-relay-production-same-cap`'s gate runs no `gcloud`, so its first acquire happens
+in the first cell job.
+
+```yaml
+- uses: google-github-actions/auth@v2
+    with: { workload_identity_provider: ..., service_account: ... }
+- uses: google-github-actions/setup-gcloud@v2
+- uses: ./.github/actions/cloud-sql-rollout-lease
+  with:
+    bucket: onorca-cloud-terraform-state
+    object: terraform/state/cloud-sql-rollout/production.lock
+```
+
+Buckets and objects in use:
+
+| Environment | Bucket                                 | Object                                              |
+| ----------- | -------------------------------------- | --------------------------------------------------- |
+| production  | `onorca-cloud-terraform-state`         | `terraform/state/cloud-sql-rollout/production.lock` |
+| staging     | `onorca-cloud-staging-terraform-state` | `terraform/state/cloud-sql-rollout/staging.lock`    |
+
+Workflows that serve both environments (`deploy-relay-asia-topology`,
+`operate-relay-asia-admission`) select the pair with an `inputs.environment == 'production'`
+ternary on both `bucket` and `object`. `deploy-staging` keeps its own `deploy-artifacts-staging`
+concurrency group but takes the staging lease, because it rolls the staging API revision.
+
+The object sits beside `terraform/state/relay-fence-broker/<env>.lock`. The IAM grant names both the
+relay and app service accounts, so it is a **foundation-root** resource: `roles/storage.objectAdmin`
+conditioned on the `terraform/state/cloud-sql-rollout/` prefix, **plus** an unconditioned
+`roles/storage.legacyBucketReader`. Without the second role the generation-matched write fails in a
+way that looks like a permissions flake.
+
+## One lease per run, not per job
+
+`deploy-relay-production-capacity` calls its reusable job six times and
+`deploy-relay-production-same-cap` four times. Each call is a separate job on a separate runner, so
+a naive per-job acquire/release would leave the object free between waves — for runs that have taken
+up to 85 minutes.
+
+The lease is therefore keyed to the **run**, not the job. `holder-key` defaults to
+`${{ github.repository }}/${{ github.run_id }}`, and a job that finds its own holder key on a live
+lease **re-enters** it: the record is refreshed, not rejected. Every job in the chain acquires; only
+the last one releases.
+
+```yaml
+jobs:
+  gate:
+    steps:
+      - uses: ./.github/actions/cloud-sql-rollout-lease
+        with: { bucket: ..., object: ..., release: 'false' } # intermediate
+
+  wave-1: # ... release: 'false' on every wave job
+
+  release_lease:
+    needs: [gate, wave-1, wave-2, wave-3, wave-4]
+    if: always()
+    steps:
+      - uses: google-github-actions/auth@v2
+      - uses: google-github-actions/setup-gcloud@v2
+      - uses: ./.github/actions/cloud-sql-rollout-lease
+        with: { bucket: ..., object: ..., release: 'true' } # final
+```
+
+`release: 'false'` still acquires and still runs its `post` step; `post` only skips the delete. A
+single-job workflow leaves `release` at its `true` default and needs no extra job. So does a
+workflow whose several jobs can never hold the lease at once: `prove-relay-staging-capacity`'s two
+lease-holding jobs are guarded by complementary `inputs.mode` conditions, and the contract test
+checks that exclusivity rather than assuming it.
+
+If the final job never runs (runner killed, run cancelled hard), the lease expires on its TTL.
+
+## Timing
+
+- **TTL 35 minutes**, matching `apps/relay-fence-broker/src/mutation-lease.ts`.
+- **Renewal every 5 minutes.** `main` spawns a detached background Node process that rewrites
+  `expires_at` on the same generation-matched path; `post` kills it by pid read back from
+  `$GITHUB_STATE`. The renewer stops on its own the moment the object stops being ours, and has a
+  six-hour backstop in case `post` never runs. Its log is written to
+  `$RUNNER_TEMP/cloud-sql-rollout-lease-renewer.log` and echoed by `post`.
+- Renewal is mandatory, not optional: capacity runs have taken 85 minutes, well past any sane TTL.
+
+## Failure behaviour
+
+| Situation                        | Behaviour                                                                                                         |
+| -------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| Object absent                    | Acquire with `ifGenerationMatch: 0`.                                                                              |
+| Live lease, our own holder key   | Re-enter. Refresh `expires_at`, keep `acquired_at`. Never fails.                                                  |
+| Live lease, another holder       | **Fail the job immediately**, printing the holder's repository, workflow and run URL. Never queues, never steals. |
+| Expired lease                    | Take over with the observed generation and emit `::warning::` naming the stale holder.                            |
+| `412` on write                   | Someone raced us. Fail as a conflict.                                                                             |
+| Bucket unreachable, `403`, `5xx` | **Fail closed.**                                                                                                  |
+| Record present but unparseable   | **Fail closed.** A record we cannot read is never treated as free; an operator must inspect and delete it.        |
+| Release finds a foreign holder   | Warn and leave it alone. Our lease had already expired.                                                           |
+| Release fails                    | Warn only. `post` never fails a job over a release; the TTL bounds the damage.                                    |
+
+## Why `monitor-relay-production` must not use this
+
+`monitor-relay-production` is in the `production-cloud-sql-rollout` concurrency group but is
+**read-only**: its identity holds only monitoring, logging, Cloud SQL and compute _viewer_ roles,
+and it runs `gcloud sql instances describe`, never a mutation. It consumes no connection budget.
+Putting it on the durable lease would let a monitoring run block a real rollout, and a rollout block
+monitoring exactly when an operator most needs it. Keep its same-repo concurrency group; keep it off
+the lease. The lock census contract test records it in the not-a-candidate map with this reason.
+
+## Token acquisition
+
+`gcloud auth print-access-token`, not a hand-rolled exchange of the `external_account` credentials
+file. Every consuming workflow already runs `setup-gcloud`, gcloud already handles every ADC flavour
+including the service-account impersonation leg, and this action must stay zero-dependency because
+it is duplicated by hand into the public repo. The GCE metadata server that
+`apps/relay-fence-broker/src/google-metadata.ts` uses does **not** exist on GitHub or Blacksmith
+runners; only the compare-and-swap algorithm is shared with the fence broker.
+
+## Duplication
+
+This directory is copied verbatim into `stablyai/orca`. It has no `package.json`, no
+`node_modules`, and imports nothing outside itself — `action-contract.test.mjs` enforces all three.
+Cross-repo consumption via `uses: stablyai/orca/.github/actions/...@<sha>` was rejected: it would
+put public-repo code inside private app deploys that hold a production credential, and neither
+repository protects `main` today.
+
+## Tests
+
+```
+node --test .github/actions/cloud-sql-rollout-lease/
+```
+
+`storage-lease.test.mjs` drives the real compare-and-swap path against an in-memory Cloud Storage
+fake that enforces generations. No network.

.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs

@@ -0,0 +1,52 @@
+import assert from 'node:assert/strict'
+import { readFileSync, readdirSync } from 'node:fs'
+import { test } from 'node:test'
+
+const here = new URL('./', import.meta.url)
+const action = readFileSync(new URL('action.yml', here), 'utf8')
+const modules = readdirSync(here).filter((name) => name.endsWith('.mjs'))
+const shipped = modules.filter((name) => !name.endsWith('.test.mjs'))
+
+test('is a node24 JavaScript action with an always-run post step', () => {
+  // A composite action has no `post:`, so the lease could never be released on cancel or failure.
+  assert.match(action, /^ {2}using: node24$/m)
+  assert.doesNotMatch(action, /using: composite/)
+  assert.match(action, /^ {2}main: main\.mjs$/m)
+  assert.match(action, /^ {2}post: post\.mjs$/m)
+  assert.match(action, /^ {2}post-if: always\(\)$/m)
+})
+
+test('declares the inputs the wave-chain callers depend on', () => {
+  for (const input of ['bucket:', 'object:', 'holder-key:', 'release:']) {
+    assert.match(action, new RegExp(`^ {2}${input}$`, 'm'), input)
+  }
+  assert.match(action, /default: \$\{\{ github\.repository \}\}\/\$\{\{ github\.run_id \}\}/)
+  assert.match(action, /default: 'true'/)
+})
+
+test('stays self-contained so it can be duplicated into the public repo', () => {
+  assert.deepEqual(
+    readdirSync(here).filter((name) => name === 'package.json' || name === 'node_modules'),
+    [],
+    'the action must run with zero installed dependencies'
+  )
+  for (const name of modules) {
+    const source = readFileSync(new URL(name, here), 'utf8')
+    for (const match of source.matchAll(/^import\b[\s\S]*?from '([^']+)'/gm)) {
+      const specifier = match[1]
+      const local = specifier.startsWith('.')
+      assert.ok(
+        specifier.startsWith('node:') || (local && !specifier.includes('..')),
+        `${name} imports ${specifier}; only node: builtins and same-directory modules are allowed`
+      )
+    }
+  }
+})
+
+test('never reaches for the GCE metadata server', () => {
+  // Runners have no metadata.google.internal; the fence broker's token path must not be copied.
+  for (const name of shipped) {
+    const source = readFileSync(new URL(name, here), 'utf8')
+    assert.doesNotMatch(source, /metadata\.google\.internal/, name)
+  }
+})

.github/actions/cloud-sql-rollout-lease/action.yml

@@ -0,0 +1,41 @@
+name: Cloud SQL rollout lease
+description: >-
+  Serialize Cloud SQL connection-budget rollouts across repositories with a compare-and-swap lease
+  on a Cloud Storage object. Fails immediately when another run holds the lease; never queues,
+  never steals.
+
+inputs:
+  bucket:
+    description: Terraform state bucket that holds the lease object.
+    required: true
+  object:
+    description: Lease object name, e.g. terraform/state/cloud-sql-rollout/production.lock
+    required: true
+  holder-key:
+    description: >-
+      Identity that owns the lease. Every job in one run must pass the same value; a job that finds
+      its own holder key on a live lease re-enters it instead of failing.
+    required: false
+    default: ${{ github.repository }}/${{ github.run_id }}
+  release:
+    description: >-
+      Release the lease in the post step. Set to "false" on every job of a multi-job wave except the
+      final always() job, which sets "true".
+    required: false
+    default: 'true'
+
+outputs:
+  holder-key:
+    description: The holder key written to the lease object.
+  generation:
+    description: Cloud Storage generation of the lease object after acquisition.
+  expires-at:
+    description: ISO-8601 instant at which the lease expires without renewal.
+  reentrant:
+    description: '"true" when this job re-entered a lease its own run already held.'
+
+runs:
+  using: node24
+  main: main.mjs
+  post: post.mjs
+  post-if: always()

.github/actions/cloud-sql-rollout-lease/gcloud-access-token.mjs

@@ -0,0 +1,46 @@
+import { execFileSync } from 'node:child_process'
+
+// DESIGN CHOICE: shell out to `gcloud auth print-access-token` instead of exchanging the
+// external_account credentials file that google-github-actions/auth writes.
+//
+// Every workflow on the Cloud SQL rollout lease already runs google-github-actions/setup-gcloud
+// right after auth (verified across all 11 mutating members), so gcloud is on PATH and already
+// bound to the federated identity. Doing the exchange ourselves would mean reimplementing the STS
+// token swap plus the service-account impersonation leg, in an action that must stay
+// zero-dependency and is duplicated by hand into a second repo. gcloud already handles every ADC
+// flavour and refreshes on its own. The metadata server is not an option: it does not exist on
+// GitHub or Blacksmith runners.
+//
+// Consequence, documented in the README: the lease step MUST come after setup-gcloud.
+
+const TOKEN_REUSE_MS = 40 * 60 * 1_000 // GCP access tokens live ~60 min; re-mint well before that.
+
+export function createAccessTokenSource({ run = runGcloud, now = Date.now } = {}) {
+  let cached = null
+  return () => {
+    if (cached && cached.mintedAt + TOKEN_REUSE_MS > now()) {
+      return cached.token
+    }
+    const token = run()
+    if (!token) {
+      throw new Error('gcloud auth print-access-token returned an empty token')
+    }
+    cached = { token, mintedAt: now() }
+    return token
+  }
+}
+
+function runGcloud() {
+  const binary = process.platform === 'win32' ? 'gcloud.cmd' : 'gcloud'
+  try {
+    return execFileSync(binary, ['auth', 'print-access-token'], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      timeout: 60_000
+    }).trim()
+  } catch (error) {
+    // Never surface stdout; it is the token on success and noise on failure.
+    const detail = String(error?.stderr ?? '').trim() || error?.message || 'unknown failure'
+    throw new Error(`could not mint a GCP access token via gcloud: ${detail}`)
+  }
+}

.github/actions/cloud-sql-rollout-lease/holder-identity.mjs

@@ -0,0 +1,20 @@
+// Who we claim to be on the lease object. Shared by main and the detached renewer so both agree
+// on the holder key without re-deriving it from a different set of environment variables.
+
+export function holderIdentity(explicitHolderKey) {
+  const repository = process.env.GITHUB_REPOSITORY ?? 'unknown'
+  const runId = process.env.GITHUB_RUN_ID ?? 'unknown'
+  const server = process.env.GITHUB_SERVER_URL ?? 'https://github.com'
+  const holderKey = explicitHolderKey || `${repository}/${runId}`
+  if (/[\r\n]/.test(holderKey)) {
+    throw new Error('holder-key must be single-line')
+  }
+  return {
+    holderKey,
+    repository,
+    workflow: process.env.GITHUB_WORKFLOW ?? 'unknown',
+    runId,
+    runUrl: `${server}/${repository}/actions/runs/${runId}`,
+    runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? 'unknown'
+  }
+}