From f80c060e0fc0c84f030198b20b9063ae17b46d3e Mon Sep 17 00:00:00 2001 From: cleanjunc Date: Thu, 23 Jul 2026 06:15:43 +0000 Subject: [PATCH] feat(control-plane): persist provisioning and failed tenant lifecycle states for the customer status surface (#7677) --- control-plane/src/http-app.ts | 46 +++++-- control-plane/src/provisioning.ts | 10 +- .../src/tenant-provisioning-driver.ts | 7 +- control-plane/test/http-app.test.ts | 130 ++++++++++++++++++ 4 files changed, 178 insertions(+), 15 deletions(-) diff --git a/control-plane/src/http-app.ts b/control-plane/src/http-app.ts index c371ce8643..2b9b3439bd 100644 --- a/control-plane/src/http-app.ts +++ b/control-plane/src/http-app.ts @@ -34,9 +34,16 @@ import { provisionTenant, type ProvisioningPagerDutyOptions, } from "./provisioning.js"; -import type { Product, TenantProvisioningDriver } from "./tenant-provisioning-driver.js"; +import type { Product, TenantLifecycleState, TenantProvisioningDriver } from "./tenant-provisioning-driver.js"; import type { AmsCycleSchedule, TenantRegistry, TenantRegistryRecord } from "./tenant-registry.js"; +/** States that do NOT block re-creating a tenant of the same name+product (or re-claiming its installation + * ID): `"torn down"` (the pre-existing rule — a terminated tenant may be recreated fresh) and, since #7677 + * persists failures, `"failed"` — "Setup failed" must invite a retry, never permanently squat on the name. */ +function isRecreatableState(state: TenantLifecycleState): boolean { + return state === "torn down" || state === "failed"; +} + export type TenantHttpAppDeps = { driver: TenantProvisioningDriver; registry: TenantRegistry; @@ -165,16 +172,18 @@ export function createTenantHttpApp(deps: TenantHttpAppDeps): Hono { // Not idempotent by design (tenant-client.ts's own doc comment: "a create is not idempotent, so it must // not be silently re-sent") -- a currently-active tenant of the same name *and product* is a real conflict, // not a no-op (#8024: ORB "acme" must not block AMS "acme"). A previously torn-down tenant may be recreated - // (its createdAt is NOT preserved -- this is a fresh provision, not a resurrection of the old one). + // (its createdAt is NOT preserved -- this is a fresh provision, not a resurrection of the old one), and so + // may a "failed" one (#7677): persisting the failed state must not turn "Setup failed" into a permanent + // block on retrying the setup -- before #7677 a failed provision left NO record, so a retry always worked. const existing = await deps.registry.get(name, product); - if (existing && existing.state !== "torn down") return c.json({ error: "tenant_already_exists" }, 409); + if (existing && !isRecreatableState(existing.state)) return c.json({ error: "tenant_already_exists" }, 409); // A GitHub installation ID must resolve to exactly one hosted container (#7181's routing depends on this // being unambiguous) -- reject before ever provisioning anything, same posture as the name+product conflict // check just above. if (orbInstallationId !== undefined) { const conflicting = await deps.registry.getByOrbInstallationId(orbInstallationId); - if (conflicting && conflicting.state !== "torn down") { + if (conflicting && !isRecreatableState(conflicting.state)) { return c.json( { error: "installation_already_claimed", message: `installation ${orbInstallationId} is already claimed by tenant "${conflicting.tenant.name}"` }, 409, @@ -182,16 +191,29 @@ export function createTenantHttpApp(deps: TenantHttpAppDeps): Hono { } } - const result = await provisionTenant({ name }, product, deps.driver, deps.pagerDuty ?? {}); - const now = new Date().toISOString(); - const record: TenantRegistryRecord = { - tenant: result.tenant, - product: result.product, - state: result.state, - createdAt: now, - updatedAt: now, + // #7677 (ratified 2026-07-21): persist the record in its transitional "provisioning" state BEFORE the + // (slow) container/DB/secret standup starts, so the customer-facing dashboard's poll of the existing + // GET /v1/tenants read path can actually observe "Setting up your instance" while it happens -- and hand + // provisionTenant the seam that flips this same record to a terminal "failed" before a step failure + // rethrows, so that poll ends at "Setup failed" instead of spinning on "provisioning" forever. + const startedAt = new Date().toISOString(); + const pending: TenantRegistryRecord = { + tenant: { name }, + product, + state: "provisioning", + createdAt: startedAt, + updatedAt: startedAt, ...(schedule ? { amsSchedule: schedule } : {}), ...(orbInstallationId !== undefined ? { orbInstallationId } : {}), + }; + await deps.registry.upsert(pending); + const result = await provisionTenant({ name }, product, deps.driver, deps.pagerDuty ?? {}, async () => { + await deps.registry.upsert({ ...pending, state: "failed", updatedAt: new Date().toISOString() }); + }); + const record: TenantRegistryRecord = { + ...pending, + state: result.state, + updatedAt: new Date().toISOString(), ...(result.secretRef !== undefined ? { secretRef: result.secretRef } : {}), }; await deps.registry.upsert(record); diff --git a/control-plane/src/provisioning.ts b/control-plane/src/provisioning.ts index 3971de6cfd..471616ede0 100644 --- a/control-plane/src/provisioning.ts +++ b/control-plane/src/provisioning.ts @@ -85,12 +85,15 @@ function pageAndRethrow( * `product` is forwarded to every step, never branched on, so ORB and AMS share one call shape. `injectSecrets` * is called with `database` already attached to the request (#8066) -- a real secret driver needs the * connection details to actually store, not just the tenant identity every other step operates on. A step - * failure pages (#7667) and always rethrows — provisioning never fails silently. */ + * failure pages (#7667) and always rethrows — provisioning never fails silently. `onFailure` (#7677, + * optional) runs first in that failure path — the caller's seam for persisting the `"failed"` lifecycle + * state — and is best-effort: its own rejection is swallowed so it can never mask the step error. */ export async function provisionTenant( tenant: Tenant, product: Product, driver: TenantProvisioningDriver, pagerDuty: ProvisioningPagerDutyOptions = {}, + onFailure?: () => Promise, ): Promise { const request: TenantProvisioningRequest = { tenant, product }; let database: DatabaseConnectionDetails; @@ -100,6 +103,11 @@ export async function provisionTenant( database = await driver.provisionDatabase(request); ({ secretRef } = await driver.injectSecrets({ ...request, database })); } catch (error) { + // #7677 (ratified 2026-07-21): give the caller its chance to transition the tenant's registry record to + // "failed" BEFORE the rethrow, so a customer polling the read path sees a terminal "Setup failed" instead + // of a record stuck at "provisioning" forever. Best-effort by design: a failure writing the failed state + // must never mask the provisioning error itself, which still pages and rethrows exactly as before. + if (onFailure) await onFailure().catch(() => undefined); pageAndRethrow(tenant, product, "provision", error, pagerDuty); } return { tenant, product, state: "active", database, ...(secretRef !== undefined ? { secretRef } : {}) }; diff --git a/control-plane/src/tenant-provisioning-driver.ts b/control-plane/src/tenant-provisioning-driver.ts index 2f0c12d225..40eee56f10 100644 --- a/control-plane/src/tenant-provisioning-driver.ts +++ b/control-plane/src/tenant-provisioning-driver.ts @@ -29,9 +29,12 @@ export type Tenant = { /** The full tenant lifecycle vocabulary the #7180 provisioning API reports, passed through verbatim by * tenant-client.ts. provisionTenant/deprovisionTenant only ever produce the terminal `"active"` / `"torn down"` - * states; `"provisioning"` (transitional) and `"suspended"` (an operator action) round out the documented set. */ + * states; `"provisioning"` (transitional — written by the create route before orchestration starts, so a + * polling customer can watch the standup, #7677) and `"failed"` (#7677: a provision-step failure, persisted + * before the rethrow so that same polling customer sees a terminal "Setup failed" instead of a record stuck + * at "provisioning" forever) and `"suspended"` (an operator action) round out the documented set. */ export type TenantLifecycleState = - "provisioning" | "active" | "suspended" | "torn down"; + "provisioning" | "active" | "suspended" | "failed" | "torn down"; /** Everything one provision/deprovision step needs. A single request type flows through every driver method so a * real and a fake driver see identical inputs, and so ORB and AMS calls are shaped identically. */ diff --git a/control-plane/test/http-app.test.ts b/control-plane/test/http-app.test.ts index cb37681870..0d47b487ea 100644 --- a/control-plane/test/http-app.test.ts +++ b/control-plane/test/http-app.test.ts @@ -13,6 +13,7 @@ import { type RouterNamespaceLike, type RouterStubLike, type TenantHttpAppDeps, + type TenantRegistryRecord, type TenantProvisioningDriver, } from "../dist/index.js"; @@ -581,6 +582,135 @@ test("a driver failure surfaces as a logged 500 via onError, not an unhandled re assert.match(errors[0]!, /cloudflare containers api unavailable/); }); +// #7677 (ratified 2026-07-21): the provisioning-status surface. The record is written as "provisioning" +// BEFORE the standup starts and transitions to a terminal "failed" before a step failure rethrows — all +// observable via the same GET /v1/tenants read path a customer's dashboard polls. + +function swallowConsoleError() { + const originalError = console.error; + console.error = () => {}; + consoleErrorRestore = () => { + console.error = originalError; + }; +} + +function failingCreateDriver(): TenantProvisioningDriver { + return { + ...createFakeTenantProvisioningDriver(), + async createContainer() { + throw new Error("container standup failed"); + }, + }; +} + +test("a provision failure transitions the record to 'failed', observable via the dashboard's own read path (#7677 acceptance)", async () => { + swallowConsoleError(); + const registry = createFakeTenantRegistry(); + const app = createTenantHttpApp(baseDeps({ driver: failingCreateDriver(), registry })); + + const res = await app.request( + "/v1/tenants", + authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb" }) }), + ); + + assert.equal(res.status, 500); // the failure itself still pages + rethrows exactly as before + assert.equal((await registry.get("acme", "orb"))?.state, "failed"); + // The SAME read path a customer's dashboard polls (GET /v1/tenants) shows the terminal failed state — + // not a record stuck at "provisioning" forever. + const list = await app.request("/v1/tenants", authed()); + const { tenants } = (await list.json()) as { tenants: Array<{ tenant: { name: string }; state: string }> }; + assert.deepEqual( + tenants.map((t) => ({ name: t.tenant.name, state: t.state })), + [{ name: "acme", state: "failed" }], + ); +}); + +test("the record is observable as 'provisioning' while the standup is still in flight (#7677 polling premise)", async () => { + const registry = createFakeTenantRegistry(); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const slowDriver: TenantProvisioningDriver = { + ...createFakeTenantProvisioningDriver(), + async createContainer() { + await gate; + }, + }; + const app = createTenantHttpApp(baseDeps({ driver: slowDriver, registry })); + + const pending = app.request( + "/v1/tenants", + authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb" }) }), + ); + // Let the route reach the awaited (gated) driver step, then poll mid-standup like the dashboard would. + await new Promise((resolve) => setImmediate(resolve)); + assert.equal((await registry.get("acme", "orb"))?.state, "provisioning"); + release(); + const res = await pending; + assert.equal(res.status, 201); + assert.equal((await registry.get("acme", "orb"))?.state, "active"); +}); + +test("a 'failed' tenant may be re-created — 'Setup failed' invites a retry, it never squats on the name (#7677)", async () => { + swallowConsoleError(); + const registry = createFakeTenantRegistry(); + const failOnce = failingCreateDriver(); + const app = createTenantHttpApp(baseDeps({ driver: failOnce, registry })); + const body = { method: "POST" as const, headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb", orbInstallationId: 77 }) }; + + assert.equal((await app.request("/v1/tenants", authed(body))).status, 500); + assert.equal((await registry.get("acme", "orb"))?.state, "failed"); + + // Retry with a healthy driver: neither the name+product conflict check nor the installation-ID claim check + // treats the failed record as a blocker (before #7677 a failure left NO record, so a retry always worked). + const healthy = createTenantHttpApp(baseDeps({ driver: createFakeTenantProvisioningDriver(), registry })); + const retry = await healthy.request("/v1/tenants", authed(body)); + assert.equal(retry.status, 201); + assert.equal((await registry.get("acme", "orb"))?.state, "active"); +}); + +test("an ACTIVE tenant still 409s a re-create — #7677 loosens only the failed/torn-down states", async () => { + const registry = createFakeTenantRegistry(); + await registry.upsert({ tenant: { name: "acme" }, product: "orb", state: "active", createdAt: "t0", updatedAt: "t0", orbInstallationId: 77 }); + const app = createTenantHttpApp(baseDeps({ registry })); + + const sameName = await app.request( + "/v1/tenants", + authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb" }) }), + ); + assert.equal(sameName.status, 409); + const sameInstallation = await app.request( + "/v1/tenants", + authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "other", product: "orb", orbInstallationId: 77 }) }), + ); + assert.equal(sameInstallation.status, 409); +}); + +test("a rejection writing the 'failed' state never masks the provisioning error itself (#7677 best-effort seam)", async () => { + swallowConsoleError(); + const registry = createFakeTenantRegistry(); + const fragileRegistry = { + ...registry, + async upsert(record: TenantRegistryRecord) { + if (record.state === "failed") throw new Error("kv write outage"); + await registry.upsert(record); + }, + }; + const app = createTenantHttpApp(baseDeps({ driver: failingCreateDriver(), registry: fragileRegistry })); + + const res = await app.request( + "/v1/tenants", + authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb" }) }), + ); + + // Still the ORIGINAL provisioning failure surfacing through onError — the failed-state write outage is + // swallowed, and the record is simply left at its pre-written "provisioning" state. + assert.equal(res.status, 500); + assert.deepEqual(await res.json(), { error: "internal_error" }); + assert.equal((await registry.get("acme", "orb"))?.state, "provisioning"); +}); + // #4898: POST /v1/tenants/rollout — pin/unpin an explicit list of tenants' pinnedVersion, all-or-nothing. // The registry-seeding style mirrors the GET /v1/tenants tests above (records seeded directly, no driver run).