feat(control-plane): persist provisioning and failed tenant lifecycle states for the customer status surface

control-plane/src/http-app.ts

@@ -34,9 +34,16 @@ import {
   provisionTenant,
   type ProvisioningPagerDutyOptions,
 } from "./provisioning.js";
-import type { Product, TenantProvisioningDriver } from "./tenant-provisioning-driver.js";
+import type { Product, TenantLifecycleState, TenantProvisioningDriver } from "./tenant-provisioning-driver.js";
 import type { AmsCycleSchedule, TenantRegistry, TenantRegistryRecord } from "./tenant-registry.js";
 
+/** States that do NOT block re-creating a tenant of the same name+product (or re-claiming its installation
+ *  ID): `"torn down"` (the pre-existing rule — a terminated tenant may be recreated fresh) and, since #7677
+ *  persists failures, `"failed"` — "Setup failed" must invite a retry, never permanently squat on the name. */
+function isRecreatableState(state: TenantLifecycleState): boolean {
+  return state === "torn down" || state === "failed";
+}
+
 export type TenantHttpAppDeps = {
   driver: TenantProvisioningDriver;
   registry: TenantRegistry;
@@ -165,33 +172,48 @@ export function createTenantHttpApp(deps: TenantHttpAppDeps): Hono {
     // Not idempotent by design (tenant-client.ts's own doc comment: "a create is not idempotent, so it must
     // not be silently re-sent") -- a currently-active tenant of the same name *and product* is a real conflict,
     // not a no-op (#8024: ORB "acme" must not block AMS "acme"). A previously torn-down tenant may be recreated
-    // (its createdAt is NOT preserved -- this is a fresh provision, not a resurrection of the old one).
+    // (its createdAt is NOT preserved -- this is a fresh provision, not a resurrection of the old one), and so
+    // may a "failed" one (#7677): persisting the failed state must not turn "Setup failed" into a permanent
+    // block on retrying the setup -- before #7677 a failed provision left NO record, so a retry always worked.
     const existing = await deps.registry.get(name, product);
-    if (existing && existing.state !== "torn down") return c.json({ error: "tenant_already_exists" }, 409);
+    if (existing && !isRecreatableState(existing.state)) return c.json({ error: "tenant_already_exists" }, 409);
 
     // A GitHub installation ID must resolve to exactly one hosted container (#7181's routing depends on this
     // being unambiguous) -- reject before ever provisioning anything, same posture as the name+product conflict
     // check just above.
     if (orbInstallationId !== undefined) {
       const conflicting = await deps.registry.getByOrbInstallationId(orbInstallationId);
-      if (conflicting && conflicting.state !== "torn down") {
+      if (conflicting && !isRecreatableState(conflicting.state)) {
         return c.json(
           { error: "installation_already_claimed", message: `installation ${orbInstallationId} is already claimed by tenant "${conflicting.tenant.name}"` },
           409,
         );
       }
     }
 
-    const result = await provisionTenant({ name }, product, deps.driver, deps.pagerDuty ?? {});
-    const now = new Date().toISOString();
-    const record: TenantRegistryRecord = {
-      tenant: result.tenant,
-      product: result.product,
-      state: result.state,
-      createdAt: now,
-      updatedAt: now,
+    // #7677 (ratified 2026-07-21): persist the record in its transitional "provisioning" state BEFORE the
+    // (slow) container/DB/secret standup starts, so the customer-facing dashboard's poll of the existing
+    // GET /v1/tenants read path can actually observe "Setting up your instance" while it happens -- and hand
+    // provisionTenant the seam that flips this same record to a terminal "failed" before a step failure
+    // rethrows, so that poll ends at "Setup failed" instead of spinning on "provisioning" forever.
+    const startedAt = new Date().toISOString();
+    const pending: TenantRegistryRecord = {
+      tenant: { name },
+      product,
+      state: "provisioning",
+      createdAt: startedAt,
+      updatedAt: startedAt,
       ...(schedule ? { amsSchedule: schedule } : {}),
       ...(orbInstallationId !== undefined ? { orbInstallationId } : {}),
+    };
+    await deps.registry.upsert(pending);
+    const result = await provisionTenant({ name }, product, deps.driver, deps.pagerDuty ?? {}, async () => {
+      await deps.registry.upsert({ ...pending, state: "failed", updatedAt: new Date().toISOString() });
+    });
+    const record: TenantRegistryRecord = {
+      ...pending,
+      state: result.state,
+      updatedAt: new Date().toISOString(),
       ...(result.secretRef !== undefined ? { secretRef: result.secretRef } : {}),
     };
     await deps.registry.upsert(record);

control-plane/src/provisioning.ts

@@ -85,12 +85,15 @@ function pageAndRethrow(
  *  `product` is forwarded to every step, never branched on, so ORB and AMS share one call shape. `injectSecrets`
  *  is called with `database` already attached to the request (#8066) -- a real secret driver needs the
  *  connection details to actually store, not just the tenant identity every other step operates on. A step
- *  failure pages (#7667) and always rethrows — provisioning never fails silently. */
+ *  failure pages (#7667) and always rethrows — provisioning never fails silently. `onFailure` (#7677,
+ *  optional) runs first in that failure path — the caller's seam for persisting the `"failed"` lifecycle
+ *  state — and is best-effort: its own rejection is swallowed so it can never mask the step error. */
 export async function provisionTenant(
   tenant: Tenant,
   product: Product,
   driver: TenantProvisioningDriver,
   pagerDuty: ProvisioningPagerDutyOptions = {},
+  onFailure?: () => Promise<void>,
 ): Promise<TenantProvisioningResult> {
   const request: TenantProvisioningRequest = { tenant, product };
   let database: DatabaseConnectionDetails;
@@ -100,6 +103,11 @@ export async function provisionTenant(
     database = await driver.provisionDatabase(request);
     ({ secretRef } = await driver.injectSecrets({ ...request, database }));
   } catch (error) {
+    // #7677 (ratified 2026-07-21): give the caller its chance to transition the tenant's registry record to
+    // "failed" BEFORE the rethrow, so a customer polling the read path sees a terminal "Setup failed" instead
+    // of a record stuck at "provisioning" forever. Best-effort by design: a failure writing the failed state
+    // must never mask the provisioning error itself, which still pages and rethrows exactly as before.
+    if (onFailure) await onFailure().catch(() => undefined);
     pageAndRethrow(tenant, product, "provision", error, pagerDuty);
   }
   return { tenant, product, state: "active", database, ...(secretRef !== undefined ? { secretRef } : {}) };

control-plane/src/tenant-provisioning-driver.ts

@@ -29,9 +29,12 @@ export type Tenant = {
 
 /** The full tenant lifecycle vocabulary the #7180 provisioning API reports, passed through verbatim by
  *  tenant-client.ts. provisionTenant/deprovisionTenant only ever produce the terminal `"active"` / `"torn down"`
- *  states; `"provisioning"` (transitional) and `"suspended"` (an operator action) round out the documented set. */
+ *  states; `"provisioning"` (transitional — written by the create route before orchestration starts, so a
+ *  polling customer can watch the standup, #7677) and `"failed"` (#7677: a provision-step failure, persisted
+ *  before the rethrow so that same polling customer sees a terminal "Setup failed" instead of a record stuck
+ *  at "provisioning" forever) and `"suspended"` (an operator action) round out the documented set. */
 export type TenantLifecycleState =
-  "provisioning" | "active" | "suspended" | "torn down";
+  "provisioning" | "active" | "suspended" | "failed" | "torn down";
 
 /** Everything one provision/deprovision step needs. A single request type flows through every driver method so a
  *  real and a fake driver see identical inputs, and so ORB and AMS calls are shaped identically. */

control-plane/test/http-app.test.ts

@@ -13,6 +13,7 @@ import {
   type RouterNamespaceLike,
   type RouterStubLike,
   type TenantHttpAppDeps,
+  type TenantRegistryRecord,
   type TenantProvisioningDriver,
 } from "../dist/index.js";
 
@@ -581,6 +582,135 @@ test("a driver failure surfaces as a logged 500 via onError, not an unhandled re
   assert.match(errors[0]!, /cloudflare containers api unavailable/);
 });
 
+// #7677 (ratified 2026-07-21): the provisioning-status surface. The record is written as "provisioning"
+// BEFORE the standup starts and transitions to a terminal "failed" before a step failure rethrows — all
+// observable via the same GET /v1/tenants read path a customer's dashboard polls.
+
+function swallowConsoleError() {
+  const originalError = console.error;
+  console.error = () => {};
+  consoleErrorRestore = () => {
+    console.error = originalError;
+  };
+}
+
+function failingCreateDriver(): TenantProvisioningDriver {
+  return {
+    ...createFakeTenantProvisioningDriver(),
+    async createContainer() {
+      throw new Error("container standup failed");
+    },
+  };
+}
+
+test("a provision failure transitions the record to 'failed', observable via the dashboard's own read path (#7677 acceptance)", async () => {
+  swallowConsoleError();
+  const registry = createFakeTenantRegistry();
+  const app = createTenantHttpApp(baseDeps({ driver: failingCreateDriver(), registry }));
+
+  const res = await app.request(
+    "/v1/tenants",
+    authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb" }) }),
+  );
+
+  assert.equal(res.status, 500); // the failure itself still pages + rethrows exactly as before
+  assert.equal((await registry.get("acme", "orb"))?.state, "failed");
+  // The SAME read path a customer's dashboard polls (GET /v1/tenants) shows the terminal failed state —
+  // not a record stuck at "provisioning" forever.
+  const list = await app.request("/v1/tenants", authed());
+  const { tenants } = (await list.json()) as { tenants: Array<{ tenant: { name: string }; state: string }> };
+  assert.deepEqual(
+    tenants.map((t) => ({ name: t.tenant.name, state: t.state })),
+    [{ name: "acme", state: "failed" }],
+  );
+});
+
+test("the record is observable as 'provisioning' while the standup is still in flight (#7677 polling premise)", async () => {
+  const registry = createFakeTenantRegistry();
+  let release!: () => void;
+  const gate = new Promise<void>((resolve) => {
+    release = resolve;
+  });
+  const slowDriver: TenantProvisioningDriver = {
+    ...createFakeTenantProvisioningDriver(),
+    async createContainer() {
+      await gate;
+    },
+  };
+  const app = createTenantHttpApp(baseDeps({ driver: slowDriver, registry }));
+
+  const pending = app.request(
+    "/v1/tenants",
+    authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb" }) }),
+  );
+  // Let the route reach the awaited (gated) driver step, then poll mid-standup like the dashboard would.
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.equal((await registry.get("acme", "orb"))?.state, "provisioning");
+  release();
+  const res = await pending;
+  assert.equal(res.status, 201);
+  assert.equal((await registry.get("acme", "orb"))?.state, "active");
+});
+
+test("a 'failed' tenant may be re-created — 'Setup failed' invites a retry, it never squats on the name (#7677)", async () => {
+  swallowConsoleError();
+  const registry = createFakeTenantRegistry();
+  const failOnce = failingCreateDriver();
+  const app = createTenantHttpApp(baseDeps({ driver: failOnce, registry }));
+  const body = { method: "POST" as const, headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb", orbInstallationId: 77 }) };
+
+  assert.equal((await app.request("/v1/tenants", authed(body))).status, 500);
+  assert.equal((await registry.get("acme", "orb"))?.state, "failed");
+
+  // Retry with a healthy driver: neither the name+product conflict check nor the installation-ID claim check
+  // treats the failed record as a blocker (before #7677 a failure left NO record, so a retry always worked).
+  const healthy = createTenantHttpApp(baseDeps({ driver: createFakeTenantProvisioningDriver(), registry }));
+  const retry = await healthy.request("/v1/tenants", authed(body));
+  assert.equal(retry.status, 201);
+  assert.equal((await registry.get("acme", "orb"))?.state, "active");
+});
+
+test("an ACTIVE tenant still 409s a re-create — #7677 loosens only the failed/torn-down states", async () => {
+  const registry = createFakeTenantRegistry();
+  await registry.upsert({ tenant: { name: "acme" }, product: "orb", state: "active", createdAt: "t0", updatedAt: "t0", orbInstallationId: 77 });
+  const app = createTenantHttpApp(baseDeps({ registry }));
+
+  const sameName = await app.request(
+    "/v1/tenants",
+    authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb" }) }),
+  );
+  assert.equal(sameName.status, 409);
+  const sameInstallation = await app.request(
+    "/v1/tenants",
+    authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "other", product: "orb", orbInstallationId: 77 }) }),
+  );
+  assert.equal(sameInstallation.status, 409);
+});
+
+test("a rejection writing the 'failed' state never masks the provisioning error itself (#7677 best-effort seam)", async () => {
+  swallowConsoleError();
+  const registry = createFakeTenantRegistry();
+  const fragileRegistry = {
+    ...registry,
+    async upsert(record: TenantRegistryRecord) {
+      if (record.state === "failed") throw new Error("kv write outage");
+      await registry.upsert(record);
+    },
+  };
+  const app = createTenantHttpApp(baseDeps({ driver: failingCreateDriver(), registry: fragileRegistry }));
+
+  const res = await app.request(
+    "/v1/tenants",
+    authed({ method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "acme", product: "orb" }) }),
+  );
+
+  // Still the ORIGINAL provisioning failure surfacing through onError — the failed-state write outage is
+  // swallowed, and the record is simply left at its pre-written "provisioning" state.
+  assert.equal(res.status, 500);
+  assert.deepEqual(await res.json(), { error: "internal_error" });
+  assert.equal((await registry.get("acme", "orb"))?.state, "provisioning");
+});
+
 // #4898: POST /v1/tenants/rollout — pin/unpin an explicit list of tenants' pinnedVersion, all-or-nothing.
 // The registry-seeding style mirrors the GET /v1/tenants tests above (records seeded directly, no driver run).