From b6028a5509c57171a513f89f581a0d3cdd56ceca Mon Sep 17 00:00:00 2001 From: jsdevninja Date: Mon, 20 Jul 2026 16:11:21 -0500 Subject: [PATCH 1/2] feat(control-plane): add tenant provision orchestration with fake driver Co-authored-by: Cursor --- .loopover.yml | 1 + control-plane/README.md | 15 ++ control-plane/package-lock.json | 119 ++++++++++++++ control-plane/package.json | 22 +++ control-plane/src/deprovision-tenant.ts | 75 +++++++++ control-plane/src/errors.ts | 28 ++++ control-plane/src/fake-secret-broker.ts | 57 +++++++ .../src/fake-tenant-provisioning-driver.ts | 124 +++++++++++++++ control-plane/src/index.ts | 37 +++++ control-plane/src/provision-tenant.ts | 97 +++++++++++ control-plane/src/secret-broker.ts | 39 +++++ .../src/tenant-provisioning-driver.ts | 41 +++++ control-plane/src/types.ts | 50 ++++++ control-plane/src/validate-tenant.ts | 19 +++ .../test/provision-lifecycle.test.ts | 150 ++++++++++++++++++ control-plane/test/secret-broker.test.ts | 20 +++ control-plane/test/validate-tenant.test.ts | 23 +++ control-plane/tsconfig.json | 17 ++ package.json | 4 +- src/config/loopover-repo-focus-manifest.ts | 1 + test/unit/loopover-focus-manifest.test.ts | 1 + 21 files changed, 939 insertions(+), 1 deletion(-) create mode 100644 control-plane/README.md create mode 100644 control-plane/package-lock.json create mode 100644 control-plane/package.json create mode 100644 control-plane/src/deprovision-tenant.ts create mode 100644 control-plane/src/errors.ts create mode 100644 control-plane/src/fake-secret-broker.ts create mode 100644 control-plane/src/fake-tenant-provisioning-driver.ts create mode 100644 control-plane/src/index.ts create mode 100644 control-plane/src/provision-tenant.ts create mode 100644 control-plane/src/secret-broker.ts create mode 100644 control-plane/src/tenant-provisioning-driver.ts create mode 100644 control-plane/src/types.ts create mode 100644 control-plane/src/validate-tenant.ts create mode 100644 control-plane/test/provision-lifecycle.test.ts create mode 100644 control-plane/test/secret-broker.test.ts create mode 100644 control-plane/test/validate-tenant.test.ts create mode 100644 control-plane/tsconfig.json diff --git a/.loopover.yml b/.loopover.yml index 64a0651dd6..da431216ed 100644 --- a/.loopover.yml +++ b/.loopover.yml @@ -10,6 +10,7 @@ wantedPaths: - migrations/ - scripts/ - review-enrichment/ + - control-plane/ - .github/workflows/ - wrangler.jsonc - apps/loopover-ui/ diff --git a/control-plane/README.md b/control-plane/README.md new file mode 100644 index 0000000000..bb0d69a890 --- /dev/null +++ b/control-plane/README.md @@ -0,0 +1,15 @@ +# @loopover/control-plane + +Hosting control-plane package for LoopOver ORB/AMS tenant lifecycle (#7173 / #7180 / #7524). + +This package hosts **orchestration only**: `provisionTenant()` / `deprovisionTenant()` run against an +injectable `TenantProvisioningDriver`. The in-tree driver is a fake/in-memory implementation for +tests. Real Cloudflare Containers / Postgres providers are intentionally out of scope until the +Postgres-provider decision on #7180 lands. + +Secrets injection goes through the `#7174` broker-shaped `TenantSecretBroker` seam (typed +`secretType`), not a hand-rolled credentials path. + +Codecov note: this package is a standalone (non-workspace) tree like `review-enrichment/`. It is +**not** under root `vitest.config.ts` `coverage.include` (`src/**`), so Codecov's patch gate does +not apply numerically here — coverage is enforced by this package's own `npm test` suite. diff --git a/control-plane/package-lock.json b/control-plane/package-lock.json new file mode 100644 index 0000000000..dd58e0f506 --- /dev/null +++ b/control-plane/package-lock.json @@ -0,0 +1,119 @@ +{ + "name": "@loopover/control-plane", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@loopover/control-plane", + "version": "0.1.0", + "dependencies": { + "loopover": "file:.." + }, + "devDependencies": { + "@types/node": "^22.20.0", + "typescript": "^5.9.3" + }, + "engines": { + "node": ">=20" + } + }, + "..": { + "name": "loopover", + "version": "0.1.0", + "license": "AGPL-3.0-only", + "workspaces": [ + "apps/*", + "packages/*" + ], + "dependencies": { + "@asteasolutions/zod-to-openapi": "^8.5.0", + "@cloudflare/puppeteer": "^1.1.0", + "@hono/node-server": "^2.0.8", + "@loopover/engine": "*", + "@modelcontextprotocol/sdk": "1.29.0", + "@octokit/core": "^7.0.6", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/exporter-trace-otlp-http": "^0.220.0", + "@opentelemetry/resources": "^2.9.0", + "@opentelemetry/sdk-trace-node": "^2.9.0", + "@sentry/cloudflare": "^10.63.0", + "@sentry/hono": "^10.63.0", + "@sentry/node": "^10.63.0", + "@sentry/opentelemetry": "^10.63.0", + "agents": "^0.17.3", + "aws4fetch": "^1.0.20", + "drizzle-orm": "^0.45.2", + "hono": "^4.12.27", + "ioredis": "^5.11.1", + "pg": "^8.22.0", + "posthog-node": "^5.44.0", + "sharp": "^0.34.5", + "yaml": "^2.9.0", + "zod": "^4.4.3" + }, + "devDependencies": { + "@cloudflare/vitest-pool-workers": "^0.18.0", + "@tktco/node-actionlint": "^1.6.0", + "@types/node": "^24.13.2", + "@types/pg": "^8.20.0", + "@types/pixelmatch": "^5.2.6", + "@types/pngjs": "^6.0.5", + "@vitest/coverage-v8": "^4.1.9", + "ajv": "^8.20.0", + "drizzle-kit": "^0.31.10", + "gifenc": "^1.0.3", + "git-cliff": "^2.13.1", + "github-actionlint": "^1.7.12", + "node-addon-api": "^8.9.0", + "node-gyp": "^12.4.0", + "pixelmatch": "^7.2.0", + "playwright": "^1.61.1", + "pngjs": "^7.0.0", + "tsx": "^4.22.5", + "turbo": "^2.10.5", + "typescript": "^5.9.3", + "vitest": "^4.1.9", + "wrangler": "^4.107.0" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@types/node": { + "version": "22.20.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.1.tgz", + "integrity": "sha512-EANqOCF9QFyra+4pfxUcX9STKJpCLjMbObVzljIJomAWSnuSIEAvyzEU53GaajbXJEgdh0iEcPL+DGvpUd4k1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/loopover": { + "resolved": "..", + "link": true + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/control-plane/package.json b/control-plane/package.json new file mode 100644 index 0000000000..9a28f9c470 --- /dev/null +++ b/control-plane/package.json @@ -0,0 +1,22 @@ +{ + "name": "@loopover/control-plane", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "LoopOver hosting control-plane — tenant provision/deprovision orchestration behind an injectable driver (#7524 / #7180). Fake/in-memory driver only in this package today; no live Cloudflare Containers or Postgres SDKs.", + "engines": { + "node": ">=20" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "test": "npm run build && npm run test:node", + "test:node": "node --test --experimental-strip-types \"test/**/*.test.ts\"" + }, + "devDependencies": { + "@types/node": "^22.20.0", + "typescript": "^5.9.3" + }, + "dependencies": { + "loopover": "file:.." + } +} diff --git a/control-plane/src/deprovision-tenant.ts b/control-plane/src/deprovision-tenant.ts new file mode 100644 index 0000000000..2fc04e128d --- /dev/null +++ b/control-plane/src/deprovision-tenant.ts @@ -0,0 +1,75 @@ +// `deprovisionTenant` orchestration (#7524 / #7180). Reverse of provision: revoke secrets → destroy DB → +// destroy container. Destroy of a nonexistent tenant is idempotent (`alreadyAbsent: true`). + +import { ControlPlaneError, isControlPlaneError } from "./errors.js"; +import type { TenantProvisioningDriver } from "./tenant-provisioning-driver.js"; +import type { DeprovisionedTenant, SecretsHandle, TenantRef } from "./types.js"; +import { assertValidTenantRef } from "./validate-tenant.js"; + +export type DeprovisionTenantOptions = { + driver: TenantProvisioningDriver; + /** + * Optional handles from a prior `provisionTenant` result. When omitted, the driver is asked to + * destroy using best-effort absent-tolerant calls (fake driver returns `"absent"`). + */ + provisioned?: + | { + containerId: string; + databaseId: string; + secrets: SecretsHandle; + } + | undefined; +}; + +/** + * Tear down a hosted tenant through the driver seam. Idempotent when the tenant is already gone. + */ +export async function deprovisionTenant( + tenantInput: TenantRef, + options: DeprovisionTenantOptions, +): Promise { + const tenant = assertValidTenantRef(tenantInput); + const { driver, provisioned } = options; + + let secretsResult: "revoked" | "absent" = "absent"; + let databaseResult: "destroyed" | "absent" = "absent"; + let containerResult: "destroyed" | "absent" = "absent"; + + if (provisioned) { + try { + secretsResult = await driver.revokeSecrets(tenant, provisioned.secrets.enrollId); + } catch (error) { + if (isControlPlaneError(error)) throw error; + throw new ControlPlaneError( + "secrets_revoke_failed", + `Secrets revoke failed for ${tenant.product}/${tenant.tenantId}`, + error, + ); + } + try { + databaseResult = await driver.destroyDatabase(tenant, provisioned.databaseId); + } catch (error) { + if (isControlPlaneError(error)) throw error; + throw new ControlPlaneError( + "database_destroy_failed", + `Database destroy failed for ${tenant.product}/${tenant.tenantId}`, + error, + ); + } + try { + containerResult = await driver.destroyContainer(tenant, provisioned.containerId); + } catch (error) { + if (isControlPlaneError(error)) throw error; + throw new ControlPlaneError( + "container_destroy_failed", + `Container destroy failed for ${tenant.product}/${tenant.tenantId}`, + error, + ); + } + } + + const alreadyAbsent = + secretsResult === "absent" && databaseResult === "absent" && containerResult === "absent"; + + return { tenant, state: "gone", alreadyAbsent }; +} diff --git a/control-plane/src/errors.ts b/control-plane/src/errors.ts new file mode 100644 index 0000000000..9f44c43b80 --- /dev/null +++ b/control-plane/src/errors.ts @@ -0,0 +1,28 @@ +// Control-plane error types (#7524). Keep messages operator-safe: no secrets, wallets, or raw tokens. + +export type ControlPlaneErrorCode = + | "invalid_tenant" + | "invalid_product" + | "container_create_failed" + | "database_provision_failed" + | "secrets_inject_failed" + | "container_destroy_failed" + | "database_destroy_failed" + | "secrets_revoke_failed" + | "rollback_failed"; + +export class ControlPlaneError extends Error { + readonly code: ControlPlaneErrorCode; + readonly causeError?: unknown; + + constructor(code: ControlPlaneErrorCode, message: string, causeError?: unknown) { + super(message); + this.name = "ControlPlaneError"; + this.code = code; + this.causeError = causeError; + } +} + +export function isControlPlaneError(value: unknown): value is ControlPlaneError { + return value instanceof ControlPlaneError; +} diff --git a/control-plane/src/fake-secret-broker.ts b/control-plane/src/fake-secret-broker.ts new file mode 100644 index 0000000000..7b059a56c0 --- /dev/null +++ b/control-plane/src/fake-secret-broker.ts @@ -0,0 +1,57 @@ +// In-memory `TenantSecretBroker` for control-plane tests (#7524). Records enrollments by id and +// secretType the same way #7174's broker column distinguishes mint strategies — without Worker Env. + +import { + type TenantSecretBroker, + type TenantSecretBrokerIssueInput, + type TenantSecretBrokerIssueResult, + type TenantSecretBrokerRevokeResult, +} from "./secret-broker.js"; + +export type FakeSecretBrokerEnrollment = { + enrollId: string; + tenantId: string; + product: string; + secretType: string; + revoked: boolean; +}; + +export type FakeTenantSecretBroker = TenantSecretBroker & { + readonly enrollments: ReadonlyMap; + reset(): void; +}; + +export function createFakeTenantSecretBroker( + options: { idPrefix?: string; sequence?: { next: number } } = {}, +): FakeTenantSecretBroker { + const enrollments = new Map(); + const sequence = options.sequence ?? { next: 1 }; + const idPrefix = options.idPrefix ?? "enr"; + + return { + get enrollments() { + return enrollments; + }, + reset() { + enrollments.clear(); + sequence.next = 1; + }, + async issueEnrollment(input: TenantSecretBrokerIssueInput): Promise { + const enrollId = `${idPrefix}_${sequence.next++}`; + enrollments.set(enrollId, { + enrollId, + tenantId: input.tenantId, + product: input.product, + secretType: input.secretType, + revoked: false, + }); + return { enrollId, secretShownOnce: true }; + }, + async revokeEnrollment(enrollId: string): Promise { + const row = enrollments.get(enrollId); + if (!row || row.revoked) return "not_found"; + row.revoked = true; + return "revoked"; + }, + }; +} diff --git a/control-plane/src/fake-tenant-provisioning-driver.ts b/control-plane/src/fake-tenant-provisioning-driver.ts new file mode 100644 index 0000000000..a837ccf1bd --- /dev/null +++ b/control-plane/src/fake-tenant-provisioning-driver.ts @@ -0,0 +1,124 @@ +// In-memory `TenantProvisioningDriver` (#7524). Mirrors `createFakeCodingAgentDriver`: maps stand in for +// "a container exists" / "a DB exists," toggled by the same create/destroy calls. injectSecrets always +// goes through the injected `TenantSecretBroker` (#7174 seam) — never a parallel secrets store. + +import { createFakeTenantSecretBroker } from "./fake-secret-broker.js"; +import { TENANT_SECRET_TYPE_DB_CREDENTIAL } from "./secret-broker.js"; +import type { TenantProvisioningDriver, TenantProvisioningDriverDeps } from "./tenant-provisioning-driver.js"; +import type { + ContainerHandle, + DatabaseHandle, + SecretsHandle, + TenantRef, +} from "./types.js"; + +function tenantKey(tenant: TenantRef): string { + return `${tenant.product}::${tenant.tenantId}`; +} + +export type FakeTenantProvisioningDriver = TenantProvisioningDriver & { + readonly containers: ReadonlyMap; + readonly databases: ReadonlyMap; + readonly secretsByTenant: ReadonlyMap; + containerExists(tenant: TenantRef): boolean; + databaseExists(tenant: TenantRef): boolean; + reset(): void; +}; + +export function createFakeTenantProvisioningDriver( + deps: Partial & { broker?: TenantProvisioningDriverDeps["broker"] } = {}, +): FakeTenantProvisioningDriver { + const broker = deps.broker ?? createFakeTenantSecretBroker(); + const secretType = deps.secretType ?? TENANT_SECRET_TYPE_DB_CREDENTIAL; + const nowMs = deps.nowMs ?? (() => Date.now()); + const containers = new Map(); + const databases = new Map(); + const secretsByTenant = new Map(); + let seq = 0; + + return { + get containers() { + return containers; + }, + get databases() { + return databases; + }, + get secretsByTenant() { + return secretsByTenant; + }, + containerExists(tenant) { + return containers.has(tenantKey(tenant)); + }, + databaseExists(tenant) { + return databases.has(tenantKey(tenant)); + }, + reset() { + containers.clear(); + databases.clear(); + secretsByTenant.clear(); + seq = 0; + }, + async createContainer(tenant) { + const key = tenantKey(tenant); + const existing = containers.get(key); + if (existing) return existing; + seq += 1; + const handle: ContainerHandle = { + containerId: `ctr_${tenant.product}_${tenant.tenantId}_${seq}_${nowMs()}`, + endpoint: `fake://${tenant.product}/${tenant.tenantId}`, + }; + containers.set(key, handle); + return handle; + }, + async provisionDatabase(tenant, container) { + const key = tenantKey(tenant); + const existing = databases.get(key); + if (existing) return existing; + seq += 1; + const handle: DatabaseHandle = { + databaseId: `db_${tenant.product}_${tenant.tenantId}_${seq}`, + connectionRef: `fake-db://${container.containerId}`, + }; + databases.set(key, handle); + return handle; + }, + async injectSecrets(tenant, _container, _database): Promise { + const key = tenantKey(tenant); + const existing = secretsByTenant.get(key); + if (existing) return existing; + const issued = await broker.issueEnrollment({ + tenantId: tenant.tenantId, + product: tenant.product, + secretType, + }); + const handle: SecretsHandle = { enrollId: issued.enrollId, secretType }; + secretsByTenant.set(key, handle); + return handle; + }, + async destroyContainer(tenant, containerId) { + const key = tenantKey(tenant); + const current = containers.get(key); + if (!current || current.containerId !== containerId) return "absent"; + containers.delete(key); + return "destroyed"; + }, + async destroyDatabase(tenant, databaseId) { + const key = tenantKey(tenant); + const current = databases.get(key); + if (!current || current.databaseId !== databaseId) return "absent"; + databases.delete(key); + return "destroyed"; + }, + async revokeSecrets(tenant, enrollId) { + const key = tenantKey(tenant); + const current = secretsByTenant.get(key); + if (!current || current.enrollId !== enrollId) { + const brokerResult = await broker.revokeEnrollment(enrollId); + return brokerResult === "revoked" ? "revoked" : "absent"; + } + const brokerResult = await broker.revokeEnrollment(enrollId); + secretsByTenant.delete(key); + return brokerResult === "revoked" ? "revoked" : "absent"; + }, + }; +} diff --git a/control-plane/src/index.ts b/control-plane/src/index.ts new file mode 100644 index 0000000000..954c14bbb8 --- /dev/null +++ b/control-plane/src/index.ts @@ -0,0 +1,37 @@ +// Public exports for `@loopover/control-plane` (#7524). + +export { ControlPlaneError, isControlPlaneError, type ControlPlaneErrorCode } from "./errors.js"; +export { + createFakeTenantSecretBroker, + type FakeSecretBrokerEnrollment, + type FakeTenantSecretBroker, +} from "./fake-secret-broker.js"; +export { + createFakeTenantProvisioningDriver, + type FakeTenantProvisioningDriver, +} from "./fake-tenant-provisioning-driver.js"; +export { deprovisionTenant, type DeprovisionTenantOptions } from "./deprovision-tenant.js"; +export { provisionTenant, type ProvisionTenantOptions } from "./provision-tenant.js"; +export { + TENANT_SECRET_TYPE_DB_CREDENTIAL, + TENANT_SECRET_TYPE_GITHUB_TOKEN, + type TenantSecretBroker, + type TenantSecretBrokerIssueInput, + type TenantSecretBrokerIssueResult, + type TenantSecretBrokerRevokeResult, +} from "./secret-broker.js"; +export type { + TenantProvisioningDriver, + TenantProvisioningDriverDeps, +} from "./tenant-provisioning-driver.js"; +export type { + ContainerHandle, + DatabaseHandle, + DeprovisionedTenant, + ProvisionedTenant, + SecretsHandle, + TenantId, + TenantProduct, + TenantRef, +} from "./types.js"; +export { assertValidTenantRef } from "./validate-tenant.js"; diff --git a/control-plane/src/provision-tenant.ts b/control-plane/src/provision-tenant.ts new file mode 100644 index 0000000000..e20a12b61c --- /dev/null +++ b/control-plane/src/provision-tenant.ts @@ -0,0 +1,97 @@ +// `provisionTenant` orchestration (#7524 / #7180). Product-agnostic: same steps for ORB and AMS. +// Runs create-container → provision-DB → inject-secrets against the injectable driver only. +// On mid-flight failure, rolls back earlier steps best-effort (destroy DB / container). + +import { ControlPlaneError, isControlPlaneError } from "./errors.js"; +import type { TenantProvisioningDriver } from "./tenant-provisioning-driver.js"; +import type { ProvisionedTenant, TenantRef } from "./types.js"; +import { assertValidTenantRef } from "./validate-tenant.js"; + +export type ProvisionTenantOptions = { + driver: TenantProvisioningDriver; +}; + +async function rollbackAfterSecretsFailure( + driver: TenantProvisioningDriver, + tenant: TenantRef, + containerId: string, + databaseId: string, + primary: unknown, +): Promise { + const destroyDb = await driver.destroyDatabase(tenant, databaseId).catch((error: unknown) => error); + const destroyCtr = await driver.destroyContainer(tenant, containerId).catch((error: unknown) => error); + if (destroyDb instanceof Error || destroyCtr instanceof Error) { + throw new ControlPlaneError( + "rollback_failed", + `Secrets inject failed and rollback also failed for ${tenant.product}/${tenant.tenantId}`, + { primary, destroyDb, destroyCtr }, + ); + } + if (isControlPlaneError(primary)) throw primary; + throw new ControlPlaneError( + "secrets_inject_failed", + `Secrets inject failed for ${tenant.product}/${tenant.tenantId}`, + primary, + ); +} + +async function rollbackAfterDatabaseFailure( + driver: TenantProvisioningDriver, + tenant: TenantRef, + containerId: string, + primary: unknown, +): Promise { + const destroyCtr = await driver.destroyContainer(tenant, containerId).catch((error: unknown) => error); + if (destroyCtr instanceof Error) { + throw new ControlPlaneError( + "rollback_failed", + `Database provision failed and container rollback also failed for ${tenant.product}/${tenant.tenantId}`, + { primary, destroyCtr }, + ); + } + if (isControlPlaneError(primary)) throw primary; + throw new ControlPlaneError( + "database_provision_failed", + `Database provision failed for ${tenant.product}/${tenant.tenantId}`, + primary, + ); +} + +/** + * Provision a hosted tenant through the driver seam. Does not call Cloudflare or Postgres directly. + */ +export async function provisionTenant( + tenantInput: TenantRef, + options: ProvisionTenantOptions, +): Promise { + const tenant = assertValidTenantRef(tenantInput); + const { driver } = options; + + let container; + try { + container = await driver.createContainer(tenant); + } catch (error) { + if (isControlPlaneError(error)) throw error; + throw new ControlPlaneError( + "container_create_failed", + `Container create failed for ${tenant.product}/${tenant.tenantId}`, + error, + ); + } + + let database; + try { + database = await driver.provisionDatabase(tenant, container); + } catch (error) { + return rollbackAfterDatabaseFailure(driver, tenant, container.containerId, error); + } + + let secrets; + try { + secrets = await driver.injectSecrets(tenant, container, database); + } catch (error) { + return rollbackAfterSecretsFailure(driver, tenant, container.containerId, database.databaseId, error); + } + + return { tenant, container, database, secrets, state: "ready" }; +} diff --git a/control-plane/src/secret-broker.ts b/control-plane/src/secret-broker.ts new file mode 100644 index 0000000000..d9aafe80ab --- /dev/null +++ b/control-plane/src/secret-broker.ts @@ -0,0 +1,39 @@ +// #7174 broker-shaped secret injection seam for tenant provisioning (#7524 / #7180). +// +// The Orb token broker already generalized enrollments with a `secret_type` column so a future +// AI-provider-key / DB-credential mint strategy can record what a row is FOR without a second table +// (`src/orb/broker.ts`). This control-plane seam calls into that model — it does not invent a +// parallel secrets store. Real wiring to `issueOrbEnrollment` / revoke paths lives in a follow-up +// driver; this package only defines the injectable contract + an in-memory fake for tests. + +/** The only type the live Orb broker mints today (#7174). */ +export const TENANT_SECRET_TYPE_GITHUB_TOKEN = "github_token"; + +/** + * Placeholder type for a future DB-credential mint strategy called out in #7174 / #7180. + * The fake broker accepts it so orchestration can prove the inject-secrets step records the type. + */ +export const TENANT_SECRET_TYPE_DB_CREDENTIAL = "db_credential"; + +export type TenantSecretBrokerIssueInput = { + tenantId: string; + product: string; + secretType: string; +}; + +export type TenantSecretBrokerIssueResult = { + enrollId: string; + /** Plaintext is shown once in the real broker; fakes only acknowledge the contract. */ + secretShownOnce: true; +}; + +export type TenantSecretBrokerRevokeResult = "revoked" | "not_found"; + +/** + * Injectable broker used by `TenantProvisioningDriver.injectSecrets` / revoke paths. + * Mirrors the #7174 enrollment issue/revoke shape without depending on Worker `Env`. + */ +export interface TenantSecretBroker { + issueEnrollment(input: TenantSecretBrokerIssueInput): Promise; + revokeEnrollment(enrollId: string): Promise; +} diff --git a/control-plane/src/tenant-provisioning-driver.ts b/control-plane/src/tenant-provisioning-driver.ts new file mode 100644 index 0000000000..8b38ef8951 --- /dev/null +++ b/control-plane/src/tenant-provisioning-driver.ts @@ -0,0 +1,41 @@ +// `TenantProvisioningDriver` injectable seam (#7524). Mirrors `CodingAgentDriver` +// (`packages/loopover-engine/src/miner/coding-agent-driver.ts`): a small interface, provider-agnostic +// handles, and concrete fakes that hold in-memory state so orchestration can be tested without IO. +// +// The three #7180 steps are explicit methods — create-container, provision-DB, inject-secrets — plus +// matching destroy/revoke for deprovision. Real Cloudflare Containers / Postgres SDKs must NOT appear +// in implementations shipped with this issue. + +import type { TenantSecretBroker } from "./secret-broker.js"; +import type { + ContainerHandle, + DatabaseHandle, + SecretsHandle, + TenantRef, +} from "./types.js"; + +export type TenantProvisioningDriver = { + createContainer(tenant: TenantRef): Promise; + provisionDatabase(tenant: TenantRef, container: ContainerHandle): Promise; + /** + * Inject tenant credentials via the #7174 broker seam (`TenantSecretBroker`), not a hand-rolled + * secrets path. Drivers receive the broker at construction time. + */ + injectSecrets( + tenant: TenantRef, + container: ContainerHandle, + database: DatabaseHandle, + ): Promise; + destroyContainer(tenant: TenantRef, containerId: string): Promise<"destroyed" | "absent">; + destroyDatabase(tenant: TenantRef, databaseId: string): Promise<"destroyed" | "absent">; + revokeSecrets(tenant: TenantRef, enrollId: string): Promise<"revoked" | "absent">; +}; + +export type TenantProvisioningDriverDeps = { + /** Required — inject-secrets must call the broker (#7174 / #7524). */ + broker: TenantSecretBroker; + /** Secret type recorded on enrollment; defaults to the DB-credential placeholder type. */ + secretType?: string | undefined; + /** Optional clock for deterministic fake IDs in tests. */ + nowMs?: (() => number) | undefined; +}; diff --git a/control-plane/src/types.ts b/control-plane/src/types.ts new file mode 100644 index 0000000000..e93c2b481f --- /dev/null +++ b/control-plane/src/types.ts @@ -0,0 +1,50 @@ +// Shared tenant / product types for the control-plane provisioning core (#7524 / #7180). +// Product-agnostic on purpose: the same call shape serves an ORB tenant and an AMS tenant. + +/** Stable tenant identifier within a product plane (not a GitHub login, not a wallet). */ +export type TenantId = string; + +/** + * Product key for the hosted plane. Open string so new products do not require orchestration + * changes — callers today use `"orb"` / `"ams"`. + */ +export type TenantProduct = string; + +/** Tenant identity passed through every provisioning step. */ +export type TenantRef = { + tenantId: TenantId; + product: TenantProduct; +}; + +export type ContainerHandle = { + containerId: string; + /** Opaque reachability hint used by fakes/tests; real drivers may omit. */ + endpoint?: string | undefined; +}; + +export type DatabaseHandle = { + databaseId: string; + /** Opaque connection reference — never a live password in this package. */ + connectionRef?: string | undefined; +}; + +export type SecretsHandle = { + enrollId: string; + /** Broker secret type from the #7174 generalization (e.g. github_token / db_credential). */ + secretType: string; +}; + +export type ProvisionedTenant = { + tenant: TenantRef; + container: ContainerHandle; + database: DatabaseHandle; + secrets: SecretsHandle; + state: "ready"; +}; + +export type DeprovisionedTenant = { + tenant: TenantRef; + state: "gone"; + /** True when the driver reported the tenant was already absent (idempotent destroy). */ + alreadyAbsent: boolean; +}; diff --git a/control-plane/src/validate-tenant.ts b/control-plane/src/validate-tenant.ts new file mode 100644 index 0000000000..d5f896719a --- /dev/null +++ b/control-plane/src/validate-tenant.ts @@ -0,0 +1,19 @@ +// Tenant ref validation shared by provision / deprovision (#7524). + +import { ControlPlaneError } from "./errors.js"; +import type { TenantRef } from "./types.js"; + +const TENANT_ID_RE = /^[a-z0-9][a-z0-9_-]{0,62}$/i; +const PRODUCT_RE = /^[a-z][a-z0-9_-]{0,31}$/i; + +export function assertValidTenantRef(tenant: TenantRef): TenantRef { + const tenantId = String(tenant.tenantId ?? "").trim(); + const product = String(tenant.product ?? "").trim().toLowerCase(); + if (!tenantId || !TENANT_ID_RE.test(tenantId)) { + throw new ControlPlaneError("invalid_tenant", `Invalid tenantId: ${JSON.stringify(tenant.tenantId)}`); + } + if (!product || !PRODUCT_RE.test(product)) { + throw new ControlPlaneError("invalid_product", `Invalid product: ${JSON.stringify(tenant.product)}`); + } + return { tenantId, product }; +} diff --git a/control-plane/test/provision-lifecycle.test.ts b/control-plane/test/provision-lifecycle.test.ts new file mode 100644 index 0000000000..9dcc54725a --- /dev/null +++ b/control-plane/test/provision-lifecycle.test.ts @@ -0,0 +1,150 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + ControlPlaneError, + createFakeTenantProvisioningDriver, + createFakeTenantSecretBroker, + deprovisionTenant, + isControlPlaneError, + provisionTenant, + TENANT_SECRET_TYPE_DB_CREDENTIAL, + TENANT_SECRET_TYPE_GITHUB_TOKEN, +} from "../dist/index.js"; + +test("#7524: provisionTenant create → container exists → deprovision → container gone (fake driver)", async () => { + const broker = createFakeTenantSecretBroker(); + const driver = createFakeTenantProvisioningDriver({ broker, nowMs: () => 1_700_000_000_000 }); + const tenant = { tenantId: "acme-1", product: "orb" }; + + const provisioned = await provisionTenant(tenant, { driver }); + assert.equal(provisioned.state, "ready"); + assert.equal(provisioned.tenant.product, "orb"); + assert.equal(driver.containerExists(tenant), true); + assert.equal(driver.databaseExists(tenant), true); + assert.equal(provisioned.secrets.secretType, TENANT_SECRET_TYPE_DB_CREDENTIAL); + assert.ok(broker.enrollments.has(provisioned.secrets.enrollId)); + + const gone = await deprovisionTenant(tenant, { + driver, + provisioned: { + containerId: provisioned.container.containerId, + databaseId: provisioned.database.databaseId, + secrets: provisioned.secrets, + }, + }); + assert.equal(gone.state, "gone"); + assert.equal(gone.alreadyAbsent, false); + assert.equal(driver.containerExists(tenant), false); + assert.equal(driver.databaseExists(tenant), false); + assert.equal(broker.enrollments.get(provisioned.secrets.enrollId)?.revoked, true); +}); + +test("#7524: product-agnostic — AMS uses the same provision/deprovision call shape", async () => { + const driver = createFakeTenantProvisioningDriver({ nowMs: () => 42 }); + const tenant = { tenantId: "ams-tenant", product: "AMS" }; + const provisioned = await provisionTenant(tenant, { driver }); + assert.equal(provisioned.tenant.product, "ams"); + assert.equal(driver.containerExists({ tenantId: "ams-tenant", product: "ams" }), true); + + await deprovisionTenant( + { tenantId: "ams-tenant", product: "ams" }, + { + driver, + provisioned: { + containerId: provisioned.container.containerId, + databaseId: provisioned.database.databaseId, + secrets: provisioned.secrets, + }, + }, + ); + assert.equal(driver.containerExists({ tenantId: "ams-tenant", product: "ams" }), false); +}); + +test("#7524: deprovision of a nonexistent tenant is idempotent (alreadyAbsent)", async () => { + const driver = createFakeTenantProvisioningDriver(); + const tenant = { tenantId: "missing", product: "orb" }; + const gone = await deprovisionTenant(tenant, { + driver, + provisioned: { + containerId: "ctr_missing", + databaseId: "db_missing", + secrets: { enrollId: "enr_missing", secretType: TENANT_SECRET_TYPE_DB_CREDENTIAL }, + }, + }); + assert.equal(gone.state, "gone"); + assert.equal(gone.alreadyAbsent, true); +}); + +test("#7524: deprovision without provisioned handles reports alreadyAbsent", async () => { + const driver = createFakeTenantProvisioningDriver(); + const gone = await deprovisionTenant({ tenantId: "x", product: "orb" }, { driver }); + assert.equal(gone.alreadyAbsent, true); +}); + +test("#7524: inject-secrets records #7174 secret_type via the broker seam", async () => { + const broker = createFakeTenantSecretBroker(); + const driver = createFakeTenantProvisioningDriver({ + broker, + secretType: TENANT_SECRET_TYPE_GITHUB_TOKEN, + nowMs: () => 1, + }); + const provisioned = await provisionTenant({ tenantId: "sec", product: "orb" }, { driver }); + const row = broker.enrollments.get(provisioned.secrets.enrollId); + assert.equal(row?.secretType, TENANT_SECRET_TYPE_GITHUB_TOKEN); + assert.equal(provisioned.secrets.secretType, TENANT_SECRET_TYPE_GITHUB_TOKEN); +}); + +test("#7524: rejects invalid tenantId / product before any driver call", async () => { + const driver = createFakeTenantProvisioningDriver(); + await assert.rejects( + () => provisionTenant({ tenantId: "", product: "orb" }, { driver }), + (error: unknown) => isControlPlaneError(error) && error.code === "invalid_tenant", + ); + await assert.rejects( + () => provisionTenant({ tenantId: "ok", product: "!!" }, { driver }), + (error: unknown) => isControlPlaneError(error) && error.code === "invalid_product", + ); + assert.equal(driver.containers.size, 0); +}); + +test("#7524: rolls back container when database provision throws", async () => { + const driver = createFakeTenantProvisioningDriver({ nowMs: () => 9 }); + driver.provisionDatabase = async () => { + throw new Error("db boom"); + }; + await assert.rejects( + () => provisionTenant({ tenantId: "roll-db", product: "orb" }, { driver }), + (error: unknown) => error instanceof ControlPlaneError && error.code === "database_provision_failed", + ); + assert.equal(driver.containerExists({ tenantId: "roll-db", product: "orb" }), false); + assert.equal(driver.databaseExists({ tenantId: "roll-db", product: "orb" }), false); +}); + +test("#7524: rolls back database + container when secrets inject throws", async () => { + const driver = createFakeTenantProvisioningDriver({ nowMs: () => 8 }); + driver.injectSecrets = async () => { + throw new Error("broker boom"); + }; + await assert.rejects( + () => provisionTenant({ tenantId: "roll-sec", product: "orb" }, { driver }), + (error: unknown) => error instanceof ControlPlaneError && error.code === "secrets_inject_failed", + ); + assert.equal(driver.containerExists({ tenantId: "roll-sec", product: "orb" }), false); + assert.equal(driver.databaseExists({ tenantId: "roll-sec", product: "orb" }), false); +}); + +test("#7524: fake driver create is idempotent for the same tenant key", async () => { + const driver = createFakeTenantProvisioningDriver({ nowMs: () => 3 }); + const tenant = { tenantId: "same", product: "orb" }; + const a = await driver.createContainer(tenant); + const b = await driver.createContainer(tenant); + assert.equal(a.containerId, b.containerId); + assert.equal(driver.containers.size, 1); +}); + +test("isControlPlaneError narrows only ControlPlaneError instances", () => { + assert.equal(isControlPlaneError(new ControlPlaneError("invalid_tenant", "x")), true); + assert.equal(isControlPlaneError(new Error("x")), false); + assert.equal(isControlPlaneError(null), false); +}); diff --git a/control-plane/test/secret-broker.test.ts b/control-plane/test/secret-broker.test.ts new file mode 100644 index 0000000000..96b7ef44d1 --- /dev/null +++ b/control-plane/test/secret-broker.test.ts @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { createFakeTenantSecretBroker } from "../dist/fake-secret-broker.js"; +import { TENANT_SECRET_TYPE_DB_CREDENTIAL } from "../dist/secret-broker.js"; + +test("fake broker issue + revoke lifecycle", async () => { + const broker = createFakeTenantSecretBroker({ idPrefix: "t" }); + const issued = await broker.issueEnrollment({ + tenantId: "a", + product: "orb", + secretType: TENANT_SECRET_TYPE_DB_CREDENTIAL, + }); + assert.equal(issued.secretShownOnce, true); + assert.equal(await broker.revokeEnrollment(issued.enrollId), "revoked"); + assert.equal(await broker.revokeEnrollment(issued.enrollId), "not_found"); + assert.equal(await broker.revokeEnrollment("missing"), "not_found"); + broker.reset(); + assert.equal(broker.enrollments.size, 0); +}); diff --git a/control-plane/test/validate-tenant.test.ts b/control-plane/test/validate-tenant.test.ts new file mode 100644 index 0000000000..d09269d20b --- /dev/null +++ b/control-plane/test/validate-tenant.test.ts @@ -0,0 +1,23 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { assertValidTenantRef } from "../dist/validate-tenant.js"; +import { ControlPlaneError } from "../dist/errors.js"; + +test("assertValidTenantRef normalizes product case", () => { + assert.deepEqual(assertValidTenantRef({ tenantId: "T1", product: "ORB" }), { + tenantId: "T1", + product: "orb", + }); +}); + +test("assertValidTenantRef rejects empty / illegal ids", () => { + assert.throws( + () => assertValidTenantRef({ tenantId: "bad id", product: "orb" }), + (error: unknown) => error instanceof ControlPlaneError && error.code === "invalid_tenant", + ); + assert.throws( + () => assertValidTenantRef({ tenantId: "ok", product: "" }), + (error: unknown) => error instanceof ControlPlaneError && error.code === "invalid_product", + ); +}); diff --git a/control-plane/tsconfig.json b/control-plane/tsconfig.json new file mode 100644 index 0000000000..9da9cf5b20 --- /dev/null +++ b/control-plane/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "strict": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": true, + "sourceMap": true, + "inlineSources": true + }, + "include": ["src"] +} diff --git a/package.json b/package.json index 6cd4729b10..29d9fa0a01 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,8 @@ "test:miner-deployment-docs-audit": "node scripts/check-miner-deployment-docs.mjs", "rees:install": "npm ci --prefix review-enrichment --prefer-offline --no-audit --no-fund", "rees:test": "npm run rees:install && npm --prefix review-enrichment test", + "control-plane:install": "npm ci --prefix control-plane --prefer-offline --no-audit --no-fund", + "control-plane:test": "npm run control-plane:install && npm --prefix control-plane test", "rees:metadata": "npm --prefix review-enrichment run metadata", "rees:metadata:check": "npm --prefix review-enrichment run metadata:check", "rees:validate-sourcemaps": "npm --prefix review-enrichment run validate:sourcemaps", @@ -100,7 +102,7 @@ "test:smoke:observability:metrics": "node scripts/smoke-observability-metrics.mjs", "test:smoke:browser:install": "playwright install chromium", "test:smoke:browser": "node scripts/smoke-ui-browser.mjs", - "test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run test --workspace @loopover/engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run release-manifest:sync:check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", + "test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run test --workspace @loopover/engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run control-plane:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run release-manifest:sync:check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", "test:release": "npm run test:ci && npm run changelog:check", "test:release:mcp": "npm run test:ci", "test:watch": "vitest", diff --git a/src/config/loopover-repo-focus-manifest.ts b/src/config/loopover-repo-focus-manifest.ts index b027c8a8ff..dda8dec216 100644 --- a/src/config/loopover-repo-focus-manifest.ts +++ b/src/config/loopover-repo-focus-manifest.ts @@ -14,6 +14,7 @@ wantedPaths: - migrations/ - scripts/ - review-enrichment/ + - control-plane/ - .github/workflows/ - wrangler.jsonc - apps/loopover-ui/ diff --git a/test/unit/loopover-focus-manifest.test.ts b/test/unit/loopover-focus-manifest.test.ts index 4dd0ee10d9..4025ccbf10 100644 --- a/test/unit/loopover-focus-manifest.test.ts +++ b/test/unit/loopover-focus-manifest.test.ts @@ -42,6 +42,7 @@ describe("LoopOver repo focus manifest", () => { expect(manifest.present).toBe(true); expect(manifest.wantedPaths).toContain("src/"); expect(manifest.wantedPaths).toContain("review-enrichment/"); + expect(manifest.wantedPaths).toContain("control-plane/"); expect(manifest.wantedPaths).toContain("apps/loopover-ui/"); expect(manifest.issueDiscoveryPolicy).toBe("discouraged"); From 14e765f30962fae69064f1042372d2dc276e11c7 Mon Sep 17 00:00:00 2001 From: jsdevninja Date: Mon, 20 Jul 2026 16:28:11 -0500 Subject: [PATCH 2/2] fix(control-plane): always invoke driver teardown when deprovisioning Co-authored-by: Cursor --- control-plane/src/deprovision-tenant.ts | 73 ++++++++++--------- .../src/fake-tenant-provisioning-driver.ts | 16 +++- .../src/tenant-provisioning-driver.ts | 10 ++- .../test/provision-lifecycle.test.ts | 59 ++++++++++++++- 4 files changed, 115 insertions(+), 43 deletions(-) diff --git a/control-plane/src/deprovision-tenant.ts b/control-plane/src/deprovision-tenant.ts index 2fc04e128d..5cfae17237 100644 --- a/control-plane/src/deprovision-tenant.ts +++ b/control-plane/src/deprovision-tenant.ts @@ -1,5 +1,8 @@ // `deprovisionTenant` orchestration (#7524 / #7180). Reverse of provision: revoke secrets → destroy DB → // destroy container. Destroy of a nonexistent tenant is idempotent (`alreadyAbsent: true`). +// +// #7539 follow-up: when `provisioned` handles are omitted (e.g. caller restarted and lost in-memory IDs), +// still ask the driver to tear down by tenant — never return `{ state: "gone" }` without driver calls. import { ControlPlaneError, isControlPlaneError } from "./errors.js"; import type { TenantProvisioningDriver } from "./tenant-provisioning-driver.js"; @@ -10,7 +13,8 @@ export type DeprovisionTenantOptions = { driver: TenantProvisioningDriver; /** * Optional handles from a prior `provisionTenant` result. When omitted, the driver is asked to - * destroy using best-effort absent-tolerant calls (fake driver returns `"absent"`). + * destroy/revoke by tenant with absent-tolerant calls (IDs left undefined so the driver resolves + * whatever it still has recorded for that tenant). */ provisioned?: | { @@ -23,6 +27,7 @@ export type DeprovisionTenantOptions = { /** * Tear down a hosted tenant through the driver seam. Idempotent when the tenant is already gone. + * Always invokes revoke/destroy on the driver — even when `provisioned` is omitted. */ export async function deprovisionTenant( tenantInput: TenantRef, @@ -31,41 +36,39 @@ export async function deprovisionTenant( const tenant = assertValidTenantRef(tenantInput); const { driver, provisioned } = options; - let secretsResult: "revoked" | "absent" = "absent"; - let databaseResult: "destroyed" | "absent" = "absent"; - let containerResult: "destroyed" | "absent" = "absent"; + let secretsResult: "revoked" | "absent"; + let databaseResult: "destroyed" | "absent"; + let containerResult: "destroyed" | "absent"; - if (provisioned) { - try { - secretsResult = await driver.revokeSecrets(tenant, provisioned.secrets.enrollId); - } catch (error) { - if (isControlPlaneError(error)) throw error; - throw new ControlPlaneError( - "secrets_revoke_failed", - `Secrets revoke failed for ${tenant.product}/${tenant.tenantId}`, - error, - ); - } - try { - databaseResult = await driver.destroyDatabase(tenant, provisioned.databaseId); - } catch (error) { - if (isControlPlaneError(error)) throw error; - throw new ControlPlaneError( - "database_destroy_failed", - `Database destroy failed for ${tenant.product}/${tenant.tenantId}`, - error, - ); - } - try { - containerResult = await driver.destroyContainer(tenant, provisioned.containerId); - } catch (error) { - if (isControlPlaneError(error)) throw error; - throw new ControlPlaneError( - "container_destroy_failed", - `Container destroy failed for ${tenant.product}/${tenant.tenantId}`, - error, - ); - } + try { + secretsResult = await driver.revokeSecrets(tenant, provisioned?.secrets.enrollId); + } catch (error) { + if (isControlPlaneError(error)) throw error; + throw new ControlPlaneError( + "secrets_revoke_failed", + `Secrets revoke failed for ${tenant.product}/${tenant.tenantId}`, + error, + ); + } + try { + databaseResult = await driver.destroyDatabase(tenant, provisioned?.databaseId); + } catch (error) { + if (isControlPlaneError(error)) throw error; + throw new ControlPlaneError( + "database_destroy_failed", + `Database destroy failed for ${tenant.product}/${tenant.tenantId}`, + error, + ); + } + try { + containerResult = await driver.destroyContainer(tenant, provisioned?.containerId); + } catch (error) { + if (isControlPlaneError(error)) throw error; + throw new ControlPlaneError( + "container_destroy_failed", + `Container destroy failed for ${tenant.product}/${tenant.tenantId}`, + error, + ); } const alreadyAbsent = diff --git a/control-plane/src/fake-tenant-provisioning-driver.ts b/control-plane/src/fake-tenant-provisioning-driver.ts index a837ccf1bd..78922644aa 100644 --- a/control-plane/src/fake-tenant-provisioning-driver.ts +++ b/control-plane/src/fake-tenant-provisioning-driver.ts @@ -98,25 +98,33 @@ export function createFakeTenantProvisioningDriver( async destroyContainer(tenant, containerId) { const key = tenantKey(tenant); const current = containers.get(key); - if (!current || current.containerId !== containerId) return "absent"; + if (!current) return "absent"; + // When containerId is omitted, destroy whatever container this tenant still holds (lost-handle path). + if (containerId !== undefined && current.containerId !== containerId) return "absent"; containers.delete(key); return "destroyed"; }, async destroyDatabase(tenant, databaseId) { const key = tenantKey(tenant); const current = databases.get(key); - if (!current || current.databaseId !== databaseId) return "absent"; + if (!current) return "absent"; + if (databaseId !== undefined && current.databaseId !== databaseId) return "absent"; databases.delete(key); return "destroyed"; }, async revokeSecrets(tenant, enrollId) { const key = tenantKey(tenant); const current = secretsByTenant.get(key); - if (!current || current.enrollId !== enrollId) { + if (!current) { + if (enrollId === undefined) return "absent"; const brokerResult = await broker.revokeEnrollment(enrollId); return brokerResult === "revoked" ? "revoked" : "absent"; } - const brokerResult = await broker.revokeEnrollment(enrollId); + if (enrollId !== undefined && current.enrollId !== enrollId) { + const brokerResult = await broker.revokeEnrollment(enrollId); + return brokerResult === "revoked" ? "revoked" : "absent"; + } + const brokerResult = await broker.revokeEnrollment(current.enrollId); secretsByTenant.delete(key); return brokerResult === "revoked" ? "revoked" : "absent"; }, diff --git a/control-plane/src/tenant-provisioning-driver.ts b/control-plane/src/tenant-provisioning-driver.ts index 8b38ef8951..d0c364fa4e 100644 --- a/control-plane/src/tenant-provisioning-driver.ts +++ b/control-plane/src/tenant-provisioning-driver.ts @@ -26,9 +26,13 @@ export type TenantProvisioningDriver = { container: ContainerHandle, database: DatabaseHandle, ): Promise; - destroyContainer(tenant: TenantRef, containerId: string): Promise<"destroyed" | "absent">; - destroyDatabase(tenant: TenantRef, databaseId: string): Promise<"destroyed" | "absent">; - revokeSecrets(tenant: TenantRef, enrollId: string): Promise<"revoked" | "absent">; + destroyContainer(tenant: TenantRef, containerId?: string): Promise<"destroyed" | "absent">; + destroyDatabase(tenant: TenantRef, databaseId?: string): Promise<"destroyed" | "absent">; + /** + * Revoke broker enrollments for the tenant. When `enrollId` is omitted, destroy whatever secrets + * the driver has recorded for this tenant (best-effort recovery after lost in-memory handles). + */ + revokeSecrets(tenant: TenantRef, enrollId?: string): Promise<"revoked" | "absent">; }; export type TenantProvisioningDriverDeps = { diff --git a/control-plane/test/provision-lifecycle.test.ts b/control-plane/test/provision-lifecycle.test.ts index 9dcc54725a..995e448739 100644 --- a/control-plane/test/provision-lifecycle.test.ts +++ b/control-plane/test/provision-lifecycle.test.ts @@ -76,10 +76,67 @@ test("#7524: deprovision of a nonexistent tenant is idempotent (alreadyAbsent)", assert.equal(gone.alreadyAbsent, true); }); -test("#7524: deprovision without provisioned handles reports alreadyAbsent", async () => { +test("REGRESSION (#7539): deprovision without provisioned handles still asks the driver to tear down live resources", async () => { + const broker = createFakeTenantSecretBroker(); + const driver = createFakeTenantProvisioningDriver({ broker, nowMs: () => 99 }); + const tenant = { tenantId: "lost-handles", product: "orb" }; + const provisioned = await provisionTenant(tenant, { driver }); + assert.equal(driver.containerExists(tenant), true); + assert.equal(driver.databaseExists(tenant), true); + + // Caller lost in-memory handles (e.g. process restart) but the driver still knows the tenant. + const calls = { revoke: 0, destroyDb: 0, destroyCtr: 0 }; + const baseRevoke = driver.revokeSecrets.bind(driver); + const baseDestroyDb = driver.destroyDatabase.bind(driver); + const baseDestroyCtr = driver.destroyContainer.bind(driver); + driver.revokeSecrets = async (t, enrollId) => { + calls.revoke += 1; + return baseRevoke(t, enrollId); + }; + driver.destroyDatabase = async (t, databaseId) => { + calls.destroyDb += 1; + return baseDestroyDb(t, databaseId); + }; + driver.destroyContainer = async (t, containerId) => { + calls.destroyCtr += 1; + return baseDestroyCtr(t, containerId); + }; + + const gone = await deprovisionTenant(tenant, { driver }); + assert.equal(gone.state, "gone"); + assert.equal(gone.alreadyAbsent, false); + assert.equal(calls.revoke, 1); + assert.equal(calls.destroyDb, 1); + assert.equal(calls.destroyCtr, 1); + assert.equal(driver.containerExists(tenant), false); + assert.equal(driver.databaseExists(tenant), false); + assert.equal(broker.enrollments.get(provisioned.secrets.enrollId)?.revoked, true); +}); + +test("REGRESSION (#7539): deprovision without handles on an empty tenant is absent-tolerant but still calls the driver", async () => { const driver = createFakeTenantProvisioningDriver(); + const calls = { revoke: 0, destroyDb: 0, destroyCtr: 0 }; + const baseRevoke = driver.revokeSecrets.bind(driver); + const baseDestroyDb = driver.destroyDatabase.bind(driver); + const baseDestroyCtr = driver.destroyContainer.bind(driver); + driver.revokeSecrets = async (t, enrollId) => { + calls.revoke += 1; + return baseRevoke(t, enrollId); + }; + driver.destroyDatabase = async (t, databaseId) => { + calls.destroyDb += 1; + return baseDestroyDb(t, databaseId); + }; + driver.destroyContainer = async (t, containerId) => { + calls.destroyCtr += 1; + return baseDestroyCtr(t, containerId); + }; + const gone = await deprovisionTenant({ tenantId: "x", product: "orb" }, { driver }); assert.equal(gone.alreadyAbsent, true); + assert.equal(calls.revoke, 1); + assert.equal(calls.destroyDb, 1); + assert.equal(calls.destroyCtr, 1); }); test("#7524: inject-secrets records #7174 secret_type via the broker seam", async () => {