Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 21 additions & 6 deletions control-plane/src/container-driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,16 +67,31 @@ function bindingFor(config: ContainerDriverConfig, product: Product): ContainerN
* `pinnedVersion` rides into the container, whose entrypoint resolves the versioned artifact itself. */
export const PINNED_VERSION_ENV_VAR = "LOOPOVER_PINNED_VERSION";

/** The env var a tenant's container reads its one-time secret-bootstrap credential from at cold boot (#8202).
* Deliberately product-agnostic (no `ORB_`/`AMS_` prefix), same reasoning as {@link PINNED_VERSION_ENV_VAR}:
* both `OrbTenantContainer` and `AmsTenantContainer` (#8246) read the identical name. The value itself is a
* one-time secret from `injectSecrets` (`TenantProvisioningRequest.bootstrapSecret`) the container exchanges
* via `POST /v1/orb/token` (`src/orb/broker-client.ts`'s `fetchBrokeredStoredSecret`) for whatever the broker
* actually has custodied -- this driver never sees or needs to know what that is. */
export const TENANT_SECRET_ENV_VAR = "LOOPOVER_TENANT_SECRET_TOKEN";

/** Idempotent: an already-provisioned tenant's container is left running as-is, never restarted -- a repeat
* create must not interrupt a container mid-work. A tenant with a `pinnedVersion` (#4898) starts with that
* version in {@link PINNED_VERSION_ENV_VAR}; an unpinned tenant gets the exact pre-#4898 `start()` call, so
* every existing tenant's behavior is byte-identical until a rollout pins it. */
* create must not interrupt a container mid-work. This is also the ONLY point in a tenant's lifecycle where
* `envVars` actually reach the container (confirmed against the real `@cloudflare/containers` SDK: a `start()`
* call against an already-running/starting instance is a no-op or throws, never re-applies `envVars`) -- so
* both of the values below must already be known by the time this runs, not supplied later. A tenant with a
* `pinnedVersion` (#4898) starts with that version in {@link PINNED_VERSION_ENV_VAR}; one with a
* `bootstrapSecret` (#8202, set on `request` by `provisionTenant` from `injectSecrets`' result) starts with it
* in {@link TENANT_SECRET_ENV_VAR}; a tenant with neither gets the exact pre-#4898 `start()` call, so every
* existing tenant's behavior is byte-identical until either rollout applies. */
export async function createTenantContainer(config: ContainerDriverConfig, request: TenantProvisioningRequest): Promise<void> {
const stub = bindingFor(config, request.product).getByName(instanceNameFor(request));
if (await stub.isProvisioned()) return;
const pinnedVersion = request.tenant.pinnedVersion;
if (pinnedVersion) {
await stub.start({ envVars: { [PINNED_VERSION_ENV_VAR]: pinnedVersion } });
const envVars: Record<string, string> = {};
if (request.tenant.pinnedVersion) envVars[PINNED_VERSION_ENV_VAR] = request.tenant.pinnedVersion;
if (request.bootstrapSecret) envVars[TENANT_SECRET_ENV_VAR] = request.bootstrapSecret;
if (Object.keys(envVars).length > 0) {
await stub.start({ envVars });
} else {
await stub.start();
}
Expand Down
1 change: 1 addition & 0 deletions control-plane/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ export {
destroyTenantContainer,
instanceNameFor,
PINNED_VERSION_ENV_VAR,
TENANT_SECRET_ENV_VAR,
tenantContainerExists,
type ContainerDriver,
type ContainerDriverConfig,
Expand Down
33 changes: 21 additions & 12 deletions control-plane/src/provisioning.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
// provisionTenant / deprovisionTenant orchestration (#7524) over the injectable `TenantProvisioningDriver`.
// Product-agnostic: an ORB tenant and an AMS tenant take the identical call shape — `product` is forwarded to
// every driver step but never branched on. Provision runs #7180's three steps in order (create-container,
// provision-DB, inject-secrets); deprovision tears them down in REVERSE (revoke-secrets, drop-DB,
// destroy-container) so a secret is never left addressable after the DB/container it belonged to is gone.
// every driver step but never branched on. Provision runs #7180's three steps as provision-DB, inject-secrets,
// create-container (#8202 reordered this from the original create-container-first sequence: a tenant's
// bootstrap secret, produced by inject-secrets, must exist BEFORE create-container's one real `stub.start()`
// call, since Cloudflare Containers only ever apply `envVars` at a container's actual cold (re)start -- never
// as a live update to one already running or starting, confirmed against the real `@cloudflare/containers` SDK).
// Deprovision tears down in the order revoke-secrets, drop-DB, destroy-container -- REVERSE of the ORIGINAL
// #7180 order, kept deliberately unchanged by #8202's reorder: revoking a secret before the DB/container it
// belonged to is gone is the security property that matters here, not exact step-order symmetry with provision.
//
// #7667: a driver-step failure in EITHER direction also pages, via the same PagerDuty Events API v2 contract
// ORB uses in `src/services/notify-pagerduty.ts` (see ./pagerduty-notify.ts for the mirrored contract and why
Expand Down Expand Up @@ -81,13 +86,16 @@ function pageAndRethrow(
throw error;
}

/** Provision a tenant by running #7180's three steps in order against the injected driver. Product-agnostic:
* `product` is forwarded to every step, never branched on, so ORB and AMS share one call shape. `injectSecrets`
* is called with `database` already attached to the request (#8066) -- a real secret driver needs the
* connection details to actually store, not just the tenant identity every other step operates on. A step
* failure pages (#7667) and always rethrows — provisioning never fails silently. `onFailure` (#7677,
* optional) runs first in that failure path — the caller's seam for persisting the `"failed"` lifecycle
* state — and is best-effort: its own rejection is swallowed so it can never mask the step error. */
/** Provision a tenant by running #7180's three steps against the injected driver, in the order database, secrets,
* container (#8202 -- see this module's header for why). Product-agnostic: `product` is forwarded to every step,
* never branched on, so ORB and AMS share one call shape. `injectSecrets` is called with `database` already
* attached to the request (#8066) -- a real secret driver needs the connection details to actually store, not
* just the tenant identity every other step operates on. `createContainer` is in turn called with `database`
* still attached AND `bootstrapSecret` newly attached (#8202) whenever `injectSecrets` returned one -- a real
* container driver delivers it into the container's own cold-boot environment. A step failure pages (#7667) and
* always rethrows — provisioning never fails silently. `onFailure` (#7677, optional) runs first in that failure
* path — the caller's seam for persisting the `"failed"` lifecycle state — and is best-effort: its own
* rejection is swallowed so it can never mask the step error. */
export async function provisionTenant(
tenant: Tenant,
product: Product,
Expand All @@ -99,9 +107,10 @@ export async function provisionTenant(
let database: DatabaseConnectionDetails;
let secretRef: string | undefined;
try {
await driver.createContainer(request);
database = await driver.provisionDatabase(request);
({ secretRef } = await driver.injectSecrets({ ...request, database }));
const injected = await driver.injectSecrets({ ...request, database });
secretRef = injected.secretRef;
await driver.createContainer({ ...request, database, ...(injected.bootstrapSecret !== undefined ? { bootstrapSecret: injected.bootstrapSecret } : {}) });
} catch (error) {
// #7677 (ratified 2026-07-21): give the caller its chance to transition the tenant's registry record to
// "failed" BEFORE the rethrow, so a customer polling the read path sees a terminal "Setup failed" instead
Expand Down
22 changes: 13 additions & 9 deletions control-plane/src/secret-driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
// calls the SAME two routes, just to STORE a tenant's DB credential rather than mint a GitHub token (#8064's
// `tenant_db_credential` secret type), plus a third route (#8064) to revoke it on teardown.
//
// Scope, deliberately narrow: this ONLY stores/revokes custody of the credential in the broker. It does NOT
// deliver the secret into a running container's environment -- that's separate, not-yet-built infrastructure
// (a container's own bootstrap would need to independently exchange its own enrollment secret, the same way a
// self-hosted container already does against `/v1/orb/token` today). #8066's own boundary excludes it.
// Scope, deliberately narrow: this ONLY stores custody of the credential in the broker and hands back the
// one-time exchange secret as `bootstrapSecret` -- it does NOT itself deliver anything into a running
// container's environment. That delivery is provisioning.ts's + container-driver.ts's job (#8202): provisioning
// threads `bootstrapSecret` from this driver's `injectSecrets` result into the SAME tenant's `createContainer`
// call, which is where it actually reaches `stub.start({envVars})`. #8066's own boundary excluded delivery
// entirely; #8202 is precisely the "separate, not-yet-built infrastructure" that comment pointed at.
//
// Deliberately does NOT implement the full `TenantProvisioningDriver` interface -- only injectSecrets/
// revokeSecrets (see `SecretDriver` below). `withRealSecretDriver` (driver-factory.ts) composes this onto an
Expand Down Expand Up @@ -42,7 +44,7 @@ export type SecretDriverConfig = {
/** The secret-only slice of `TenantProvisioningDriver` this module actually implements. Composed onto a full
* driver by `withRealSecretDriver` (driver-factory.ts), never used standalone against `provisionTenant`. */
export type SecretDriver = {
injectSecrets(request: TenantProvisioningRequest): Promise<{ secretRef?: string }>;
injectSecrets(request: TenantProvisioningRequest): Promise<{ secretRef?: string; bootstrapSecret?: string }>;
revokeSecrets(request: TenantProvisioningRequest): Promise<void>;
};

Expand Down Expand Up @@ -71,9 +73,11 @@ async function mainAppFetch<T>(config: SecretDriverConfig, method: string, path:
* object is stored (JSON-encoded), not just the bare `connectionString` -- a later reader gets every field
* back, not just what it can re-parse out of a URI, mirroring that type's own "kept alongside the parts"
* rationale. Returns the enrollment's `enrollId` as this driver's `secretRef` -- the caller (`provisionTenant`,
* via its own result) must persist this to revoke it later; the one-time exchange `secret` is intentionally
* discarded here, since this driver's job ends at custody, not consumption (see this file's header comment). */
export async function injectTenantSecrets(config: SecretDriverConfig, request: TenantProvisioningRequest): Promise<{ secretRef?: string }> {
* via its own result) must persist this to revoke it later -- AND the one-time exchange `secret` as
* `bootstrapSecret` (#8202): the caller threads this into the tenant's container at its next `createContainer`
* call, so the container can itself present it to `/v1/orb/token` and get this exact value back. Previously
* discarded here (see this file's former header comment); #8202 is what actually consumes it now. */
export async function injectTenantSecrets(config: SecretDriverConfig, request: TenantProvisioningRequest): Promise<{ secretRef?: string; bootstrapSecret?: string }> {
if (!request.database) {
throw new Error(`injectTenantSecrets: no database connection details on the request for tenant "${request.tenant.name}"`);
}
Expand All @@ -86,7 +90,7 @@ export async function injectTenantSecrets(config: SecretDriverConfig, request: T
"/v1/internal/orb/enrollments",
{ secretType: SECRET_TYPE_TENANT_DB_CREDENTIAL, secretValue: JSON.stringify(request.database) },
);
return { secretRef: result.enrollId };
return { secretRef: result.enrollId, bootstrapSecret: result.secret };
}

/** Idempotent teardown: a request with no `secretRef` (never provisioned with a real secret driver, or already
Expand Down
31 changes: 23 additions & 8 deletions control-plane/src/tenant-provisioning-driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,21 @@ export type TenantLifecycleState =
export type TenantProvisioningRequest = {
tenant: Tenant;
product: Product;
/** The tenant's already-provisioned database connection details (#7653) -- populated ONLY for the
* `injectSecrets` call, by `provisionTenant`'s own orchestration right after `provisionDatabase` resolves
* (#8066). Every other step (createContainer, destroyContainer, etc.) never sees this field. */
/** The tenant's already-provisioned database connection details (#7653) -- populated for the `injectSecrets`
* call (and, from there on, every step after it -- see `createContainer` below) by `provisionTenant`'s own
* orchestration right after `provisionDatabase` resolves (#8066). */
database?: DatabaseConnectionDetails;
/** An opaque, driver-specific reference to a previously injected secret (#8066) -- whatever `injectSecrets`
* returned as `secretRef`, threaded back in by `deprovisionTenant` so `revokeSecrets` knows what to revoke.
* Absent when a tenant was never provisioned with a real secret driver configured (idempotent revoke of an
* unconfigured tenant, matching every other driver's teardown contract). */
secretRef?: string;
/** A one-time credential the tenant's OWN container can later exchange for a real custodied secret (#8202) --
* whatever `injectSecrets` returned as `bootstrapSecret`, threaded by `provisionTenant` into the SAME
* `createContainer` call that follows it (#8202 reordered provisioning so this is possible -- see
* provisioning.ts). Populated ONLY for that one `createContainer` call; no other step ever sees it, and it is
* never itself the delivered secret -- just the key the container uses to fetch one. */
bootstrapSecret?: string;
};

/** What `provisionDatabase` hands back (#7653): everything a caller needs to actually reach the tenant's
Expand All @@ -69,7 +75,12 @@ export type DatabaseConnectionDetails = {
};

export interface TenantProvisioningDriver {
/** Step 1 (#7180): stand up the tenant's isolated container. Real driver → Cloudflare Containers API. */
/** Step 1 in call order (#7180), but the LAST of the three to run within `provisionTenant` as of #8202: stand
* up the tenant's isolated container. Real driver → Cloudflare Containers API. May see `request.bootstrapSecret`
* (#8202, set when `injectSecrets` returned one) to deliver into the container's own process environment at
* this, its actual cold-boot `start()` call -- the only point in a container's lifecycle Cloudflare Containers
* actually applies `envVars` (confirmed against the real `@cloudflare/containers` SDK: a repeat `start()` on
* an already-running/starting instance is a no-op or throws, never re-applies `envVars`). */
createContainer(request: TenantProvisioningRequest): Promise<void>;
/** Step 2 (#7180): provision the tenant's database, returning its connection details (#7653) -- a freshly
* created role's password is typically retrievable from the provider only at creation time, so the caller
Expand All @@ -78,10 +89,14 @@ export interface TenantProvisioningDriver {
provisionDatabase(request: TenantProvisioningRequest): Promise<DatabaseConnectionDetails>;
/** Step 3 (#7180): inject the tenant's secrets, given its database connection details (`request.database`,
* #8066). Returns an opaque `secretRef` the caller must persist and thread back into a later `revokeSecrets`
* call via `request.secretRef` -- `undefined` when the driver has nothing to track (e.g. the fake). A real
* driver delegates to #7174's generalized broker (src/orb/broker.ts, via #8064's stored-secret type); the
* fake only records the call. */
injectSecrets(request: TenantProvisioningRequest): Promise<{ secretRef?: string }>;
* call via `request.secretRef` -- `undefined` when the driver has nothing to track (e.g. the fake). Also
* returns `bootstrapSecret` (#8202): a one-time credential the caller threads into the SAME tenant's next
* `createContainer` call (provisioning.ts runs this step BEFORE createContainer specifically so this is
* possible), so the running container can itself exchange it later for the real secret this step just
* custodied -- `undefined` when the driver has nothing for a container to bootstrap with. A real driver
* delegates to #7174's generalized broker (src/orb/broker.ts, via #8064's stored-secret type); the fake only
* records the call. */
injectSecrets(request: TenantProvisioningRequest): Promise<{ secretRef?: string; bootstrapSecret?: string }>;
/** Teardown inverse of createContainer. MUST be idempotent — safe to call when the container was never
* created — so deprovisioning a nonexistent tenant is a no-op, never a throw. */
destroyContainer(request: TenantProvisioningRequest): Promise<void>;
Expand Down
36 changes: 36 additions & 0 deletions control-plane/test/container-driver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
createTenantContainer,
destroyTenantContainer,
PINNED_VERSION_ENV_VAR,
TENANT_SECRET_ENV_VAR,
tenantContainerExists,
type ContainerDriverConfig,
type ContainerNamespaceLike,
Expand Down Expand Up @@ -188,3 +189,38 @@ test("a repeat create of an already-provisioned pinned tenant never restarts it

assert.deepEqual(stub.startOptions, []);
});

// #8202: a tenant's one-time secret-bootstrap credential rides into its container at cold boot the same way
// pinnedVersion does above -- the only point in a container's lifecycle envVars are actually applied.
test("a tenant with a bootstrap secret starts with TENANT_SECRET_ENV_VAR carrying it", async () => {
const stub = optionCapturingStub();

await createTenantContainer(configFor(stub), { tenant: { name: "acme" }, product: "orb", bootstrapSecret: "orbsec_xyz" });

assert.deepEqual(stub.startOptions, [{ envVars: { [TENANT_SECRET_ENV_VAR]: "orbsec_xyz" } }]);
});

test("a tenant with both a pinned version and a bootstrap secret starts with both env vars merged into one call", async () => {
const stub = optionCapturingStub();

await createTenantContainer(configFor(stub), { tenant: { name: "acme", pinnedVersion: "v1.4.2" }, product: "orb", bootstrapSecret: "orbsec_xyz" });

assert.deepEqual(stub.startOptions, [{ envVars: { [PINNED_VERSION_ENV_VAR]: "v1.4.2", [TENANT_SECRET_ENV_VAR]: "orbsec_xyz" } }]);
});

test("a tenant with neither a pinned version nor a bootstrap secret still gets the exact pre-#4898 call (no options at all)", async () => {
const stub = optionCapturingStub();

await createTenantContainer(configFor(stub), { tenant: { name: "acme" }, product: "orb", bootstrapSecret: undefined });

assert.deepEqual(stub.startOptions, [undefined]);
});

test("a repeat create of an already-provisioned tenant with a bootstrap secret never restarts it (idempotence contract holds here too)", async () => {
const stub = optionCapturingStub();
await stub.markProvisioned();

await createTenantContainer(configFor(stub), { tenant: { name: "acme" }, product: "orb", bootstrapSecret: "orbsec_xyz" });

assert.deepEqual(stub.startOptions, []);
});
4 changes: 2 additions & 2 deletions control-plane/test/driver-factory.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ test("createTenantProvisioningDriver: selects the real secret driver when both M
const driver = createTenantProvisioningDriver({ MAIN_APP_BASE_URL: "https://api.loopover.test", INTERNAL_JOB_TOKEN: "internal-test-token" });

const result = await driver.injectSecrets({ ...REQUEST, database: { host: "h", port: 5432, database: "d", user: "u", password: "p", connectionString: "postgres://u:p@h:5432/d" } });
assert.deepEqual(result, { secretRef: "orbenr_abc" });
assert.deepEqual(result, { secretRef: "orbenr_abc", bootstrapSecret: "orbsec_xyz" });
assert.ok(calls.some((url) => url.includes("api.loopover.test")));
});

Expand All @@ -270,7 +270,7 @@ test("createTenantProvisioningDriver: composes the real database, container, AND
assert.equal(await driver.containerExists(REQUEST), true);
await assert.rejects(driver.provisionDatabase(REQUEST));
const injected = await driver.injectSecrets({ ...REQUEST, database: { host: "h", port: 5432, database: "d", user: "u", password: "p", connectionString: "postgres://u:p@h:5432/d" } });
assert.deepEqual(injected, { secretRef: "orbenr_abc" });
assert.deepEqual(injected, { secretRef: "orbenr_abc", bootstrapSecret: "orbsec_xyz" });
});

test("createTenantProvisioningDriver: defaults env to process.env when no override is passed", async () => {
Expand Down
Loading