fix(orb): stale-token grace when the broker mint blips in broker mode

src/github/app.ts

@@ -4,6 +4,7 @@ import { makeInstallationOctokit } from "./client";
 import { maintainerControlPanelUrl } from "./footer";
 import type { AgentActionMode } from "../settings/agent-execution";
 import { signRs256Jwt } from "../utils/crypto";
+import { errorMessage } from "../utils/json";
 import { evaluateGateCheck, formatCheckRunOutput, formatGateCheckOutput, type CheckRunAnnotationContext, type CheckRunOutput, type GateCheckConclusion, type GateCheckEvaluation, type GateCheckPolicy } from "../rules/advisory";
 
 type CheckRunResponse = {
@@ -57,9 +58,23 @@ export async function createInstallationToken(env: Env, installationId: number):
   // secret, so this branch is inert there → byte-identical. The token caches the same way (the install id is the
   // self-host's single bound install). See src/orb/broker-client.
   if (isOrbBrokerMode(env)) {
-    const brokered = await fetchBrokeredInstallationToken(env);
-    installationTokenCache.set(installationId, { token: brokered.token, expiresAtMs: brokered.expiresAtMs });
-    return brokered.token;
+    try {
+      const brokered = await fetchBrokeredInstallationToken(env);
+      installationTokenCache.set(installationId, { token: brokered.token, expiresAtMs: brokered.expiresAtMs });
+      return brokered.token;
+    } catch (error) {
+      // Stale-token grace (#2): a brokered self-host holds no App key, so without this a single Orb mint failure
+      // fails the review (→ retry/DLQ) and an Orb blip during the re-mint window stalls the fleet. If the cached
+      // token is STILL within its real expiry, serve it — a valid token beats a stalled review (NO dangerous reuse:
+      // an actually-expired token is never served). Otherwise emit an alertable structured log and rethrow so the
+      // queue's retry/DLQ handles a genuine outage.
+      if (cached && cached.expiresAtMs > Date.now()) {
+        console.warn(JSON.stringify({ level: "warn", event: "orb_broker_degraded_serving_cached_token", installationId, expiresInMs: cached.expiresAtMs - Date.now(), error: errorMessage(error) }));
+        return cached.token;
+      }
+      console.error(JSON.stringify({ level: "error", event: "orb_broker_unavailable", installationId, error: errorMessage(error) }));
+      throw error;
+    }
   }
   const jwt = await createAppJwt(env);
   const response = await timeoutFetch(`https://api.github.com/app/installations/${installationId}/access_tokens`, {

test/unit/github-app.test.ts

@@ -147,6 +147,49 @@ describe("GitHub check runs", () => {
     expect(brokerCalls).toBe(1);
   });
 
+  it("#2: serves a still-valid cached token when the Orb mint fails (stale-token grace, no fleet stall)", async () => {
+    let calls = 0;
+    vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
+      const url = input.toString();
+      if (url.includes("/v1/orb/token")) {
+        calls += 1;
+        // First mint returns a token expiring within the 2-min safety margin → the next call re-mints; that re-mint fails.
+        if (calls === 1) return Response.json({ token: "tok-1", installationId: 1001, expiresAt: new Date(Date.now() + 90_000).toISOString() });
+        return new Response("orb down", { status: 503 });
+      }
+      return new Response("nf", { status: 404 });
+    });
+    const env = createTestEnv({ ORB_ENROLLMENT_SECRET: "orbsec_test" });
+    expect(await createInstallationToken(env, 1001)).toBe("tok-1"); // caches a near-expiry token
+    expect(await createInstallationToken(env, 1001)).toBe("tok-1"); // re-mint fails → grace serves the still-valid cached token
+    expect(calls).toBe(2); // the second call DID attempt a re-mint, then fell back to the cache
+  });
+
+  it("#2: rethrows when the broker is down and there is no still-valid cached token", async () => {
+    vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
+      const url = input.toString();
+      if (url.includes("/v1/orb/token")) return new Response("orb down", { status: 503 });
+      return new Response("nf", { status: 404 });
+    });
+    await expect(createInstallationToken(createTestEnv({ ORB_ENROLLMENT_SECRET: "orbsec_test" }), 1002)).rejects.toThrow();
+  });
+
+  it("#2: rethrows when the only cached token has actually expired (no dangerous reuse)", async () => {
+    let calls = 0;
+    vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
+      const url = input.toString();
+      if (url.includes("/v1/orb/token")) {
+        calls += 1;
+        if (calls === 1) return Response.json({ token: "tok-old", installationId: 1003, expiresAt: new Date(Date.now() - 1_000).toISOString() });
+        return new Response("orb down", { status: 503 });
+      }
+      return new Response("nf", { status: 404 });
+    });
+    const env = createTestEnv({ ORB_ENROLLMENT_SECRET: "orbsec_test" });
+    expect(await createInstallationToken(env, 1003)).toBe("tok-old"); // caches an already-expired token
+    await expect(createInstallationToken(env, 1003)).rejects.toThrow(); // re-mint fails + cached expired → rethrow
+  });
+
   it("fetches repository collaborator permissions with installation credentials", async () => {
     const privateKey = await generatePrivateKeyPem();
     const calls: string[] = [];