feat(selfhost): expand Sentry observability context for self-host runtime

apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx

@@ -115,6 +115,31 @@ review_context_fetch_failed`}
         log for the same subsystem.
       </p>
 
+      <h2>Sentry alert classes</h2>
+      <FeatureRow
+        items={[
+          {
+            title: "Page on persistent stoppage",
+            description:
+              "Alert on missed scheduled-loop or relay-drain monitors after two consecutive failures, not one late tick.",
+          },
+          {
+            title: "Ticket operational faults",
+            description:
+              "Open an issue for dead-letter growth, repeated check-run permission gaps, or repeated relay/register failures.",
+          },
+          {
+            title: "Warn on degradations",
+            description:
+              "AI provider exhaustion, broker fallback, and backup advisories should warn first and page only when they persist.",
+          },
+        ]}
+      />
+      <CodeBlock
+        code={`Tags: subsystem, operation, reasonCode, repo, pullNumber, jobType, backend, provider, model, effort, mode
+Do not tag: raw diff, prompt, review text, auth headers, installation IDs, delivery IDs, or local auth paths`}
+      />
+
       <h2>Routine checks</h2>
       <ul>
         <li>Queue pending count is not growing without processing.</li>

apps/gittensory-ui/src/routes/docs.self-hosting-security.tsx

@@ -90,6 +90,13 @@ function SelfHostingSecurity() {
         private scoring context, or maintainer-only notes. For hosted and self-host boundaries, keep
         <Link to="/docs/privacy-security"> Privacy and security</Link> nearby.
       </p>
+
+      <h2>Sentry data boundary</h2>
+      <p>
+        Self-host Sentry is opt-in only. When enabled, capture should keep tags low-cardinality and
+        scrub request bodies, raw diffs, prompts, review text, auth headers, private repo config,
+        and local auth paths before an event leaves the box.
+      </p>
     </DocsPage>
   );
 }

src/github/app.ts

@@ -224,6 +224,9 @@ async function mintInstallationToken(
         JSON.stringify({
           level: "error",
           event: "orb_broker_unavailable",
+          subsystem: "github",
+          operation: "broker_installation_token",
+          reasonCode: "orb_broker_unavailable",
           installationId,
           error: errorMessage(error),
         }),
@@ -733,6 +736,9 @@ async function createOrUpdateNamedCheckRun(
           JSON.stringify({
             level: "error",
             event: "check_run_post_denied",
+            subsystem: "github",
+            operation: "check_run_publish",
+            reasonCode: "permission_denied",
             repository: `${owner}/${repo}`,
             status: e.status ?? null,
             message: (e.message ?? "Resource not accessible by integration").slice(

src/github/client.ts

@@ -179,20 +179,27 @@ function responseFromCached(hit: CachedGitHubResponse, replayKind: "hit" | "coal
   });
 }
 
-async function fetchWithGitHubRetry(input: RequestInfo | URL, init?: RequestInit): Promise<Response> {
+async function fetchWithGitHubRetry(
+  input: RequestInfo | URL,
+  init?: RequestInit,
+): Promise<GitHubFetchResult> {
   let response: Response;
   for (let attempt = 0; ; attempt += 1) {
-    response = init?.signal
-      ? await fetch(input, init)
-      : await fetch(input, {
-          ...(init ?? {}),
-          signal: AbortSignal.timeout(GITHUB_FETCH_TIMEOUT_MS),
-        });
+    try {
+      response = init?.signal
+        ? await fetch(input, init)
+        : await fetch(input, {
+            ...(init ?? {}),
+            signal: AbortSignal.timeout(GITHUB_FETCH_TIMEOUT_MS),
+          });
+    } catch (error) {
+      return { ok: false, error };
+    }
     // Retry a transient rate-limit (with backoff) instead of surfacing it; stop once exhausted or it's not a limit.
     if (attempt >= GITHUB_RATE_LIMIT_MAX_RETRIES || !(await isRateLimitedResponse(response))) break;
     await sleep(rateLimitRetryMs(response, attempt));
   }
-  return response;
+  return { ok: true, response };
 }
 
 async function fetchAndMaybeCacheGitHubGet(
@@ -201,9 +208,11 @@ async function fetchAndMaybeCacheGitHubGet(
   url: string,
   cacheKey: string,
   cls: GitHubCacheClass,
-): Promise<{ response: Response; cached: CachedGitHubResponse | null }> {
-  const response = await fetchWithGitHubRetry(input, init);
-  if (response.status !== 200) return { response, cached: null };
+): Promise<InFlightCacheableGet> {
+  const fetched = await fetchWithGitHubRetry(input, init);
+  if (!fetched.ok) return { ok: false, error: fetched.error };
+  const { response } = fetched;
+  if (response.status !== 200) return { ok: true, response, cached: null };
   try {
     const cached = {
       status: 200,
@@ -215,16 +224,37 @@ async function fetchAndMaybeCacheGitHubGet(
     };
     await responseCache!.set(cacheKey, cached, githubResponseCacheTtlSeconds(cls));
     recordGitHubCacheMetric("set", cls);
-    return { response, cached };
+    return { ok: true, response, cached };
   } catch {
     recordGitHubCacheMetric("error", cls);
-    return { response, cached: null };
+    return { ok: true, response, cached: null };
   }
 }
 
+type InFlightCacheableGet =
+  | {
+      ok: true;
+      response: Response;
+      cached: CachedGitHubResponse | null;
+    }
+  | {
+      ok: false;
+      error: unknown;
+    };
+
+type GitHubFetchResult =
+  | {
+      ok: true;
+      response: Response;
+    }
+  | {
+      ok: false;
+      error: unknown;
+    };
+
 // Single-flight cacheable GETs inside one isolate: a webhook burst often asks for the same metadata
 // before Redis has been populated. Join those cold misses so GitHub sees one request, then replay the cached body.
-const inFlightCacheableGets = new Map<string, Promise<CachedGitHubResponse | null>>();
+const inFlightCacheableGets = new Map<string, Promise<InFlightCacheableGet>>();
 
 // A 12s hard cap on every GitHub request. Centralised here so the app token/installation raw fetches plus comment /
 // label / check-run / pr-action Octokit helpers all inherit the cache boundary, retry, and timeout behavior.
@@ -237,7 +267,9 @@ export async function timeoutFetch(input: RequestInfo | URL, init?: RequestInit)
   const useCache = responseCache !== null && cls !== null;
   if (!useCache) {
     recordGitHubCacheMetric("bypassed", cacheBypassClass(method, url, headers));
-    return fetchWithGitHubRetry(input, init);
+    const fetched = await fetchWithGitHubRetry(input, init);
+    if (!fetched.ok) throw fetched.error;
+    return fetched.response;
   }
 
   const cacheKey = await responseCacheKey(url, headers);
@@ -257,19 +289,30 @@ export async function timeoutFetch(input: RequestInfo | URL, init?: RequestInit)
   if (existing) {
     recordGitHubCacheMetric("coalesced", cls);
     const replay = await existing;
-    if (replay) return responseFromCached(replay, "coalesced");
+    if (!replay.ok) throw replay.error;
+    if (replay.ok && replay.cached)
+      return responseFromCached(replay.cached, "coalesced");
   }
 
-  const request = fetchAndMaybeCacheGitHubGet(input, init, url, cacheKey, cls).then(
-    (result) => ({ ok: true as const, result }),
-    (error: unknown) => ({ ok: false as const, error }),
-  );
-  const shared = request.then((settled) => (settled.ok ? settled.result.cached : null));
-  const sharedWithCleanup = shared.finally(() => inFlightCacheableGets.delete(cacheKey));
-  inFlightCacheableGets.set(cacheKey, sharedWithCleanup);
+  const request = (async (): Promise<InFlightCacheableGet> => {
+    try {
+      return await fetchAndMaybeCacheGitHubGet(
+        input,
+        init,
+        url,
+        cacheKey,
+        cls,
+      );
+    } catch (error) {
+      return { ok: false, error };
+    } finally {
+      inFlightCacheableGets.delete(cacheKey);
+    }
+  })();
+  inFlightCacheableGets.set(cacheKey, request);
   const result = await request;
   if (!result.ok) throw result.error;
-  return result.result.response;
+  return result.response;
 }
 
 /** Test-only: reset shared GitHub response cache state between tests. */

src/queue/processors.ts

@@ -4252,7 +4252,19 @@ async function auditGateCheckPermissionMissing(
   });
   // Surface the install-wide Checks:write gap to Sentry — until the scope is granted the required gate check-run
   // silently never posts on ANY PR for this install; an operator must SEE this config fault, not just the ledger.
-  console.error(JSON.stringify({ level: "error", event: "gate_check_permission_missing", message: warning, repository: repoFullName, pullNumber, deliveryId }));
+  console.error(
+    JSON.stringify({
+      level: "error",
+      event: "gate_check_permission_missing",
+      subsystem: "github",
+      operation: "gate_check_publish",
+      reasonCode: "permission_missing",
+      message: warning,
+      repository: repoFullName,
+      pullNumber,
+      deliveryId,
+    }),
+  );
 }
 
 /**
@@ -5149,7 +5161,19 @@ async function maybePublishPrPublicSurface(
           detail: checkRunResult.warning,
           metadata: { deliveryId: webhook.deliveryId, repoFullName },
         });
-        console.error(JSON.stringify({ level: "error", event: "check_run_permission_missing", message: checkRunResult.warning, repository: repoFullName, pullNumber: pr.number, deliveryId: webhook.deliveryId }));
+        console.error(
+          JSON.stringify({
+            level: "error",
+            event: "check_run_permission_missing",
+            subsystem: "github",
+            operation: "check_run_publish",
+            reasonCode: "permission_missing",
+            message: checkRunResult.warning,
+            repository: repoFullName,
+            pullNumber: pr.number,
+            deliveryId: webhook.deliveryId,
+          }),
+        );
       } else if (checkRunResult?.kind === "published") {
         publishedOutputs.push("check_run");
       }

src/selfhost/ai.ts

@@ -483,6 +483,9 @@ function logSelfHostAiProviderFailed(input: {
     JSON.stringify({
       level: "error",
       event: "selfhost_ai_provider_failed",
+      subsystem: "ai",
+      operation: "provider_attempt",
+      reasonCode: "provider_error",
       provider: input.provider,
       model: input.model || "default",
       effort: input.effort,
@@ -599,13 +602,26 @@ export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }>
         } catch (error) {
           lastError = error;
           failures.push({ provider: p.name, error: errorMessage(error) });
-          console.error(JSON.stringify({ level: "warn", event: "selfhost_ai_provider_failed_in_chain", provider: p.name, error: errorMessage(error) }));
+          console.error(
+            JSON.stringify({
+              level: "warn",
+              event: "selfhost_ai_provider_failed_in_chain",
+              subsystem: "ai",
+              operation: "provider_chain",
+              reasonCode: "provider_error",
+              provider: p.name,
+              error: errorMessage(error),
+            }),
+          );
         }
       }
       console.error(
         JSON.stringify({
           level: "error",
           event: "selfhost_ai_providers_exhausted",
+          subsystem: "ai",
+          operation: "provider_chain",
+          reasonCode: "providers_exhausted",
           provider: failures.length === 1 ? failures[0]?.provider : undefined,
           model: model || "default",
           providers: failures.map((failure) => failure.provider),

src/selfhost/monitored-work.ts

@@ -22,7 +22,12 @@ export async function runScheduledLoopWithMonitor<T>(
 ): Promise<T> {
   return withSentryMonitor(
     "scheduled-loop",
-    { jobType: "scheduled-loop", cron },
+    {
+      subsystem: "scheduler",
+      operation: "scheduled_loop",
+      jobType: "scheduled-loop",
+      cron,
+    },
     () => Promise.resolve(scheduled()),
   );
 }
@@ -31,11 +36,15 @@ export async function runOrbExportWithMonitor(
   exportBatch: () => Promise<number>,
   log: (line: string) => void = console.log,
 ): Promise<void> {
-  await withSentryMonitor("orb-export", { jobType: "orb-export" }, async () => {
-    const exported = await exportBatch();
-    if (exported > 0)
-      log(JSON.stringify({ event: "selfhost_orb_export", exported }));
-  });
+  await withSentryMonitor(
+    "orb-export",
+    { subsystem: "orb", operation: "orb_export", jobType: "orb-export" },
+    async () => {
+      const exported = await exportBatch();
+      if (exported > 0)
+        log(JSON.stringify({ event: "selfhost_orb_export", exported }));
+    },
+  );
 }
 
 export async function drainOrbRelayWithMonitor(args: {
@@ -53,7 +62,12 @@ export async function drainOrbRelayWithMonitor(args: {
 }): Promise<void> {
   await withSentryMonitor(
     "orb-relay-drain",
-    { jobType: "orb-relay-drain", pendingAckCount: args.state.pendingAck.length },
+    {
+      subsystem: "orb",
+      operation: "relay_drain",
+      jobType: "orb-relay-drain",
+      pendingAckCount: args.state.pendingAck.length,
+    },
     async () => {
       const events = await args.drain(args.relayEnv, args.state.pendingAck);
       args.state.pendingAck = [];

src/selfhost/pg-queue.ts

@@ -306,8 +306,11 @@ export function createPgQueue(
         }),
       );
       captureError(new Error("self-host queue processing lease expired"), {
+        subsystem: "queue",
+        operation: "job_claim",
         kind: "job_recovered",
         reason: "processing_timeout",
+        queueBackend: "postgres",
         recovered,
         timeoutMs: processingTimeoutMs,
       });
@@ -335,8 +338,11 @@ export function createPgQueue(
           error: "unparseable payload",
         });
         captureError(new Error("unparseable queue payload"), {
+          subsystem: "queue",
+          operation: "job_process",
           kind: "job_dead",
           reason: "unparseable_payload",
+          queueBackend: "postgres",
           jobId: job.id,
         });
         return true;
@@ -437,6 +443,11 @@ export function createPgQueue(
             JSON.stringify({
               level: "error",
               event: "selfhost_job_dead",
+              subsystem: "queue",
+              operation: "job_process",
+              reasonCode: "max_retries_exhausted",
+              backend: "postgres",
+              jobType: extractPayloadType(job.payload),
               id: job.id,
               attempts,
               error: errMsg,
@@ -452,8 +463,11 @@ export function createPgQueue(
             error: errMsg,
           }, jobTraceParent);
           captureError(error, {
+            subsystem: "queue",
+            operation: "job_process",
             kind: "job_dead",
             reason: "max_retries_exhausted",
+            queueBackend: "postgres",
             jobType: extractPayloadType(job.payload),
             jobId: job.id,
             attempts,