feat(observability): add self-host Sentry monitors

apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx

@@ -85,6 +85,36 @@ review_context_fetch_failed`}
       </p>
       <CodeBlock lang="bash" code={`docker compose --profile observability up -d`} />
 
+      <h2>Sentry cron monitors</h2>
+      <p>
+        When <code>SENTRY_DSN</code> is set, the self-host runtime emits Sentry monitor check-ins
+        for the recurring loops where silent stoppage matters most. Leaving <code>SENTRY_DSN</code>{" "}
+        unset keeps monitor reporting off.
+      </p>
+      <FeatureRow
+        items={[
+          {
+            title: "scheduled loop",
+            description:
+              "The two-minute maintenance tick that fans out sweeps, backfills, and refresh jobs.",
+          },
+          {
+            title: "Orb export",
+            description: "The hourly outcome export loop used by brokered self-host deployments.",
+          },
+          {
+            title: "Orb relay drain",
+            description:
+              "The pull-mode relay loop for installations that receive events outbound from Orb.",
+          },
+        ]}
+      />
+      <p>
+        A missed monitor means the process may still be alive but the recurring work is not checking
+        in on schedule. Pair the monitor with queue depth, dead-job counts, and the structured error
+        log for the same subsystem.
+      </p>
+
       <h2>Routine checks</h2>
       <ul>
         <li>Queue pending count is not growing without processing.</li>

src/selfhost/monitored-work.ts

@@ -0,0 +1,75 @@
+import type { EnqueueWebhookResult } from "../github/webhook";
+import { withSentryMonitor } from "./sentry";
+
+export type OrbRelayEvent = {
+  deliveryId: string;
+  eventName: string;
+  rawBody: string;
+};
+
+export type OrbRelayDrainState = {
+  pendingAck: string[];
+};
+
+type OrbRelayEnv = {
+  ORB_ENROLLMENT_SECRET?: string | undefined;
+  ORB_BROKER_URL?: string | undefined;
+};
+
+export async function runScheduledLoopWithMonitor<T>(
+  cron: string,
+  scheduled: () => T | Promise<T>,
+): Promise<T> {
+  return withSentryMonitor(
+    "scheduled-loop",
+    { jobType: "scheduled-loop", cron },
+    () => Promise.resolve(scheduled()),
+  );
+}
+
+export async function runOrbExportWithMonitor(
+  exportBatch: () => Promise<number>,
+  log: (line: string) => void = console.log,
+): Promise<void> {
+  await withSentryMonitor("orb-export", { jobType: "orb-export" }, async () => {
+    const exported = await exportBatch();
+    if (exported > 0)
+      log(JSON.stringify({ event: "selfhost_orb_export", exported }));
+  });
+}
+
+export async function drainOrbRelayWithMonitor(args: {
+  state: OrbRelayDrainState;
+  relayEnv: OrbRelayEnv;
+  env: Env;
+  drain: (env: OrbRelayEnv, ack: string[]) => Promise<OrbRelayEvent[]>;
+  enqueue: (
+    env: Env,
+    deliveryId: string,
+    eventName: string,
+    rawBody: string,
+  ) => Promise<EnqueueWebhookResult>;
+  log?: (line: string) => void;
+}): Promise<void> {
+  await withSentryMonitor(
+    "orb-relay-drain",
+    { jobType: "orb-relay-drain", pendingAckCount: args.state.pendingAck.length },
+    async () => {
+      const events = await args.drain(args.relayEnv, args.state.pendingAck);
+      args.state.pendingAck = [];
+      for (const ev of events) {
+        const result = await args.enqueue(
+          args.env,
+          ev.deliveryId,
+          ev.eventName,
+          ev.rawBody,
+        );
+        if (result !== "enqueue_failed") args.state.pendingAck.push(ev.deliveryId);
+      }
+      if (events.length > 0)
+        (args.log ?? console.log)(
+          JSON.stringify({ event: "orb_relay_drained", count: events.length }),
+        );
+    },
+  );
+}

src/selfhost/sentry.ts

@@ -5,12 +5,15 @@
 import { currentOtelTraceIds } from "./otel";
 
 type SentryNs = typeof import("@sentry/node");
+type SentryMonitorConfig = NonNullable<Parameters<SentryNs["captureCheckIn"]>[1]>;
+export type SentryMonitorName = "scheduled-loop" | "orb-export" | "orb-relay-drain";
 type SentryScope = {
   setContext(name: string, context: Record<string, unknown>): void;
   setTag(key: string, value: string): void;
 };
 let Sentry: SentryNs | undefined;
 let active = false;
+let sentryEnvironment = "production";
 
 const SECRET_KEY =
   /(token|secret|key|password|passwd|authorization|auth|dsn|cookie|bearer|credential|private)/i;
@@ -20,6 +23,72 @@ function nonBlank(value: string | undefined): string | undefined {
   return trimmed ? trimmed : undefined;
 }
 
+const SENTRY_MONITORS: Record<SentryMonitorName, { slug: string; config: SentryMonitorConfig }> = {
+  "scheduled-loop": {
+    slug: "scheduled-loop",
+    config: {
+      schedule: { type: "interval", value: 2, unit: "minute" },
+      checkinMargin: 3,
+      maxRuntime: 2,
+      failureIssueThreshold: 2,
+      recoveryThreshold: 1,
+    },
+  },
+  "orb-export": {
+    slug: "orb-export",
+    config: {
+      schedule: { type: "interval", value: 1, unit: "hour" },
+      checkinMargin: 10,
+      maxRuntime: 10,
+      failureIssueThreshold: 2,
+      recoveryThreshold: 1,
+    },
+  },
+  "orb-relay-drain": {
+    slug: "orb-relay-drain",
+    config: {
+      schedule: { type: "interval", value: 1, unit: "minute" },
+      checkinMargin: 2,
+      maxRuntime: 1,
+      failureIssueThreshold: 3,
+      recoveryThreshold: 1,
+    },
+  },
+};
+
+function slugPart(value: string | undefined): string {
+  const slug = nonBlank(value)
+    ?.toLowerCase()
+    .replace(/[^a-z0-9-]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 48);
+  return slug || "production";
+}
+
+export function resolveSentryMonitorSlug(
+  name: SentryMonitorName,
+  environment = sentryEnvironment,
+): string {
+  return `gittensory-selfhost-${slugPart(environment)}-${SENTRY_MONITORS[name].slug}`;
+}
+
+function safeMonitorContext(
+  name: SentryMonitorName,
+  monitorSlug: string,
+  context: Record<string, unknown> | undefined,
+): Record<string, unknown> {
+  const safe: Record<string, unknown> = { monitor: name, monitorSlug };
+  if (!context) return safe;
+  for (const [key, value] of Object.entries(context)) {
+    if (SECRET_KEY.test(key) || value === null || value === undefined) continue;
+    if (typeof value === "string")
+      safe[key] = value.length > 160 ? `${value.slice(0, 157)}...` : value;
+    else if (typeof value === "number" && Number.isFinite(value)) safe[key] = value;
+    else if (typeof value === "boolean") safe[key] = value;
+  }
+  return safe;
+}
+
 function setOtelTraceScope(scope: SentryScope): void {
   const trace = currentOtelTraceIds();
   if (!trace) return;
@@ -65,9 +134,10 @@ export async function initSentry(env: NodeJS.ProcessEnv): Promise<boolean> {
   if (!env.SENTRY_DSN) return false;
   Sentry = await import("@sentry/node");
   const release = resolveSentryRelease(env);
+  sentryEnvironment = nonBlank(env.SENTRY_ENVIRONMENT) ?? "production";
   Sentry.init({
     dsn: env.SENTRY_DSN,
-    environment: env.SENTRY_ENVIRONMENT ?? "production",
+    environment: sentryEnvironment,
     ...(release ? { release } : {}),
     tracesSampleRate: Number(env.SENTRY_TRACES_SAMPLE_RATE ?? "0"),
     serverName: env.PUBLIC_API_ORIGIN,
@@ -242,6 +312,47 @@ export function forwardStructuredLogToSentry(line: unknown, fromErrorSink = fals
   });
 }
 
+/** Wrap recurring self-host work with Sentry cron check-ins. No-op when Sentry is disabled. */
+export async function withSentryMonitor<T>(
+  name: SentryMonitorName,
+  context: Record<string, unknown> | undefined,
+  callback: () => Promise<T>,
+): Promise<T> {
+  if (!active || !Sentry) return callback();
+  const monitorSlug = resolveSentryMonitorSlug(name);
+  const checkInId = Sentry.captureCheckIn(
+    { monitorSlug, status: "in_progress" },
+    SENTRY_MONITORS[name].config,
+  );
+  const startedAt = Date.now();
+  try {
+    const result = await callback();
+    Sentry.captureCheckIn({
+      monitorSlug,
+      status: "ok",
+      checkInId,
+      duration: (Date.now() - startedAt) / 1000,
+    });
+    return result;
+  } catch (error) {
+    Sentry.captureCheckIn({
+      monitorSlug,
+      status: "error",
+      checkInId,
+      duration: (Date.now() - startedAt) / 1000,
+    });
+    Sentry.withScope((scope) => {
+      scope.setLevel("error");
+      setOtelTraceScope(scope);
+      scope.setContext("sentry_monitor", safeMonitorContext(name, monitorSlug, context));
+      scope.setTag("monitor", monitorSlug);
+      scope.setFingerprint(["gittensory-sentry-monitor", name]);
+      Sentry!.captureException(error instanceof Error ? error : new Error(String(error)));
+    });
+    throw error;
+  }
+}
+
 /** Flush buffered events before exit. No-op when off. */
 export async function flushSentry(timeoutMs = 2000): Promise<void> {
   if (!active || !Sentry) return;
@@ -252,6 +363,7 @@ export async function flushSentry(timeoutMs = 2000): Promise<void> {
 export function resetSentryForTest(): void {
   Sentry = undefined;
   active = false;
+  sentryEnvironment = "production";
 }
 
 interface StructuredLogConsole {

src/server.ts

@@ -58,6 +58,11 @@ import {
   initSentry,
   installStructuredLogForwarding,
 } from "./selfhost/sentry";
+import {
+  drainOrbRelayWithMonitor,
+  runOrbExportWithMonitor,
+  runScheduledLoopWithMonitor,
+} from "./selfhost/monitored-work";
 import {
   currentOtelTraceParent,
   initOpenTelemetry,
@@ -741,13 +746,16 @@ async function main(): Promise<void> {
 
   // Cron — gittensory ticks ~every 2 minutes; drive the SAME scheduled handler.
   const intervalMs = Number(process.env.CRON_INTERVAL_MS ?? 120_000);
+  /* v8 ignore start -- self-host entrypoint timers start a live server; monitor semantics are covered in selfhost tests. */
   const cron = setInterval(() => {
     const controller = {
       scheduledTime: Date.now(),
       cron: "*/2 * * * *",
       noRetry: () => undefined,
     } as unknown as ScheduledController;
-    Promise.resolve(worker.scheduled(controller, env, ctx)).catch((error) =>
+    runScheduledLoopWithMonitor(controller.cron, () =>
+      worker.scheduled(controller, env, ctx),
+    ).catch((error) =>
       console.error(
         JSON.stringify({
           level: "error",
@@ -757,28 +765,24 @@ async function main(): Promise<void> {
       ),
     );
   }, intervalMs);
+  /* v8 ignore stop */
 
   // Orb fleet-telemetry export — ALWAYS ON (the fleet-calibration contract of self-hosting). Self-gates
   // inside exportOrbBatch: a no-op until the GitHub App is configured, or when ORB_AIR_GAP=true.
+  /* v8 ignore start -- self-host entrypoint timers start a live server; monitor semantics are covered in selfhost tests. */
   const runOrbExport = () =>
-    exportOrbBatch(backend.db)
-      .then((n) => {
-        if (n > 0)
-          console.log(
-            JSON.stringify({ event: "selfhost_orb_export", exported: n }),
-          );
-      })
-      .catch((error) =>
-        console.error(
-          JSON.stringify({
-            level: "error",
-            event: "selfhost_orb_export_error",
-            error: error instanceof Error ? error.message : "unknown error",
-          }),
-        ),
-      );
+    runOrbExportWithMonitor(() => exportOrbBatch(backend.db)).catch((error) =>
+      console.error(
+        JSON.stringify({
+          level: "error",
+          event: "selfhost_orb_export_error",
+          error: error instanceof Error ? error.message : "unknown error",
+        }),
+      ),
+    );
   void runOrbExport(); // flush any pending events at startup
   setInterval(runOrbExport, 3_600_000); // then hourly
+  /* v8 ignore stop */
 
   // Brokered self-host: register our relay target with the central Orb (best-effort, fire-and-forget). PUSH mode
   // (default) registers a public relay URL the Orb POSTs to; PULL mode (ORB_RELAY_MODE=pull) registers no URL and
@@ -817,23 +821,29 @@ async function main(): Promise<void> {
   if (process.env.ORB_RELAY_MODE === "pull" && process.env.ORB_ENROLLMENT_SECRET) {
     const { drainOrbRelay } = await import("./orb/broker-client");
     const { enqueueWebhookByEnv } = await import("./github/webhook");
-    let pendingAck: string[] = [];
+    const relayDrainState = { pendingAck: [] as string[] };
+    /* v8 ignore start -- pull-mode relay loop is a live self-host timer; monitor semantics are covered in selfhost tests. */
     const drainRelay = async (): Promise<void> => {
-      const events = await drainOrbRelay(
-        { ORB_ENROLLMENT_SECRET: process.env.ORB_ENROLLMENT_SECRET, ORB_BROKER_URL: process.env.ORB_BROKER_URL },
-        pendingAck,
-      );
-      pendingAck = [];
-      for (const ev of events) {
-        const result = await enqueueWebhookByEnv(env, ev.deliveryId, ev.eventName, ev.rawBody);
-        // Ack everything durably handled (queued / duplicate / invalid_json) so the Orb deletes it; retry only a
-        // real enqueue failure on the next pull (don't ack → the Orb keeps it).
-        if (result !== "enqueue_failed") pendingAck.push(ev.deliveryId);
-      }
-      if (events.length > 0) console.log(JSON.stringify({ event: "orb_relay_drained", count: events.length }));
+      await drainOrbRelayWithMonitor({
+        state: relayDrainState,
+        relayEnv: {
+          ORB_ENROLLMENT_SECRET: process.env.ORB_ENROLLMENT_SECRET,
+          ORB_BROKER_URL: process.env.ORB_BROKER_URL,
+        },
+        env,
+        drain: drainOrbRelay,
+        enqueue: enqueueWebhookByEnv,
+      });
     };
     void drainRelay();
-    setInterval(() => void drainRelay().catch((error) => captureError(error, { kind: "orb_relay_drain" })), 15_000);
+    setInterval(
+      () =>
+        void drainRelay().catch((error) =>
+          captureError(error, { kind: "orb_relay_drain" }),
+        ),
+      15_000,
+    );
+    /* v8 ignore stop */
   }
 
   // Graceful shutdown: stop accepting HTTP, let the queue finish, close the backend.