JSONbored · JSONbored · Jul 3, 2026 · Jul 3, 2026
@@ -269,13 +269,21 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # TRICKLE (MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS) force-admits it once it's waited long enough regardless of
 # pressure, so sustained load can slow maintenance down but never starve it forever. All defaults are sane for
 # a small single-node box; every value is optional.
+# DRAIN (MAINTENANCE_ADMISSION_DRAIN_AGE_MS, #selfhost-maintenance-self-pin): a second, much shorter age escape
+# scoped ONLY to `maintenance_pending_high` -- the aggregate lane-backlog check has no feedback loop back to the
+# count as jobs individually age out via the trickle above, so without this a backed-up lane can deny every
+# claim for hours even though the trickle exists. The drain lets the OLDEST jobs in that same backlog through
+# well before the full trickle ceiling, so the backlog can actually shrink (further bounded by
+# QUEUE_BACKGROUND_CONCURRENCY). Still fully blocked by host_load_high -- draining more work onto an overloaded
+# box is exactly what that check exists to prevent.
 # MAINTENANCE_ADMISSION_ENABLED=true         # set false/0/off to fully disable this policy (old always-run behavior)
 # MAINTENANCE_ADMISSION_MAX_LIVE_PENDING=5   # defer maintenance once this many live (webhook/regate) jobs are queued
 # MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS=120000     # defer maintenance once the oldest live job has waited this long (2m)
 # MAINTENANCE_ADMISSION_MAX_PENDING=15       # defer NEW maintenance admissions once this many maintenance jobs are already queued
 # MAINTENANCE_ADMISSION_MAX_HOST_LOAD=1.5    # defer once 1-min load average per CPU core exceeds this (best-effort; see host-pressure.ts)
 # MAINTENANCE_ADMISSION_DEFER_MS=180000      # base defer duration on denial, before jitter (3m)
 # MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS=14400000  # trickle ceiling: force-admit a maintenance job that has waited this long (4h)
+# MAINTENANCE_ADMISSION_DRAIN_AGE_MS=600000  # drain ceiling: admit a job despite a backed-up lane once it has waited this long (10m); clamped to MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS
 
 # --- Caddy HTTPS terminator (#1203; requires --profile caddy) ---
 # DOMAIN=gittensory.example.com             # fully-qualified domain; Caddy auto-obtains a Let's Encrypt cert

@@ -11,11 +11,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "AI_EMBED_API_KEY",
-    firstReference: "src/server.ts:422",
+    firstReference: "src/server.ts:423",
   },
   {
     name: "AI_EMBED_BASE_URL",
-    firstReference: "src/server.ts:419",
+    firstReference: "src/server.ts:420",
   },
   {
     name: "AI_EMBED_MODEL",
@@ -43,7 +43,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "BACKUP_ACKNOWLEDGED",
-    firstReference: "src/server.ts:361",
+    firstReference: "src/server.ts:362",
   },
   {
     name: "BROWSER_WS_ENDPOINT",
@@ -79,11 +79,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "CRON_INTERVAL_MS",
-    firstReference: "src/server.ts:865",
+    firstReference: "src/server.ts:872",
   },
   {
     name: "DATABASE_PATH",
-    firstReference: "src/server.ts:244",
+    firstReference: "src/server.ts:245",
   },
   {
     name: "DATABASE_URL",
@@ -107,11 +107,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "GITHUB_CACHE_TTL_SECONDS",
-    firstReference: "src/server.ts:490",
+    firstReference: "src/server.ts:491",
   },
   {
     name: "GITTENSORY_REPO_CONFIG_DIR",
-    firstReference: "src/server.ts:278",
+    firstReference: "src/server.ts:279",
   },
   {
     name: "GITTENSORY_VERSION",
@@ -123,11 +123,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "MAINTENANCE_ADMISSION_ENABLED",
-    firstReference: "src/selfhost/maintenance-admission.ts:83",
+    firstReference: "src/selfhost/maintenance-admission.ts:99",
   },
   {
     name: "MIGRATIONS_DIR",
-    firstReference: "src/server.ts:374",
+    firstReference: "src/server.ts:375",
   },
   {
     name: "OBSERVABILITY_SMOKE_POLL_MS",
@@ -187,7 +187,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "ORB_BROKER_URL",
-    firstReference: "src/server.ts:914",
+    firstReference: "src/server.ts:921",
   },
   {
     name: "ORB_COLLECTOR_TOKEN",
@@ -203,7 +203,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "ORB_RELAY_MODE",
-    firstReference: "src/server.ts:916",
+    firstReference: "src/server.ts:923",
   },
   {
     name: "OTEL_EXPORTER_OTLP_ENDPOINT",
@@ -235,11 +235,11 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "PGVECTOR_ENABLED",
-    firstReference: "src/server.ts:224",
+    firstReference: "src/server.ts:225",
   },
   {
     name: "PORT",
-    firstReference: "src/server.ts:664",
+    firstReference: "src/server.ts:671",
   },
   {
     name: "PUBLIC_API_ORIGIN",
@@ -255,7 +255,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "QDRANT_URL",
-    firstReference: "src/server.ts:509",
+    firstReference: "src/server.ts:510",
   },
   {
     name: "QUEUE_BACKGROUND_CONCURRENCY",
@@ -267,7 +267,7 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "REVIEW_AUDIT_DIR",
-    firstReference: "src/server.ts:554",
+    firstReference: "src/server.ts:555",
   },
   {
     name: "SELFHOST_BUNDLE_ALL",
@@ -303,23 +303,23 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
   },
   {
     name: "SETUP_OUTPUT_PATH",
-    firstReference: "src/server.ts:781",
+    firstReference: "src/server.ts:788",
   },
 ];
 
 export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| Name | First reference |",
   "| --- | --- |",
   "| `AI_COMBINE` | `src/selfhost/ai.ts:930` |",
-  "| `AI_EMBED_API_KEY` | `src/server.ts:422` |",
-  "| `AI_EMBED_BASE_URL` | `src/server.ts:419` |",
+  "| `AI_EMBED_API_KEY` | `src/server.ts:423` |",
+  "| `AI_EMBED_BASE_URL` | `src/server.ts:420` |",
   "| `AI_EMBED_MODEL` | `src/selfhost/ai.ts:826` |",
   "| `AI_ON_MERGE` | `src/selfhost/ai.ts:932` |",
   "| `AI_PROVIDER` | `src/selfhost/ai-config.ts:43` |",
   "| `ANTHROPIC_AI_BASE_URL` | `src/selfhost/ai.ts:830` |",
   "| `ANTHROPIC_AI_MODEL` | `src/selfhost/ai.ts:57` |",
   "| `ANTHROPIC_API_KEY` | `src/selfhost/ai.ts:829` |",
-  "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:361` |",
+  "| `BACKUP_ACKNOWLEDGED` | `src/server.ts:362` |",
   "| `BROWSER_WS_ENDPOINT` | `src/selfhost/stubs/puppeteer.ts:11` |",
   "| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts:108` |",
   "| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts:49` |",
@@ -328,19 +328,19 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts:53` |",
   "| `CODEX_AI_TIMEOUT_MS` | `src/selfhost/ai.ts:112` |",
   "| `CODEX_HOME` | `src/selfhost/ai.ts:274` |",
-  "| `CRON_INTERVAL_MS` | `src/server.ts:865` |",
-  "| `DATABASE_PATH` | `src/server.ts:244` |",
+  "| `CRON_INTERVAL_MS` | `src/server.ts:872` |",
+  "| `DATABASE_PATH` | `src/server.ts:245` |",
   "| `DATABASE_URL` | `src/selfhost/preflight.ts:201` |",
   "| `DISCORD_REPO_WEBHOOKS` | `src/selfhost/discord-notify.ts:31` |",
   "| `DISCORD_WEBHOOK_URL` | `src/selfhost/discord-notify.ts:40` |",
   "| `GITHUB_APP_ID` | `src/selfhost/orb-collector.ts:59` |",
   "| `GITHUB_APP_PRIVATE_KEY` | `src/selfhost/orb-collector.ts:166` |",
-  "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:490` |",
-  "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:278` |",
+  "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:491` |",
+  "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:279` |",
   "| `GITTENSORY_VERSION` | `src/selfhost/health.ts:29` |",
   "| `HOME` | `src/selfhost/ai.ts:274` |",
-  "| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:83` |",
-  "| `MIGRATIONS_DIR` | `src/server.ts:374` |",
+  "| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:99` |",
+  "| `MIGRATIONS_DIR` | `src/server.ts:375` |",
   "| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.mjs:8` |",
   "| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.mjs:6` |",
   "| `OLLAMA_AI_API_KEY` | `src/selfhost/ai.ts:823` |",
@@ -355,27 +355,27 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `ORB_AIR_GAP` | `src/selfhost/orb-collector.ts:161` |",
   "| `ORB_ANONYMIZE` | `src/selfhost/orb-collector.ts:174` |",
   "| `ORB_APP_ID` | `src/selfhost/orb-collector.ts:59` |",
-  "| `ORB_BROKER_URL` | `src/server.ts:914` |",
+  "| `ORB_BROKER_URL` | `src/server.ts:921` |",
   "| `ORB_COLLECTOR_TOKEN` | `src/selfhost/orb-collector.ts:205` |",
   "| `ORB_COLLECTOR_URL` | `src/selfhost/orb-collector.ts:172` |",
   "| `ORB_ENROLLMENT_SECRET` | `src/selfhost/orb-collector.ts:165` |",
-  "| `ORB_RELAY_MODE` | `src/server.ts:916` |",
+  "| `ORB_RELAY_MODE` | `src/server.ts:923` |",
   "| `OTEL_EXPORTER_OTLP_ENDPOINT` | `src/selfhost/otel.ts:47` |",
   "| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `src/selfhost/otel.ts:45` |",
   "| `OTEL_SERVICE_ENVIRONMENT` | `src/selfhost/otel.ts:60` |",
   "| `OTEL_SERVICE_NAME` | `src/selfhost/otel.ts:59` |",
   "| `OTEL_TRACES_EXPORTER` | `src/selfhost/otel.ts:40` |",
   "| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts:74` |",
   "| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts:76` |",
-  "| `PGVECTOR_ENABLED` | `src/server.ts:224` |",
-  "| `PORT` | `src/server.ts:664` |",
+  "| `PGVECTOR_ENABLED` | `src/server.ts:225` |",
+  "| `PORT` | `src/server.ts:671` |",
   "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts:192` |",
   "| `QDRANT_API_KEY` | `src/selfhost/qdrant-vectorize.ts:50` |",
   "| `QDRANT_DIM` | `src/selfhost/qdrant-vectorize.ts:71` |",
-  "| `QDRANT_URL` | `src/server.ts:509` |",
+  "| `QDRANT_URL` | `src/server.ts:510` |",
   "| `QUEUE_BACKGROUND_CONCURRENCY` | `src/selfhost/queue-common.ts:102` |",
   "| `REDIS_URL` | `src/selfhost/preflight.ts:144` |",
-  "| `REVIEW_AUDIT_DIR` | `src/server.ts:554` |",
+  "| `REVIEW_AUDIT_DIR` | `src/server.ts:555` |",
   "| `SELFHOST_BUNDLE_ALL` | `scripts/build-selfhost.mjs:13` |",
   "| `SELFHOST_SERVICE` | `scripts/smoke-observability-traces.mjs:5` |",
   "| `SELFHOST_SETUP_TOKEN` | `src/selfhost/preflight.ts:186` |",
@@ -384,5 +384,5 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `SENTRY_RELEASE` | `src/selfhost/otel.ts:62` |",
   "| `SENTRY_SERVER_NAME` | `src/selfhost/sentry.ts:383` |",
   "| `SENTRY_TRACES_SAMPLE_RATE` | `src/selfhost/sentry.ts:171` |",
-  "| `SETUP_OUTPUT_PATH` | `src/server.ts:781` |",
+  "| `SETUP_OUTPUT_PATH` | `src/server.ts:788` |",
 ].join("\n");
@@ -2396,6 +2396,33 @@
           "refId": "A"
         }
       ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": { "lineWidth": 2, "fillOpacity": 10 },
+          "unit": "ops"
+        }
+      },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 149 },
+      "id": 137,
+      "options": {
+        "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "title": "Maintenance Admission Granted Under Pressure (trickle/drain, #selfhost-maintenance-self-pin)",
+      "description": "A maintenance job admitted DESPITE active pressure -- the trickle (maxDeferAgeMs) or drain (maintenanceDrainAgeMs) age escapes firing. Healthy when this tracks alongside a high 'Maintenance Queue Pending' (the backlog is actively draining); zero activity here while that gauge stays high for a long stretch means the escapes aren't reaching their age thresholds yet.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "sum by (reason, job_type) (rate(gittensory_jobs_maintenance_admission_granted_under_pressure_total[5m])) or vector(0)",
+          "legendFormat": "{{reason}} {{job_type}}",
+          "refId": "A"
+        }
+      ]
     }
   ],
   "refresh": "30s",

@@ -134,19 +134,21 @@ groups:
           runbook: "Open the Runtime Pressure & Maintenance row. If gittensory_host_load_avg1_per_core is elevated, a co-located CI runner or other host process is starving the app -- see docker-compose.yml's runner isolation guidance."
 
       - alert: GittensoryMaintenanceStarved
-        # Maintenance admission (maintenance-admission.ts) force-admits a maintenance job once it has waited
-        # MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS (default 4h) regardless of pressure -- so under correct
-        # operation this should never sit much past that trickle ceiling. A value well beyond it means
-        # either the trickle isn't triggering (a bug) or the host is so overloaded even the trickle-forced
-        # job can't be claimed/processed.
+        # Maintenance admission (maintenance-admission.ts) has TWO age escapes (#selfhost-maintenance-self-pin):
+        # a short maintenanceDrainAgeMs trickle that lets old jobs through even while `maintenance_pending_high`
+        # is breached (bounded further by the queue's own background concurrency cap), and the ultimate
+        # MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS (default 4h) force-admit that applies regardless of pressure --
+        # so under correct operation this should never sit much past that outer ceiling, and in practice the
+        # drain escape should keep it well under it. A value well beyond 4h means either an escape isn't
+        # triggering (a bug) or the host is so overloaded even a force-admitted job can't be claimed/processed.
         expr: gittensory_queue_oldest_maintenance_pending_age_seconds > 21600
         for: 15m
         labels:
           severity: warning
         annotations:
           summary: "gittensory maintenance work has not run in over 6h"
           description: "The oldest maintenance-lane queue job has been pending for {{ $value | printf \"%.0f\" }}s, past the default trickle ceiling. Contributor evidence, RAG indexing, drift scans, and similar sweeps are stale."
-          runbook: "Check gittensory_jobs_maintenance_admission_deferred_by_reason_total for the dominant defer reason, and confirm MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS wasn't raised. Sustained host_load_high suggests the box itself (not just this app) is overloaded."
+          runbook: "Check gittensory_jobs_maintenance_admission_deferred_by_reason_total for the dominant defer reason and gittensory_jobs_maintenance_admission_granted_under_pressure_total to confirm the escapes are actually firing, and confirm MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS / MAINTENANCE_ADMISSION_DRAIN_AGE_MS weren't raised. Sustained host_load_high suggests the box itself (not just this app) is overloaded -- host_load_high also blocks the drain escape specifically, see maintenance-admission.ts."
 
   # ── GitHub API budget / queue admission pressure ──────────────────────────
   - name: gittensory-github-rate-limits

@@ -890,7 +890,15 @@ export async function processJob(env: Env, message: JobMessage): Promise<void> {
       await executeAgentRun(env, message.runId);
       return;
     case "notify-evaluate": {
-      const deliveries = await evaluateNotificationEvent(env, message.event);
+      // Legacy payload compat: a row enqueued before the batched-events deploy (#selfhost-maintenance-self-pin)
+      // still carries the OLD singular `event` field on disk, not `events` -- a rolling deploy can process such
+      // a row after the new code ships, so normalize both shapes rather than assuming every persisted payload
+      // already matches the current type (which only the type checker, not the durable queue, enforces).
+      const legacyMessage = message as unknown as { events?: DetectedNotificationEvent[]; event?: DetectedNotificationEvent };
+      const events = Array.isArray(legacyMessage.events) ? legacyMessage.events : legacyMessage.event ? [legacyMessage.event] : [];
+      const deliveries = (
+        await mapWithConcurrency(events, NOTIFY_EVALUATE_EVENT_CONCURRENCY, (event) => evaluateNotificationEvent(env, event))
+      ).flat();
       await Promise.all(
         deliveries.map((delivery) =>
           env.JOBS.send({
@@ -4144,6 +4152,29 @@ async function countLiveOpenWithConcurrencyUntil(
   return confirmedOpenCount;
 }
 
+// A batched notify-evaluate job (#selfhost-maintenance-self-pin) can carry many events from one webhook (a
+// popular newly-opened issue can have dozens of watchers) -- an unbounded Promise.all over all of them would
+// let a single job spend as many concurrent DB/eval calls as it likes, bypassing the queue's own
+// backgroundConcurrency cap (which defaults to 1) entirely from inside one job's execution. Bounded worker-pool
+// fan-out, same shape as GLOBAL_OPEN_ITEM_LIVE_CHECK_CONCURRENCY above.
+const NOTIFY_EVALUATE_EVENT_CONCURRENCY = 5;
+
+async function mapWithConcurrency<T, R>(items: T[], concurrency: number, mapper: (item: T) => Promise<R>): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let nextIndex = 0;
+  const workerCount = Math.max(1, Math.min(concurrency, items.length || 1));
+  await Promise.all(
+    Array.from({ length: workerCount }, async () => {
+      while (nextIndex < items.length) {
+        const index = nextIndex;
+        nextIndex += 1;
+        results[index] = await mapper(items[index] as T);
+      }
+    }),
+  );
+  return results;
+}
+
 /**
  * Install-wide contributor open-item count, LIVE-VERIFIED (#2562 gate-review follow-up): every OTHER counted
  * item is confirmed still-open via a live GET before counting toward the cap (mirrors the existing per-repo
@@ -5022,10 +5053,8 @@ async function processGitHubWebhook(
       payload.installation?.id,
       detectNotificationEvents(eventName, payload),
     );
-    for (const notificationEvent of [
-      ...trustedReviewEvents,
-      ...issueWatchEvents,
-    ]) {
+    const notificationEvents = [...trustedReviewEvents, ...issueWatchEvents];
+    for (const notificationEvent of notificationEvents) {
       await recordAuditEvent(env, {
         eventType: "notification.event_detected",
         actor: notificationEvent.actorLogin,
@@ -5042,10 +5071,16 @@ async function processGitHubWebhook(
           deeplink: notificationEvent.deeplink,
         },
       });
+    }
+    // Batched (#selfhost-maintenance-self-pin): every event this ONE webhook delivery detected rides in a
+    // single notify-evaluate job instead of one job per event -- the audit trail above still records each
+    // event individually, so nothing about observability changes, only how many maintenance-lane rows a
+    // multi-watcher issue (or a review event landing alongside issue-watch matches) creates.
+    if (notificationEvents.length > 0) {
       await env.JOBS.send({
         type: "notify-evaluate",
         requestedBy: "webhook",
-        event: notificationEvent,
+        events: notificationEvents,
       });
     }