feat(selfhost): add per-installation GitHub-fetch concurrency admission

.env.example

@@ -323,6 +323,19 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 #                                                   #   released job re-attempting GitHub at once and immediately
 #                                                   #   re-tripping the same rate-limit bucket it was deferred for
 
+# --- Per-installation GitHub-fetch concurrency (#selfhost-installation-concurrency) ---
+# QUEUE_BACKGROUND_CONCURRENCY caps how many background jobs run AT ALL, globally -- it has no notion of WHICH
+# installation those jobs belong to, so raising it above the default of 1 lets one installation's background
+# sweep/backfill claim every available background slot at once, starving every OTHER installation's background
+# work even though neither is anywhere near GitHub-rate-limit exhaustion. This is a THIRD, per-installation
+# claim-time admission check (alongside GitHub rate-limit admission and maintenance-lane admission above),
+# checked only for background jobs that make GitHub calls -- never for foreground live-PR work
+# (github-webhook/agent-regate-pr). A denied job is deferred with jitter, never dropped. Per instance, per
+# process (same single-process-per-deployment model as QUEUE_CONCURRENCY/QUEUE_BACKGROUND_CONCURRENCY above).
+# GITHUB_INSTALLATION_CONCURRENCY_ENABLED=true       # set false/0/off to fully disable this check
+# GITHUB_INSTALLATION_CONCURRENCY_LIMIT=2            # max concurrent GitHub-fetching background jobs per installation
+# GITHUB_INSTALLATION_CONCURRENCY_DEFER_MS=15000     # base defer duration on denial, before jitter (15s)
+
 # --- Caddy HTTPS terminator (#1203; requires --profile caddy) ---
 # DOMAIN=gittensory.example.com             # fully-qualified domain; Caddy auto-obtains a Let's Encrypt cert
 

apps/gittensory-ui/src/lib/selfhost-env-reference.ts

@@ -97,10 +97,22 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "DISCORD_WEBHOOK_URL",
     firstReference: "src/services/notify-discord.ts:78",
   },
+  {
+    name: "FOREGROUND_LIVENESS_CHECK_INTERVAL_MS",
+    firstReference: "src/selfhost/foreground-liveness.ts:52",
+  },
   {
     name: "FOREGROUND_LIVENESS_ENABLED",
     firstReference: "src/selfhost/foreground-liveness.ts:41",
   },
+  {
+    name: "FOREGROUND_LIVENESS_MAX_DEFER_MS",
+    firstReference: "src/selfhost/foreground-liveness.ts:51",
+  },
+  {
+    name: "FOREGROUND_LIVENESS_MAX_RELEASE_PER_SWEEP",
+    firstReference: "src/selfhost/foreground-liveness.ts:53",
+  },
   {
     name: "GITHUB_APP_ID",
     firstReference: "src/selfhost/orb-collector.ts:59",
@@ -113,6 +125,18 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "GITHUB_CACHE_TTL_SECONDS",
     firstReference: "src/server.ts:508",
   },
+  {
+    name: "GITHUB_INSTALLATION_CONCURRENCY_DEFER_MS",
+    firstReference: "src/selfhost/installation-concurrency-admission.ts:47",
+  },
+  {
+    name: "GITHUB_INSTALLATION_CONCURRENCY_ENABLED",
+    firstReference: "src/selfhost/installation-concurrency-admission.ts:34",
+  },
+  {
+    name: "GITHUB_INSTALLATION_CONCURRENCY_LIMIT",
+    firstReference: "src/selfhost/installation-concurrency-admission.ts:43",
+  },
   {
     name: "GITTENSORY_REPO_CONFIG_DIR",
     firstReference: "src/server.ts:288",
@@ -125,10 +149,38 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "HOME",
     firstReference: "src/selfhost/ai.ts:302",
   },
+  {
+    name: "MAINTENANCE_ADMISSION_DEFER_MS",
+    firstReference: "src/selfhost/maintenance-admission.ts:171",
+  },
+  {
+    name: "MAINTENANCE_ADMISSION_DRAIN_AGE_MS",
+    firstReference: "src/selfhost/maintenance-admission.ts:145",
+  },
   {
     name: "MAINTENANCE_ADMISSION_ENABLED",
     firstReference: "src/selfhost/maintenance-admission.ts:126",
   },
+  {
+    name: "MAINTENANCE_ADMISSION_MAX_BACKLOG_CONVERGENCE_PENDING",
+    firstReference: "src/selfhost/maintenance-admission.ts:167",
+  },
+  {
+    name: "MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS",
+    firstReference: "src/selfhost/maintenance-admission.ts:141",
+  },
+  {
+    name: "MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS",
+    firstReference: "src/selfhost/maintenance-admission.ts:155",
+  },
+  {
+    name: "MAINTENANCE_ADMISSION_MAX_LIVE_PENDING",
+    firstReference: "src/selfhost/maintenance-admission.ts:151",
+  },
+  {
+    name: "MAINTENANCE_ADMISSION_MAX_PENDING",
+    firstReference: "src/selfhost/maintenance-admission.ts:159",
+  },
   {
     name: "MIGRATIONS_DIR",
     firstReference: "src/server.ts:392",
@@ -237,6 +289,10 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "OTEL_TRACES_SAMPLER_ARG",
     firstReference: "src/selfhost/otel.ts:76",
   },
+  {
+    name: "PGPOOL_MAX",
+    firstReference: "src/selfhost/queue-common.ts:710",
+  },
   {
     name: "PGVECTOR_ENABLED",
     firstReference: "src/server.ts:229",
@@ -265,6 +321,18 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "QUEUE_BACKGROUND_CONCURRENCY",
     firstReference: "src/selfhost/queue-common.ts:130",
   },
+  {
+    name: "QUEUE_CONCURRENCY",
+    firstReference: "src/selfhost/pg-queue.ts:285",
+  },
+  {
+    name: "QUEUE_DEAD_LETTER_AUTO_RETRY_MAX_EXTRA_ATTEMPTS",
+    firstReference: "src/selfhost/queue-common.ts:718",
+  },
+  {
+    name: "QUEUE_STARTUP_JITTER_MIN_JOBS",
+    firstReference: "src/selfhost/queue-common.ts:699",
+  },
   {
     name: "REDIS_URL",
     firstReference: "src/selfhost/preflight.ts:144",
@@ -341,14 +409,27 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `DATABASE_URL` | `src/selfhost/preflight.ts:201` |",
   "| `DISCORD_REPO_WEBHOOKS` | `src/services/notify-discord.ts:41` |",
   "| `DISCORD_WEBHOOK_URL` | `src/services/notify-discord.ts:78` |",
+  "| `FOREGROUND_LIVENESS_CHECK_INTERVAL_MS` | `src/selfhost/foreground-liveness.ts:52` |",
   "| `FOREGROUND_LIVENESS_ENABLED` | `src/selfhost/foreground-liveness.ts:41` |",
+  "| `FOREGROUND_LIVENESS_MAX_DEFER_MS` | `src/selfhost/foreground-liveness.ts:51` |",
+  "| `FOREGROUND_LIVENESS_MAX_RELEASE_PER_SWEEP` | `src/selfhost/foreground-liveness.ts:53` |",
   "| `GITHUB_APP_ID` | `src/selfhost/orb-collector.ts:59` |",
   "| `GITHUB_APP_PRIVATE_KEY` | `src/selfhost/orb-collector.ts:166` |",
   "| `GITHUB_CACHE_TTL_SECONDS` | `src/server.ts:508` |",
+  "| `GITHUB_INSTALLATION_CONCURRENCY_DEFER_MS` | `src/selfhost/installation-concurrency-admission.ts:47` |",
+  "| `GITHUB_INSTALLATION_CONCURRENCY_ENABLED` | `src/selfhost/installation-concurrency-admission.ts:34` |",
+  "| `GITHUB_INSTALLATION_CONCURRENCY_LIMIT` | `src/selfhost/installation-concurrency-admission.ts:43` |",
   "| `GITTENSORY_REPO_CONFIG_DIR` | `src/server.ts:288` |",
   "| `GITTENSORY_VERSION` | `src/selfhost/otel.ts:62` |",
   "| `HOME` | `src/selfhost/ai.ts:302` |",
+  "| `MAINTENANCE_ADMISSION_DEFER_MS` | `src/selfhost/maintenance-admission.ts:171` |",
+  "| `MAINTENANCE_ADMISSION_DRAIN_AGE_MS` | `src/selfhost/maintenance-admission.ts:145` |",
   "| `MAINTENANCE_ADMISSION_ENABLED` | `src/selfhost/maintenance-admission.ts:126` |",
+  "| `MAINTENANCE_ADMISSION_MAX_BACKLOG_CONVERGENCE_PENDING` | `src/selfhost/maintenance-admission.ts:167` |",
+  "| `MAINTENANCE_ADMISSION_MAX_DEFER_AGE_MS` | `src/selfhost/maintenance-admission.ts:141` |",
+  "| `MAINTENANCE_ADMISSION_MAX_LIVE_AGE_MS` | `src/selfhost/maintenance-admission.ts:155` |",
+  "| `MAINTENANCE_ADMISSION_MAX_LIVE_PENDING` | `src/selfhost/maintenance-admission.ts:151` |",
+  "| `MAINTENANCE_ADMISSION_MAX_PENDING` | `src/selfhost/maintenance-admission.ts:159` |",
   "| `MIGRATIONS_DIR` | `src/server.ts:392` |",
   "| `OBSERVABILITY_SMOKE_POLL_MS` | `scripts/smoke-observability-traces.mjs:8` |",
   "| `OBSERVABILITY_SMOKE_TIMEOUT_MS` | `scripts/smoke-observability-traces.mjs:6` |",
@@ -376,13 +457,17 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `OTEL_TRACES_EXPORTER` | `src/selfhost/otel.ts:40` |",
   "| `OTEL_TRACES_SAMPLER` | `src/selfhost/otel.ts:74` |",
   "| `OTEL_TRACES_SAMPLER_ARG` | `src/selfhost/otel.ts:76` |",
+  "| `PGPOOL_MAX` | `src/selfhost/queue-common.ts:710` |",
   "| `PGVECTOR_ENABLED` | `src/server.ts:229` |",
   "| `PORT` | `src/server.ts:715` |",
   "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts:192` |",
   "| `QDRANT_API_KEY` | `src/selfhost/qdrant-vectorize.ts:50` |",
   "| `QDRANT_DIM` | `src/selfhost/qdrant-vectorize.ts:71` |",
   "| `QDRANT_URL` | `src/server.ts:527` |",
   "| `QUEUE_BACKGROUND_CONCURRENCY` | `src/selfhost/queue-common.ts:130` |",
+  "| `QUEUE_CONCURRENCY` | `src/selfhost/pg-queue.ts:285` |",
+  "| `QUEUE_DEAD_LETTER_AUTO_RETRY_MAX_EXTRA_ATTEMPTS` | `src/selfhost/queue-common.ts:718` |",
+  "| `QUEUE_STARTUP_JITTER_MIN_JOBS` | `src/selfhost/queue-common.ts:699` |",
   "| `REDIS_URL` | `src/selfhost/preflight.ts:144` |",
   "| `REVIEW_AUDIT_DIR` | `src/server.ts:572` |",
   "| `SELFHOST_BUNDLE_ALL` | `scripts/build-selfhost.mjs:13` |",

scripts/gen-selfhost-env-reference.mjs

@@ -72,6 +72,8 @@ function collectEnvReads(source, fileName) {
       }
     } else if (ts.isCallExpression(node) && isStaticEnvHelperCall(node)) {
       addRead(node.arguments[1].text, node.arguments[1]);
+    } else if (ts.isCallExpression(node) && isProcessEnvNameHelperCall(node)) {
+      addRead(node.arguments[0].text, node.arguments[0]);
     }
     ts.forEachChild(node, visit);
   };
@@ -89,6 +91,21 @@ function isStaticEnvHelperCall(node) {
   );
 }
 
+// Some self-host helpers read `process.env` internally by name rather than taking an env container argument --
+// e.g. `parsePositiveIntEnv("QUEUE_CONCURRENCY", { min: 1, fallback: 4 })`. Recognized separately from
+// isStaticEnvHelperCall above (envString) because these take the var NAME as arg[0], not arg[1] after a
+// container.
+const PROCESS_ENV_NAME_HELPERS = new Set(["parsePositiveIntEnv"]);
+
+function isProcessEnvNameHelperCall(node) {
+  return (
+    ts.isIdentifier(node.expression) &&
+    PROCESS_ENV_NAME_HELPERS.has(node.expression.text) &&
+    node.arguments.length >= 1 &&
+    ts.isStringLiteralLike(node.arguments[0])
+  );
+}
+
 function bindingElementName(element) {
   const candidate = element.propertyName ?? element.name;
   if (ts.isIdentifier(candidate) || ts.isStringLiteralLike(candidate)) return candidate.text;

src/selfhost/installation-concurrency-admission.ts

@@ -0,0 +1,109 @@
+// Per-installation GitHub-fetch concurrency admission (#selfhost-installation-concurrency). The queue's own
+// QUEUE_BACKGROUND_CONCURRENCY caps how many background jobs run AT ALL, globally -- it has no notion of WHICH
+// installation those jobs belong to, so once an operator raises that cap above its default of 1, one
+// installation's background sweep/backfill can claim every available background slot at once and starve every
+// OTHER installation's background work, even though GitHub's rate-limit admission (queue-common.ts) is nowhere
+// near exhausted for either installation. This module adds an ORTHOGONAL signal, checked at claim time
+// alongside GitHub rate-limit admission and maintenance-lane pressure admission: is THIS installation already
+// running its share of concurrent GitHub-fetching background jobs right now? A denied job is pushed back to
+// 'pending' with a jittered future run_after, same as the other two admission layers -- never dropped.
+//
+// Deliberately in-process, not DB-backed: the queue's existing `active`/`activeBackground` counters (pg-queue.ts
+// / sqlite-queue.ts) are already per-process scalars with no cross-process aggregation, and maintenance-
+// admission.ts's own hostLoadAvg1PerCore() is inherently per-box too -- single-process-per-deployment is already
+// the supported topology for the whole admission system (the SQLite backend structurally cannot share state
+// across processes at all). A DB-backed live COUNT(*) query would need a new indexed installation column on
+// every job row just to answer a question this in-process tracker answers for free in that topology.
+//
+// Deliberately NEVER applied to foreground jobs (github-webhook, agent-regate-pr): this policy only ever runs
+// for a job where isGitHubBudgetBackgroundJob() is true, mirroring exactly how maintenance-admission.ts's
+// evaluateMaintenanceAdmission is only invoked for a background-priority job -- "reserve headroom for live PR
+// work" is satisfied structurally, not via a headroom calculation.
+import { deterministicJitterMs, parsePositiveIntEnv } from "./queue-common";
+
+const DEFAULT_MAX_CONCURRENT_PER_INSTALLATION = 2;
+const DEFAULT_DEFER_MS = 15_000;
+
+export interface InstallationConcurrencyConfig {
+  enabled: boolean;
+  maxConcurrentPerInstallation: number;
+  deferMs: number;
+}
+
+function installationConcurrencyEnabled(): boolean {
+  const raw = (process.env.GITHUB_INSTALLATION_CONCURRENCY_ENABLED ?? "").trim().toLowerCase();
+  return raw !== "0" && raw !== "false" && raw !== "off" && raw !== "no";
+}
+
+/** Reads every GITHUB_INSTALLATION_CONCURRENCY_* knob from process.env, each with a sane, protective default.
+ *  Resolved ONCE per queue instance (mirrors resolveMaintenanceAdmissionConfig) rather than per job. */
+export function resolveInstallationConcurrencyConfig(): InstallationConcurrencyConfig {
+  return {
+    enabled: installationConcurrencyEnabled(),
+    maxConcurrentPerInstallation: parsePositiveIntEnv("GITHUB_INSTALLATION_CONCURRENCY_LIMIT", {
+      min: 1,
+      fallback: DEFAULT_MAX_CONCURRENT_PER_INSTALLATION,
+    }),
+    deferMs: parsePositiveIntEnv("GITHUB_INSTALLATION_CONCURRENCY_DEFER_MS", {
+      min: 1_000,
+      fallback: DEFAULT_DEFER_MS,
+    }),
+  };
+}
+
+export type InstallationConcurrencyReason = "disabled" | "concurrency_high" | "clear";
+
+export interface InstallationConcurrencyDecision {
+  admit: boolean;
+  reason: InstallationConcurrencyReason;
+}
+
+/** PURE policy decision: is this installation allowed one more concurrent GitHub-budget-background job right
+ *  now? `currentInFlightCount` is the caller's own live read of the InstallationConcurrencyTracker below for
+ *  this exact admission key, taken immediately before this call. */
+export function evaluateInstallationConcurrencyAdmission(
+  config: InstallationConcurrencyConfig,
+  currentInFlightCount: number,
+): InstallationConcurrencyDecision {
+  if (!config.enabled) return { admit: true, reason: "disabled" };
+  if (currentInFlightCount >= config.maxConcurrentPerInstallation) {
+    return { admit: false, reason: "concurrency_high" };
+  }
+  return { admit: true, reason: "clear" };
+}
+
+/** Jittered defer duration for a denied background job -- the base `deferMs` plus up to another `deferMs` of
+ *  deterministic jitter (seeded by the job's own identity) so a cohort of denied jobs for the same installation
+ *  doesn't wake up on the same tick and immediately re-trip this same check (mirrors
+ *  maintenanceAdmissionDeferMs). Its own, shorter default (15s vs. maintenance's 3min) reflects that a
+ *  background-fetch burst for one installation settles on the order of seconds, not minutes. */
+export function installationConcurrencyDeferMs(config: InstallationConcurrencyConfig, jitterSeed: string): number {
+  return config.deferMs + deterministicJitterMs(jitterSeed, config.deferMs);
+}
+
+/** The ONLY stateful piece in this module -- a plain in-process in-flight counter keyed by GitHub rate-limit
+ *  admission key (installation:<id>). Constructed once per queue backend at module scope, mirroring how
+ *  `active`/`activeBackground` are module-scope scalars in pg-queue.ts/sqlite-queue.ts -- never exported as a
+ *  shared singleton, so it can only be mutated from the claim path that owns it. */
+export class InstallationConcurrencyTracker {
+  private readonly counts = new Map<string, number>();
+
+  currentCount(admissionKey: string): number {
+    return this.counts.get(admissionKey) ?? 0;
+  }
+
+  increment(admissionKey: string): void {
+    this.counts.set(admissionKey, this.currentCount(admissionKey) + 1);
+  }
+
+  /** Floors at 0 and deletes the key once it reaches 0, so a busy deployment with many distinct installations
+   *  never grows this Map unboundedly with stale zero entries. */
+  decrement(admissionKey: string): void {
+    const next = Math.max(0, this.currentCount(admissionKey) - 1);
+    if (next === 0) {
+      this.counts.delete(admissionKey);
+    } else {
+      this.counts.set(admissionKey, next);
+    }
+  }
+}

src/selfhost/metrics.ts

@@ -77,6 +77,8 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
   ["gittensory_jobs_rate_limit_budget_deferred_total", { help: "Jobs deferred by rate-limit budget checks.", type: "counter" }],
   ["gittensory_jobs_rate_limited_by_type_total", { help: "Jobs rate-limited by job type.", type: "counter" }],
   ["gittensory_jobs_maintenance_admission_deferred_by_reason_total", { help: "Maintenance jobs deferred by reason.", type: "counter" }],
+  ["gittensory_jobs_installation_concurrency_deferred_total", { help: "Background jobs deferred by per-installation GitHub-fetch concurrency admission.", type: "counter" }],
+  ["gittensory_jobs_installation_concurrency_deferred_by_reason_total", { help: "Per-installation GitHub-fetch concurrency deferrals by reason and job type.", type: "counter" }],
   ["gittensory_jobs_dead_letter_revived_total", { help: "Dead-letter jobs revived for retry.", type: "counter" }],
   ["gittensory_jobs_foreground_liveness_released_total", { help: "Foreground-priority jobs force-released from a stale deferral by the liveness sweep.", type: "counter" }],
   ["gittensory_jobs_foreground_liveness_released_by_reason_total", { help: "Foreground liveness releases by reason (age vs rate_limit_cleared).", type: "counter" }],