diff --git a/src/index.ts b/src/index.ts index 5af8faadf6..e98ec8e532 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,11 +7,16 @@ import { isOrbBrokerEnabled } from "./orb/broker"; import { isOpsEnabled } from "./review/ops-wire"; import { isRagEnabled } from "./review/rag-wire"; import { isSelfTuneEnabled } from "./review/selftune-wire"; -import { isGitHubBudgetBackgroundJob } from "./selfhost/queue-common"; +import { + isGitHubBudgetBackgroundJob, + queueSnapshotBacklog, + queueSnapshotFromBinding, +} from "./selfhost/queue-common"; import { isReviewExecutionJob, isSelfHostedReviewRuntime } from "./selfhost/review-runtime"; import type { JobMessage } from "./types"; const app = createApp(); +const REGATE_BACKPRESSURE_TYPES = ["agent-regate-pr", "agent-regate-sweep"] as const; export { RateLimiter }; @@ -99,11 +104,26 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // tick (~2 min) retries, and after the bucket resets the sweep resumes. Webhooks never pre-yield. const jobs: JobMessage[] = []; const selfHostedReviews = isSelfHostedReviewRuntime(env); + const queueSnapshot = selfHostedReviews + ? await queueSnapshotFromBinding(env.JOBS).catch((error) => { + console.warn( + JSON.stringify({ + level: "warn", + event: "selfhost_queue_snapshot_failed", + error: error instanceof Error ? error.message : "unknown error", + }), + ); + return null; + }) + : null; + const regateBacklog = queueSnapshotBacklog(queueSnapshot, REGATE_BACKPRESSURE_TYPES); let sweepThrottledUntil: string | undefined; if (selfHostedReviews) { sweepThrottledUntil = await shouldWaitForGitHubRateLimit(env, MAINTENANCE_RESERVED_HEADROOM); if (sweepThrottledUntil) { console.log(JSON.stringify({ event: "regate_sweep_throttled", resetAt: sweepThrottledUntil })); + } else if (regateBacklog > 0) { + console.log(JSON.stringify({ event: "regate_sweep_backlog_deferred", backlog: regateBacklog })); } else { jobs.push({ type: "agent-regate-sweep", requestedBy: "schedule" }); } @@ -120,8 +140,10 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // 30-min tick retries, and after the bucket resets the backfill resumes. The cheap single-call health jobs // (repair-data-fidelity, refresh-installation-health) stay unconditional — they cost ~one call and keep // installation/health state fresh even while the budget is reserved. - if (selfHostedReviews && !sweepThrottledUntil) { + if (selfHostedReviews && !sweepThrottledUntil && regateBacklog === 0) { jobs.push({ type: "backfill-registered-repos", requestedBy: "schedule", mode: isFullSyncWindow ? "full" : "light" }); + } else if (selfHostedReviews && regateBacklog > 0) { + console.log(JSON.stringify({ event: "backfill_backlog_deferred", backlog: regateBacklog })); } else if (selfHostedReviews) { console.log(JSON.stringify({ event: "backfill_throttled", resetAt: sweepThrottledUntil })); } diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 23415e09df..953ef3093c 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -209,6 +209,10 @@ import { delayUntil, shouldWaitForGitHubRateLimit, } from "../github/rate-limit"; +import { + queueSnapshotBacklog, + queueSnapshotFromBinding, +} from "../selfhost/queue-common"; import { downgradeCloseToHold, downgradeMergeToHold, @@ -408,6 +412,7 @@ import { errorMessage, nowIso } from "../utils/json"; const OFFICIAL_MINER_DETECTION_TTL_MS = 5 * 60 * 1000; const OFFICIAL_MINER_DETECTION_UNAVAILABLE_TTL_MS = 60 * 1000; +const PER_PR_REGATE_BACKPRESSURE_TYPES = ["agent-regate-pr"] as const; const PR_PUBLIC_SURFACE_ACTIONS = new Set([ "opened", "reopened", @@ -791,7 +796,7 @@ export async function processJob(env: Env, message: JobMessage): Promise { await fanOutAgentRegateSweepJobs(env, message.requestedBy); return; } - await sweepRepoRegate(env, message.repoFullName); + await sweepRepoRegate(env, message.repoFullName, message.requestedBy); return; case "agent-regate-pr": // One bounded re-gate unit fanned out by the sweep (#audit-sweep-fanout): re-review + stamp a single PR. @@ -1020,6 +1025,11 @@ async function fanOutAgentRegateSweepJobs( }); } +async function currentRegateBacklog(env: Env): Promise { + const snapshot = await queueSnapshotFromBinding(env.JOBS).catch(() => null); + return queueSnapshotBacklog(snapshot, PER_PR_REGATE_BACKPRESSURE_TYPES); +} + // Convergence (RAG / codebase index, flag GITTENSORY_REVIEW_RAG). The dispatch for the `rag-index-repo` job. // Caller already gated on isRagEnabled(env). // - No repoFullName → cron fan-out: enqueue one FULL re-index job per registered + cutover-allowlisted repo. @@ -1142,6 +1152,7 @@ async function maybeEnqueueRagReindexForMergedPr( async function sweepRepoRegate( env: Env, repoFullName: string | undefined, + requestedBy: "schedule" | "api" | "test", ): Promise { if (!repoFullName) return; const settings = await resolveRepositorySettings(env, repoFullName); @@ -1171,6 +1182,19 @@ async function sweepRepoRegate( }); return; } + const regateBacklog = requestedBy === "schedule" ? await currentRegateBacklog(env) : 0; + if (regateBacklog > 0) { + await recordAuditEvent(env, { + eventType: "agent.sweep.regate", + actor: "gittensory", + targetKey: repoFullName, + outcome: "queued", + detail: + "re-gate sweep deferred: prior scheduled re-gate work is still pending or processing", + metadata: { repoFullName, mode, deferred: true, regateBacklog }, + }); + return; + } const [repo, openPullRequests] = await Promise.all([ getRepository(env, repoFullName), listOpenPullRequests(env, repoFullName), diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index 39e9c5fe76..5df8f442d2 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -15,6 +15,7 @@ import { githubRateLimitAdmissionTargetForJob, githubRateLimitMetricContext, githubRateLimitRetryDelayMs, + buildSelfHostQueueSnapshot, jobCoalesceKey, jobPriority, queueBackgroundConcurrency, @@ -25,6 +26,7 @@ import { rateLimitRetryDelayWithJitter, matchesGitHubRateLimitAdmissionTarget, type GitHubRateLimitAdmissionTarget, + type SelfHostQueueSnapshot, } from "./queue-common"; import type { JobMessage } from "../types"; @@ -60,6 +62,7 @@ export interface PgDurableQueue { size(): Promise; deadCount(): Promise; stats(): Promise>; + snapshot(): Promise; } interface JobRow { @@ -570,6 +573,14 @@ export function createPgQueue( async stats() { return readQueueStats(); }, + async snapshot() { + const res = await pool.query( + `SELECT payload, status, run_after FROM ${TABLE} WHERE status IN ('pending','processing','dead')`, + ); + return buildSelfHostQueueSnapshot( + res.rows as Array<{ payload: string; status: string; run_after: string | number }>, + ); + }, }; async function reclaimExpiredProcessingJobs(): Promise { diff --git a/src/selfhost/queue-common.ts b/src/selfhost/queue-common.ts index 63fdd9a33a..2c4615bd18 100644 --- a/src/selfhost/queue-common.ts +++ b/src/selfhost/queue-common.ts @@ -20,6 +20,24 @@ const DEFAULT_PROCESSING_TIMEOUT_MS = 30 * 60_000; const DEFAULT_BACKGROUND_CONCURRENCY = 1; export const FOREGROUND_QUEUE_PRIORITY_FLOOR = 8; +export type SelfHostQueueJobStatus = "pending" | "processing" | "dead"; + +export type SelfHostQueueSnapshotRow = { + type: string; + status: SelfHostQueueJobStatus; + count: number; + due: number; +}; + +export type SelfHostQueueSnapshot = { + totals: Record & { due: number }; + byType: SelfHostQueueSnapshotRow[]; +}; + +export interface SelfHostQueueIntrospection { + snapshot(): SelfHostQueueSnapshot | Promise; +} + // Webhook-driven work (a fresh PR -> its review) jumps ahead of heavy background jobs. Per-PR review refreshes // sit just below real webhooks, and sweep fan-out sits below those so stale surfaces are repaired during bursts. // Bot-generated comment edits are background noise; keeping them with real webhooks lets panel edits starve repair. @@ -95,6 +113,61 @@ export function isGitHubBudgetBackgroundJob(message: JobMessage): boolean { return GITHUB_BUDGET_BACKGROUND_TYPES.has(message.type); } +export function buildSelfHostQueueSnapshot( + rows: Iterable<{ payload?: unknown; status?: unknown; run_after?: unknown; runAfter?: unknown }>, + nowMs = Date.now(), +): SelfHostQueueSnapshot { + const totals = { pending: 0, processing: 0, dead: 0, due: 0 }; + const byKey = new Map(); + for (const row of rows) { + const status = queueStatus(row.status); + if (!status) continue; + const type = typeof row.payload === "string" ? (extractPayloadType(row.payload) ?? "unknown") : "unknown"; + const runAfter = queueRunAfterMs(row.run_after ?? row.runAfter); + const due = status === "pending" && (runAfter === null || runAfter <= nowMs) ? 1 : 0; + const key = `${type}\0${status}`; + const current = byKey.get(key) ?? { type, status, count: 0, due: 0 }; + current.count += 1; + current.due += due; + byKey.set(key, current); + totals[status] += 1; + totals.due += due; + } + return { + totals, + byType: [...byKey.values()].sort((a, b) => a.type.localeCompare(b.type) || a.status.localeCompare(b.status)), + }; +} + +export function queueSnapshotBacklog( + snapshot: SelfHostQueueSnapshot | null | undefined, + types: readonly string[], + statuses: readonly SelfHostQueueJobStatus[] = ["pending", "processing"], +): number { + if (!snapshot) return 0; + const typeSet = new Set(types); + const statusSet = new Set(statuses); + return snapshot.byType.reduce( + (sum, row) => sum + (typeSet.has(row.type) && statusSet.has(row.status) ? row.count : 0), + 0, + ); +} + +export async function queueSnapshotFromBinding(binding: Queue): Promise { + const snapshot = (binding as Queue & Partial).snapshot; + if (typeof snapshot !== "function") return null; + return snapshot.call(binding); +} + +function queueStatus(value: unknown): SelfHostQueueJobStatus | null { + return value === "pending" || value === "processing" || value === "dead" ? value : null; +} + +function queueRunAfterMs(value: unknown): number | null { + const parsed = typeof value === "number" ? value : typeof value === "string" ? Number(value) : null; + return parsed !== null && Number.isFinite(parsed) ? parsed : null; +} + function githubObservedRateLimitDelayMs( observation: | { remaining?: unknown; reset_at?: unknown; resetAt?: unknown } diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index d623a155f4..cefcc36bbb 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -16,6 +16,7 @@ import { githubRateLimitAdmissionTargetForJob, githubRateLimitMetricContext, githubRateLimitRetryDelayMs, + buildSelfHostQueueSnapshot, jobCoalesceKey, jobPriority, queueBackgroundConcurrency, @@ -26,6 +27,7 @@ import { rateLimitRetryDelayWithJitter, matchesGitHubRateLimitAdmissionTarget, type GitHubRateLimitAdmissionTarget, + type SelfHostQueueSnapshot, } from "./queue-common"; import type { JobMessage } from "../types"; @@ -62,6 +64,7 @@ export interface DurableQueue { size(): number; deadCount(): number; stats(): Record; + snapshot(): SelfHostQueueSnapshot; } interface JobRow { @@ -518,6 +521,14 @@ export function createSqliteQueue( stats() { return readQueueStats(driver); }, + snapshot() { + return buildSelfHostQueueSnapshot( + driver.query( + `SELECT payload, status, run_after FROM ${TABLE} WHERE status IN ('pending','processing','dead')`, + [], + ).rows as Array<{ payload: string; status: string; run_after: number }>, + ); + }, }; } diff --git a/src/settings/agent-sweep.ts b/src/settings/agent-sweep.ts index ec18c533c1..0aef6efb78 100644 --- a/src/settings/agent-sweep.ts +++ b/src/settings/agent-sweep.ts @@ -12,12 +12,11 @@ import type { PullRequestRecord } from "../types"; // REST bucket. Each fanned-out per-PR re-review costs ~9 REST GETs (1 resync `GET /pulls/{n}`, then required- // contexts + CI aggregate in prReadyForReview, then required-contexts + merge-state + CI aggregate + files in // auto-maintain). The sweep re-arms every ~2 min (≈30 ticks/hr) and fans out per repo, so the worst-case hourly -// sweep cost is `SWEEP_MAX_PRS × repos × 9 × 30`. At the old cap of 25 over 3 self-host repos that is ~200k/hr — -// far over budget, the cause of the `regate_sweep_throttled` exhaustion. A cap of 6 bounds the worst case to -// `6 × 3 × 9 × 30 ≈ 4.9k/hr` (and steady state is far lower — the freshness skip + the in-flight drain guard stop -// re-regating just-touched PRs), reserving headroom for webhooks. A 30-PR backlog still fully converges in -// ceil(30/6)=5 sweeps (~10 min), so no PR's merge/close is meaningfully delayed. -export const SWEEP_MAX_PRS = 6; +// sweep cost is `SWEEP_MAX_PRS × repos × 9 × 30`. At the old cap of 25 over 3 self-host repos that is ~200k/hr. +// A cap of 6 still consumed nearly the whole REST bucket across three active repos, so scheduled sweeps now run +// at a smaller source budget and also skip while prior regate work is queued. A cap of 3 bounds the raw worst case +// to `3 × 3 × 9 × 30 ≈ 2.4k/hr`, leaving budget for live webhooks, cache misses, and branch-protection reads. +export const SWEEP_MAX_PRS = 3; // Skip-if-fresh window: a PR touched within this span was almost certainly just gated by its webhook, so the // sweep leaves it alone for that brief moment to avoid racing the in-flight webhook review. Kept SHORT (2 min) diff --git a/test/unit/agent-sweep.test.ts b/test/unit/agent-sweep.test.ts index 58f99adf38..7fed6f40ca 100644 --- a/test/unit/agent-sweep.test.ts +++ b/test/unit/agent-sweep.test.ts @@ -132,7 +132,7 @@ describe("selectRegateCandidates (#777 re-gate sweep selection)", () => { it("defaults: freshness window is two minutes and the cap is bounded for the shared REST budget (#audit-rate-headroom)", () => { expect(SWEEP_FRESHNESS_MS).toBe(2 * 60 * 1000); - expect(SWEEP_MAX_PRS).toBe(6); // lowered from 25: 6 × 3 repos × 9 GETs × 30 ticks/hr ≈ 4.9k/hr ≤ the 5000 bucket + expect(SWEEP_MAX_PRS).toBe(3); // 3 × 3 repos × 9 GETs × 30 ticks/hr ≈ 2.4k/hr, leaving headroom for webhooks const pulls = Array.from({ length: 40 }, (_, i) => pr({ number: i + 1, createdAt: minutesAgo(120 + i) })); expect(selectRegateCandidates({ pulls, now: NOW })).toHaveLength(SWEEP_MAX_PRS); }); diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 1c4df56eb0..d4731924e7 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -276,6 +276,55 @@ describe("worker entrypoint", () => { expect(sent).toEqual([{ type: "agent-regate-sweep", requestedBy: "schedule" }]); }); + it("does not enqueue scheduled sweep/backfill work while prior regate jobs are still queued", async () => { + const sent: Array = []; + const env = createTestEnv({ + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + snapshot: async () => ({ + totals: { pending: 2, processing: 1, dead: 0, due: 2 }, + byType: [ + { type: "agent-regate-pr", status: "pending", count: 2, due: 2 }, + { type: "agent-regate-sweep", status: "processing", count: 1, due: 0 }, + ], + }), + } as unknown as Queue, + }); + const waitUntil: Promise[] = []; + + await worker.scheduled(controllerFor("2026-05-25T05:30:00.000Z"), env, executionContext(waitUntil)); + await Promise.all(waitUntil); + + expect(sent).toEqual([ + { type: "repair-data-fidelity", requestedBy: "schedule" }, + { type: "refresh-installation-health", requestedBy: "schedule" }, + ]); + }); + + it("fails open when queue introspection is unavailable so scheduled maintenance still runs", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const sent: Array = []; + const env = createTestEnv({ + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + snapshot: async () => { + throw new Error("snapshot unavailable"); + }, + } as unknown as Queue, + }); + const waitUntil: Promise[] = []; + + await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await Promise.all(waitUntil); + + expect(sent).toEqual([{ type: "agent-regate-sweep", requestedBy: "schedule" }]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("selfhost_queue_snapshot_failed")); + }); + it("does not enqueue review sweeps from a broker-only Cloudflare runtime", async () => { const sent: Array = []; const env = createTestEnv({ diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 002bb35c41..093e65288a 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -2050,6 +2050,100 @@ describe("queue processors", () => { expect(JSON.parse(audit?.metadata_json ?? "{}")).toMatchObject({ deferred: true }); }); + it("REGRESSION: a scheduled repo sweep does not fan out more per-PR regates while prior regate work is queued", async () => { + const sent: import("../../src/types").JobMessage[] = []; + const env = createTestEnv({ + JOBS: { + async send(m: import("../../src/types").JobMessage) { + sent.push(m); + }, + snapshot: async () => ({ + totals: { pending: 1, processing: 0, dead: 0, due: 1 }, + byType: [{ type: "agent-regate-pr", status: "pending", count: 1, due: 1 }], + }), + } as unknown as Queue, + }); + await upsertInstallation(env, { action: "created", installation: { id: 9201, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9201); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" } }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 7, title: "PR7", state: "open", user: { login: "c" }, head: { sha: "a7" }, labels: [], body: "" }); + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + const getRepo = vi.spyOn(repositoriesModule, "getRepository"); + const listOpen = vi.spyOn(repositoriesModule, "listOpenPullRequests"); + + await processJob(env, { type: "agent-regate-sweep", requestedBy: "schedule", repoFullName: "owner/agent-repo" }); + + expect(sent.filter((m) => m.type === "agent-regate-pr")).toEqual([]); + expect(getRepo).not.toHaveBeenCalled(); + expect(listOpen).not.toHaveBeenCalled(); + const audit = await env.DB.prepare("select outcome, metadata_json from audit_events where event_type = ?").bind("agent.sweep.regate").first<{ outcome: string; metadata_json: string }>(); + expect(audit?.outcome).toBe("queued"); + expect(JSON.parse(audit?.metadata_json ?? "{}")).toMatchObject({ deferred: true, regateBacklog: 1 }); + }); + + it("REGRESSION: a scheduled repo sweep ignores sweep rows when deciding per-PR regate backlog", async () => { + const sent: import("../../src/types").JobMessage[] = []; + const env = createTestEnv({ + JOBS: { + async send(m: import("../../src/types").JobMessage) { + sent.push(m); + }, + snapshot: async () => ({ + totals: { pending: 0, processing: 1, dead: 0, due: 0 }, + byType: [{ type: "agent-regate-sweep", status: "processing", count: 1, due: 0 }], + }), + } as unknown as Queue, + }); + await upsertInstallation(env, { action: "created", installation: { id: 9203, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9203); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" } }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 9, title: "PR9", state: "open", user: { login: "c" }, head: { sha: "a9" }, labels: [], body: "" }); + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + + await processJob(env, { type: "agent-regate-sweep", requestedBy: "schedule", repoFullName: "owner/agent-repo" }); + + expect(sent.filter((m) => m.type === "agent-regate-pr")).toEqual([ + expect.objectContaining({ + type: "agent-regate-pr", + deliveryId: "regate-sweep:owner/agent-repo#9", + repoFullName: "owner/agent-repo", + prNumber: 9, + installationId: 9203, + }), + ]); + }); + + it("INVARIANT: a scheduled repo sweep fails open when queue introspection throws", async () => { + const sent: import("../../src/types").JobMessage[] = []; + const env = createTestEnv({ + JOBS: { + async send(m: import("../../src/types").JobMessage) { + sent.push(m); + }, + snapshot: async () => { + throw new Error("snapshot unavailable"); + }, + } as unknown as Queue, + }); + await upsertInstallation(env, { action: "created", installation: { id: 9202, account: { login: "owner", id: 1, type: "Organization" }, target_type: "Organization", repository_selection: "selected", permissions: {}, events: [] } }); + await upsertRepositoryFromGitHub(env, { name: "agent-repo", full_name: "owner/agent-repo", private: false, owner: { login: "owner" } }, 9202); + await upsertRepositorySettings(env, { repoFullName: "owner/agent-repo", autonomy: { merge: "auto" } }); + await upsertPullRequestFromGitHub(env, "owner/agent-repo", { number: 8, title: "PR8", state: "open", user: { login: "c" }, head: { sha: "a8" }, labels: [], body: "" }); + vi.setSystemTime(new Date("2026-05-28T02:00:00.000Z")); + + await processJob(env, { type: "agent-regate-sweep", requestedBy: "schedule", repoFullName: "owner/agent-repo" }); + + expect(sent.filter((m) => m.type === "agent-regate-pr")).toEqual([ + { + type: "agent-regate-pr", + deliveryId: "regate-sweep:owner/agent-repo#8", + repoFullName: "owner/agent-repo", + prNumber: 8, + installationId: 9202, + }, + ]); + }); + it("REGRESSION: a per-PR re-gate job DEFERS (re-queues, no re-review/stamp) when the REST budget is below the maintenance floor (#audit-rate-headroom)", async () => { const sent: import("../../src/types").JobMessage[] = []; const env = createTestEnv({ JOBS: { async send(m: import("../../src/types").JobMessage) { sent.push(m); } } as unknown as Queue }); diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index 66bb5b7192..e4f68b76f5 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -1323,4 +1323,32 @@ describe("createPgQueue (durable #977)", () => { gittensory_jobs_dead_total: 0, }); }); + + it("snapshot() reports pending/processing/dead queue depth by job type", async () => { + const m = makePool(); + const q = createPgQueue(m.pool, async () => undefined); + await q.init(); + const now = Date.now(); + m.fn.mockResolvedValueOnce({ + rows: [ + { payload: JSON.stringify(msg("agent-regate-pr")), status: "pending", run_after: String(now - 1) }, + { payload: JSON.stringify(msg("agent-regate-pr")), status: "processing", run_after: String(now - 1) }, + { payload: JSON.stringify(msg("github-webhook")), status: "pending", run_after: String(now + 60_000) }, + { payload: JSON.stringify(msg("rag-index-repo")), status: "dead", run_after: String(now - 1) }, + ], + rowCount: 4, + }); + + const snapshot = await q.snapshot(); + + expect(snapshot.totals).toMatchObject({ pending: 2, processing: 1, dead: 1 }); + expect(snapshot.byType).toEqual( + expect.arrayContaining([ + { type: "agent-regate-pr", status: "pending", count: 1, due: 1 }, + { type: "agent-regate-pr", status: "processing", count: 1, due: 0 }, + { type: "github-webhook", status: "pending", count: 1, due: 0 }, + { type: "rag-index-repo", status: "dead", count: 1, due: 0 }, + ]), + ); + }); }); diff --git a/test/unit/selfhost-queue-common.test.ts b/test/unit/selfhost-queue-common.test.ts index 8623b652cf..9917a2617f 100644 --- a/test/unit/selfhost-queue-common.test.ts +++ b/test/unit/selfhost-queue-common.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { FOREGROUND_QUEUE_PRIORITY_FLOOR, + buildSelfHostQueueSnapshot, consumingRetryDelayMs, githubRateLimitAdmissionDelayMs, githubRateLimitAdmissionKeyScope, @@ -18,6 +19,8 @@ import { queueBackgroundConcurrency, queueProcessingTimeoutMs, queueRecoveryJitterMs, + queueSnapshotBacklog, + queueSnapshotFromBinding, queueStartupJitterMinJobs, queueStartupJitterMs, } from "../../src/selfhost/queue-common"; @@ -61,6 +64,49 @@ describe("self-host queue common helpers", () => { expect(queueBackgroundConcurrency(4, "")).toBe(1); }); + it("builds queue snapshots by job type/status and only marks due pending jobs", () => { + const now = 1_000; + const snapshot = buildSelfHostQueueSnapshot( + [ + { payload: payload({ type: "agent-regate-pr" }), status: "pending", run_after: 999 }, + { payload: payload({ type: "agent-regate-pr" }), status: "pending", run_after: "1001" }, + { payload: payload({ type: "agent-regate-pr" }), status: "processing", run_after: 1 }, + { payload: payload({ type: "github-webhook" }), status: "processing", run_after: 1 }, + { payload: "not-json", status: "dead", run_after: 1 }, + { payload: payload({ type: "ignored" }), status: "done", run_after: 1 }, + { payload: null, status: "pending", runAfter: "not-a-number" }, + ], + now, + ); + + expect(snapshot.totals).toEqual({ pending: 3, processing: 2, dead: 1, due: 2 }); + expect(snapshot.byType).toEqual([ + { type: "agent-regate-pr", status: "pending", count: 2, due: 1 }, + { type: "agent-regate-pr", status: "processing", count: 1, due: 0 }, + { type: "github-webhook", status: "processing", count: 1, due: 0 }, + { type: "unknown", status: "dead", count: 1, due: 0 }, + { type: "unknown", status: "pending", count: 1, due: 1 }, + ]); + expect(queueSnapshotBacklog(snapshot, ["agent-regate-pr"])).toBe(3); + expect(queueSnapshotBacklog(snapshot, ["agent-regate-pr"], ["processing"])).toBe(1); + expect(queueSnapshotBacklog(snapshot, ["agent-regate-pr"], ["dead"])).toBe(0); + expect(queueSnapshotBacklog(null, ["agent-regate-pr"])).toBe(0); + }); + + it("reads queue snapshots only from self-host bindings that expose introspection", async () => { + const snapshot = buildSelfHostQueueSnapshot([ + { payload: payload({ type: "agent-regate-sweep" }), status: "pending", run_after: 0 }, + ]); + const binding = { + async send() {}, + async sendBatch() {}, + snapshot: () => snapshot, + } as unknown as Queue; + + await expect(queueSnapshotFromBinding(binding)).resolves.toBe(snapshot); + await expect(queueSnapshotFromBinding({ async send() {}, async sendBatch() {} } as unknown as Queue)).resolves.toBeNull(); + }); + it("identifies GitHub-budget background jobs without pre-yielding fresh webhooks or manual re-gates", () => { expect(isGitHubBudgetBackgroundJob({ type: "github-webhook", deliveryId: "d1", eventName: "pull_request", payload: {} })).toBe(false); expect(isGitHubBudgetBackgroundJob({ type: "recapture-preview", deliveryId: "r1", repoFullName: "owner/repo", prNumber: 1, installationId: 2, attempt: 1 })).toBe(false); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index f111b93194..c51174970f 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -819,6 +819,36 @@ describe("createSqliteQueue (durable #980)", () => { }); }); + it("snapshot() reports pending/processing/dead queue depth by job type", async () => { + const driver = makeDriver(); + const q = createSqliteQueue(driver, async () => undefined); + await q.binding.send(msg("agent-regate-pr"), { delaySeconds: 60 }); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'pending', 0, ?, 0, 10)", + [JSON.stringify(msg("github-webhook")), Date.now() - 1], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'processing', 0, ?, 0, 9)", + [JSON.stringify(msg("agent-regate-pr")), Date.now()], + ); + driver.query( + "INSERT INTO _selfhost_jobs (payload, status, attempts, run_after, created_at, priority) VALUES (?, 'dead', 0, ?, 0, 0)", + [JSON.stringify(msg("rag-index-repo")), Date.now()], + ); + + const snapshot = q.snapshot(); + + expect(snapshot.totals).toMatchObject({ pending: 2, processing: 1, dead: 1 }); + expect(snapshot.byType).toEqual( + expect.arrayContaining([ + { type: "agent-regate-pr", status: "pending", count: 1, due: 0 }, + { type: "agent-regate-pr", status: "processing", count: 1, due: 0 }, + { type: "github-webhook", status: "pending", count: 1, due: 1 }, + { type: "rag-index-repo", status: "dead", count: 1, due: 0 }, + ]), + ); + }); + it("coalesces recurring maintenance jobs by semantic scope and keeps distinct scopes separate", async () => { const driver = makeDriver(); const q = createSqliteQueue(driver, async () => undefined);