Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 31 additions & 26 deletions src/db/repositories.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4127,8 +4127,10 @@ export async function listSignalSnapshots(env: Env, signalType: string, targetKe
return rows.map(toSignalSnapshotRecord);
}

const SIGNAL_SNAPSHOT_TARGET_KEY_SQL_BATCH = 90;

/** Bulk variant of `listSignalSnapshots` for callers that need the LATEST snapshot per target key across many
* keys in one round trip (#3202 review finding: a per-repo loop here made the daily repo-doc refresh sweep
* keys in bounded round trips (#3202 review finding: a per-repo loop here made the daily repo-doc refresh sweep
* scale linearly in DB round trips with the installed-repo count). Keyed by the exact `targetKey` string, same
* casing convention as `listSignalSnapshots` -- callers that key by lowercased repo name must lowercase both
* the input and the returned map's keys themselves. */
Expand All @@ -4139,31 +4141,34 @@ export async function listLatestSignalSnapshotsForTargets(
): Promise<Map<string, SignalSnapshotRecord>> {
const result = new Map<string, SignalSnapshotRecord>();
if (targetKeys.length === 0) return result;
const placeholders = targetKeys.map(() => "?").join(", ");
const { results } = await env.DB.prepare(
`
SELECT id, signal_type, target_key, repo_full_name, generated_at
FROM (
SELECT
id, signal_type, target_key, repo_full_name, generated_at,
row_number() OVER (PARTITION BY target_key ORDER BY generated_at DESC, id DESC) AS snapshot_rank
FROM signal_snapshots
WHERE signal_type = ? AND target_key IN (${placeholders})
)
WHERE snapshot_rank = 1
`,
)
.bind(signalType, ...targetKeys)
.all<{ id: string; signal_type: string; target_key: string; repo_full_name: string | null; generated_at: string }>();
for (const row of results) {
result.set(row.target_key, {
id: row.id,
signalType: row.signal_type,
targetKey: row.target_key,
repoFullName: row.repo_full_name,
payload: {},
generatedAt: row.generated_at,
});
for (let i = 0; i < targetKeys.length; i += SIGNAL_SNAPSHOT_TARGET_KEY_SQL_BATCH) {
const batch = targetKeys.slice(i, i + SIGNAL_SNAPSHOT_TARGET_KEY_SQL_BATCH);
const placeholders = batch.map(() => "?").join(", ");
const { results } = await env.DB.prepare(
`
SELECT id, signal_type, target_key, repo_full_name, generated_at
FROM (
SELECT
id, signal_type, target_key, repo_full_name, generated_at,
row_number() OVER (PARTITION BY target_key ORDER BY generated_at DESC, id DESC) AS snapshot_rank
FROM signal_snapshots
WHERE signal_type = ? AND target_key IN (${placeholders})
)
WHERE snapshot_rank = 1
`,
)
.bind(signalType, ...batch)
.all<{ id: string; signal_type: string; target_key: string; repo_full_name: string | null; generated_at: string }>();
for (const row of results) {
result.set(row.target_key, {
id: row.id,
signalType: row.signal_type,
targetKey: row.target_key,
repoFullName: row.repo_full_name,
payload: {},
generatedAt: row.generated_at,
});
}
}
return result;
}
Expand Down
39 changes: 39 additions & 0 deletions test/unit/data-spine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -512,4 +512,43 @@ describe("listLatestSignalSnapshotsForTargets (#3202 — bulk latest-per-target
expect(result.get("owner/b")).toMatchObject({ id: "b-only", generatedAt: "2026-03-01T00:00:00.000Z" });
expect(result.has("owner/c")).toBe(false);
});

it("batches target-key lookups to stay under D1 bound-parameter limits", async () => {
const env = createTestEnv();
const targetKeys = Array.from({ length: 95 }, (_, index) => `owner/repo-${index}`);
for (const [index, targetKey] of targetKeys.entries()) {
await persistSignalSnapshot(env, {
id: `batched-${index}`,
signalType: "repo-doc-refresh-attempt",
targetKey,
payload: {},
generatedAt: `2026-06-01T00:${String(index).padStart(2, "0")}:00.000Z`,
});
}
const db = env.DB;
const boundCounts: number[] = [];
env.DB = {
...db,
prepare(sql: string) {
const statement = db.prepare(sql);
return {
...statement,
bind(...values: Parameters<typeof statement.bind>) {
if (sql.includes("FROM signal_snapshots") && sql.includes("target_key IN")) {
boundCounts.push(values.length);
if (values.length > 91) throw new Error(`too many bound parameters: ${values.length}`);
}
return statement.bind(...values);
},
};
},
} as D1Database;

const result = await listLatestSignalSnapshotsForTargets(env, "repo-doc-refresh-attempt", targetKeys);

expect(result.size).toBe(95);
expect(result.get("owner/repo-0")?.id).toBe("batched-0");
expect(result.get("owner/repo-94")?.id).toBe("batched-94");
expect(boundCounts).toEqual([91, 6]);
});
});
Loading