From a68589b631c620a6241474181b634d8c0f2b3d92 Mon Sep 17 00:00:00 2001 From: Derek Trimm <275381468+derektrimm@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:46:44 -0500 Subject: [PATCH 1/3] fix(usage): skip unchanged Antigravity databases on repeat scans Every usage summary decoded every Antigravity conversation database again, blocking the server long enough for clients to drop the connection. Memoise each database's parsed candidates by the (size, mtime) of the file and its WAL sidecar, and keep merging aliases before the date filter. --- apps/server/src/usage/UsageService.ts | 9 ++- .../src/usage/antigravityUsageReader.ts | 47 +++++++++++- .../src/usage/usageTranscriptReader.test.ts | 72 ++++++++++++++++++- 3 files changed, 122 insertions(+), 6 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 1e5cb6db20fe..a496fb86bb3b 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -8,7 +8,9 @@ * `(size, mtime)`. A cold 30-day scan of ~1.4 GB lands around 2-3 seconds; warm * scans only reparse files that changed, and a file that merely grew resumes * from its cached parse position so only the appended bytes are read. - * SQLite readers query live databases each scan so WAL writes remain visible. + * OpenCode's SQLite reader queries the live database each scan so WAL writes + * remain visible. Antigravity databases are memoised in memory while the + * database and its WAL keep the same `(size, mtime)`. * * @module UsageService */ @@ -51,7 +53,7 @@ import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; import { resolveAntigravityInstanceDirectories } from "../provider/antigravityAuthSupport.ts"; import { mergeProviderInstanceEnvironment } from "../provider/ProviderInstanceEnvironment.ts"; import { readOpenCodeUsage } from "./opencodeUsageReader.ts"; -import { readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; import { readCursorAccountUsage } from "./cursorUsageReader.ts"; import { UsageAggregator } from "./usageAggregation.ts"; import { createOverrideRateTable, parseRateTable, type RateTable } from "./usagePricing.ts"; @@ -160,6 +162,7 @@ export const make = Effect.gen(function* () { const platform = yield* HostProcessPlatform; const fileCache: ScanCache = new Map(); + const antigravityCache = makeAntigravityUsageCache(); const sourceCache = new Map(); let cacheDirty = false; const isWithinDirectory = (filePath: string, dir: string) => { @@ -564,7 +567,7 @@ export const make = Effect.gen(function* () { antigravityDirs.add(yield* fileSystem.realPath(dir).pipe(Effect.orElseSucceed(() => dir))); } const antigravity = yield* Effect.promise(() => - readAntigravityUsage([...antigravityDirs], windowStartMs), + readAntigravityUsage([...antigravityDirs], windowStartMs, antigravityCache), ); for (const dir of antigravityDirs) { const exists = yield* fileSystem diff --git a/apps/server/src/usage/antigravityUsageReader.ts b/apps/server/src/usage/antigravityUsageReader.ts index 54c00b804b7a..1c4eeabf74f1 100644 --- a/apps/server/src/usage/antigravityUsageReader.ts +++ b/apps/server/src/usage/antigravityUsageReader.ts @@ -257,10 +257,30 @@ async function readDatabase(path: string, fallbackTimestamp: number): Promise new Map(); + +/** + * Reads and merges aliases across every configured Antigravity store before date + * filtering. With a cache, databases unchanged since the previous read are not + * decoded again. + */ export async function readAntigravityUsage( conversationsDirectories: string | readonly string[], sinceMs: number, + cache?: Map, ) { const roots = typeof conversationsDirectories === "string" @@ -351,7 +371,27 @@ export async function readAntigravityUsage( if (visited.has(canonical)) continue; visited.add(canonical); const stat = await NodeFSP.stat(path); - const candidates = await readDatabase(path, stat.mtimeMs); + const wal = await NodeFSP.stat(`${path}-wal`).catch(() => null); + const cached = cache?.get(canonical); + let candidates: readonly UsageCandidate[]; + if ( + cached !== undefined && + cached.size === stat.size && + cached.mtimeMs === stat.mtimeMs && + cached.walSize === (wal?.size ?? null) && + cached.walMtimeMs === (wal?.mtimeMs ?? null) + ) { + candidates = cached.candidates; + } else { + candidates = await readDatabase(path, stat.mtimeMs); + cache?.set(canonical, { + size: stat.size, + mtimeMs: stat.mtimeMs, + walSize: wal?.size ?? null, + walMtimeMs: wal?.mtimeMs ?? null, + candidates, + }); + } const fileIndex = files.length; files.push({ root, path, records: [] }); for (const [index, candidate] of candidates.entries()) { @@ -365,6 +405,9 @@ export async function readAntigravityUsage( } }; for (const root of roots) await walk(root, root); + if (cache !== undefined) { + for (const key of cache.keys()) if (!visited.has(key)) cache.delete(key); + } for (const [index, group] of groups.entries()) { if (group.parent === index && group.record.timestampMs >= sinceMs) { files[group.fileIndex]!.records.push(group.record); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 6c95a36df19b..b3242cd160cc 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -11,7 +11,7 @@ import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest"; import { readTranscriptRecords } from "./usageTranscriptReader.ts"; import { readOpenCodeUsage } from "./opencodeUsageReader.ts"; import { readCursorAccountUsage } from "./cursorUsageReader.ts"; -import { readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; let dir: string; @@ -725,6 +725,76 @@ describe("SQLite usage readers", () => { ); }); + const antigravityGeneration = (responseId: string) => + new Uint8Array( + protoBytes(1, [ + ...protoBytes(4, [ + ...protoNumber(2, 100), + ...protoNumber(3, 40), + ...protoText(11, responseId), + ]), + ...protoText(19, "Gemini 3 Pro"), + ...protoBytes(9, protoBytes(4, protoNumber(1, 1780000000))), + ]), + ); + + it("reuses an unchanged Antigravity database instead of decoding it again", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)"); + db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1")); + } finally { + db.close(); + } + await NodeFSP.utimes(path, 1780000000, 1780000000); + const cache = makeAntigravityUsageCache(); + const first = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(first.errors, []); + + // Same size and mtime, but no longer a readable database: only a cache hit + // can still return the earlier records. + const { size } = await NodeFSP.stat(path); + await NodeFSP.writeFile(path, Buffer.alloc(size)); + await NodeFSP.utimes(path, 1780000000, 1780000000); + assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1); + const cached = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(cached.errors, []); + assert.deepStrictEqual( + cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["antigravity:11:r-1"], + ); + }); + + it("rereads an Antigravity database when only its WAL changed", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec( + "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0; CREATE TABLE gen_metadata (idx INTEGER, data BLOB)", + ); + const insert = db.prepare("INSERT INTO gen_metadata VALUES (?, ?)"); + insert.run(0, antigravityGeneration("r-1")); + const cache = makeAntigravityUsageCache(); + const first = await readAntigravityUsage(dir, 0, cache); + assert.strictEqual(first.files.flatMap((file) => file.records).length, 1); + + const before = await NodeFSP.stat(path); + insert.run(1, antigravityGeneration("r-2")); + const after = await NodeFSP.stat(path); + assert.strictEqual(after.size, before.size); + assert.strictEqual(after.mtimeMs, before.mtimeMs); + + const next = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual( + next.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["antigravity:11:r-1", "antigravity:11:r-2"], + ); + } finally { + db.close(); + } + }); + it("reads Antigravity step-only stores and reports malformed databases", async () => { const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "steps.db")); try { From c993d62ada193fd53cabc03280970bd4fff7249e Mon Sep 17 00:00:00 2001 From: Derek Trimm <275381468+derektrimm@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:26:45 -0500 Subject: [PATCH 2/3] fix(usage): include ctime in the Antigravity cache fingerprint A rewrite that restores size and mtime still moves ctime, so the cache no longer serves stale candidates in that case. The reuse test now proves a cache hit with an exclusive lock instead of a forged rewrite, and the forged rewrite gets its own test that expects a fresh read. --- apps/server/src/usage/UsageService.ts | 2 +- .../src/usage/antigravityUsageReader.ts | 29 +++++--------- .../src/usage/usageTranscriptReader.test.ts | 39 ++++++++++++++----- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index a496fb86bb3b..c6f381a4934e 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -10,7 +10,7 @@ * from its cached parse position so only the appended bytes are read. * OpenCode's SQLite reader queries the live database each scan so WAL writes * remain visible. Antigravity databases are memoised in memory while the - * database and its WAL keep the same `(size, mtime)`. + * database and its WAL keep the same `(size, mtime, ctime)`. * * @module UsageService */ diff --git a/apps/server/src/usage/antigravityUsageReader.ts b/apps/server/src/usage/antigravityUsageReader.ts index 1c4eeabf74f1..a34f70300a2d 100644 --- a/apps/server/src/usage/antigravityUsageReader.ts +++ b/apps/server/src/usage/antigravityUsageReader.ts @@ -258,17 +258,15 @@ async function readDatabase(path: string, fallbackTimestamp: number): Promise new Map(); @@ -372,25 +370,16 @@ export async function readAntigravityUsage( visited.add(canonical); const stat = await NodeFSP.stat(path); const wal = await NodeFSP.stat(`${path}-wal`).catch(() => null); + const fingerprint = [stat, wal] + .map((file) => (file ? `${file.size}:${file.mtimeMs}:${file.ctimeMs}` : "-")) + .join("/"); const cached = cache?.get(canonical); let candidates: readonly UsageCandidate[]; - if ( - cached !== undefined && - cached.size === stat.size && - cached.mtimeMs === stat.mtimeMs && - cached.walSize === (wal?.size ?? null) && - cached.walMtimeMs === (wal?.mtimeMs ?? null) - ) { + if (cached?.fingerprint === fingerprint) { candidates = cached.candidates; } else { candidates = await readDatabase(path, stat.mtimeMs); - cache?.set(canonical, { - size: stat.size, - mtimeMs: stat.mtimeMs, - walSize: wal?.size ?? null, - walMtimeMs: wal?.mtimeMs ?? null, - candidates, - }); + cache?.set(canonical, { fingerprint, candidates }); } const fileIndex = files.length; files.push({ root, path, records: [] }); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index b3242cd160cc..9e5dc49e27d3 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -739,6 +739,31 @@ describe("SQLite usage readers", () => { ); it("reuses an unchanged Antigravity database instead of decoding it again", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)"); + db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1")); + const cache = makeAntigravityUsageCache(); + assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []); + + // An exclusive lock makes a fresh read fail without touching the file, so + // only a cache hit can still return the earlier records. + db.exec("BEGIN EXCLUSIVE"); + assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1); + const cached = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(cached.errors, []); + assert.deepStrictEqual( + cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["antigravity:11:r-1"], + ); + db.exec("ROLLBACK"); + } finally { + db.close(); + } + }); + + it("rereads an Antigravity database rewritten with its size and mtime restored", async () => { const path = NodePath.join(dir, "session-1.db"); const db = new NodeSqlite.DatabaseSync(path); try { @@ -749,20 +774,16 @@ describe("SQLite usage readers", () => { } await NodeFSP.utimes(path, 1780000000, 1780000000); const cache = makeAntigravityUsageCache(); - const first = await readAntigravityUsage(dir, 0, cache); - assert.deepStrictEqual(first.errors, []); + assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []); - // Same size and mtime, but no longer a readable database: only a cache hit - // can still return the earlier records. const { size } = await NodeFSP.stat(path); await NodeFSP.writeFile(path, Buffer.alloc(size)); await NodeFSP.utimes(path, 1780000000, 1780000000); - assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1); - const cached = await readAntigravityUsage(dir, 0, cache); - assert.deepStrictEqual(cached.errors, []); + const next = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(next.errors, [path]); assert.deepStrictEqual( - cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey), - ["antigravity:11:r-1"], + next.files.flatMap((file) => file.records), + [], ); }); From f1246e3d2542b7d3834002bc27f1a4dec95650c1 Mon Sep 17 00:00:00 2001 From: Derek Trimm <275381468+derektrimm@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:30:51 -0500 Subject: [PATCH 3/3] test(usage): land the forged rewrite on a later timestamp tick On kernels with coarse timestamps a write in the same tick as the cached read keeps the same ctime, so the test repeats the rewrite until ctime moves, as any real rewrite does. --- .../server/src/usage/usageTranscriptReader.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 9e5dc49e27d3..37d92a6ace20 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -776,9 +776,17 @@ describe("SQLite usage readers", () => { const cache = makeAntigravityUsageCache(); assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []); - const { size } = await NodeFSP.stat(path); - await NodeFSP.writeFile(path, Buffer.alloc(size)); - await NodeFSP.utimes(path, 1780000000, 1780000000); + // ctime has the kernel's timestamp granularity, which can be a few + // milliseconds, so repeat the forged rewrite until it lands on a later tick + // than the cached read, as any real rewrite does. + const cached = await NodeFSP.stat(path); + do { + await NodeFSP.writeFile(path, Buffer.alloc(cached.size)); + await NodeFSP.utimes(path, 1780000000, 1780000000); + } while ((await NodeFSP.stat(path)).ctimeMs === cached.ctimeMs); + const restored = await NodeFSP.stat(path); + assert.strictEqual(restored.size, cached.size); + assert.strictEqual(restored.mtimeMs, cached.mtimeMs); const next = await readAntigravityUsage(dir, 0, cache); assert.deepStrictEqual(next.errors, [path]); assert.deepStrictEqual(