From 809cf3f04be8f694adad5cd1aab41a0b863c5b82 Mon Sep 17 00:00:00 2001 From: Derek Trimm <275381468+derektrimm@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:46:44 -0500 Subject: [PATCH 1/3] fix(usage): skip unchanged Antigravity databases on repeat scans Every usage summary decoded every Antigravity conversation database again, blocking the server long enough for clients to drop the connection. Memoise each database's parsed candidates by the (size, mtime) of the file and its WAL sidecar, and keep merging aliases before the date filter. --- apps/server/src/usage/UsageService.ts | 9 ++- .../src/usage/antigravityUsageReader.ts | 47 +++++++++++- .../src/usage/usageTranscriptReader.test.ts | 72 ++++++++++++++++++- 3 files changed, 122 insertions(+), 6 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index c2206a53d33c..38245e74678e 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -8,7 +8,9 @@ * `(size, mtime)`. A cold 30-day scan of ~1.4 GB lands around 2-3 seconds; warm * scans only reparse files that changed, and a file that merely grew resumes * from its cached parse position so only the appended bytes are read. - * SQLite readers query live databases each scan so WAL writes remain visible. + * OpenCode's SQLite reader queries the live database each scan so WAL writes + * remain visible. Antigravity databases are memoised in memory while the + * database and its WAL keep the same `(size, mtime)`. * * @module UsageService */ @@ -52,7 +54,7 @@ import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; import { resolveAntigravityInstanceDirectories } from "../provider/antigravityAuthSupport.ts"; import { mergeProviderInstanceEnvironment } from "../provider/ProviderInstanceEnvironment.ts"; import { readOpenCodeUsage } from "./opencodeUsageReader.ts"; -import { readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; import { readCursorAccountUsage } from "./cursorUsageReader.ts"; import { resolveModelAliases, UsageAggregator } from "./usageAggregation.ts"; import { createOverrideRateTable, parseRateTable, type RateTable } from "./usagePricing.ts"; @@ -195,6 +197,7 @@ export const make = Effect.gen(function* () { const platform = yield* HostProcessPlatform; const fileCache: ScanCache = new Map(); + const antigravityCache = makeAntigravityUsageCache(); const sourceCache = new Map(); let cacheDirty = false; const isWithinDirectory = (filePath: string, dir: string) => { @@ -661,7 +664,7 @@ export const make = Effect.gen(function* () { antigravityDirs.add(yield* fileSystem.realPath(dir).pipe(Effect.orElseSucceed(() => dir))); } const antigravity = yield* Effect.promise(() => - readAntigravityUsage([...antigravityDirs], windowStartMs), + readAntigravityUsage([...antigravityDirs], windowStartMs, antigravityCache), ); for (const dir of antigravityDirs) { const exists = yield* fileSystem diff --git a/apps/server/src/usage/antigravityUsageReader.ts b/apps/server/src/usage/antigravityUsageReader.ts index 34a88114f42e..2ac0d0cefba5 100644 --- a/apps/server/src/usage/antigravityUsageReader.ts +++ b/apps/server/src/usage/antigravityUsageReader.ts @@ -257,10 +257,30 @@ async function readDatabase(path: string, fallbackTimestamp: number): Promise new Map(); + +/** + * Reads and merges aliases across every configured Antigravity store before date + * filtering. With a cache, databases unchanged since the previous read are not + * decoded again. + */ export async function readAntigravityUsage( conversationsDirectories: string | readonly string[], sinceMs: number, + cache?: Map, ) { const roots = typeof conversationsDirectories === "string" @@ -351,7 +371,27 @@ export async function readAntigravityUsage( if (visited.has(canonical)) continue; visited.add(canonical); const stat = await NodeFSP.stat(path); - const candidates = await readDatabase(path, stat.mtimeMs); + const wal = await NodeFSP.stat(`${path}-wal`).catch(() => null); + const cached = cache?.get(canonical); + let candidates: readonly UsageCandidate[]; + if ( + cached !== undefined && + cached.size === stat.size && + cached.mtimeMs === stat.mtimeMs && + cached.walSize === (wal?.size ?? null) && + cached.walMtimeMs === (wal?.mtimeMs ?? null) + ) { + candidates = cached.candidates; + } else { + candidates = await readDatabase(path, stat.mtimeMs); + cache?.set(canonical, { + size: stat.size, + mtimeMs: stat.mtimeMs, + walSize: wal?.size ?? null, + walMtimeMs: wal?.mtimeMs ?? null, + candidates, + }); + } const fileIndex = files.length; files.push({ root, path, records: [] }); for (const [index, candidate] of candidates.entries()) { @@ -365,6 +405,9 @@ export async function readAntigravityUsage( } }; for (const root of roots) await walk(root, root); + if (cache !== undefined) { + for (const key of cache.keys()) if (!visited.has(key)) cache.delete(key); + } for (const [index, group] of groups.entries()) { if (group.parent === index && group.record.timestampMs >= sinceMs) { files[group.fileIndex]!.records.push(group.record); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index e55c01ad4919..8282a3cc9bb9 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -11,7 +11,7 @@ import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest"; import { readTranscriptRecords } from "./usageTranscriptReader.ts"; import { readOpenCodeUsage } from "./opencodeUsageReader.ts"; import { readCursorAccountUsage } from "./cursorUsageReader.ts"; -import { readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; let dir: string; @@ -738,6 +738,76 @@ describe("SQLite usage readers", () => { ); }); + const antigravityGeneration = (responseId: string) => + new Uint8Array( + protoBytes(1, [ + ...protoBytes(4, [ + ...protoNumber(2, 100), + ...protoNumber(3, 40), + ...protoText(11, responseId), + ]), + ...protoText(19, "Gemini 3 Pro"), + ...protoBytes(9, protoBytes(4, protoNumber(1, 1780000000))), + ]), + ); + + it("reuses an unchanged Antigravity database instead of decoding it again", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)"); + db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1")); + } finally { + db.close(); + } + await NodeFSP.utimes(path, 1780000000, 1780000000); + const cache = makeAntigravityUsageCache(); + const first = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(first.errors, []); + + // Same size and mtime, but no longer a readable database: only a cache hit + // can still return the earlier records. + const { size } = await NodeFSP.stat(path); + await NodeFSP.writeFile(path, Buffer.alloc(size)); + await NodeFSP.utimes(path, 1780000000, 1780000000); + assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1); + const cached = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(cached.errors, []); + assert.deepStrictEqual( + cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["antigravity:11:r-1"], + ); + }); + + it("rereads an Antigravity database when only its WAL changed", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec( + "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0; CREATE TABLE gen_metadata (idx INTEGER, data BLOB)", + ); + const insert = db.prepare("INSERT INTO gen_metadata VALUES (?, ?)"); + insert.run(0, antigravityGeneration("r-1")); + const cache = makeAntigravityUsageCache(); + const first = await readAntigravityUsage(dir, 0, cache); + assert.strictEqual(first.files.flatMap((file) => file.records).length, 1); + + const before = await NodeFSP.stat(path); + insert.run(1, antigravityGeneration("r-2")); + const after = await NodeFSP.stat(path); + assert.strictEqual(after.size, before.size); + assert.strictEqual(after.mtimeMs, before.mtimeMs); + + const next = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual( + next.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["antigravity:11:r-1", "antigravity:11:r-2"], + ); + } finally { + db.close(); + } + }); + it("reads Antigravity step-only stores and reports malformed databases", async () => { const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "steps.db")); try { From c17258b9850eef98a7ae7813b4de0c7b22d7825e Mon Sep 17 00:00:00 2001 From: Derek Trimm <275381468+derektrimm@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:26:45 -0500 Subject: [PATCH 2/3] fix(usage): include ctime in the Antigravity cache fingerprint A rewrite that restores size and mtime still moves ctime, so the cache no longer serves stale candidates in that case. The reuse test now proves a cache hit with an exclusive lock instead of a forged rewrite, and the forged rewrite gets its own test that expects a fresh read. --- apps/server/src/usage/UsageService.ts | 2 +- .../src/usage/antigravityUsageReader.ts | 29 +++++--------- .../src/usage/usageTranscriptReader.test.ts | 39 ++++++++++++++----- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index 38245e74678e..059d4e176bca 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -10,7 +10,7 @@ * from its cached parse position so only the appended bytes are read. * OpenCode's SQLite reader queries the live database each scan so WAL writes * remain visible. Antigravity databases are memoised in memory while the - * database and its WAL keep the same `(size, mtime)`. + * database and its WAL keep the same `(size, mtime, ctime)`. * * @module UsageService */ diff --git a/apps/server/src/usage/antigravityUsageReader.ts b/apps/server/src/usage/antigravityUsageReader.ts index 2ac0d0cefba5..8b1559cfedd7 100644 --- a/apps/server/src/usage/antigravityUsageReader.ts +++ b/apps/server/src/usage/antigravityUsageReader.ts @@ -258,17 +258,15 @@ async function readDatabase(path: string, fallbackTimestamp: number): Promise new Map(); @@ -372,25 +370,16 @@ export async function readAntigravityUsage( visited.add(canonical); const stat = await NodeFSP.stat(path); const wal = await NodeFSP.stat(`${path}-wal`).catch(() => null); + const fingerprint = [stat, wal] + .map((file) => (file ? `${file.size}:${file.mtimeMs}:${file.ctimeMs}` : "-")) + .join("/"); const cached = cache?.get(canonical); let candidates: readonly UsageCandidate[]; - if ( - cached !== undefined && - cached.size === stat.size && - cached.mtimeMs === stat.mtimeMs && - cached.walSize === (wal?.size ?? null) && - cached.walMtimeMs === (wal?.mtimeMs ?? null) - ) { + if (cached?.fingerprint === fingerprint) { candidates = cached.candidates; } else { candidates = await readDatabase(path, stat.mtimeMs); - cache?.set(canonical, { - size: stat.size, - mtimeMs: stat.mtimeMs, - walSize: wal?.size ?? null, - walMtimeMs: wal?.mtimeMs ?? null, - candidates, - }); + cache?.set(canonical, { fingerprint, candidates }); } const fileIndex = files.length; files.push({ root, path, records: [] }); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 8282a3cc9bb9..4e354039b7b0 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -752,6 +752,31 @@ describe("SQLite usage readers", () => { ); it("reuses an unchanged Antigravity database instead of decoding it again", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)"); + db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1")); + const cache = makeAntigravityUsageCache(); + assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []); + + // An exclusive lock makes a fresh read fail without touching the file, so + // only a cache hit can still return the earlier records. + db.exec("BEGIN EXCLUSIVE"); + assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1); + const cached = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(cached.errors, []); + assert.deepStrictEqual( + cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["antigravity:11:r-1"], + ); + db.exec("ROLLBACK"); + } finally { + db.close(); + } + }); + + it("rereads an Antigravity database rewritten with its size and mtime restored", async () => { const path = NodePath.join(dir, "session-1.db"); const db = new NodeSqlite.DatabaseSync(path); try { @@ -762,20 +787,16 @@ describe("SQLite usage readers", () => { } await NodeFSP.utimes(path, 1780000000, 1780000000); const cache = makeAntigravityUsageCache(); - const first = await readAntigravityUsage(dir, 0, cache); - assert.deepStrictEqual(first.errors, []); + assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []); - // Same size and mtime, but no longer a readable database: only a cache hit - // can still return the earlier records. const { size } = await NodeFSP.stat(path); await NodeFSP.writeFile(path, Buffer.alloc(size)); await NodeFSP.utimes(path, 1780000000, 1780000000); - assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1); - const cached = await readAntigravityUsage(dir, 0, cache); - assert.deepStrictEqual(cached.errors, []); + const next = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(next.errors, [path]); assert.deepStrictEqual( - cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey), - ["antigravity:11:r-1"], + next.files.flatMap((file) => file.records), + [], ); }); From 19c271ff50df7419092cf5a5926aaacfca1d82ec Mon Sep 17 00:00:00 2001 From: Derek Trimm <275381468+derektrimm@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:30:51 -0500 Subject: [PATCH 3/3] test(usage): land the forged rewrite on a later timestamp tick On kernels with coarse timestamps a write in the same tick as the cached read keeps the same ctime, so the test repeats the rewrite until ctime moves, as any real rewrite does. --- .../server/src/usage/usageTranscriptReader.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index 4e354039b7b0..e6f19d50718a 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -789,9 +789,17 @@ describe("SQLite usage readers", () => { const cache = makeAntigravityUsageCache(); assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []); - const { size } = await NodeFSP.stat(path); - await NodeFSP.writeFile(path, Buffer.alloc(size)); - await NodeFSP.utimes(path, 1780000000, 1780000000); + // ctime has the kernel's timestamp granularity, which can be a few + // milliseconds, so repeat the forged rewrite until it lands on a later tick + // than the cached read, as any real rewrite does. + const cached = await NodeFSP.stat(path); + do { + await NodeFSP.writeFile(path, Buffer.alloc(cached.size)); + await NodeFSP.utimes(path, 1780000000, 1780000000); + } while ((await NodeFSP.stat(path)).ctimeMs === cached.ctimeMs); + const restored = await NodeFSP.stat(path); + assert.strictEqual(restored.size, cached.size); + assert.strictEqual(restored.mtimeMs, cached.mtimeMs); const next = await readAntigravityUsage(dir, 0, cache); assert.deepStrictEqual(next.errors, [path]); assert.deepStrictEqual(