Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 24 additions & 8 deletions packages/loopover-miner/lib/policy-verdict-cache.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import type { AiPolicyVerdict } from "@loopover/engine";
import { normalizeLocalStoreDbPath, openLocalStoreAdapter, resolveLocalStoreDbPath } from "./local-store.js";
import { applySchemaMigrations } from "./schema-version.js";
import { POLICY_VERDICT_CACHE_PURGE_SPEC, purgeStoreByRepo } from "./store-maintenance.js";
import { hostScopedRepoSuffixPattern } from "./store-maintenance.js";

// Local cache of resolved AI-usage-policy verdicts (#4843). Even with #4842's conditional-GET doc cache, the small
// but non-zero cost of resolving `resolveAiPolicyVerdict` from raw doc text was still paid on every discover run.
Expand Down Expand Up @@ -40,8 +40,13 @@ export type PolicyVerdictCacheStore = {
etag: string,
verdict: AiPolicyVerdict,
): PolicyVerdictCacheWrite;
/** Delete every cached verdict row for one repo scope (#6987); returns the number of rows removed. */
purgeByRepo(repoScope: string): number;
/** Delete every cached verdict row for one repo, across every forge host it was ever cached against (#6987,
* #10001). Takes a plain `owner/repo` (`repoFullName`) -- NOT a host-scoped repo SCOPE like `get`/`put` --
* matching the only value its production caller (purge-cli.js's `purgeOneStore`) ever passes; a real row's
* `repo_scope` is `<apiBaseUrl>::owner/repo` (`policyVerdictCacheKey`, opportunity-fanout.js), so this
* matches the `owner/repo` SUFFIX after each row's `::` separator, across every host. Returns the number of
* rows removed. */
purgeByRepo(repoFullName: string): number;
close(): void;
};

Expand Down Expand Up @@ -122,6 +127,11 @@ export function initPolicyVerdictCacheStore(dbPath: string = resolvePolicyVerdic
verdict = excluded.verdict,
updated_at = excluded.updated_at
`;
// Suffix match, not equality (#10001): repo_scope is `<apiBaseUrl>::owner/repo`, never a bare `owner/repo`, so
// `repo_scope = ?` can never match a real row. hostScopedRepoSuffixPattern escapes `_`/`%` in the repo value
// so this is the SAME pattern countStoreByRepo's hostScopedSuffixMatch branch builds for the dry-run count --
// the two share that one helper instead of each hand-rolling the escaping, so they can never diverge.
const purgeByRepoSql = "DELETE FROM policy_verdict_cache WHERE repo_scope LIKE ? ESCAPE '\\'";

return {
dbPath: resolvedPath,
Expand All @@ -146,12 +156,18 @@ export function initPolicyVerdictCacheStore(dbPath: string = resolvePolicyVerdic
return { repoScope: normalizedRepoScope, decisiveDoc: normalizedDecisiveDoc, etag: normalizedEtag, verdict, updatedAt };
},
/**
* Delete every cached verdict row for one repo scope (#6987) -- the right-to-be-forgotten path
* `loopover-miner purge` invokes. Returns the number of rows removed. Reuses store-maintenance.js's
* identifier-guarded purgeStoreByRepo, exactly like the other repo-scoped stores.
* Delete every cached verdict row for one repo, across every forge host it was ever cached against
* (#6987, #10001) -- the right-to-be-forgotten path `loopover-miner purge` invokes. Takes a plain
* `owner/repo`, matching purge-cli.js's only caller; a real row's `repo_scope` is `<apiBaseUrl>::owner/
* repo`, so this matches the `owner/repo` SUFFIX after each row's `::` separator rather than reusing
* store-maintenance.js's generic purgeStoreByRepo, whose `repoColumn = ?` equality can never match such a
* row. Own hand-written delete, same house-style split as WORKTREE_ALLOCATOR_PURGE_SPEC's own custom
* purgeByRepo (store-maintenance.js's countStoreByRepo mirrors it, not the other way around). Returns the
* number of rows removed.
*/
purgeByRepo(repoScope) {
return purgeStoreByRepo(db, POLICY_VERDICT_CACHE_PURGE_SPEC, normalizeRepoScope(repoScope));
purgeByRepo(repoFullName) {
const info = db.prepare(purgeByRepoSql).run(hostScopedRepoSuffixPattern(normalizeRepoScope(repoFullName)));
return Number(info.changes);
},
close() {
db.close();
Expand Down
49 changes: 45 additions & 4 deletions packages/loopover-miner/lib/store-maintenance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,17 @@ export const PREDICTION_LEDGER_RETENTION_SPEC: LedgerRetentionSpec = { table: "p
* real-delete path, `purgeStoreByRepo`, is never used for such a store (it does an unconditional `DELETE`,
* wrong for a row that must be preserved and merely blanked) — its own custom `purgeByRepo` method is used
* instead, and `--dry-run` must count using the identical condition so its preview never overstates what a
* real purge would remove. */
export type LedgerPurgeSpec = { table: string; repoColumn: string; extraWhereSql?: string };
* real purge would remove.
*
* `hostScopedSuffixMatch` is the same "custom real purge, `countStoreByRepo` mirrors it" split for a store
* whose `repoColumn` is not a bare `owner/repo` but a composite `<apiBaseUrl>::owner/repo` scope (#10001) —
* `repoColumn = ?` can never match such a row for a plain `owner/repo` argument, so an exact-equality purge
* silently purges nothing, always. When set, `countStoreByRepo` matches the `owner/repo` SUFFIX after each
* row's `::` separator instead, across every recorded host prefix, via `hostScopedRepoSuffixPattern`'s
* escaped `LIKE` pattern — the store's own real purge (e.g. `policy-verdict-cache.ts`'s `purgeByRepo`) builds
* the identical pattern with the same helper so the two can never diverge. Mutually exclusive with
* `extraWhereSql` in practice: no spec needs both. */
export type LedgerPurgeSpec = { table: string; repoColumn: string; extraWhereSql?: string; hostScopedSuffixMatch?: boolean };

/** Fixed purge specs (#5564, #6599) for the six stores whose rows are directly scoped by a `repoColumn`. Same
* internal-constant-only discipline as the retention specs above. `attempt-log.js` is deliberately absent: its
Expand Down Expand Up @@ -62,8 +71,15 @@ export const GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC: LedgerPurgeSpec = { table: "go
/** policy-verdict-cache (#6987), another repo-scoped store the earlier sweeps missed. Its `repo_scope TEXT
* PRIMARY KEY` is the per-repo column (a tenant forge host + `owner/repo`), the same `repoColumn` shape and
* internal-constant-only discipline as the specs above. `policy-doc-cache.js` stays out (keyed by URL, no repo
* column, exactly like `attempt-log.js`). */
export const POLICY_VERDICT_CACHE_PURGE_SPEC: LedgerPurgeSpec = { table: "policy_verdict_cache", repoColumn: "repo_scope" };
* column, exactly like `attempt-log.js`). `hostScopedSuffixMatch: true` (#10001): `repo_scope` rows are keyed
* `<apiBaseUrl>::owner/repo` (`policyVerdictCacheKey`, opportunity-fanout.js), never a bare `owner/repo` — the
* only value `purge-cli.js` ever calls `purgeByRepo`/`countStoreByRepo` with — so an exact-equality match here
* matched zero rows, always. See `LedgerPurgeSpec`'s own doc for the suffix-match replacement. */
export const POLICY_VERDICT_CACHE_PURGE_SPEC: LedgerPurgeSpec = {
table: "policy_verdict_cache",
repoColumn: "repo_scope",
hostScopedSuffixMatch: true,
};

/** Three more repo-scoped stores the #5564/#7091/#6987 sweeps missed (#8009), same `repoColumn` shape and same
* internal-constant-only discipline. ranked-candidates is a wholesale-replaced snapshot, but its rows persist
Expand Down Expand Up @@ -212,6 +228,25 @@ export function purgeStoreByRepo(db: DatabaseSync, spec: LedgerPurgeSpec, repoFu
return Number(info.changes);
}

/** Escape SQL `LIKE` wildcards (`_`, `%`) and the escape character itself in a caller-supplied value before it
* is embedded in a `LIKE` pattern, so a literal `_`/`%` (both valid in a GitHub `owner/repo` segment — see
* `REPO_SEGMENT_PATTERN`, repo-clone.js) is matched literally instead of as a wildcard. Same convention as
* `src/db/repositories.ts`'s own `escapeSqlLikePattern`. */
function escapeSqlLikePattern(value: string): string {
return value.replace(/[\\%_]/g, "\\$&");
}

/** Build the exact-suffix `LIKE` pattern for a `hostScopedSuffixMatch` spec's composite `<apiBaseUrl>::owner/
* repo` column: matches a row whose column value ends with exactly `::` + `repoFullName`, across every forge
* host prefix. Exported so a store's own hand-written real-purge SQL (`policy-verdict-cache.ts`'s
* `purgeByRepo`) builds the identical pattern `countStoreByRepo` uses for its dry-run count — the two share
* this one escaping path instead of each hand-rolling their own, so they can never diverge (#10001). No
* trailing wildcard: a longer name sharing the same prefix (e.g. `acme/my_repo-extra` when purging
* `acme/my_repo`) does not match, since the pattern requires the column to end exactly at the escaped value. */
export function hostScopedRepoSuffixPattern(repoFullName: string): string {
return `%::${escapeSqlLikePattern(repoFullName)}`;
}

/**
* Count rows for one repo in a store without deleting anything (#5564) — the read-only counterpart to
* `purgeStoreByRepo`, used by `purge-cli.js --dry-run` to report what a real purge would remove.
Expand All @@ -220,6 +255,12 @@ export function countStoreByRepo(db: DatabaseSync, spec: LedgerPurgeSpec, repoFu
for (const identifier of [spec.table, spec.repoColumn]) {
if (!SQL_IDENTIFIER.test(identifier)) throw new Error(`unsafe SQL identifier: ${identifier}`);
}
if (spec.hostScopedSuffixMatch) {
const row = db
.prepare(`SELECT COUNT(*) AS count FROM ${spec.table} WHERE ${spec.repoColumn} LIKE ? ESCAPE '\\'`)
.get(hostScopedRepoSuffixPattern(repoFullName));
return Number(row?.count);
}
// extraWhereSql is only ever one of this file's own internal constants (never caller/user text), so it is
// ANDed in verbatim rather than parsed as an identifier — see LedgerPurgeSpec's doc comment (#8320).
const extraWhere = spec.extraWhereSql ? ` AND (${spec.extraWhereSql})` : "";
Expand Down
29 changes: 22 additions & 7 deletions test/unit/miner-policy-verdict-cache.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,16 +136,31 @@ describe("loopover-miner policy-verdict cache store (#4843)", () => {
expect(() => initPolicyVerdictCacheStore("")).toThrow("invalid_policy_verdict_cache_db_path");
});

it("purgeByRepo deletes only the given repo scope's row and returns the count (#6987)", () => {
it("purgeByRepo takes a plain owner/repo and deletes every row scoped to it across every forge host, leaving other repos intact (#6987, #10001)", () => {
// Real repo_scope shape: `<apiBaseUrl>::owner/repo` -- purgeByRepo's argument is the bare owner/repo
// (matching purge-cli.js's only caller), never the full scope get/put take.
const store = openStore();
store.put("acme/widgets", "AI-USAGE.md", '"v1"', VERDICT);
store.put("acme/other", "AI-USAGE.md", '"v2"', VERDICT);
expect(store.purgeByRepo("acme/widgets")).toBe(1);
expect(store.get("acme/widgets")).toBeNull();
expect(store.get("acme/other")).not.toBeNull();
store.put("https://api.github.com::acme/widgets", "AI-USAGE.md", '"v1"', VERDICT);
store.put("https://forge.example.com::acme/widgets", "AI-USAGE.md", '"v2"', VERDICT);
store.put("https://api.github.com::acme/other", "AI-USAGE.md", '"v3"', VERDICT);
expect(store.purgeByRepo("acme/widgets")).toBe(2);
expect(store.get("https://api.github.com::acme/widgets")).toBeNull();
expect(store.get("https://forge.example.com::acme/widgets")).toBeNull();
expect(store.get("https://api.github.com::acme/other")).not.toBeNull();
});

it("purgeByRepo returns 0 when the repo scope has no cached verdict (#6987)", () => {
it("purgeByRepo returns 0 when the repo has no cached verdict under any host (#6987)", () => {
expect(openStore().purgeByRepo("acme/widgets")).toBe(0);
});

it("REGRESSION (#10001): purgeByRepo does not over-match a `_` in the repo name as a LIKE wildcard, and does not match a longer name sharing the same prefix", () => {
const store = openStore();
store.put("https://api.github.com::acme/my_repo", "AI-USAGE.md", '"v1"', VERDICT);
store.put("https://api.github.com::acme/myXrepo", "AI-USAGE.md", '"v2"', VERDICT);
store.put("https://api.github.com::acme/my_repo-extra", "AI-USAGE.md", '"v3"', VERDICT);
expect(store.purgeByRepo("acme/my_repo")).toBe(1);
expect(store.get("https://api.github.com::acme/my_repo")).toBeNull();
expect(store.get("https://api.github.com::acme/myXrepo")).not.toBeNull();
expect(store.get("https://api.github.com::acme/my_repo-extra")).not.toBeNull();
});
});
99 changes: 93 additions & 6 deletions test/unit/miner-purge-cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,10 @@ describe("runPurge --dry-run (#5564, #6599)", () => {
cache.put(emptyContributionProfile("acme/other", "2026-07-17T00:00:00.000Z"));
cache.close();

// Real repo_scope shape (#10001): `<apiBaseUrl>::owner/repo`, never a bare `owner/repo`.
const policyVerdictCache = initPolicyVerdictCacheStore(policyVerdictCacheDbPath);
policyVerdictCache.put("acme/widgets", "AI-USAGE.md", '"v1"', POLICY_VERDICT);
policyVerdictCache.put("acme/other", "AI-USAGE.md", '"v2"', POLICY_VERDICT);
policyVerdictCache.put("https://api.github.com::acme/widgets", "AI-USAGE.md", '"v1"', POLICY_VERDICT);
policyVerdictCache.put("https://api.github.com::acme/other", "AI-USAGE.md", '"v2"', POLICY_VERDICT);
policyVerdictCache.close();

// governor-state's two repo-scoped tables. reputation history for acme/widgets is recorded under TWO
Expand Down Expand Up @@ -846,9 +847,10 @@ describe("runPurge (real, #5564, #6599)", () => {
const root = tempDir();
const policyDbPath = join(root, "policy-verdict-cache.sqlite3");

// Real repo_scope shape (#10001): `<apiBaseUrl>::owner/repo`, never a bare `owner/repo`.
const seeded = initPolicyVerdictCacheStore(policyDbPath);
seeded.put("acme/widgets", "AI-USAGE.md", '"v1"', POLICY_VERDICT);
seeded.put("acme/other", "AI-USAGE.md", '"v2"', POLICY_VERDICT);
seeded.put("https://api.github.com::acme/widgets", "AI-USAGE.md", '"v1"', POLICY_VERDICT);
seeded.put("https://api.github.com::acme/other", "AI-USAGE.md", '"v2"', POLICY_VERDICT);
seeded.close();

const policyStore = initPolicyVerdictCacheStore(policyDbPath);
Expand All @@ -874,8 +876,93 @@ describe("runPurge (real, #5564, #6599)", () => {
).toBe(0);
const summary = JSON.parse(String(log.mock.calls[0]?.[0]));
expect(summary.stores).toContainEqual({ store: "policy-verdict-cache", purged: 1 });
expect(policyStore.get("acme/widgets")).toBeNull();
expect(policyStore.get("acme/other")).not.toBeNull();
expect(policyStore.get("https://api.github.com::acme/widgets")).toBeNull();
expect(policyStore.get("https://api.github.com::acme/other")).not.toBeNull();
});

it("REGRESSION (#10001): purge deletes policy-verdict-cache rows keyed by the real apiBaseUrl::owner/repo scope, across every host, and dry-run previews the identical count", () => {
const root = tempDir();
const policyDbPath = join(root, "policy-verdict-cache.sqlite3");

const seeded = initPolicyVerdictCacheStore(policyDbPath);
// Same repo cached under TWO forge hosts -- both must be swept.
seeded.put("https://api.github.com::acme/widgets", "AI-USAGE.md", '"v1"', POLICY_VERDICT);
seeded.put("https://forge.example.com::acme/widgets", "AI-USAGE.md", '"v2"', POLICY_VERDICT);
// A different repo on the same host must survive.
seeded.put("https://api.github.com::acme/other", "AI-USAGE.md", '"v3"', POLICY_VERDICT);
seeded.close();

const otherStoresResolveDbPaths = {
"claim-ledger": () => join(root, "claim-ledger.sqlite3"),
"event-ledger": () => join(root, "event-ledger.sqlite3"),
"governor-ledger": () => join(root, "governor-ledger.sqlite3"),
"prediction-ledger": () => join(root, "prediction-ledger.sqlite3"),
"portfolio-queue": () => join(root, "portfolio-queue.sqlite3"),
"run-state": () => join(root, "run-state.sqlite3"),
"contribution-profile-cache": () => join(root, "contribution-profile-cache.sqlite3"),
"policy-verdict-cache": () => policyDbPath,
"governor-state": () => join(root, "governor-state.sqlite3"),
"ranked-candidates": () => join(root, "ranked-candidates.sqlite3"),
"replay-snapshot": () => join(root, "replay-snapshot.sqlite3"),
"deny-hook-synthesis": () => join(root, "deny-hook-synthesis.sqlite3"),
"worktree-allocator": () => join(root, "worktree-allocator.sqlite3"),
"attempt-log": () => join(root, "attempt-log.sqlite3"),
};

const dryRunLog = vi.spyOn(console, "log").mockImplementation(() => undefined);
expect(runPurge(["--repo", "acme/widgets", "--dry-run", "--json"], { resolveDbPaths: otherStoresResolveDbPaths })).toBe(0);
const dryRunResult = JSON.parse(String(dryRunLog.mock.calls[0]?.[0]));
expect(dryRunResult.stores).toContainEqual({ store: "policy-verdict-cache", wouldPurge: 2 });
dryRunLog.mockRestore();

const policyStore = initPolicyVerdictCacheStore(policyDbPath);
closeables.push(policyStore);

const log = vi.spyOn(console, "log").mockImplementation(() => undefined);
expect(
runPurge(["--repo", "acme/widgets", "--json"], {
openClaimLedger: () => fakeStore(0),
initEventLedger: () => fakeStore(0),
initGovernorLedger: () => fakeStore(0),
initPredictionLedger: () => fakeStore(0),
initPortfolioQueueStore: () => fakeStore(0),
initRunStateStore: () => fakeStore(0),
initContributionProfileCache: () => fakeStore(0),
openGovernorState: () => fakeStore(0),
initPolicyVerdictCacheStore: () => policyStore,
initRankedCandidatesStore: () => fakeStore(0),
openReplaySnapshotStore: () => fakeStore(0),
initDenyHookSynthesisStore: () => fakeStore(0),
openWorktreeAllocator: () => fakeStore(0),
} as never),
).toBe(0);
const summary = JSON.parse(String(log.mock.calls[0]?.[0]));
// Both host-scoped rows counted (dry-run and real purge agree), the other repo is untouched.
expect(summary.stores).toContainEqual({ store: "policy-verdict-cache", purged: 2 });
expect(policyStore.get("https://api.github.com::acme/widgets")).toBeNull();
expect(policyStore.get("https://forge.example.com::acme/widgets")).toBeNull();
expect(policyStore.get("https://api.github.com::acme/other")).not.toBeNull();
});

it("REGRESSION (#10001): a `_` or `%` in the repo name is matched literally, never as a LIKE wildcard", () => {
const root = tempDir();
const policyDbPath = join(root, "policy-verdict-cache.sqlite3");

const seeded = initPolicyVerdictCacheStore(policyDbPath);
seeded.put("https://api.github.com::acme/my_repo", "AI-USAGE.md", '"v1"', POLICY_VERDICT);
// `_` is a LIKE single-character wildcard -- an unescaped match would also hit this row.
seeded.put("https://api.github.com::acme/myXrepo", "AI-USAGE.md", '"v2"', POLICY_VERDICT);
// A longer name sharing the same `owner/repo` prefix must survive too (suffix match, not prefix match).
seeded.put("https://api.github.com::acme/my_repo-extra", "AI-USAGE.md", '"v3"', POLICY_VERDICT);
seeded.close();

const policyStore = initPolicyVerdictCacheStore(policyDbPath);
closeables.push(policyStore);

expect(policyStore.purgeByRepo("acme/my_repo")).toBe(1);
expect(policyStore.get("https://api.github.com::acme/my_repo")).toBeNull();
expect(policyStore.get("https://api.github.com::acme/myXrepo")).not.toBeNull();
expect(policyStore.get("https://api.github.com::acme/my_repo-extra")).not.toBeNull();
});

it("REGRESSION (#8009): really deletes ranked-candidates, replay-snapshot, and deny-hook-synthesis rows across api_base_urls, leaving other repos intact", () => {
Expand Down
Loading