From b0523ac3ec72896e50b8239fd0d32ce72f7b8e22 Mon Sep 17 00:00:00 2001 From: Andriy Polanski Date: Sun, 19 Jul 2026 16:18:09 +0000 Subject: [PATCH] chore(miner): migrate batch 4.7 foundational lib modules to TypeScript (#7315) Convert eight foundational loopover-miner lib modules from plain .js + hand-maintained .d.ts to real TypeScript under the existing in-place tsc emit pipeline, with zero behavior change. Co-authored-by: Cursor --- packages/loopover-miner/lib/claim-ledger.d.ts | 138 ++--- packages/loopover-miner/lib/claim-ledger.js | 506 ++++++++---------- packages/loopover-miner/lib/claim-ledger.ts | 464 ++++++++++++++++ packages/loopover-miner/lib/forge-config.d.ts | 36 +- packages/loopover-miner/lib/forge-config.js | 36 +- packages/loopover-miner/lib/forge-config.ts | 51 ++ .../loopover-miner/lib/governor-state.d.ts | 91 ++-- packages/loopover-miner/lib/governor-state.js | 497 ++++++++--------- packages/loopover-miner/lib/governor-state.ts | 482 +++++++++++++++++ packages/loopover-miner/lib/local-store.d.ts | 57 +- packages/loopover-miner/lib/local-store.js | 85 ++- packages/loopover-miner/lib/local-store.ts | 99 ++++ .../loopover-miner/lib/prediction-ledger.d.ts | 74 ++- .../loopover-miner/lib/prediction-ledger.js | 285 +++++----- .../loopover-miner/lib/prediction-ledger.ts | 285 ++++++++++ packages/loopover-miner/lib/run-state.d.ts | 60 +-- packages/loopover-miner/lib/run-state.js | 201 ++++--- packages/loopover-miner/lib/run-state.ts | 210 ++++++++ .../loopover-miner/lib/schema-version.d.ts | 21 +- packages/loopover-miner/lib/schema-version.js | 89 ++- packages/loopover-miner/lib/schema-version.ts | 74 +++ .../loopover-miner/lib/store-maintenance.d.ts | 137 +++-- .../loopover-miner/lib/store-maintenance.js | 170 +++--- .../loopover-miner/lib/store-maintenance.ts | 198 +++++++ 24 files changed, 3043 insertions(+), 1303 deletions(-) create mode 100644 packages/loopover-miner/lib/claim-ledger.ts create mode 100644 packages/loopover-miner/lib/forge-config.ts create mode 100644 packages/loopover-miner/lib/governor-state.ts create mode 100644 packages/loopover-miner/lib/local-store.ts create mode 100644 packages/loopover-miner/lib/prediction-ledger.ts create mode 100644 packages/loopover-miner/lib/run-state.ts create mode 100644 packages/loopover-miner/lib/schema-version.ts create mode 100644 packages/loopover-miner/lib/store-maintenance.ts diff --git a/packages/loopover-miner/lib/claim-ledger.d.ts b/packages/loopover-miner/lib/claim-ledger.d.ts index 97625f7151..69a8ffd46b 100644 --- a/packages/loopover-miner/lib/claim-ledger.d.ts +++ b/packages/loopover-miner/lib/claim-ledger.d.ts @@ -1,82 +1,84 @@ export type ClaimStatus = "active" | "released" | "expired"; - export type ClaimEntry = { - id: number; - apiBaseUrl: string; - repoFullName: string; - issueNumber: number; - claimedAt: string; - status: ClaimStatus; - note: string | null; + id: number; + apiBaseUrl: string; + repoFullName: string; + issueNumber: number; + claimedAt: string; + status: ClaimStatus; + note: string | null; }; - export type RecordClaimInput = { - repoFullName: string; - issueNumber: number; - note?: string; - apiBaseUrl?: string; + repoFullName: string; + issueNumber: number; + note?: string; + apiBaseUrl?: string; }; - export type ListClaimsFilter = { - repoFullName?: string | null; - status?: ClaimStatus | null; + repoFullName?: string | null; + status?: ClaimStatus | null; }; - /** Result of an atomic, concurrency-capped claim (#6758). `claimed` discriminates success (a recorded claim) * from a cap rejection (`claim: null`); both carry the pre-insert active count and the resolved cap so a * rejected caller can still log the violation. */ -export type ClaimWithinCapResult = - | { claimed: true; claim: ClaimEntry; activeClaimCount: number; maxConcurrentClaims: number } - | { claimed: false; claim: null; activeClaimCount: number; maxConcurrentClaims: number }; - +export type ClaimWithinCapResult = { + claimed: true; + claim: ClaimEntry; + activeClaimCount: number; + maxConcurrentClaims: number; +} | { + claimed: false; + claim: null; + activeClaimCount: number; + maxConcurrentClaims: number; +}; export type ClaimLedger = { - dbPath: string; - recordClaim(claim: RecordClaimInput): ClaimEntry; - /** Claims the issue, expiring any claim orphaned by a dead process first (#6156). */ - claimIssue(repoFullName: string, issueNumber: number, note?: string, apiBaseUrl?: string): ClaimEntry; - /** Atomically records the claim only while this repo's active-claim count is under `maxConcurrentClaims`, - * counting and inserting in one transaction so racing sibling processes can't exceed the cap (#6758). */ - claimIssueWithinCap( - repoFullName: string, - issueNumber: number, - note: string | undefined, - apiBaseUrl: string | undefined, - maxConcurrentClaims: number, - ): ClaimWithinCapResult; - /** Expire claims orphaned by a crashed/killed process, returning the transitioned rows (#6156). */ - reclaimExpiredClaims(maxAgeMs?: number): ClaimEntry[]; - releaseClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; - expireClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; - listClaims(filter?: ListClaimsFilter): ClaimEntry[]; - listActiveClaims(repoFullName?: string): ClaimEntry[]; - purgeByRepo(repoFullName: string): number; - close(): void; + dbPath: string; + recordClaim(claim: RecordClaimInput): ClaimEntry; + /** Claims the issue, expiring any claim orphaned by a dead process first (#6156). */ + claimIssue(repoFullName: string, issueNumber: number, note?: string, apiBaseUrl?: string): ClaimEntry; + /** Atomically records the claim only while this repo's active-claim count is under `maxConcurrentClaims`, + * counting and inserting in one transaction so racing sibling processes can't exceed the cap (#6758). */ + claimIssueWithinCap(repoFullName: string, issueNumber: number, note: string | undefined, apiBaseUrl: string | undefined, maxConcurrentClaims: number): ClaimWithinCapResult; + /** Expire claims orphaned by a crashed/killed process, returning the transitioned rows (#6156). */ + reclaimExpiredClaims(maxAgeMs?: number): ClaimEntry[]; + releaseClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; + expireClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; + listClaims(filter?: ListClaimsFilter): ClaimEntry[]; + listActiveClaims(repoFullName?: string): ClaimEntry[]; + purgeByRepo(repoFullName: string): number; + close(): void; }; - -export const CLAIM_STATUSES: readonly ClaimStatus[]; - -export function resolveClaimLedgerDbPath(env?: Record): string; - -export function openClaimLedger(dbPath?: string): ClaimLedger; - export type ReadOnlyClaimLedger = { - dbPath: string; - listActiveClaims(repoFullName: string): ClaimEntry[]; - close(): void; + dbPath: string; + listActiveClaims(repoFullName: string): ClaimEntry[]; + close(): void; }; - -export function openClaimLedgerReadOnly(dbPath: string): ReadOnlyClaimLedger; - -export function recordClaim(claim: RecordClaimInput): ClaimEntry; - -export function releaseClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; - -export function expireClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; - -export function listClaims(filter?: ListClaimsFilter): ClaimEntry[]; - -export function claimIssue(repoFullName: string, issueNumber: number, note?: string, apiBaseUrl?: string): ClaimEntry; - -export function listActiveClaims(repoFullName?: string): ClaimEntry[]; - -export function closeDefaultClaimLedger(): void; +export declare const CLAIM_STATUSES: readonly ClaimStatus[]; +export declare function resolveClaimLedgerDbPath(env?: Record): string; +/** + * Opens the local claim ledger, creating the table on first use. `UNIQUE(api_base_url, repo_full_name, + * issue_number)` keeps ONE row per claimed issue per forge host, and `recordClaim` is a single atomic + * INSERT…ON CONFLICT statement (no read-then-write), so concurrent claims cannot duplicate a row. (#2314, #5563) + */ +export declare function openClaimLedger(dbPath?: string): ClaimLedger; +/** + * Strictly read-only ledger access for advisory-only callers (#5157) that must never write anything -- + * not even the schema-creation DDL and schema-version stamp {@link openClaimLedger} always runs on open. + * Opens the DB file in SQLite's own `readonly` mode (driver-enforced: an attempted write throws, this isn't + * just a by-convention guarantee) and touches the filesystem in no other way -- no `mkdirSync`/`chmodSync`, + * no `CREATE TABLE IF NOT EXISTS`, no migrations. The caller MUST only call this against a path it has + * already confirmed exists (e.g. via `existsSync`); a read-only connection to a nonexistent file throws. + * Throws if the expected table is missing too (a file exists at this path but isn't a real claim ledger) -- + * callers should treat that identically to any other open/query failure. + */ +export declare function openClaimLedgerReadOnly(dbPath: string): ReadOnlyClaimLedger; +export declare function recordClaim(claim: RecordClaimInput): ClaimEntry; +export declare function releaseClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; +export declare function expireClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; +export declare function listClaims(filter?: ListClaimsFilter): ClaimEntry[]; +/** Foundation-phase alias for `recordClaim({ repoFullName, issueNumber, note, apiBaseUrl })`. (#3351) */ +export declare function claimIssue(repoFullName: string, issueNumber: number, note?: string, apiBaseUrl?: string): ClaimEntry; +/** List only `active` claims, optionally scoped to one repo. (#3351) */ +export declare function listActiveClaims(repoFullName?: string): ClaimEntry[]; +export declare function closeDefaultClaimLedger(): void; diff --git a/packages/loopover-miner/lib/claim-ledger.js b/packages/loopover-miner/lib/claim-ledger.js index a7136ded66..d17593342e 100644 --- a/packages/loopover-miner/lib/claim-ledger.js +++ b/packages/loopover-miner/lib/claim-ledger.js @@ -5,76 +5,67 @@ import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } import { isValidRepoSegment } from "./repo-clone.js"; import { applySchemaMigrations } from "./schema-version.js"; import { CLAIM_LEDGER_PURGE_SPEC, purgeStoreByRepo } from "./store-maintenance.js"; - -// The miner's local soft-claim ledger (#2314): a 100% client-side record of "I'm working on issue #N in repo X", -// so Phase 2's soft-claim adjudication (sibling issues) has somewhere to persist claims. Schema + CRUD only — no -// adjudication logic, no network calls, no autonomous writes. The database only lives on this machine; this module -// never uploads, syncs, or phones home. Mirrors the package's existing local-store pattern (run-state.js, -// portfolio-queue.js, event-ledger.js) — plain JS + node:sqlite, not the hosted Worker's shared D1 `migrations/`. - export const CLAIM_STATUSES = Object.freeze(["active", "released", "expired"]); - const defaultDbFileName = "claim-ledger.sqlite3"; let defaultClaimLedger = null; - export function resolveClaimLedgerDbPath(env = process.env) { - return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_CLAIM_LEDGER_DB", env); + return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_CLAIM_LEDGER_DB", env); } - function normalizeDbPath(dbPath) { - return normalizeLocalStoreDbPath(dbPath, resolveClaimLedgerDbPath(), "invalid_claim_ledger_db_path"); + return normalizeLocalStoreDbPath(dbPath, resolveClaimLedgerDbPath(), "invalid_claim_ledger_db_path"); } - function normalizeRepoFullName(repoFullName) { - if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name"); - const [owner, repo, extra] = repoFullName.trim().split("/"); - if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name"); - if (!isValidRepoSegment(owner) || !isValidRepoSegment(repo)) throw new Error("invalid_repo_full_name"); - return `${owner}/${repo}`; + if (typeof repoFullName !== "string") + throw new Error("invalid_repo_full_name"); + const [owner, repo, extra] = repoFullName.trim().split("/"); + if (!owner || !repo || extra !== undefined) + throw new Error("invalid_repo_full_name"); + if (!isValidRepoSegment(owner) || !isValidRepoSegment(repo)) + throw new Error("invalid_repo_full_name"); + return `${owner}/${repo}`; } - function normalizeIssueNumber(issueNumber) { - if (!Number.isInteger(issueNumber) || issueNumber < 1) throw new Error("invalid_issue_number"); - return issueNumber; + if (!Number.isInteger(issueNumber) || issueNumber < 1) + throw new Error("invalid_issue_number"); + return issueNumber; } - // The per-repo concurrent-claim cap the atomic count-and-claim gates on (#6758). Always an already-validated // positive integer from the caller's MinerGoalSpec, but re-checked here because a bad value must fail loudly // rather than silently disable the cap (a comparison against `undefined` is always false). function normalizeMaxConcurrentClaims(maxConcurrentClaims) { - if (!Number.isInteger(maxConcurrentClaims) || maxConcurrentClaims < 1) { - throw new Error("invalid_max_concurrent_claims"); - } - return maxConcurrentClaims; + if (!Number.isInteger(maxConcurrentClaims) || maxConcurrentClaims < 1) { + throw new Error("invalid_max_concurrent_claims"); + } + return maxConcurrentClaims; } - /** Optional forge host, scoping rows so two hosts serving the same owner/repo name never collide (#5563). * Omitted/nullish → the github.com default, so every pre-existing single-forge caller is unaffected. */ function normalizeApiBaseUrl(apiBaseUrl) { - if (apiBaseUrl === undefined || apiBaseUrl === null) return DEFAULT_FORGE_CONFIG.apiBaseUrl; - if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) throw new Error("invalid_api_base_url"); - return apiBaseUrl.trim(); + if (apiBaseUrl === undefined || apiBaseUrl === null) + return DEFAULT_FORGE_CONFIG.apiBaseUrl; + if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) + throw new Error("invalid_api_base_url"); + return apiBaseUrl.trim(); } - /** Optional free-text note: omitted/nullish → null; a string is kept as-is; anything else is rejected. */ function normalizeNote(note) { - if (note === undefined || note === null) return null; - if (typeof note !== "string") throw new Error("invalid_note"); - return note; + if (note === undefined || note === null) + return null; + if (typeof note !== "string") + throw new Error("invalid_note"); + return note; } - function rowToClaim(row) { - return { - id: row.id, - apiBaseUrl: row.api_base_url, - repoFullName: row.repo_full_name, - issueNumber: row.issue_number, - claimedAt: row.claimed_at, - status: row.status, - note: row.note, - }; + return { + id: row.id, + apiBaseUrl: row.api_base_url, + repoFullName: row.repo_full_name, + issueNumber: row.issue_number, + claimedAt: row.claimed_at, + status: row.status, + note: row.note, + }; } - // v1 -> v2 (#5563): scope the UNIQUE constraint by (api_base_url, repo_full_name, issue_number) instead of bare // (repo_full_name, issue_number) -- two different forge hosts serving a same-named repo/issue must not collide // in this ledger. SQLite cannot ALTER a UNIQUE constraint in place, so this rebuilds the table: create the new @@ -82,7 +73,7 @@ function rowToClaim(row) { // table, rename the new one in. Runs inside applySchemaMigrations' own transaction, so a mid-rebuild failure // leaves the file at v1 and retries cleanly on next open. function addApiBaseUrlScope(db) { - db.exec(` + db.exec(` CREATE TABLE miner_claims_v2 ( id INTEGER PRIMARY KEY AUTOINCREMENT, api_base_url TEXT NOT NULL, @@ -94,42 +85,39 @@ function addApiBaseUrlScope(db) { UNIQUE (api_base_url, repo_full_name, issue_number) ) `); - // OR IGNORE: a row this store's own read path already treats as unusable garbage (an unrecognized `status`, - // e.g. from a hand-edited or otherwise corrupted file) would violate the CHECK constraint above and abort the - // whole migration. Skipping it here is consistent with that same fail-closed posture, rather than turning one - // bad row into a permanently unmigratable file. - db.prepare( - `INSERT OR IGNORE INTO miner_claims_v2 (id, api_base_url, repo_full_name, issue_number, claimed_at, status, note) - SELECT id, ?, repo_full_name, issue_number, claimed_at, status, note FROM miner_claims`, - ).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); - db.exec("DROP TABLE miner_claims"); - db.exec("ALTER TABLE miner_claims_v2 RENAME TO miner_claims"); + // OR IGNORE: a row this store's own read path already treats as unusable garbage (an unrecognized `status`, + // e.g. from a hand-edited or otherwise corrupted file) would violate the CHECK constraint above and abort the + // whole migration. Skipping it here is consistent with that same fail-closed posture, rather than turning one + // bad row into a permanently unmigratable file. + db.prepare(`INSERT OR IGNORE INTO miner_claims_v2 (id, api_base_url, repo_full_name, issue_number, claimed_at, status, note) + SELECT id, ?, repo_full_name, issue_number, claimed_at, status, note FROM miner_claims`).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); + db.exec("DROP TABLE miner_claims"); + db.exec("ALTER TABLE miner_claims_v2 RENAME TO miner_claims"); } - // v2 -> v3 (#4939): additive tenant-scoping column, a prerequisite for any hosted, multi-tenant use of this // same store's logic. NULL for every row today -- self-host behavior is byte-identical, since nothing reads or // writes it yet (no consumer exists until a future hosted deployment populates it). Same defensive // column-presence guard as this file's own v1->v2 migration's sibling in portfolio-queue.js. function addTenantIdColumn(db) { - const hasTenantIdColumn = db - .prepare("PRAGMA table_info(miner_claims)") - .all() - .some((column) => column.name === "tenant_id"); - if (!hasTenantIdColumn) db.exec("ALTER TABLE miner_claims ADD COLUMN tenant_id TEXT"); + const hasTenantIdColumn = db + .prepare("PRAGMA table_info(miner_claims)") + .all() + .some((column) => column.name === "tenant_id"); + if (!hasTenantIdColumn) + db.exec("ALTER TABLE miner_claims ADD COLUMN tenant_id TEXT"); } - /** * Opens the local claim ledger, creating the table on first use. `UNIQUE(api_base_url, repo_full_name, * issue_number)` keeps ONE row per claimed issue per forge host, and `recordClaim` is a single atomic * INSERT…ON CONFLICT statement (no read-then-write), so concurrent claims cannot duplicate a row. (#2314, #5563) */ export function openClaimLedger(dbPath = resolveClaimLedgerDbPath()) { - const resolvedPath = normalizeDbPath(dbPath); - const db = openLocalStoreDb(resolvedPath); - // LOCAL bookkeeping only: this table records which issues this miner instance has soft-claimed on this - // machine. It does NOT adjudicate contested duplicates — sibling miners claiming the same issue are - // resolved elsewhere via `isDuplicateClusterWinnerByClaim` from `@loopover/engine` (#3355). - db.exec(` + const resolvedPath = normalizeDbPath(dbPath); + const db = openLocalStoreDb(resolvedPath); + // LOCAL bookkeeping only: this table records which issues this miner instance has soft-claimed on this + // machine. It does NOT adjudicate contested duplicates — sibling miners claiming the same issue are + // resolved elsewhere via `isDuplicateClusterWinnerByClaim` from `@loopover/engine` (#3355). + db.exec(` CREATE TABLE IF NOT EXISTS miner_claims ( id INTEGER PRIMARY KEY AUTOINCREMENT, repo_full_name TEXT NOT NULL, @@ -140,13 +128,12 @@ export function openClaimLedger(dbPath = resolveClaimLedgerDbPath()) { UNIQUE (repo_full_name, issue_number) ) `); - // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. - applySchemaMigrations(db, [addApiBaseUrlScope, addTenantIdColumn]); - - // Idempotent claim in ONE atomic statement: insert a new active claim, or — only if the existing row is NOT - // already active — re-activate it (a released/expired claim can be re-claimed). The `WHERE status <> 'active'` - // guard makes re-claiming an already-active issue a true no-op (no row churn), never a duplicate row. - const recordStatement = db.prepare(` + // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. + applySchemaMigrations(db, [addApiBaseUrlScope, addTenantIdColumn]); + // Idempotent claim in ONE atomic statement: insert a new active claim, or — only if the existing row is NOT + // already active — re-activate it (a released/expired claim can be re-claimed). The `WHERE status <> 'active'` + // guard makes re-claiming an already-active issue a true no-op (no row churn), never a duplicate row. + const recordStatement = db.prepare(` INSERT INTO miner_claims (api_base_url, repo_full_name, issue_number, claimed_at, status, note) VALUES (?, ?, ?, ?, 'active', ?) ON CONFLICT(api_base_url, repo_full_name, issue_number) DO UPDATE SET @@ -155,154 +142,144 @@ export function openClaimLedger(dbPath = resolveClaimLedgerDbPath()) { status = 'active' WHERE miner_claims.status <> 'active' `); - const getStatement = db.prepare( - "SELECT * FROM miner_claims WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ?", - ); - // RETURNING (matching portfolio-queue.js's own claim/release statements) makes the "nothing to release/expire" - // case observable directly from ONE atomic statement, rather than a separate post-UPDATE SELECT whose "row - // went missing" branch would be structurally unreachable (nothing else runs between the UPDATE and a SELECT - // on the same key within one synchronous call). - const releaseStatement = db.prepare( - "UPDATE miner_claims SET status = 'released' WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ? AND status = 'active' RETURNING *", - ); - const expireStatement = db.prepare( - "UPDATE miner_claims SET status = 'expired' WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ? AND status = 'active' RETURNING *", - ); - const listAllStatement = db.prepare("SELECT * FROM miner_claims ORDER BY id ASC"); - const listRepoStatement = db.prepare( - "SELECT * FROM miner_claims WHERE repo_full_name = ? ORDER BY id ASC", - ); - const listStatusStatement = db.prepare( - "SELECT * FROM miner_claims WHERE status = ? ORDER BY id ASC", - ); - const listRepoStatusStatement = db.prepare( - "SELECT * FROM miner_claims WHERE repo_full_name = ? AND status = ? ORDER BY id ASC", - ); - // Repo-wide active-claim tally for the atomic concurrency cap (#6758). Scoped by repo_full_name only (not - // api_base_url), matching the cross-forge counting that listActiveClaims(repoFullName) -- and the prior - // attempt-cli.js pre-check built on it -- already did, so the cap's MEANING is unchanged; only its atomicity is. - const countActiveRepoStatement = db.prepare( - "SELECT COUNT(*) AS count FROM miner_claims WHERE repo_full_name = ? AND status = 'active'", - ); - - function normalizeListRepoFilter(repoFullName) { - if (repoFullName === undefined || repoFullName === null) return undefined; - return normalizeRepoFullName(repoFullName); - } - - function normalizeStatusFilter(status) { - if (status === undefined || status === null) return undefined; - if (!CLAIM_STATUSES.includes(status)) throw new Error("invalid_status"); - return status; - } - - const ledger = { - dbPath: resolvedPath, - recordClaim(claim) { - const apiBaseUrl = normalizeApiBaseUrl(claim?.apiBaseUrl); - const repoFullName = normalizeRepoFullName(claim?.repoFullName); - const issueNumber = normalizeIssueNumber(claim?.issueNumber); - const note = normalizeNote(claim?.note); - const claimedAt = new Date().toISOString(); - recordStatement.run(apiBaseUrl, repoFullName, issueNumber, claimedAt, note); - return rowToClaim(getStatement.get(apiBaseUrl, repoFullName, issueNumber)); - }, - releaseClaim(repoFullName, issueNumber, apiBaseUrl) { - const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); - const normalizedRepo = normalizeRepoFullName(repoFullName); - const normalizedIssue = normalizeIssueNumber(issueNumber); - const row = releaseStatement.get(normalizedForge, normalizedRepo, normalizedIssue); - return row ? rowToClaim(row) : null; - }, - expireClaim(repoFullName, issueNumber, apiBaseUrl) { - const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); - const normalizedRepo = normalizeRepoFullName(repoFullName); - const normalizedIssue = normalizeIssueNumber(issueNumber); - const row = expireStatement.get(normalizedForge, normalizedRepo, normalizedIssue); - return row ? rowToClaim(row) : null; - }, - listClaims(filter = {}) { - const repoFullName = normalizeListRepoFilter(filter.repoFullName); - const status = normalizeStatusFilter(filter.status); - - let rows; - if (repoFullName !== undefined && status !== undefined) { - rows = listRepoStatusStatement.all(repoFullName, status); - } else if (repoFullName !== undefined) { - rows = listRepoStatement.all(repoFullName); - } else if (status !== undefined) { - rows = listStatusStatement.all(status); - } else { - rows = listAllStatement.all(); - } - return rows.map(rowToClaim); - }, - /** Expire claims orphaned by a crashed/killed process, returning the transitioned rows (#6156). The explicit - * counterpart to the sweep claimIssue runs on its own, mirroring reclaimStuckItems (portfolio-queue-manager.js). */ - reclaimExpiredClaims(maxAgeMs = DEFAULT_MAX_CLAIM_AGE_MS) { - return sweepExpiredClaims(ledger, Date.now(), maxAgeMs); - }, - claimIssue(repoFullName, issueNumber, note, apiBaseUrl) { - // Expire orphaned claims first, so an issue stranded 'active' by a dead process becomes claimable again - // instead of blocking indefinitely (#6156). Without this, recordClaim's `WHERE status <> 'active'` guard - // makes re-claiming an active row a no-op, so a claim whose owning process died keeps winning forever -- - // there is no other path to expireClaim. Mirrors claimNextBatch's sweep-then-claim - // (portfolio-queue-manager.js), where a lease stranded by a dead process would otherwise starve the queue. - sweepExpiredClaims(ledger, Date.now(), DEFAULT_MAX_CLAIM_AGE_MS); - return ledger.recordClaim({ repoFullName, issueNumber, note, apiBaseUrl }); - }, - /** - * Atomic, concurrency-capped claim (#6758). Sweeps orphaned claims, counts this repo's ACTIVE claims, and - * records the new claim ONLY while still strictly under `maxConcurrentClaims` -- all inside ONE `BEGIN - * IMMEDIATE` transaction. The prior enforcement split the count (attempt-cli.js's listActiveClaims) from the - * insert (claimIssue) across two statements with no shared transaction, so two sibling miner processes racing - * the same repo could both read the same sub-cap count and both claim, exceeding the cap. Fusing count + - * insert under an IMMEDIATE write lock -- with node:sqlite's shared `busy_timeout`, so the loser WAITS for the - * winner's commit rather than erroring -- closes that window: the second process sees the committed count and - * is cleanly rejected with `claimed: false` (never silently dropped), so the caller can log the cap violation. - * Returns the pre-insert `activeClaimCount` and the resolved `maxConcurrentClaims` on both paths. - */ - claimIssueWithinCap(repoFullName, issueNumber, note, apiBaseUrl, maxConcurrentClaims) { - const cap = normalizeMaxConcurrentClaims(maxConcurrentClaims); - // Normalize the repo up front: the count query keys on it, and a bad value must throw BEFORE `BEGIN` so it - // can never strand an open transaction. `issueNumber`/`note`/`apiBaseUrl` are validated by recordClaim - // INSIDE the transaction -- a bad value there is rolled back whole via the catch below. - const normalizedRepo = normalizeRepoFullName(repoFullName); - db.exec("BEGIN IMMEDIATE"); - try { - sweepExpiredClaims(ledger, Date.now(), DEFAULT_MAX_CLAIM_AGE_MS); - const activeClaimCount = countActiveRepoStatement.get(normalizedRepo).count; - if (activeClaimCount >= cap) { - // COMMIT, not ROLLBACK: a claim the sweep just expired is a legitimate transition that must persist even - // though THIS claim is rejected -- rolling back would resurrect a dead process's stale claim. - db.exec("COMMIT"); - return { claimed: false, claim: null, activeClaimCount, maxConcurrentClaims: cap }; - } - const claim = ledger.recordClaim({ repoFullName, issueNumber, note, apiBaseUrl }); - db.exec("COMMIT"); - return { claimed: true, claim, activeClaimCount, maxConcurrentClaims: cap }; - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } - }, - listActiveClaims(repoFullName) { - const filter = { status: "active" }; - if (repoFullName !== undefined) filter.repoFullName = repoFullName; - return ledger.listClaims(filter); - }, - // Explicit, operator-invoked right-to-be-forgotten purge (#5564) — never runs automatically. Distinct from - // this store's normal claim/release/expire lifecycle: deletes every row for a repo outright. - purgeByRepo(repoFullName) { - return purgeStoreByRepo(db, CLAIM_LEDGER_PURGE_SPEC, normalizeRepoFullName(repoFullName)); - }, - close() { - db.close(); - }, - }; - return ledger; + const getStatement = db.prepare("SELECT * FROM miner_claims WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ?"); + // RETURNING (matching portfolio-queue.js's own claim/release statements) makes the "nothing to release/expire" + // case observable directly from ONE atomic statement, rather than a separate post-UPDATE SELECT whose "row + // went missing" branch would be structurally unreachable (nothing else runs between the UPDATE and a SELECT + // on the same key within one synchronous call). + const releaseStatement = db.prepare("UPDATE miner_claims SET status = 'released' WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ? AND status = 'active' RETURNING *"); + const expireStatement = db.prepare("UPDATE miner_claims SET status = 'expired' WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ? AND status = 'active' RETURNING *"); + const listAllStatement = db.prepare("SELECT * FROM miner_claims ORDER BY id ASC"); + const listRepoStatement = db.prepare("SELECT * FROM miner_claims WHERE repo_full_name = ? ORDER BY id ASC"); + const listStatusStatement = db.prepare("SELECT * FROM miner_claims WHERE status = ? ORDER BY id ASC"); + const listRepoStatusStatement = db.prepare("SELECT * FROM miner_claims WHERE repo_full_name = ? AND status = ? ORDER BY id ASC"); + // Repo-wide active-claim tally for the atomic concurrency cap (#6758). Scoped by repo_full_name only (not + // api_base_url), matching the cross-forge counting that listActiveClaims(repoFullName) -- and the prior + // attempt-cli.js pre-check built on it -- already did, so the cap's MEANING is unchanged; only its atomicity is. + const countActiveRepoStatement = db.prepare("SELECT COUNT(*) AS count FROM miner_claims WHERE repo_full_name = ? AND status = 'active'"); + function normalizeListRepoFilter(repoFullName) { + if (repoFullName === undefined || repoFullName === null) + return undefined; + return normalizeRepoFullName(repoFullName); + } + function normalizeStatusFilter(status) { + if (status === undefined || status === null) + return undefined; + if (!CLAIM_STATUSES.includes(status)) + throw new Error("invalid_status"); + return status; + } + const ledger = { + dbPath: resolvedPath, + recordClaim(claim) { + const apiBaseUrl = normalizeApiBaseUrl(claim?.apiBaseUrl); + const repoFullName = normalizeRepoFullName(claim?.repoFullName); + const issueNumber = normalizeIssueNumber(claim?.issueNumber); + const note = normalizeNote(claim?.note); + const claimedAt = new Date().toISOString(); + recordStatement.run(apiBaseUrl, repoFullName, issueNumber, claimedAt, note); + return rowToClaim(getStatement.get(apiBaseUrl, repoFullName, issueNumber)); + }, + releaseClaim(repoFullName, issueNumber, apiBaseUrl) { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const normalizedIssue = normalizeIssueNumber(issueNumber); + const row = releaseStatement.get(normalizedForge, normalizedRepo, normalizedIssue); + return row ? rowToClaim(row) : null; + }, + expireClaim(repoFullName, issueNumber, apiBaseUrl) { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const normalizedIssue = normalizeIssueNumber(issueNumber); + const row = expireStatement.get(normalizedForge, normalizedRepo, normalizedIssue); + return row ? rowToClaim(row) : null; + }, + listClaims(filter = {}) { + const repoFullName = normalizeListRepoFilter(filter.repoFullName); + const status = normalizeStatusFilter(filter.status); + let rows; + if (repoFullName !== undefined && status !== undefined) { + rows = listRepoStatusStatement.all(repoFullName, status); + } + else if (repoFullName !== undefined) { + rows = listRepoStatement.all(repoFullName); + } + else if (status !== undefined) { + rows = listStatusStatement.all(status); + } + else { + rows = listAllStatement.all(); + } + return rows.map((row) => rowToClaim(row)); + }, + /** Expire claims orphaned by a crashed/killed process, returning the transitioned rows (#6156). The explicit + * counterpart to the sweep claimIssue runs on its own, mirroring reclaimStuckItems (portfolio-queue-manager.js). */ + reclaimExpiredClaims(maxAgeMs = DEFAULT_MAX_CLAIM_AGE_MS) { + return sweepExpiredClaims(ledger, Date.now(), maxAgeMs); + }, + claimIssue(repoFullName, issueNumber, note, apiBaseUrl) { + // Expire orphaned claims first, so an issue stranded 'active' by a dead process becomes claimable again + // instead of blocking indefinitely (#6156). Without this, recordClaim's `WHERE status <> 'active'` guard + // makes re-claiming an active row a no-op, so a claim whose owning process died keeps winning forever -- + // there is no other path to expireClaim. Mirrors claimNextBatch's sweep-then-claim + // (portfolio-queue-manager.js), where a lease stranded by a dead process would otherwise starve the queue. + sweepExpiredClaims(ledger, Date.now(), DEFAULT_MAX_CLAIM_AGE_MS); + return ledger.recordClaim({ repoFullName, issueNumber, note, apiBaseUrl }); + }, + /** + * Atomic, concurrency-capped claim (#6758). Sweeps orphaned claims, counts this repo's ACTIVE claims, and + * records the new claim ONLY while still strictly under `maxConcurrentClaims` -- all inside ONE `BEGIN + * IMMEDIATE` transaction. The prior enforcement split the count (attempt-cli.js's listActiveClaims) from the + * insert (claimIssue) across two statements with no shared transaction, so two sibling miner processes racing + * the same repo could both read the same sub-cap count and both claim, exceeding the cap. Fusing count + + * insert under an IMMEDIATE write lock -- with node:sqlite's shared `busy_timeout`, so the loser WAITS for the + * winner's commit rather than erroring -- closes that window: the second process sees the committed count and + * is cleanly rejected with `claimed: false` (never silently dropped), so the caller can log the cap violation. + * Returns the pre-insert `activeClaimCount` and the resolved `maxConcurrentClaims` on both paths. + */ + claimIssueWithinCap(repoFullName, issueNumber, note, apiBaseUrl, maxConcurrentClaims) { + const cap = normalizeMaxConcurrentClaims(maxConcurrentClaims); + // Normalize the repo up front: the count query keys on it, and a bad value must throw BEFORE `BEGIN` so it + // can never strand an open transaction. `issueNumber`/`note`/`apiBaseUrl` are validated by recordClaim + // INSIDE the transaction -- a bad value there is rolled back whole via the catch below. + const normalizedRepo = normalizeRepoFullName(repoFullName); + db.exec("BEGIN IMMEDIATE"); + try { + sweepExpiredClaims(ledger, Date.now(), DEFAULT_MAX_CLAIM_AGE_MS); + const activeClaimCount = countActiveRepoStatement.get(normalizedRepo).count; + if (activeClaimCount >= cap) { + // COMMIT, not ROLLBACK: a claim the sweep just expired is a legitimate transition that must persist even + // though THIS claim is rejected -- rolling back would resurrect a dead process's stale claim. + db.exec("COMMIT"); + return { claimed: false, claim: null, activeClaimCount, maxConcurrentClaims: cap }; + } + const claim = ledger.recordClaim({ repoFullName, issueNumber, note, apiBaseUrl }); + db.exec("COMMIT"); + return { claimed: true, claim, activeClaimCount, maxConcurrentClaims: cap }; + } + catch (error) { + db.exec("ROLLBACK"); + throw error; + } + }, + listActiveClaims(repoFullName) { + const filter = { + status: "active", + ...(repoFullName !== undefined ? { repoFullName } : {}), + }; + return ledger.listClaims(filter); + }, + // Explicit, operator-invoked right-to-be-forgotten purge (#5564) — never runs automatically. Distinct from + // this store's normal claim/release/expire lifecycle: deletes every row for a repo outright. + purgeByRepo(repoFullName) { + return purgeStoreByRepo(db, CLAIM_LEDGER_PURGE_SPEC, normalizeRepoFullName(repoFullName)); + }, + close() { + db.close(); + }, + }; + return ledger; } - /** * Strictly read-only ledger access for advisory-only callers (#5157) that must never write anything -- * not even the schema-creation DDL and schema-version stamp {@link openClaimLedger} always runs on open. @@ -314,67 +291,60 @@ export function openClaimLedger(dbPath = resolveClaimLedgerDbPath()) { * callers should treat that identically to any other open/query failure. */ export function openClaimLedgerReadOnly(dbPath) { - const resolvedPath = normalizeDbPath(dbPath); - // `readOnly` (camelCase) -- node:sqlite silently IGNORES `readonly` (lowercase) as an unrecognized option - // and opens read-write anyway, defeating the entire point of this function. Verified empirically: a write - // via a `{ readonly: true }` connection succeeds with no error. - const db = new DatabaseSync(resolvedPath, { readOnly: true }); - let listActiveStatement; - try { - listActiveStatement = db.prepare( - "SELECT * FROM miner_claims WHERE repo_full_name = ? AND status = 'active' ORDER BY id ASC", - ); - } catch (error) { - // The table doesn't exist (a file exists at this path but isn't a real claim ledger) -- close the - // connection we already opened before rethrowing, so this never leaks a file handle. - db.close(); - throw error; - } - return { - dbPath: resolvedPath, - listActiveClaims(repoFullName) { - const normalizedRepo = normalizeRepoFullName(repoFullName); - return listActiveStatement.all(normalizedRepo).map(rowToClaim); - }, - close() { - db.close(); - }, - }; + const resolvedPath = normalizeDbPath(dbPath); + // `readOnly` (camelCase) -- node:sqlite silently IGNORES `readonly` (lowercase) as an unrecognized option + // and opens read-write anyway, defeating the entire point of this function. Verified empirically: a write + // via a `{ readonly: true }` connection succeeds with no error. + const db = new DatabaseSync(resolvedPath, { readOnly: true }); + let listActiveStatement; + try { + listActiveStatement = db.prepare("SELECT * FROM miner_claims WHERE repo_full_name = ? AND status = 'active' ORDER BY id ASC"); + } + catch (error) { + // The table doesn't exist (a file exists at this path but isn't a real claim ledger) -- close the + // connection we already opened before rethrowing, so this never leaks a file handle. + db.close(); + throw error; + } + return { + dbPath: resolvedPath, + listActiveClaims(repoFullName) { + const normalizedRepo = normalizeRepoFullName(repoFullName); + return listActiveStatement.all(normalizedRepo).map((row) => rowToClaim(row)); + }, + close() { + db.close(); + }, + }; } - function getDefaultClaimLedger() { - defaultClaimLedger ??= openClaimLedger(); - return defaultClaimLedger; + defaultClaimLedger ??= openClaimLedger(); + return defaultClaimLedger; } - export function recordClaim(claim) { - return getDefaultClaimLedger().recordClaim(claim); + return getDefaultClaimLedger().recordClaim(claim); } - export function releaseClaim(repoFullName, issueNumber, apiBaseUrl) { - return getDefaultClaimLedger().releaseClaim(repoFullName, issueNumber, apiBaseUrl); + return getDefaultClaimLedger().releaseClaim(repoFullName, issueNumber, apiBaseUrl); } - export function expireClaim(repoFullName, issueNumber, apiBaseUrl) { - return getDefaultClaimLedger().expireClaim(repoFullName, issueNumber, apiBaseUrl); + return getDefaultClaimLedger().expireClaim(repoFullName, issueNumber, apiBaseUrl); } - export function listClaims(filter) { - return getDefaultClaimLedger().listClaims(filter); + return getDefaultClaimLedger().listClaims(filter); } - /** Foundation-phase alias for `recordClaim({ repoFullName, issueNumber, note, apiBaseUrl })`. (#3351) */ export function claimIssue(repoFullName, issueNumber, note, apiBaseUrl) { - return getDefaultClaimLedger().claimIssue(repoFullName, issueNumber, note, apiBaseUrl); + return getDefaultClaimLedger().claimIssue(repoFullName, issueNumber, note, apiBaseUrl); } - /** List only `active` claims, optionally scoped to one repo. (#3351) */ export function listActiveClaims(repoFullName) { - return getDefaultClaimLedger().listActiveClaims(repoFullName); + return getDefaultClaimLedger().listActiveClaims(repoFullName); } - export function closeDefaultClaimLedger() { - if (!defaultClaimLedger) return; - defaultClaimLedger.close(); - defaultClaimLedger = null; + if (!defaultClaimLedger) + return; + defaultClaimLedger.close(); + defaultClaimLedger = null; } +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xhaW0tbGVkZ2VyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiY2xhaW0tbGVkZ2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDM0MsT0FBTyxFQUFFLHdCQUF3QixFQUFFLGtCQUFrQixFQUFFLE1BQU0sMEJBQTBCLENBQUM7QUFDeEYsT0FBTyxFQUFFLG9CQUFvQixFQUFFLE1BQU0sbUJBQW1CLENBQUM7QUFDekQsT0FBTyxFQUFFLHlCQUF5QixFQUFFLGdCQUFnQixFQUFFLHVCQUF1QixFQUFFLE1BQU0sa0JBQWtCLENBQUM7QUFDeEcsT0FBTyxFQUFFLGtCQUFrQixFQUFFLE1BQU0saUJBQWlCLENBQUM7QUFDckQsT0FBTyxFQUFFLHFCQUFxQixFQUFFLE1BQU0scUJBQXFCLENBQUM7QUFDNUQsT0FBTyxFQUFFLHVCQUF1QixFQUFFLGdCQUFnQixFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFvRm5GLE1BQU0sQ0FBQyxNQUFNLGNBQWMsR0FBRyxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUMsUUFBUSxFQUFFLFVBQVUsRUFBRSxTQUFTLENBQUMsQ0FBMkIsQ0FBQztBQUV6RyxNQUFNLGlCQUFpQixHQUFHLHNCQUFzQixDQUFDO0FBQ2pELElBQUksa0JBQWtCLEdBQXVCLElBQUksQ0FBQztBQUVsRCxNQUFNLFVBQVUsd0JBQXdCLENBQUMsTUFBMEMsT0FBTyxDQUFDLEdBQUc7SUFDNUYsT0FBTyx1QkFBdUIsQ0FBQyxpQkFBaUIsRUFBRSxnQ0FBZ0MsRUFBRSxHQUFHLENBQUMsQ0FBQztBQUMzRixDQUFDO0FBRUQsU0FBUyxlQUFlLENBQUMsTUFBaUM7SUFDeEQsT0FBTyx5QkFBeUIsQ0FBQyxNQUFNLEVBQUUsd0JBQXdCLEVBQUUsRUFBRSw4QkFBOEIsQ0FBQyxDQUFDO0FBQ3ZHLENBQUM7QUFFRCxTQUFTLHFCQUFxQixDQUFDLFlBQXFCO0lBQ2xELElBQUksT0FBTyxZQUFZLEtBQUssUUFBUTtRQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsd0JBQXdCLENBQUMsQ0FBQztJQUNoRixNQUFNLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxLQUFLLENBQUMsR0FBRyxZQUFZLENBQUMsSUFBSSxFQUFFLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBQzVELElBQUksQ0FBQyxLQUFLLElBQUksQ0FBQyxJQUFJLElBQUksS0FBSyxLQUFLLFNBQVM7UUFBRSxNQUFNLElBQUksS0FBSyxDQUFDLHdCQUF3QixDQUFDLENBQUM7SUFDdEYsSUFBSSxDQUFDLGtCQUFrQixDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsa0JBQWtCLENBQUMsSUFBSSxDQUFDO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyx3QkFBd0IsQ0FBQyxDQUFDO0lBQ3ZHLE9BQU8sR0FBRyxLQUFLLElBQUksSUFBSSxFQUFFLENBQUM7QUFDNUIsQ0FBQztBQUVELFNBQVMsb0JBQW9CLENBQUMsV0FBb0I7SUFDaEQsSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsV0FBVyxDQUFDLElBQUssV0FBc0IsR0FBRyxDQUFDO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyxzQkFBc0IsQ0FBQyxDQUFDO0lBQzNHLE9BQU8sV0FBcUIsQ0FBQztBQUMvQixDQUFDO0FBRUQsNkdBQTZHO0FBQzdHLDZHQUE2RztBQUM3RywyRkFBMkY7QUFDM0YsU0FBUyw0QkFBNEIsQ0FBQyxtQkFBNEI7SUFDaEUsSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsbUJBQW1CLENBQUMsSUFBSyxtQkFBOEIsR0FBRyxDQUFDLEVBQUUsQ0FBQztRQUNsRixNQUFNLElBQUksS0FBSyxDQUFDLCtCQUErQixDQUFDLENBQUM7SUFDbkQsQ0FBQztJQUNELE9BQU8sbUJBQTZCLENBQUM7QUFDdkMsQ0FBQztBQUVEO3lHQUN5RztBQUN6RyxTQUFTLG1CQUFtQixDQUFDLFVBQW1CO0lBQzlDLElBQUksVUFBVSxLQUFLLFNBQVMsSUFBSSxVQUFVLEtBQUssSUFBSTtRQUFFLE9BQU8sb0JBQW9CLENBQUMsVUFBVSxDQUFDO0lBQzVGLElBQUksT0FBTyxVQUFVLEtBQUssUUFBUSxJQUFJLENBQUMsVUFBVSxDQUFDLElBQUksRUFBRTtRQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsc0JBQXNCLENBQUMsQ0FBQztJQUNsRyxPQUFPLFVBQVUsQ0FBQyxJQUFJLEVBQUUsQ0FBQztBQUMzQixDQUFDO0FBRUQsMEdBQTBHO0FBQzFHLFNBQVMsYUFBYSxDQUFDLElBQWE7SUFDbEMsSUFBSSxJQUFJLEtBQUssU0FBUyxJQUFJLElBQUksS0FBSyxJQUFJO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDckQsSUFBSSxPQUFPLElBQUksS0FBSyxRQUFRO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyxjQUFjLENBQUMsQ0FBQztJQUM5RCxPQUFPLElBQUksQ0FBQztBQUNkLENBQUM7QUFFRCxTQUFTLFVBQVUsQ0FBQyxHQUFhO0lBQy9CLE9BQU87UUFDTCxFQUFFLEVBQUUsR0FBRyxDQUFDLEVBQUU7UUFDVixVQUFVLEVBQUUsR0FBRyxDQUFDLFlBQVk7UUFDNUIsWUFBWSxFQUFFLEdBQUcsQ0FBQyxjQUFjO1FBQ2hDLFdBQVcsRUFBRSxHQUFHLENBQUMsWUFBWTtRQUM3QixTQUFTLEVBQUUsR0FBRyxDQUFDLFVBQVU7UUFDekIsTUFBTSxFQUFFLEdBQUcsQ0FBQyxNQUFNO1FBQ2xCLElBQUksRUFBRSxHQUFHLENBQUMsSUFBSTtLQUNmLENBQUM7QUFDSixDQUFDO0FBRUQsZ0hBQWdIO0FBQ2hILCtHQUErRztBQUMvRywrR0FBK0c7QUFDL0csMkdBQTJHO0FBQzNHLDZHQUE2RztBQUM3RywwREFBMEQ7QUFDMUQsU0FBUyxrQkFBa0IsQ0FBQyxFQUFnQjtJQUMxQyxFQUFFLENBQUMsSUFBSSxDQUFDOzs7Ozs7Ozs7OztHQVdQLENBQUMsQ0FBQztJQUNILDRHQUE0RztJQUM1Ryw4R0FBOEc7SUFDOUcsOEdBQThHO0lBQzlHLGdEQUFnRDtJQUNoRCxFQUFFLENBQUMsT0FBTyxDQUNSOzRGQUN3RixDQUN6RixDQUFDLEdBQUcsQ0FBQyxvQkFBb0IsQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUN2QyxFQUFFLENBQUMsSUFBSSxDQUFDLHlCQUF5QixDQUFDLENBQUM7SUFDbkMsRUFBRSxDQUFDLElBQUksQ0FBQyxvREFBb0QsQ0FBQyxDQUFDO0FBQ2hFLENBQUM7QUFFRCw0R0FBNEc7QUFDNUcsK0dBQStHO0FBQy9HLG1HQUFtRztBQUNuRyw2RkFBNkY7QUFDN0YsU0FBUyxpQkFBaUIsQ0FBQyxFQUFnQjtJQUN6QyxNQUFNLGlCQUFpQixHQUFHLEVBQUU7U0FDekIsT0FBTyxDQUFDLGlDQUFpQyxDQUFDO1NBQzFDLEdBQUcsRUFBRTtTQUNMLElBQUksQ0FBQyxDQUFDLE1BQU0sRUFBRSxFQUFFLENBQUUsTUFBdUIsQ0FBQyxJQUFJLEtBQUssV0FBVyxDQUFDLENBQUM7SUFDbkUsSUFBSSxDQUFDLGlCQUFpQjtRQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsb0RBQW9ELENBQUMsQ0FBQztBQUN4RixDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILE1BQU0sVUFBVSxlQUFlLENBQUMsU0FBaUIsd0JBQXdCLEVBQUU7SUFDekUsTUFBTSxZQUFZLEdBQUcsZUFBZSxDQUFDLE1BQU0sQ0FBQyxDQUFDO0lBQzdDLE1BQU0sRUFBRSxHQUFHLGdCQUFnQixDQUFDLFlBQVksQ0FBQyxDQUFDO0lBQzFDLHVHQUF1RztJQUN2RyxvR0FBb0c7SUFDcEcsNEZBQTRGO0lBQzVGLEVBQUUsQ0FBQyxJQUFJLENBQUM7Ozs7Ozs7Ozs7R0FVUCxDQUFDLENBQUM7SUFDSCw4RkFBOEY7SUFDOUYscUJBQXFCLENBQUMsRUFBRSxFQUFFLENBQUMsa0JBQWtCLEVBQUUsaUJBQWlCLENBQUMsQ0FBQyxDQUFDO0lBRW5FLDRHQUE0RztJQUM1RywrR0FBK0c7SUFDL0csc0dBQXNHO0lBQ3RHLE1BQU0sZUFBZSxHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUM7Ozs7Ozs7O0dBUWxDLENBQUMsQ0FBQztJQUNILE1BQU0sWUFBWSxHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQzdCLCtGQUErRixDQUNoRyxDQUFDO0lBQ0YsK0dBQStHO0lBQy9HLDJHQUEyRztJQUMzRyw0R0FBNEc7SUFDNUcsZ0RBQWdEO0lBQ2hELE1BQU0sZ0JBQWdCLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FDakMsa0pBQWtKLENBQ25KLENBQUM7SUFDRixNQUFNLGVBQWUsR0FBRyxFQUFFLENBQUMsT0FBTyxDQUNoQyxpSkFBaUosQ0FDbEosQ0FBQztJQUNGLE1BQU0sZ0JBQWdCLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FBQyw0Q0FBNEMsQ0FBQyxDQUFDO0lBQ2xGLE1BQU0saUJBQWlCLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FDbEMscUVBQXFFLENBQ3RFLENBQUM7SUFDRixNQUFNLG1CQUFtQixHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQ3BDLDZEQUE2RCxDQUM5RCxDQUFDO0lBQ0YsTUFBTSx1QkFBdUIsR0FBRyxFQUFFLENBQUMsT0FBTyxDQUN4QyxvRkFBb0YsQ0FDckYsQ0FBQztJQUNGLDBHQUEwRztJQUMxRyx3R0FBd0c7SUFDeEcsaUhBQWlIO0lBQ2pILE1BQU0sd0JBQXdCLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FDekMsMkZBQTJGLENBQzVGLENBQUM7SUFFRixTQUFTLHVCQUF1QixDQUFDLFlBQXVDO1FBQ3RFLElBQUksWUFBWSxLQUFLLFNBQVMsSUFBSSxZQUFZLEtBQUssSUFBSTtZQUFFLE9BQU8sU0FBUyxDQUFDO1FBQzFFLE9BQU8scUJBQXFCLENBQUMsWUFBWSxDQUFDLENBQUM7SUFDN0MsQ0FBQztJQUVELFNBQVMscUJBQXFCLENBQUMsTUFBK0M7UUFDNUUsSUFBSSxNQUFNLEtBQUssU0FBUyxJQUFJLE1BQU0sS0FBSyxJQUFJO1lBQUUsT0FBTyxTQUFTLENBQUM7UUFDOUQsSUFBSSxDQUFFLGNBQW9DLENBQUMsUUFBUSxDQUFDLE1BQU0sQ0FBQztZQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsZ0JBQWdCLENBQUMsQ0FBQztRQUMvRixPQUFPLE1BQXFCLENBQUM7SUFDL0IsQ0FBQztJQUVELE1BQU0sTUFBTSxHQUFnQjtRQUMxQixNQUFNLEVBQUUsWUFBWTtRQUNwQixXQUFXLENBQUMsS0FBdUI7WUFDakMsTUFBTSxVQUFVLEdBQUcsbUJBQW1CLENBQUMsS0FBSyxFQUFFLFVBQVUsQ0FBQyxDQUFDO1lBQzFELE1BQU0sWUFBWSxHQUFHLHFCQUFxQixDQUFDLEtBQUssRUFBRSxZQUFZLENBQUMsQ0FBQztZQUNoRSxNQUFNLFdBQVcsR0FBRyxvQkFBb0IsQ0FBQyxLQUFLLEVBQUUsV0FBVyxDQUFDLENBQUM7WUFDN0QsTUFBTSxJQUFJLEdBQUcsYUFBYSxDQUFDLEtBQUssRUFBRSxJQUFJLENBQUMsQ0FBQztZQUN4QyxNQUFNLFNBQVMsR0FBRyxJQUFJLElBQUksRUFBRSxDQUFDLFdBQVcsRUFBRSxDQUFDO1lBQzNDLGVBQWUsQ0FBQyxHQUFHLENBQUMsVUFBVSxFQUFFLFlBQVksRUFBRSxXQUFXLEVBQUUsU0FBUyxFQUFFLElBQUksQ0FBQyxDQUFDO1lBQzVFLE9BQU8sVUFBVSxDQUFDLFlBQVksQ0FBQyxHQUFHLENBQUMsVUFBVSxFQUFFLFlBQVksRUFBRSxXQUFXLENBQWEsQ0FBQyxDQUFDO1FBQ3pGLENBQUM7UUFDRCxZQUFZLENBQUMsWUFBb0IsRUFBRSxXQUFtQixFQUFFLFVBQW1CO1lBQ3pFLE1BQU0sZUFBZSxHQUFHLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxDQUFDO1lBQ3hELE1BQU0sY0FBYyxHQUFHLHFCQUFxQixDQUFDLFlBQVksQ0FBQyxDQUFDO1lBQzNELE1BQU0sZUFBZSxHQUFHLG9CQUFvQixDQUFDLFdBQVcsQ0FBQyxDQUFDO1lBQzFELE1BQU0sR0FBRyxHQUFHLGdCQUFnQixDQUFDLEdBQUcsQ0FBQyxlQUFlLEVBQUUsY0FBYyxFQUFFLGVBQWUsQ0FBeUIsQ0FBQztZQUMzRyxPQUFPLEdBQUcsQ0FBQyxDQUFDLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUM7UUFDdEMsQ0FBQztRQUNELFdBQVcsQ0FBQyxZQUFvQixFQUFFLFdBQW1CLEVBQUUsVUFBbUI7WUFDeEUsTUFBTSxlQUFlLEdBQUcsbUJBQW1CLENBQUMsVUFBVSxDQUFDLENBQUM7WUFDeEQsTUFBTSxjQUFjLEdBQUcscUJBQXFCLENBQUMsWUFBWSxDQUFDLENBQUM7WUFDM0QsTUFBTSxlQUFlLEdBQUcsb0JBQW9CLENBQUMsV0FBVyxDQUFDLENBQUM7WUFDMUQsTUFBTSxHQUFHLEdBQUcsZUFBZSxDQUFDLEdBQUcsQ0FBQyxlQUFlLEVBQUUsY0FBYyxFQUFFLGVBQWUsQ0FBeUIsQ0FBQztZQUMxRyxPQUFPLEdBQUcsQ0FBQyxDQUFDLENBQUMsVUFBVSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUM7UUFDdEMsQ0FBQztRQUNELFVBQVUsQ0FBQyxTQUEyQixFQUFFO1lBQ3RDLE1BQU0sWUFBWSxHQUFHLHVCQUF1QixDQUFDLE1BQU0sQ0FBQyxZQUFZLENBQUMsQ0FBQztZQUNsRSxNQUFNLE1BQU0sR0FBRyxxQkFBcUIsQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLENBQUM7WUFFcEQsSUFBSSxJQUFJLENBQUM7WUFDVCxJQUFJLFlBQVksS0FBSyxTQUFTLElBQUksTUFBTSxLQUFLLFNBQVMsRUFBRSxDQUFDO2dCQUN2RCxJQUFJLEdBQUcsdUJBQXVCLENBQUMsR0FBRyxDQUFDLFlBQVksRUFBRSxNQUFNLENBQUMsQ0FBQztZQUMzRCxDQUFDO2lCQUFNLElBQUksWUFBWSxLQUFLLFNBQVMsRUFBRSxDQUFDO2dCQUN0QyxJQUFJLEdBQUcsaUJBQWlCLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDO1lBQzdDLENBQUM7aUJBQU0sSUFBSSxNQUFNLEtBQUssU0FBUyxFQUFFLENBQUM7Z0JBQ2hDLElBQUksR0FBRyxtQkFBbUIsQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLENBQUM7WUFDekMsQ0FBQztpQkFBTSxDQUFDO2dCQUNOLElBQUksR0FBRyxnQkFBZ0IsQ0FBQyxHQUFHLEVBQUUsQ0FBQztZQUNoQyxDQUFDO1lBQ0QsT0FBTyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsR0FBRyxFQUFFLEVBQUUsQ0FBQyxVQUFVLENBQUMsR0FBZSxDQUFDLENBQUMsQ0FBQztRQUN4RCxDQUFDO1FBQ0Q7NkhBQ3FIO1FBQ3JILG9CQUFvQixDQUFDLFdBQW1CLHdCQUF3QjtZQUM5RCxPQUFPLGtCQUFrQixDQUFDLE1BQU0sRUFBRSxJQUFJLENBQUMsR0FBRyxFQUFFLEVBQUUsUUFBUSxDQUFDLENBQUM7UUFDMUQsQ0FBQztRQUNELFVBQVUsQ0FBQyxZQUFvQixFQUFFLFdBQW1CLEVBQUUsSUFBYSxFQUFFLFVBQW1CO1lBQ3RGLHdHQUF3RztZQUN4Ryx5R0FBeUc7WUFDekcseUdBQXlHO1lBQ3pHLG1GQUFtRjtZQUNuRiwyR0FBMkc7WUFDM0csa0JBQWtCLENBQUMsTUFBTSxFQUFFLElBQUksQ0FBQyxHQUFHLEVBQUUsRUFBRSx3QkFBd0IsQ0FBQyxDQUFDO1lBQ2pFLE9BQU8sTUFBTSxDQUFDLFdBQVcsQ0FBQyxFQUFFLFlBQVksRUFBRSxXQUFXLEVBQUUsSUFBSSxFQUFFLFVBQVUsRUFBc0IsQ0FBQyxDQUFDO1FBQ2pHLENBQUM7UUFDRDs7Ozs7Ozs7OztXQVVHO1FBQ0gsbUJBQW1CLENBQ2pCLFlBQW9CLEVBQ3BCLFdBQW1CLEVBQ25CLElBQXdCLEVBQ3hCLFVBQThCLEVBQzlCLG1CQUEyQjtZQUUzQixNQUFNLEdBQUcsR0FBRyw0QkFBNEIsQ0FBQyxtQkFBbUIsQ0FBQyxDQUFDO1lBQzlELDJHQUEyRztZQUMzRyx1R0FBdUc7WUFDdkcsd0ZBQXdGO1lBQ3hGLE1BQU0sY0FBYyxHQUFHLHFCQUFxQixDQUFDLFlBQVksQ0FBQyxDQUFDO1lBQzNELEVBQUUsQ0FBQyxJQUFJLENBQUMsaUJBQWlCLENBQUMsQ0FBQztZQUMzQixJQUFJLENBQUM7Z0JBQ0gsa0JBQWtCLENBQUMsTUFBTSxFQUFFLElBQUksQ0FBQyxHQUFHLEVBQUUsRUFBRSx3QkFBd0IsQ0FBQyxDQUFDO2dCQUNqRSxNQUFNLGdCQUFnQixHQUFJLHdCQUF3QixDQUFDLEdBQUcsQ0FBQyxjQUFjLENBQWMsQ0FBQyxLQUFLLENBQUM7Z0JBQzFGLElBQUksZ0JBQWdCLElBQUksR0FBRyxFQUFFLENBQUM7b0JBQzVCLHlHQUF5RztvQkFDekcsOEZBQThGO29CQUM5RixFQUFFLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxDQUFDO29CQUNsQixPQUFPLEVBQUUsT0FBTyxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFFLGdCQUFnQixFQUFFLG1CQUFtQixFQUFFLEdBQUcsRUFBRSxDQUFDO2dCQUNyRixDQUFDO2dCQUNELE1BQU0sS0FBSyxHQUFHLE1BQU0sQ0FBQyxXQUFXLENBQUMsRUFBRSxZQUFZLEVBQUUsV0FBVyxFQUFFLElBQUksRUFBRSxVQUFVLEVBQXNCLENBQUMsQ0FBQztnQkFDdEcsRUFBRSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsQ0FBQztnQkFDbEIsT0FBTyxFQUFFLE9BQU8sRUFBRSxJQUFJLEVBQUUsS0FBSyxFQUFFLGdCQUFnQixFQUFFLG1CQUFtQixFQUFFLEdBQUcsRUFBRSxDQUFDO1lBQzlFLENBQUM7WUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO2dCQUNmLEVBQUUsQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUM7Z0JBQ3BCLE1BQU0sS0FBSyxDQUFDO1lBQ2QsQ0FBQztRQUNILENBQUM7UUFDRCxnQkFBZ0IsQ0FBQyxZQUFxQjtZQUNwQyxNQUFNLE1BQU0sR0FBRztnQkFDYixNQUFNLEVBQUUsUUFBaUI7Z0JBQ3pCLEdBQUcsQ0FBQyxZQUFZLEtBQUssU0FBUyxDQUFDLENBQUMsQ0FBQyxFQUFFLFlBQVksRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7YUFDN0IsQ0FBQztZQUM3QixPQUFPLE1BQU0sQ0FBQyxVQUFVLENBQUMsTUFBTSxDQUFDLENBQUM7UUFDbkMsQ0FBQztRQUNELDJHQUEyRztRQUMzRyw2RkFBNkY7UUFDN0YsV0FBVyxDQUFDLFlBQW9CO1lBQzlCLE9BQU8sZ0JBQWdCLENBQUMsRUFBRSxFQUFFLHVCQUF1QixFQUFFLHFCQUFxQixDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUM7UUFDNUYsQ0FBQztRQUNELEtBQUs7WUFDSCxFQUFFLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDYixDQUFDO0tBQ0YsQ0FBQztJQUNGLE9BQU8sTUFBTSxDQUFDO0FBQ2hCLENBQUM7QUFFRDs7Ozs7Ozs7O0dBU0c7QUFDSCxNQUFNLFVBQVUsdUJBQXVCLENBQUMsTUFBYztJQUNwRCxNQUFNLFlBQVksR0FBRyxlQUFlLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDN0MsMEdBQTBHO0lBQzFHLDBHQUEwRztJQUMxRyxnRUFBZ0U7SUFDaEUsTUFBTSxFQUFFLEdBQUcsSUFBSSxZQUFZLENBQUMsWUFBWSxFQUFFLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7SUFDOUQsSUFBSSxtQkFBbUIsQ0FBQztJQUN4QixJQUFJLENBQUM7UUFDSCxtQkFBbUIsR0FBRyxFQUFFLENBQUMsT0FBTyxDQUM5QiwyRkFBMkYsQ0FDNUYsQ0FBQztJQUNKLENBQUM7SUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1FBQ2Ysa0dBQWtHO1FBQ2xHLHFGQUFxRjtRQUNyRixFQUFFLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDWCxNQUFNLEtBQUssQ0FBQztJQUNkLENBQUM7SUFDRCxPQUFPO1FBQ0wsTUFBTSxFQUFFLFlBQVk7UUFDcEIsZ0JBQWdCLENBQUMsWUFBb0I7WUFDbkMsTUFBTSxjQUFjLEdBQUcscUJBQXFCLENBQUMsWUFBWSxDQUFDLENBQUM7WUFDM0QsT0FBTyxtQkFBbUIsQ0FBQyxHQUFHLENBQUMsY0FBYyxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsR0FBRyxFQUFFLEVBQUUsQ0FBQyxVQUFVLENBQUMsR0FBZSxDQUFDLENBQUMsQ0FBQztRQUMzRixDQUFDO1FBQ0QsS0FBSztZQUNILEVBQUUsQ0FBQyxLQUFLLEVBQUUsQ0FBQztRQUNiLENBQUM7S0FDRixDQUFDO0FBQ0osQ0FBQztBQUVELFNBQVMscUJBQXFCO0lBQzVCLGtCQUFrQixLQUFLLGVBQWUsRUFBRSxDQUFDO0lBQ3pDLE9BQU8sa0JBQWtCLENBQUM7QUFDNUIsQ0FBQztBQUVELE1BQU0sVUFBVSxXQUFXLENBQUMsS0FBdUI7SUFDakQsT0FBTyxxQkFBcUIsRUFBRSxDQUFDLFdBQVcsQ0FBQyxLQUFLLENBQUMsQ0FBQztBQUNwRCxDQUFDO0FBRUQsTUFBTSxVQUFVLFlBQVksQ0FBQyxZQUFvQixFQUFFLFdBQW1CLEVBQUUsVUFBbUI7SUFDekYsT0FBTyxxQkFBcUIsRUFBRSxDQUFDLFlBQVksQ0FBQyxZQUFZLEVBQUUsV0FBVyxFQUFFLFVBQVUsQ0FBQyxDQUFDO0FBQ3JGLENBQUM7QUFFRCxNQUFNLFVBQVUsV0FBVyxDQUFDLFlBQW9CLEVBQUUsV0FBbUIsRUFBRSxVQUFtQjtJQUN4RixPQUFPLHFCQUFxQixFQUFFLENBQUMsV0FBVyxDQUFDLFlBQVksRUFBRSxXQUFXLEVBQUUsVUFBVSxDQUFDLENBQUM7QUFDcEYsQ0FBQztBQUVELE1BQU0sVUFBVSxVQUFVLENBQUMsTUFBeUI7SUFDbEQsT0FBTyxxQkFBcUIsRUFBRSxDQUFDLFVBQVUsQ0FBQyxNQUFNLENBQUMsQ0FBQztBQUNwRCxDQUFDO0FBRUQseUdBQXlHO0FBQ3pHLE1BQU0sVUFBVSxVQUFVLENBQUMsWUFBb0IsRUFBRSxXQUFtQixFQUFFLElBQWEsRUFBRSxVQUFtQjtJQUN0RyxPQUFPLHFCQUFxQixFQUFFLENBQUMsVUFBVSxDQUFDLFlBQVksRUFBRSxXQUFXLEVBQUUsSUFBSSxFQUFFLFVBQVUsQ0FBQyxDQUFDO0FBQ3pGLENBQUM7QUFFRCx3RUFBd0U7QUFDeEUsTUFBTSxVQUFVLGdCQUFnQixDQUFDLFlBQXFCO0lBQ3BELE9BQU8scUJBQXFCLEVBQUUsQ0FBQyxnQkFBZ0IsQ0FBQyxZQUFZLENBQUMsQ0FBQztBQUNoRSxDQUFDO0FBRUQsTUFBTSxVQUFVLHVCQUF1QjtJQUNyQyxJQUFJLENBQUMsa0JBQWtCO1FBQUUsT0FBTztJQUNoQyxrQkFBa0IsQ0FBQyxLQUFLLEVBQUUsQ0FBQztJQUMzQixrQkFBa0IsR0FBRyxJQUFJLENBQUM7QUFDNUIsQ0FBQyJ9 \ No newline at end of file diff --git a/packages/loopover-miner/lib/claim-ledger.ts b/packages/loopover-miner/lib/claim-ledger.ts new file mode 100644 index 0000000000..67a5bde40c --- /dev/null +++ b/packages/loopover-miner/lib/claim-ledger.ts @@ -0,0 +1,464 @@ +import { DatabaseSync } from "node:sqlite"; +import { DEFAULT_MAX_CLAIM_AGE_MS, sweepExpiredClaims } from "./claim-ledger-expiry.js"; +import { DEFAULT_FORGE_CONFIG } from "./forge-config.js"; +import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js"; +import { isValidRepoSegment } from "./repo-clone.js"; +import { applySchemaMigrations } from "./schema-version.js"; +import { CLAIM_LEDGER_PURGE_SPEC, purgeStoreByRepo } from "./store-maintenance.js"; + +// The miner's local soft-claim ledger (#2314): a 100% client-side record of "I'm working on issue #N in repo X", +// so Phase 2's soft-claim adjudication (sibling issues) has somewhere to persist claims. Schema + CRUD only — no +// adjudication logic, no network calls, no autonomous writes. The database only lives on this machine; this module +// never uploads, syncs, or phones home. Mirrors the package's existing local-store pattern (run-state.js, +// portfolio-queue.js, event-ledger.js) — plain JS + node:sqlite, not the hosted Worker's shared D1 `migrations/`. + +export type ClaimStatus = "active" | "released" | "expired"; + +export type ClaimEntry = { + id: number; + apiBaseUrl: string; + repoFullName: string; + issueNumber: number; + claimedAt: string; + status: ClaimStatus; + note: string | null; +}; + +export type RecordClaimInput = { + repoFullName: string; + issueNumber: number; + note?: string; + apiBaseUrl?: string; +}; + +export type ListClaimsFilter = { + repoFullName?: string | null; + status?: ClaimStatus | null; +}; + +/** Result of an atomic, concurrency-capped claim (#6758). `claimed` discriminates success (a recorded claim) + * from a cap rejection (`claim: null`); both carry the pre-insert active count and the resolved cap so a + * rejected caller can still log the violation. */ +export type ClaimWithinCapResult = + | { claimed: true; claim: ClaimEntry; activeClaimCount: number; maxConcurrentClaims: number } + | { claimed: false; claim: null; activeClaimCount: number; maxConcurrentClaims: number }; + +export type ClaimLedger = { + dbPath: string; + recordClaim(claim: RecordClaimInput): ClaimEntry; + /** Claims the issue, expiring any claim orphaned by a dead process first (#6156). */ + claimIssue(repoFullName: string, issueNumber: number, note?: string, apiBaseUrl?: string): ClaimEntry; + /** Atomically records the claim only while this repo's active-claim count is under `maxConcurrentClaims`, + * counting and inserting in one transaction so racing sibling processes can't exceed the cap (#6758). */ + claimIssueWithinCap( + repoFullName: string, + issueNumber: number, + note: string | undefined, + apiBaseUrl: string | undefined, + maxConcurrentClaims: number, + ): ClaimWithinCapResult; + /** Expire claims orphaned by a crashed/killed process, returning the transitioned rows (#6156). */ + reclaimExpiredClaims(maxAgeMs?: number): ClaimEntry[]; + releaseClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; + expireClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null; + listClaims(filter?: ListClaimsFilter): ClaimEntry[]; + listActiveClaims(repoFullName?: string): ClaimEntry[]; + purgeByRepo(repoFullName: string): number; + close(): void; +}; + +export type ReadOnlyClaimLedger = { + dbPath: string; + listActiveClaims(repoFullName: string): ClaimEntry[]; + close(): void; +}; + +/** SQLite `miner_claims` row shape (StatementSync returns `Record`). */ +type ClaimRow = { + id: number; + api_base_url: string; + repo_full_name: string; + issue_number: number; + claimed_at: string; + status: ClaimStatus; + note: string | null; +}; + +type CountRow = { count: number }; + +type TableInfoRow = { name: string }; + +export const CLAIM_STATUSES = Object.freeze(["active", "released", "expired"]) as readonly ClaimStatus[]; + +const defaultDbFileName = "claim-ledger.sqlite3"; +let defaultClaimLedger: ClaimLedger | null = null; + +export function resolveClaimLedgerDbPath(env: Record = process.env): string { + return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_CLAIM_LEDGER_DB", env); +} + +function normalizeDbPath(dbPath: string | null | undefined): string { + return normalizeLocalStoreDbPath(dbPath, resolveClaimLedgerDbPath(), "invalid_claim_ledger_db_path"); +} + +function normalizeRepoFullName(repoFullName: unknown): string { + if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name"); + const [owner, repo, extra] = repoFullName.trim().split("/"); + if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name"); + if (!isValidRepoSegment(owner) || !isValidRepoSegment(repo)) throw new Error("invalid_repo_full_name"); + return `${owner}/${repo}`; +} + +function normalizeIssueNumber(issueNumber: unknown): number { + if (!Number.isInteger(issueNumber) || (issueNumber as number) < 1) throw new Error("invalid_issue_number"); + return issueNumber as number; +} + +// The per-repo concurrent-claim cap the atomic count-and-claim gates on (#6758). Always an already-validated +// positive integer from the caller's MinerGoalSpec, but re-checked here because a bad value must fail loudly +// rather than silently disable the cap (a comparison against `undefined` is always false). +function normalizeMaxConcurrentClaims(maxConcurrentClaims: unknown): number { + if (!Number.isInteger(maxConcurrentClaims) || (maxConcurrentClaims as number) < 1) { + throw new Error("invalid_max_concurrent_claims"); + } + return maxConcurrentClaims as number; +} + +/** Optional forge host, scoping rows so two hosts serving the same owner/repo name never collide (#5563). + * Omitted/nullish → the github.com default, so every pre-existing single-forge caller is unaffected. */ +function normalizeApiBaseUrl(apiBaseUrl: unknown): string { + if (apiBaseUrl === undefined || apiBaseUrl === null) return DEFAULT_FORGE_CONFIG.apiBaseUrl; + if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) throw new Error("invalid_api_base_url"); + return apiBaseUrl.trim(); +} + +/** Optional free-text note: omitted/nullish → null; a string is kept as-is; anything else is rejected. */ +function normalizeNote(note: unknown): string | null { + if (note === undefined || note === null) return null; + if (typeof note !== "string") throw new Error("invalid_note"); + return note; +} + +function rowToClaim(row: ClaimRow): ClaimEntry { + return { + id: row.id, + apiBaseUrl: row.api_base_url, + repoFullName: row.repo_full_name, + issueNumber: row.issue_number, + claimedAt: row.claimed_at, + status: row.status, + note: row.note, + }; +} + +// v1 -> v2 (#5563): scope the UNIQUE constraint by (api_base_url, repo_full_name, issue_number) instead of bare +// (repo_full_name, issue_number) -- two different forge hosts serving a same-named repo/issue must not collide +// in this ledger. SQLite cannot ALTER a UNIQUE constraint in place, so this rebuilds the table: create the new +// shape, copy every existing row with the pre-#4784 implicit single-forge default backfilled, drop the old +// table, rename the new one in. Runs inside applySchemaMigrations' own transaction, so a mid-rebuild failure +// leaves the file at v1 and retries cleanly on next open. +function addApiBaseUrlScope(db: DatabaseSync): void { + db.exec(` + CREATE TABLE miner_claims_v2 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + api_base_url TEXT NOT NULL, + repo_full_name TEXT NOT NULL, + issue_number INTEGER NOT NULL, + claimed_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'released', 'expired')), + note TEXT, + UNIQUE (api_base_url, repo_full_name, issue_number) + ) + `); + // OR IGNORE: a row this store's own read path already treats as unusable garbage (an unrecognized `status`, + // e.g. from a hand-edited or otherwise corrupted file) would violate the CHECK constraint above and abort the + // whole migration. Skipping it here is consistent with that same fail-closed posture, rather than turning one + // bad row into a permanently unmigratable file. + db.prepare( + `INSERT OR IGNORE INTO miner_claims_v2 (id, api_base_url, repo_full_name, issue_number, claimed_at, status, note) + SELECT id, ?, repo_full_name, issue_number, claimed_at, status, note FROM miner_claims`, + ).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); + db.exec("DROP TABLE miner_claims"); + db.exec("ALTER TABLE miner_claims_v2 RENAME TO miner_claims"); +} + +// v2 -> v3 (#4939): additive tenant-scoping column, a prerequisite for any hosted, multi-tenant use of this +// same store's logic. NULL for every row today -- self-host behavior is byte-identical, since nothing reads or +// writes it yet (no consumer exists until a future hosted deployment populates it). Same defensive +// column-presence guard as this file's own v1->v2 migration's sibling in portfolio-queue.js. +function addTenantIdColumn(db: DatabaseSync): void { + const hasTenantIdColumn = db + .prepare("PRAGMA table_info(miner_claims)") + .all() + .some((column) => (column as TableInfoRow).name === "tenant_id"); + if (!hasTenantIdColumn) db.exec("ALTER TABLE miner_claims ADD COLUMN tenant_id TEXT"); +} + +/** + * Opens the local claim ledger, creating the table on first use. `UNIQUE(api_base_url, repo_full_name, + * issue_number)` keeps ONE row per claimed issue per forge host, and `recordClaim` is a single atomic + * INSERT…ON CONFLICT statement (no read-then-write), so concurrent claims cannot duplicate a row. (#2314, #5563) + */ +export function openClaimLedger(dbPath: string = resolveClaimLedgerDbPath()): ClaimLedger { + const resolvedPath = normalizeDbPath(dbPath); + const db = openLocalStoreDb(resolvedPath); + // LOCAL bookkeeping only: this table records which issues this miner instance has soft-claimed on this + // machine. It does NOT adjudicate contested duplicates — sibling miners claiming the same issue are + // resolved elsewhere via `isDuplicateClusterWinnerByClaim` from `@loopover/engine` (#3355). + db.exec(` + CREATE TABLE IF NOT EXISTS miner_claims ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_full_name TEXT NOT NULL, + issue_number INTEGER NOT NULL, + claimed_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'released', 'expired')), + note TEXT, + UNIQUE (repo_full_name, issue_number) + ) + `); + // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. + applySchemaMigrations(db, [addApiBaseUrlScope, addTenantIdColumn]); + + // Idempotent claim in ONE atomic statement: insert a new active claim, or — only if the existing row is NOT + // already active — re-activate it (a released/expired claim can be re-claimed). The `WHERE status <> 'active'` + // guard makes re-claiming an already-active issue a true no-op (no row churn), never a duplicate row. + const recordStatement = db.prepare(` + INSERT INTO miner_claims (api_base_url, repo_full_name, issue_number, claimed_at, status, note) + VALUES (?, ?, ?, ?, 'active', ?) + ON CONFLICT(api_base_url, repo_full_name, issue_number) DO UPDATE SET + claimed_at = excluded.claimed_at, + note = excluded.note, + status = 'active' + WHERE miner_claims.status <> 'active' + `); + const getStatement = db.prepare( + "SELECT * FROM miner_claims WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ?", + ); + // RETURNING (matching portfolio-queue.js's own claim/release statements) makes the "nothing to release/expire" + // case observable directly from ONE atomic statement, rather than a separate post-UPDATE SELECT whose "row + // went missing" branch would be structurally unreachable (nothing else runs between the UPDATE and a SELECT + // on the same key within one synchronous call). + const releaseStatement = db.prepare( + "UPDATE miner_claims SET status = 'released' WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ? AND status = 'active' RETURNING *", + ); + const expireStatement = db.prepare( + "UPDATE miner_claims SET status = 'expired' WHERE api_base_url = ? AND repo_full_name = ? AND issue_number = ? AND status = 'active' RETURNING *", + ); + const listAllStatement = db.prepare("SELECT * FROM miner_claims ORDER BY id ASC"); + const listRepoStatement = db.prepare( + "SELECT * FROM miner_claims WHERE repo_full_name = ? ORDER BY id ASC", + ); + const listStatusStatement = db.prepare( + "SELECT * FROM miner_claims WHERE status = ? ORDER BY id ASC", + ); + const listRepoStatusStatement = db.prepare( + "SELECT * FROM miner_claims WHERE repo_full_name = ? AND status = ? ORDER BY id ASC", + ); + // Repo-wide active-claim tally for the atomic concurrency cap (#6758). Scoped by repo_full_name only (not + // api_base_url), matching the cross-forge counting that listActiveClaims(repoFullName) -- and the prior + // attempt-cli.js pre-check built on it -- already did, so the cap's MEANING is unchanged; only its atomicity is. + const countActiveRepoStatement = db.prepare( + "SELECT COUNT(*) AS count FROM miner_claims WHERE repo_full_name = ? AND status = 'active'", + ); + + function normalizeListRepoFilter(repoFullName: string | null | undefined): string | undefined { + if (repoFullName === undefined || repoFullName === null) return undefined; + return normalizeRepoFullName(repoFullName); + } + + function normalizeStatusFilter(status: ClaimStatus | string | null | undefined): ClaimStatus | undefined { + if (status === undefined || status === null) return undefined; + if (!(CLAIM_STATUSES as readonly string[]).includes(status)) throw new Error("invalid_status"); + return status as ClaimStatus; + } + + const ledger: ClaimLedger = { + dbPath: resolvedPath, + recordClaim(claim: RecordClaimInput): ClaimEntry { + const apiBaseUrl = normalizeApiBaseUrl(claim?.apiBaseUrl); + const repoFullName = normalizeRepoFullName(claim?.repoFullName); + const issueNumber = normalizeIssueNumber(claim?.issueNumber); + const note = normalizeNote(claim?.note); + const claimedAt = new Date().toISOString(); + recordStatement.run(apiBaseUrl, repoFullName, issueNumber, claimedAt, note); + return rowToClaim(getStatement.get(apiBaseUrl, repoFullName, issueNumber) as ClaimRow); + }, + releaseClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const normalizedIssue = normalizeIssueNumber(issueNumber); + const row = releaseStatement.get(normalizedForge, normalizedRepo, normalizedIssue) as ClaimRow | undefined; + return row ? rowToClaim(row) : null; + }, + expireClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const normalizedIssue = normalizeIssueNumber(issueNumber); + const row = expireStatement.get(normalizedForge, normalizedRepo, normalizedIssue) as ClaimRow | undefined; + return row ? rowToClaim(row) : null; + }, + listClaims(filter: ListClaimsFilter = {}): ClaimEntry[] { + const repoFullName = normalizeListRepoFilter(filter.repoFullName); + const status = normalizeStatusFilter(filter.status); + + let rows; + if (repoFullName !== undefined && status !== undefined) { + rows = listRepoStatusStatement.all(repoFullName, status); + } else if (repoFullName !== undefined) { + rows = listRepoStatement.all(repoFullName); + } else if (status !== undefined) { + rows = listStatusStatement.all(status); + } else { + rows = listAllStatement.all(); + } + return rows.map((row) => rowToClaim(row as ClaimRow)); + }, + /** Expire claims orphaned by a crashed/killed process, returning the transitioned rows (#6156). The explicit + * counterpart to the sweep claimIssue runs on its own, mirroring reclaimStuckItems (portfolio-queue-manager.js). */ + reclaimExpiredClaims(maxAgeMs: number = DEFAULT_MAX_CLAIM_AGE_MS): ClaimEntry[] { + return sweepExpiredClaims(ledger, Date.now(), maxAgeMs); + }, + claimIssue(repoFullName: string, issueNumber: number, note?: string, apiBaseUrl?: string): ClaimEntry { + // Expire orphaned claims first, so an issue stranded 'active' by a dead process becomes claimable again + // instead of blocking indefinitely (#6156). Without this, recordClaim's `WHERE status <> 'active'` guard + // makes re-claiming an active row a no-op, so a claim whose owning process died keeps winning forever -- + // there is no other path to expireClaim. Mirrors claimNextBatch's sweep-then-claim + // (portfolio-queue-manager.js), where a lease stranded by a dead process would otherwise starve the queue. + sweepExpiredClaims(ledger, Date.now(), DEFAULT_MAX_CLAIM_AGE_MS); + return ledger.recordClaim({ repoFullName, issueNumber, note, apiBaseUrl } as RecordClaimInput); + }, + /** + * Atomic, concurrency-capped claim (#6758). Sweeps orphaned claims, counts this repo's ACTIVE claims, and + * records the new claim ONLY while still strictly under `maxConcurrentClaims` -- all inside ONE `BEGIN + * IMMEDIATE` transaction. The prior enforcement split the count (attempt-cli.js's listActiveClaims) from the + * insert (claimIssue) across two statements with no shared transaction, so two sibling miner processes racing + * the same repo could both read the same sub-cap count and both claim, exceeding the cap. Fusing count + + * insert under an IMMEDIATE write lock -- with node:sqlite's shared `busy_timeout`, so the loser WAITS for the + * winner's commit rather than erroring -- closes that window: the second process sees the committed count and + * is cleanly rejected with `claimed: false` (never silently dropped), so the caller can log the cap violation. + * Returns the pre-insert `activeClaimCount` and the resolved `maxConcurrentClaims` on both paths. + */ + claimIssueWithinCap( + repoFullName: string, + issueNumber: number, + note: string | undefined, + apiBaseUrl: string | undefined, + maxConcurrentClaims: number, + ): ClaimWithinCapResult { + const cap = normalizeMaxConcurrentClaims(maxConcurrentClaims); + // Normalize the repo up front: the count query keys on it, and a bad value must throw BEFORE `BEGIN` so it + // can never strand an open transaction. `issueNumber`/`note`/`apiBaseUrl` are validated by recordClaim + // INSIDE the transaction -- a bad value there is rolled back whole via the catch below. + const normalizedRepo = normalizeRepoFullName(repoFullName); + db.exec("BEGIN IMMEDIATE"); + try { + sweepExpiredClaims(ledger, Date.now(), DEFAULT_MAX_CLAIM_AGE_MS); + const activeClaimCount = (countActiveRepoStatement.get(normalizedRepo) as CountRow).count; + if (activeClaimCount >= cap) { + // COMMIT, not ROLLBACK: a claim the sweep just expired is a legitimate transition that must persist even + // though THIS claim is rejected -- rolling back would resurrect a dead process's stale claim. + db.exec("COMMIT"); + return { claimed: false, claim: null, activeClaimCount, maxConcurrentClaims: cap }; + } + const claim = ledger.recordClaim({ repoFullName, issueNumber, note, apiBaseUrl } as RecordClaimInput); + db.exec("COMMIT"); + return { claimed: true, claim, activeClaimCount, maxConcurrentClaims: cap }; + } catch (error) { + db.exec("ROLLBACK"); + throw error; + } + }, + listActiveClaims(repoFullName?: string): ClaimEntry[] { + const filter = { + status: "active" as const, + ...(repoFullName !== undefined ? { repoFullName } : {}), + } satisfies ListClaimsFilter; + return ledger.listClaims(filter); + }, + // Explicit, operator-invoked right-to-be-forgotten purge (#5564) — never runs automatically. Distinct from + // this store's normal claim/release/expire lifecycle: deletes every row for a repo outright. + purgeByRepo(repoFullName: string): number { + return purgeStoreByRepo(db, CLAIM_LEDGER_PURGE_SPEC, normalizeRepoFullName(repoFullName)); + }, + close(): void { + db.close(); + }, + }; + return ledger; +} + +/** + * Strictly read-only ledger access for advisory-only callers (#5157) that must never write anything -- + * not even the schema-creation DDL and schema-version stamp {@link openClaimLedger} always runs on open. + * Opens the DB file in SQLite's own `readonly` mode (driver-enforced: an attempted write throws, this isn't + * just a by-convention guarantee) and touches the filesystem in no other way -- no `mkdirSync`/`chmodSync`, + * no `CREATE TABLE IF NOT EXISTS`, no migrations. The caller MUST only call this against a path it has + * already confirmed exists (e.g. via `existsSync`); a read-only connection to a nonexistent file throws. + * Throws if the expected table is missing too (a file exists at this path but isn't a real claim ledger) -- + * callers should treat that identically to any other open/query failure. + */ +export function openClaimLedgerReadOnly(dbPath: string): ReadOnlyClaimLedger { + const resolvedPath = normalizeDbPath(dbPath); + // `readOnly` (camelCase) -- node:sqlite silently IGNORES `readonly` (lowercase) as an unrecognized option + // and opens read-write anyway, defeating the entire point of this function. Verified empirically: a write + // via a `{ readonly: true }` connection succeeds with no error. + const db = new DatabaseSync(resolvedPath, { readOnly: true }); + let listActiveStatement; + try { + listActiveStatement = db.prepare( + "SELECT * FROM miner_claims WHERE repo_full_name = ? AND status = 'active' ORDER BY id ASC", + ); + } catch (error) { + // The table doesn't exist (a file exists at this path but isn't a real claim ledger) -- close the + // connection we already opened before rethrowing, so this never leaks a file handle. + db.close(); + throw error; + } + return { + dbPath: resolvedPath, + listActiveClaims(repoFullName: string): ClaimEntry[] { + const normalizedRepo = normalizeRepoFullName(repoFullName); + return listActiveStatement.all(normalizedRepo).map((row) => rowToClaim(row as ClaimRow)); + }, + close(): void { + db.close(); + }, + }; +} + +function getDefaultClaimLedger(): ClaimLedger { + defaultClaimLedger ??= openClaimLedger(); + return defaultClaimLedger; +} + +export function recordClaim(claim: RecordClaimInput): ClaimEntry { + return getDefaultClaimLedger().recordClaim(claim); +} + +export function releaseClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null { + return getDefaultClaimLedger().releaseClaim(repoFullName, issueNumber, apiBaseUrl); +} + +export function expireClaim(repoFullName: string, issueNumber: number, apiBaseUrl?: string): ClaimEntry | null { + return getDefaultClaimLedger().expireClaim(repoFullName, issueNumber, apiBaseUrl); +} + +export function listClaims(filter?: ListClaimsFilter): ClaimEntry[] { + return getDefaultClaimLedger().listClaims(filter); +} + +/** Foundation-phase alias for `recordClaim({ repoFullName, issueNumber, note, apiBaseUrl })`. (#3351) */ +export function claimIssue(repoFullName: string, issueNumber: number, note?: string, apiBaseUrl?: string): ClaimEntry { + return getDefaultClaimLedger().claimIssue(repoFullName, issueNumber, note, apiBaseUrl); +} + +/** List only `active` claims, optionally scoped to one repo. (#3351) */ +export function listActiveClaims(repoFullName?: string): ClaimEntry[] { + return getDefaultClaimLedger().listActiveClaims(repoFullName); +} + +export function closeDefaultClaimLedger(): void { + if (!defaultClaimLedger) return; + defaultClaimLedger.close(); + defaultClaimLedger = null; +} diff --git a/packages/loopover-miner/lib/forge-config.d.ts b/packages/loopover-miner/lib/forge-config.d.ts index d8774d18bb..5930f4be63 100644 --- a/packages/loopover-miner/lib/forge-config.d.ts +++ b/packages/loopover-miner/lib/forge-config.d.ts @@ -1,17 +1,27 @@ +/** Per-tenant forge configuration (#4784): the GitHub-specific protocol details that discovery used to hardcode, + * gathered behind one resolver so a non-github.com tenant (GitHub Enterprise, or another GitHub-compatible forge) + * can override them. loopover's own github.com conventions survive only as `DEFAULT_FORGE_CONFIG` — calling + * `resolveForgeConfig()` with no overrides is byte-identical to the pre-#4784 hardcoded fan-out behavior, which is + * what keeps the existing loopover discovery path unchanged. Executes the #4780 repo-agnostic-capability-audit + * checklist (forge abstraction, configurable credential env var, configurable user-agent). */ /** Per-tenant forge configuration (#4784). Every field is a string knob defaulting to the github.com value in * `DEFAULT_FORGE_CONFIG`; a tenant overrides only what differs for their forge. */ export type ForgeConfig = { - apiBaseUrl: string; - apiVersion: string; - apiVersionHeader: string; - acceptHeader: string; - userAgent: string; - repoPathPrefix: string; - searchEndpoint: string; - searchQualifiers: string; - tokenEnvVar: string; + apiBaseUrl: string; + apiVersion: string; + apiVersionHeader: string; + acceptHeader: string; + userAgent: string; + repoPathPrefix: string; + searchEndpoint: string; + searchQualifiers: string; + tokenEnvVar: string; }; - -export const DEFAULT_FORGE_CONFIG: Readonly; - -export function resolveForgeConfig(overrides?: Partial): ForgeConfig; +/** The github.com defaults every forge field falls back to. Frozen so a caller can't mutate the shared baseline. */ +export declare const DEFAULT_FORGE_CONFIG: Readonly; +/** + * Resolve a full forge config from partial per-tenant overrides. Every field is an independent string knob that + * falls back to its github.com default when the override is missing, non-string, or blank — so a partial override + * (say, only `apiBaseUrl` for a GitHub Enterprise host) still yields a complete, usable config. + */ +export declare function resolveForgeConfig(overrides?: Partial): ForgeConfig; diff --git a/packages/loopover-miner/lib/forge-config.js b/packages/loopover-miner/lib/forge-config.js index a772ef9f7f..beb26fe8d4 100644 --- a/packages/loopover-miner/lib/forge-config.js +++ b/packages/loopover-miner/lib/forge-config.js @@ -4,34 +4,32 @@ * `resolveForgeConfig()` with no overrides is byte-identical to the pre-#4784 hardcoded fan-out behavior, which is * what keeps the existing loopover discovery path unchanged. Executes the #4780 repo-agnostic-capability-audit * checklist (forge abstraction, configurable credential env var, configurable user-agent). */ - /** The github.com defaults every forge field falls back to. Frozen so a caller can't mutate the shared baseline. */ export const DEFAULT_FORGE_CONFIG = Object.freeze({ - apiBaseUrl: "https://api.github.com", - apiVersion: "2022-11-28", - apiVersionHeader: "x-github-api-version", - acceptHeader: "application/vnd.github+json", - userAgent: "loopover-miner", - repoPathPrefix: "/repos", - searchEndpoint: "/search/issues", - searchQualifiers: "state:open type:issue", - tokenEnvVar: "GITHUB_TOKEN", + apiBaseUrl: "https://api.github.com", + apiVersion: "2022-11-28", + apiVersionHeader: "x-github-api-version", + acceptHeader: "application/vnd.github+json", + userAgent: "loopover-miner", + repoPathPrefix: "/repos", + searchEndpoint: "/search/issues", + searchQualifiers: "state:open type:issue", + tokenEnvVar: "GITHUB_TOKEN", }); - function trimmedStringOr(value, fallback) { - return typeof value === "string" && value.trim() ? value.trim() : fallback; + return typeof value === "string" && value.trim() ? value.trim() : fallback; } - /** * Resolve a full forge config from partial per-tenant overrides. Every field is an independent string knob that * falls back to its github.com default when the override is missing, non-string, or blank — so a partial override * (say, only `apiBaseUrl` for a GitHub Enterprise host) still yields a complete, usable config. */ export function resolveForgeConfig(overrides = {}) { - const source = overrides && typeof overrides === "object" ? overrides : {}; - const resolved = {}; - for (const [key, fallback] of Object.entries(DEFAULT_FORGE_CONFIG)) { - resolved[key] = trimmedStringOr(source[key], fallback); - } - return resolved; + const source = overrides && typeof overrides === "object" ? overrides : {}; + const resolved = {}; + for (const key of Object.keys(DEFAULT_FORGE_CONFIG)) { + resolved[key] = trimmedStringOr(source[key], DEFAULT_FORGE_CONFIG[key]); + } + return resolved; } +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZm9yZ2UtY29uZmlnLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiZm9yZ2UtY29uZmlnLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOzs7Ozs4RkFLOEY7QUFnQjlGLG9IQUFvSDtBQUNwSCxNQUFNLENBQUMsTUFBTSxvQkFBb0IsR0FBMEIsTUFBTSxDQUFDLE1BQU0sQ0FBQztJQUN2RSxVQUFVLEVBQUUsd0JBQXdCO0lBQ3BDLFVBQVUsRUFBRSxZQUFZO0lBQ3hCLGdCQUFnQixFQUFFLHNCQUFzQjtJQUN4QyxZQUFZLEVBQUUsNkJBQTZCO0lBQzNDLFNBQVMsRUFBRSxnQkFBZ0I7SUFDM0IsY0FBYyxFQUFFLFFBQVE7SUFDeEIsY0FBYyxFQUFFLGdCQUFnQjtJQUNoQyxnQkFBZ0IsRUFBRSx1QkFBdUI7SUFDekMsV0FBVyxFQUFFLGNBQWM7Q0FDNUIsQ0FBQyxDQUFDO0FBRUgsU0FBUyxlQUFlLENBQUMsS0FBYyxFQUFFLFFBQWdCO0lBQ3ZELE9BQU8sT0FBTyxLQUFLLEtBQUssUUFBUSxJQUFJLEtBQUssQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQyxRQUFRLENBQUM7QUFDN0UsQ0FBQztBQUVEOzs7O0dBSUc7QUFDSCxNQUFNLFVBQVUsa0JBQWtCLENBQUMsWUFBa0MsRUFBRTtJQUNyRSxNQUFNLE1BQU0sR0FBRyxTQUFTLElBQUksT0FBTyxTQUFTLEtBQUssUUFBUSxDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQztJQUMzRSxNQUFNLFFBQVEsR0FBRyxFQUFpQixDQUFDO0lBQ25DLEtBQUssTUFBTSxHQUFHLElBQUksTUFBTSxDQUFDLElBQUksQ0FBQyxvQkFBb0IsQ0FBNkIsRUFBRSxDQUFDO1FBQ2hGLFFBQVEsQ0FBQyxHQUFHLENBQUMsR0FBRyxlQUFlLENBQUMsTUFBTSxDQUFDLEdBQUcsQ0FBQyxFQUFFLG9CQUFvQixDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUM7SUFDMUUsQ0FBQztJQUNELE9BQU8sUUFBUSxDQUFDO0FBQ2xCLENBQUMifQ== \ No newline at end of file diff --git a/packages/loopover-miner/lib/forge-config.ts b/packages/loopover-miner/lib/forge-config.ts new file mode 100644 index 0000000000..d02a694c16 --- /dev/null +++ b/packages/loopover-miner/lib/forge-config.ts @@ -0,0 +1,51 @@ +/** Per-tenant forge configuration (#4784): the GitHub-specific protocol details that discovery used to hardcode, + * gathered behind one resolver so a non-github.com tenant (GitHub Enterprise, or another GitHub-compatible forge) + * can override them. loopover's own github.com conventions survive only as `DEFAULT_FORGE_CONFIG` — calling + * `resolveForgeConfig()` with no overrides is byte-identical to the pre-#4784 hardcoded fan-out behavior, which is + * what keeps the existing loopover discovery path unchanged. Executes the #4780 repo-agnostic-capability-audit + * checklist (forge abstraction, configurable credential env var, configurable user-agent). */ + +/** Per-tenant forge configuration (#4784). Every field is a string knob defaulting to the github.com value in + * `DEFAULT_FORGE_CONFIG`; a tenant overrides only what differs for their forge. */ +export type ForgeConfig = { + apiBaseUrl: string; + apiVersion: string; + apiVersionHeader: string; + acceptHeader: string; + userAgent: string; + repoPathPrefix: string; + searchEndpoint: string; + searchQualifiers: string; + tokenEnvVar: string; +}; + +/** The github.com defaults every forge field falls back to. Frozen so a caller can't mutate the shared baseline. */ +export const DEFAULT_FORGE_CONFIG: Readonly = Object.freeze({ + apiBaseUrl: "https://api.github.com", + apiVersion: "2022-11-28", + apiVersionHeader: "x-github-api-version", + acceptHeader: "application/vnd.github+json", + userAgent: "loopover-miner", + repoPathPrefix: "/repos", + searchEndpoint: "/search/issues", + searchQualifiers: "state:open type:issue", + tokenEnvVar: "GITHUB_TOKEN", +}); + +function trimmedStringOr(value: unknown, fallback: string): string { + return typeof value === "string" && value.trim() ? value.trim() : fallback; +} + +/** + * Resolve a full forge config from partial per-tenant overrides. Every field is an independent string knob that + * falls back to its github.com default when the override is missing, non-string, or blank — so a partial override + * (say, only `apiBaseUrl` for a GitHub Enterprise host) still yields a complete, usable config. + */ +export function resolveForgeConfig(overrides: Partial = {}): ForgeConfig { + const source = overrides && typeof overrides === "object" ? overrides : {}; + const resolved = {} as ForgeConfig; + for (const key of Object.keys(DEFAULT_FORGE_CONFIG) as Array) { + resolved[key] = trimmedStringOr(source[key], DEFAULT_FORGE_CONFIG[key]); + } + return resolved; +} diff --git a/packages/loopover-miner/lib/governor-state.d.ts b/packages/loopover-miner/lib/governor-state.d.ts index e8d0ef14f9..576f0e9a55 100644 --- a/packages/loopover-miner/lib/governor-state.d.ts +++ b/packages/loopover-miner/lib/governor-state.d.ts @@ -1,65 +1,48 @@ import type { GovernorCapUsage, OwnSubmissionRecord, RepoOutcomeHistory, WriteRateLimitBackoffStore, WriteRateLimitBucketStore } from "@loopover/engine"; - export type GovernorRateLimitState = { - buckets: WriteRateLimitBucketStore; - backoffAttempts: WriteRateLimitBackoffStore; + buckets: WriteRateLimitBucketStore; + backoffAttempts: WriteRateLimitBackoffStore; }; - export type ListRecentOwnSubmissionsFilter = { - repoFullName?: string; - limit?: number; + repoFullName?: string; + limit?: number; }; - export type GovernorPauseState = { - paused: boolean; - reason: string | null; - pausedAt: string | null; + paused: boolean; + reason: string | null; + pausedAt: string | null; }; - export type GovernorPauseInput = { - paused: boolean; - reason?: string | null; + paused: boolean; + reason?: string | null; }; - export type GovernorState = { - dbPath: string; - loadRateLimitState(): GovernorRateLimitState; - saveRateLimitState(rateLimitState: GovernorRateLimitState): void; - loadCapUsage(): GovernorCapUsage; - saveCapUsage(capUsage: GovernorCapUsage): void; - loadPauseState(): GovernorPauseState; - savePauseState(pauseState: GovernorPauseInput): GovernorPauseState; - loadReputationHistory(repoFullName: string, apiBaseUrl?: string): RepoOutcomeHistory; - saveReputationHistory(repoFullName: string, history: RepoOutcomeHistory, apiBaseUrl?: string): RepoOutcomeHistory; - recordOwnSubmission(record: OwnSubmissionRecord): OwnSubmissionRecord; - listRecentOwnSubmissions(filter?: ListRecentOwnSubmissionsFilter): OwnSubmissionRecord[]; - /** Delete every repo-scoped row for one repo across both governor tables (#7091); returns total rows removed. */ - purgeByRepo(repoFullName: string): number; - close(): void; + dbPath: string; + loadRateLimitState(): GovernorRateLimitState; + saveRateLimitState(rateLimitState: GovernorRateLimitState): void; + loadCapUsage(): GovernorCapUsage; + saveCapUsage(capUsage: GovernorCapUsage): void; + loadPauseState(): GovernorPauseState; + savePauseState(pauseState: GovernorPauseInput): GovernorPauseState; + loadReputationHistory(repoFullName: string, apiBaseUrl?: string): RepoOutcomeHistory; + saveReputationHistory(repoFullName: string, history: RepoOutcomeHistory, apiBaseUrl?: string): RepoOutcomeHistory; + recordOwnSubmission(record: OwnSubmissionRecord): OwnSubmissionRecord; + listRecentOwnSubmissions(filter?: ListRecentOwnSubmissionsFilter): OwnSubmissionRecord[]; + /** Delete every repo-scoped row for one repo across both governor tables (#7091); returns total rows removed. */ + purgeByRepo(repoFullName: string): number; + close(): void; }; - -export function resolveGovernorStateDbPath(env?: Record): string; - -export function openGovernorState(dbPath?: string): GovernorState; - -export function loadRateLimitState(): GovernorRateLimitState; - -export function saveRateLimitState(rateLimitState: GovernorRateLimitState): void; - -export function loadCapUsage(): GovernorCapUsage; - -export function saveCapUsage(capUsage: GovernorCapUsage): void; - -export function loadPauseState(): GovernorPauseState; - -export function savePauseState(pauseState: GovernorPauseInput): GovernorPauseState; - -export function loadReputationHistory(repoFullName: string, apiBaseUrl?: string): RepoOutcomeHistory; - -export function saveReputationHistory(repoFullName: string, history: RepoOutcomeHistory, apiBaseUrl?: string): RepoOutcomeHistory; - -export function recordOwnSubmission(record: OwnSubmissionRecord): OwnSubmissionRecord; - -export function listRecentOwnSubmissions(filter?: ListRecentOwnSubmissionsFilter): OwnSubmissionRecord[]; - -export function closeDefaultGovernorState(): void; +export declare function resolveGovernorStateDbPath(env?: Record): string; +/** Opens the local governor-state store, creating tables on first use. */ +export declare function openGovernorState(dbPath?: string): GovernorState; +export declare function loadRateLimitState(): GovernorRateLimitState; +export declare function saveRateLimitState(rateLimitState: GovernorRateLimitState): void; +export declare function loadCapUsage(): GovernorCapUsage; +export declare function saveCapUsage(capUsage: GovernorCapUsage): void; +export declare function loadPauseState(): GovernorPauseState; +export declare function savePauseState(pauseState: GovernorPauseInput): GovernorPauseState; +export declare function loadReputationHistory(repoFullName: string, apiBaseUrl?: string): RepoOutcomeHistory; +export declare function saveReputationHistory(repoFullName: string, history: RepoOutcomeHistory, apiBaseUrl?: string): RepoOutcomeHistory; +export declare function recordOwnSubmission(record: OwnSubmissionRecord): OwnSubmissionRecord; +export declare function listRecentOwnSubmissions(filter?: ListRecentOwnSubmissionsFilter): OwnSubmissionRecord[]; +export declare function closeDefaultGovernorState(): void; diff --git a/packages/loopover-miner/lib/governor-state.js b/packages/loopover-miner/lib/governor-state.js index 4755b43ccc..997ff1fcf2 100644 --- a/packages/loopover-miner/lib/governor-state.js +++ b/packages/loopover-miner/lib/governor-state.js @@ -1,91 +1,66 @@ import { DEFAULT_FORGE_CONFIG } from "./forge-config.js"; import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js"; -import { - GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC, - GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC, - purgeStoreByRepo, -} from "./store-maintenance.js"; - -// Governor cross-attempt state persistence (#5134, Wave 3.5). Every governor-*.js wrapper -// (governor-chokepoint.js) is a pure in/out transform: it computes and RETURNS -// updated rate-limit buckets/backoff attempts, but nothing writes them to disk, so they reset to zero on -// every process start -- the mutable counters that should gate the NEXT decision never survive past one -// process. governor-ledger.js already persists the DECISION HISTORY (an append-only audit log); this module -// persists the DECISION INPUT state instead -- a second, distinct concern, not a duplicate of that log (see -// its own module doc for the ledger/state split this issue's acceptance criteria requires). -// -// This module does not alter evaluateGovernorChokepoint's precedence ladder or any pure calculator's logic -- -// it only gives their existing, already-optional input fields (rateLimitBuckets, rateLimitBackoffAttempts, -// capUsage, reputationHistory, recentOwnSubmissions) a real load-at-start/save-at-end home. Convergence input -// (packages/loopover-engine/src/portfolio/non-convergence.ts's PortfolioConvergenceInput) is NOT persisted -// here: that module's own doc comment says its counters belong on the portfolio-queue table (a pre-existing -// store this issue's boundaries don't touch) once that table grows attempt-history columns -- inventing a -// second, competing store for the same concept here would violate the same non-duplication principle the -// ledger/state split above is built on. - +import { GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC, GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC, purgeStoreByRepo, } from "./store-maintenance.js"; const defaultDbFileName = "governor-state.sqlite3"; const DEFAULT_RATE_LIMIT_BUCKETS = Object.freeze({ global: {}, perRepo: {} }); const DEFAULT_RATE_LIMIT_BACKOFF = Object.freeze({}); const DEFAULT_CAP_USAGE = Object.freeze({ budgetSpent: 0, turnsTaken: 0, elapsedMs: 0 }); const DEFAULT_REPUTATION_HISTORY = Object.freeze({ decided: 0, unfavorable: 0 }); let defaultGovernorState = null; - export function resolveGovernorStateDbPath(env = process.env) { - return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_GOVERNOR_STATE_DB", env); + return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_GOVERNOR_STATE_DB", env); } - function normalizeDbPath(dbPath) { - return normalizeLocalStoreDbPath(dbPath, resolveGovernorStateDbPath(), "invalid_governor_state_db_path"); + return normalizeLocalStoreDbPath(dbPath, resolveGovernorStateDbPath(), "invalid_governor_state_db_path"); } - function normalizeRepoFullName(repoFullName) { - if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name"); - const [owner, repo, extra] = repoFullName.trim().split("/"); - if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name"); - return `${owner}/${repo}`; + if (typeof repoFullName !== "string") + throw new Error("invalid_repo_full_name"); + const [owner, repo, extra] = repoFullName.trim().split("/"); + if (!owner || !repo || extra !== undefined) + throw new Error("invalid_repo_full_name"); + return `${owner}/${repo}`; } - /** Optional forge host, scoping rows so two hosts serving the same owner/repo name never collide (#5563). * Omitted/nullish → the github.com default, so every pre-existing single-forge caller is unaffected. */ function normalizeApiBaseUrl(apiBaseUrl) { - if (apiBaseUrl === undefined || apiBaseUrl === null) return DEFAULT_FORGE_CONFIG.apiBaseUrl; - if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) throw new Error("invalid_api_base_url"); - return apiBaseUrl.trim(); + if (apiBaseUrl === undefined || apiBaseUrl === null) + return DEFAULT_FORGE_CONFIG.apiBaseUrl; + if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) + throw new Error("invalid_api_base_url"); + return apiBaseUrl.trim(); } - function parseJsonColumn(value, fallback) { - if (typeof value !== "string") return fallback; - try { - const parsed = JSON.parse(value); - return parsed && typeof parsed === "object" ? parsed : fallback; - } catch { - return fallback; - } + if (typeof value !== "string") + return fallback; + try { + const parsed = JSON.parse(value); + return parsed && typeof parsed === "object" ? parsed : fallback; + } + catch { + return fallback; + } } - // Add the pause/resume columns (#4851) to an on-disk file created before they existed. `CREATE TABLE IF NOT // EXISTS` above is a no-op against an already-existing table, so a pre-#4851 file needs this explicit ALTER -- // guarded by a per-column presence check (rather than a single `paused`-only check) so a file that somehow // has `paused` but not `pause_reason`/`paused_at` still gets the columns it's missing, same technique as // portfolio-queue.js's own post-creation column migration. function ensurePauseColumns(db) { - const existingColumns = new Set( - db - .prepare("PRAGMA table_info(governor_scalar_state)") - .all() - .map((column) => column.name), - ); - if (!existingColumns.has("paused")) { - db.exec("ALTER TABLE governor_scalar_state ADD COLUMN paused INTEGER NOT NULL DEFAULT 0"); - } - if (!existingColumns.has("pause_reason")) { - db.exec("ALTER TABLE governor_scalar_state ADD COLUMN pause_reason TEXT"); - } - if (!existingColumns.has("paused_at")) { - db.exec("ALTER TABLE governor_scalar_state ADD COLUMN paused_at TEXT"); - } + const existingColumns = new Set(db + .prepare("PRAGMA table_info(governor_scalar_state)") + .all() + .map((column) => column.name)); + if (!existingColumns.has("paused")) { + db.exec("ALTER TABLE governor_scalar_state ADD COLUMN paused INTEGER NOT NULL DEFAULT 0"); + } + if (!existingColumns.has("pause_reason")) { + db.exec("ALTER TABLE governor_scalar_state ADD COLUMN pause_reason TEXT"); + } + if (!existingColumns.has("paused_at")) { + db.exec("ALTER TABLE governor_scalar_state ADD COLUMN paused_at TEXT"); + } } - // Rebuild governor_reputation_history's bare `repo_full_name` PRIMARY KEY into a (api_base_url, repo_full_name) // composite (#5563) -- two forge hosts serving a same-named owner/repo must not share one reputation row. // SQLite cannot ALTER a PRIMARY KEY in place, so this rebuilds the table: create the new shape, copy every @@ -93,12 +68,13 @@ function ensurePauseColumns(db) { // one in. Guarded by a column-presence check (matching ensurePauseColumns' idempotence) so this only runs once // per file, same technique as portfolio-queue.js's own post-creation migration. function ensureReputationHistoryForgeScope(db) { - const hasApiBaseUrlColumn = db - .prepare("PRAGMA table_info(governor_reputation_history)") - .all() - .some((column) => column.name === "api_base_url"); - if (hasApiBaseUrlColumn) return; - db.exec(` + const hasApiBaseUrlColumn = db + .prepare("PRAGMA table_info(governor_reputation_history)") + .all() + .some((column) => column.name === "api_base_url"); + if (hasApiBaseUrlColumn) + return; + db.exec(` CREATE TABLE governor_reputation_history_v2 ( api_base_url TEXT NOT NULL, repo_full_name TEXT NOT NULL, @@ -108,27 +84,23 @@ function ensureReputationHistoryForgeScope(db) { PRIMARY KEY (api_base_url, repo_full_name) ) `); - // OR IGNORE: a source row that somehow violates the rebuilt table's NOT NULL columns (a hand-edited or - // otherwise corrupted file) is skipped rather than aborting the whole migration -- same fail-closed posture - // as run-state.js's own #5563 migration. - db.prepare( - `INSERT OR IGNORE INTO governor_reputation_history_v2 (api_base_url, repo_full_name, decided, unfavorable, updated_at) - SELECT ?, repo_full_name, decided, unfavorable, updated_at FROM governor_reputation_history`, - ).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); - db.exec("DROP TABLE governor_reputation_history"); - db.exec("ALTER TABLE governor_reputation_history_v2 RENAME TO governor_reputation_history"); + // OR IGNORE: a source row that somehow violates the rebuilt table's NOT NULL columns (a hand-edited or + // otherwise corrupted file) is skipped rather than aborting the whole migration -- same fail-closed posture + // as run-state.js's own #5563 migration. + db.prepare(`INSERT OR IGNORE INTO governor_reputation_history_v2 (api_base_url, repo_full_name, decided, unfavorable, updated_at) + SELECT ?, repo_full_name, decided, unfavorable, updated_at FROM governor_reputation_history`).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); + db.exec("DROP TABLE governor_reputation_history"); + db.exec("ALTER TABLE governor_reputation_history_v2 RENAME TO governor_reputation_history"); } - /** Opens the local governor-state store, creating tables on first use. */ export function openGovernorState(dbPath = resolveGovernorStateDbPath()) { - const resolvedPath = normalizeDbPath(dbPath); - const db = openLocalStoreDb(resolvedPath); - - // ONE row (id=1) holding the whole-run scalar state: rate-limit buckets/backoff and budget/turn/termination - // usage have no natural per-repo key of their own beyond what's already encoded inside the JSON blob - // (WriteRateLimitBucketStore.perRepo is itself keyed by `${actionClass}:${repoFullName}`), so a single - // UPSERTed row is simpler and more honest than inventing a relational key that doesn't exist upstream. - db.exec(` + const resolvedPath = normalizeDbPath(dbPath); + const db = openLocalStoreDb(resolvedPath); + // ONE row (id=1) holding the whole-run scalar state: rate-limit buckets/backoff and budget/turn/termination + // usage have no natural per-repo key of their own beyond what's already encoded inside the JSON blob + // (WriteRateLimitBucketStore.perRepo is itself keyed by `${actionClass}:${repoFullName}`), so a single + // UPSERTed row is simpler and more honest than inventing a relational key that doesn't exist upstream. + db.exec(` CREATE TABLE IF NOT EXISTS governor_scalar_state ( id INTEGER PRIMARY KEY CHECK (id = 1), rate_limit_buckets_json TEXT NOT NULL, @@ -140,8 +112,8 @@ export function openGovernorState(dbPath = resolveGovernorStateDbPath()) { updated_at TEXT NOT NULL ) `); - ensurePauseColumns(db); - db.exec(` + ensurePauseColumns(db); + db.exec(` CREATE TABLE IF NOT EXISTS governor_reputation_history ( repo_full_name TEXT PRIMARY KEY, decided INTEGER NOT NULL, @@ -149,8 +121,8 @@ export function openGovernorState(dbPath = resolveGovernorStateDbPath()) { updated_at TEXT NOT NULL ) `); - ensureReputationHistoryForgeScope(db); - db.exec(` + ensureReputationHistoryForgeScope(db); + db.exec(` CREATE TABLE IF NOT EXISTS governor_own_submissions ( id INTEGER PRIMARY KEY AUTOINCREMENT, repo_full_name TEXT NOT NULL, @@ -160,10 +132,9 @@ export function openGovernorState(dbPath = resolveGovernorStateDbPath()) { issue_number INTEGER ) `); - db.exec("CREATE INDEX IF NOT EXISTS idx_governor_own_submissions_repo ON governor_own_submissions (repo_full_name, id)"); - - const getScalarStatement = db.prepare("SELECT * FROM governor_scalar_state WHERE id = 1"); - const upsertScalarStatement = db.prepare(` + db.exec("CREATE INDEX IF NOT EXISTS idx_governor_own_submissions_repo ON governor_own_submissions (repo_full_name, id)"); + const getScalarStatement = db.prepare("SELECT * FROM governor_scalar_state WHERE id = 1"); + const upsertScalarStatement = db.prepare(` INSERT INTO governor_scalar_state (id, rate_limit_buckets_json, rate_limit_backoff_json, cap_usage_json, paused, pause_reason, paused_at, updated_at) VALUES (1, ?, ?, ?, ?, ?, ?, ?) @@ -176,10 +147,8 @@ export function openGovernorState(dbPath = resolveGovernorStateDbPath()) { paused_at = excluded.paused_at, updated_at = excluded.updated_at `); - const getReputationStatement = db.prepare( - "SELECT * FROM governor_reputation_history WHERE api_base_url = ? AND repo_full_name = ?", - ); - const upsertReputationStatement = db.prepare(` + const getReputationStatement = db.prepare("SELECT * FROM governor_reputation_history WHERE api_base_url = ? AND repo_full_name = ?"); + const upsertReputationStatement = db.prepare(` INSERT INTO governor_reputation_history (api_base_url, repo_full_name, decided, unfavorable, updated_at) VALUES (?, ?, ?, ?, ?) ON CONFLICT(api_base_url, repo_full_name) DO UPDATE SET @@ -187,222 +156,176 @@ export function openGovernorState(dbPath = resolveGovernorStateDbPath()) { unfavorable = excluded.unfavorable, updated_at = excluded.updated_at `); - const insertSubmissionStatement = db.prepare(` + const insertSubmissionStatement = db.prepare(` INSERT INTO governor_own_submissions (repo_full_name, fingerprint, submitted_at, pull_request_number, issue_number) VALUES (?, ?, ?, ?, ?) `); - const listSubmissionsAllStatement = db.prepare( - "SELECT * FROM governor_own_submissions ORDER BY id DESC LIMIT ?", - ); - const listSubmissionsByRepoStatement = db.prepare( - "SELECT * FROM governor_own_submissions WHERE repo_full_name = ? ORDER BY id DESC LIMIT ?", - ); - - function rowToSubmission(row) { - return { - repoFullName: row.repo_full_name, - fingerprint: row.fingerprint, - submittedAt: row.submitted_at, - pullRequestNumber: row.pull_request_number, - issueNumber: row.issue_number, - }; - } - - // BEGIN IMMEDIATE takes the write lock BEFORE `fn`'s read, so two processes on the same file (the loop daemon - // saving rate-limit/cap-usage state on every gated write, and an operator's `governor pause`/`resume` CLI - // invocation racing it) cannot interleave a stale read with each other's write and silently clobber the - // scalar-state column-group they don't own -- same fix shape as event-ledger.js's appendEvent (#7221). Shared - // by all three governor_scalar_state save methods below, since they all read-then-write across the same row. - function withTransaction(fn) { - db.exec("BEGIN IMMEDIATE"); - try { - const result = fn(); - db.exec("COMMIT"); - return result; - } catch (error) { - db.exec("ROLLBACK"); - throw error; + const listSubmissionsAllStatement = db.prepare("SELECT * FROM governor_own_submissions ORDER BY id DESC LIMIT ?"); + const listSubmissionsByRepoStatement = db.prepare("SELECT * FROM governor_own_submissions WHERE repo_full_name = ? ORDER BY id DESC LIMIT ?"); + function rowToSubmission(row) { + return { + repoFullName: row.repo_full_name, + fingerprint: row.fingerprint, + submittedAt: row.submitted_at, + pullRequestNumber: row.pull_request_number, + issueNumber: row.issue_number, + }; } - } - - const state = { - dbPath: resolvedPath, - loadRateLimitState() { - const row = getScalarStatement.get(); - return { - buckets: parseJsonColumn(row?.rate_limit_buckets_json, DEFAULT_RATE_LIMIT_BUCKETS), - backoffAttempts: parseJsonColumn(row?.rate_limit_backoff_json, DEFAULT_RATE_LIMIT_BACKOFF), - }; - }, - saveRateLimitState(rateLimitState) { - withTransaction(() => { - const row = getScalarStatement.get(); - upsertScalarStatement.run( - JSON.stringify(rateLimitState?.buckets ?? DEFAULT_RATE_LIMIT_BUCKETS), - JSON.stringify(rateLimitState?.backoffAttempts ?? DEFAULT_RATE_LIMIT_BACKOFF), - row ? row.cap_usage_json : JSON.stringify(DEFAULT_CAP_USAGE), - row ? row.paused : 0, - row ? row.pause_reason : null, - row ? row.paused_at : null, - new Date().toISOString(), - ); - }); - }, - loadCapUsage() { - const row = getScalarStatement.get(); - return parseJsonColumn(row?.cap_usage_json, DEFAULT_CAP_USAGE); - }, - saveCapUsage(capUsage) { - withTransaction(() => { - const row = getScalarStatement.get(); - upsertScalarStatement.run( - row ? row.rate_limit_buckets_json : JSON.stringify(DEFAULT_RATE_LIMIT_BUCKETS), - row ? row.rate_limit_backoff_json : JSON.stringify(DEFAULT_RATE_LIMIT_BACKOFF), - JSON.stringify(capUsage ?? DEFAULT_CAP_USAGE), - row ? row.paused : 0, - row ? row.pause_reason : null, - row ? row.paused_at : null, - new Date().toISOString(), - ); - }); - }, - // The governor pause/resume control surface (#4851): a real, persisted, operator/governor-writable flag the - // loop checks before each cycle -- distinct from governor-kill-switch.js (a read-only resolver over env/YAML - // inputs the miner does not itself write) and governor-run-halt.js (a one-way, run-scoped terminal breaker). - // `pausedAt` is stamped fresh on every transition INTO paused, and cleared on resume, so a status query can - // report how long a pause has been in effect without needing a separate history table. - loadPauseState() { - const row = getScalarStatement.get(); - return { - paused: row ? Boolean(row.paused) : false, - reason: row?.pause_reason ?? null, - pausedAt: row?.paused_at ?? null, - }; - }, - savePauseState(pauseState) { - const paused = Boolean(pauseState?.paused); - const reason = - typeof pauseState?.reason === "string" && pauseState.reason.trim() ? pauseState.reason.trim() : null; - const pausedAt = paused ? new Date().toISOString() : null; - withTransaction(() => { - const row = getScalarStatement.get(); - upsertScalarStatement.run( - row ? row.rate_limit_buckets_json : JSON.stringify(DEFAULT_RATE_LIMIT_BUCKETS), - row ? row.rate_limit_backoff_json : JSON.stringify(DEFAULT_RATE_LIMIT_BACKOFF), - row ? row.cap_usage_json : JSON.stringify(DEFAULT_CAP_USAGE), - paused ? 1 : 0, - reason, - pausedAt, - new Date().toISOString(), - ); - }); - return { paused, reason, pausedAt }; - }, - loadReputationHistory(repoFullName, apiBaseUrl) { - const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); - const normalizedRepo = normalizeRepoFullName(repoFullName); - const row = getReputationStatement.get(normalizedForge, normalizedRepo); - if (!row) return { ...DEFAULT_REPUTATION_HISTORY }; - return { decided: row.decided, unfavorable: row.unfavorable }; - }, - saveReputationHistory(repoFullName, history, apiBaseUrl) { - const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); - const normalizedRepo = normalizeRepoFullName(repoFullName); - const decided = Number.isInteger(history?.decided) ? history.decided : 0; - const unfavorable = Number.isInteger(history?.unfavorable) ? history.unfavorable : 0; - upsertReputationStatement.run(normalizedForge, normalizedRepo, decided, unfavorable, new Date().toISOString()); - return { decided, unfavorable }; - }, - recordOwnSubmission(record) { - const normalized = normalizeRepoFullName(record?.repoFullName); - if (typeof record?.fingerprint !== "string" || !record.fingerprint.trim()) { - throw new Error("invalid_fingerprint"); - } - const submittedAt = typeof record.submittedAt === "string" ? record.submittedAt : new Date().toISOString(); - const pullRequestNumber = Number.isInteger(record.pullRequestNumber) ? record.pullRequestNumber : null; - const issueNumber = Number.isInteger(record.issueNumber) ? record.issueNumber : null; - insertSubmissionStatement.run(normalized, record.fingerprint, submittedAt, pullRequestNumber, issueNumber); - return { repoFullName: normalized, fingerprint: record.fingerprint, submittedAt, pullRequestNumber, issueNumber }; - }, - listRecentOwnSubmissions(filter = {}) { - const limit = Number.isInteger(filter.limit) && filter.limit > 0 ? filter.limit : 200; - const rows = - filter.repoFullName === undefined - ? listSubmissionsAllStatement.all(limit) - : listSubmissionsByRepoStatement.all(normalizeRepoFullName(filter.repoFullName), limit); - return rows.map(rowToSubmission); - }, - /** - * Delete every repo-scoped row for one repo across BOTH governor tables against this single open handle - * (#7091) — the right-to-be-forgotten path `loopover-miner purge` invokes. `governor_reputation_history` is - * purged on `repo_full_name` alone (its key is composite with `api_base_url`), so nothing survives on any - * forge host. `governor_scalar_state` is deliberately untouched — it has no repo dimension. Returns the - * total rows removed across both tables. - * - * @param {string} repoFullName - * @returns {number} rows deleted across both repo-scoped tables - */ - purgeByRepo(repoFullName) { - const normalized = normalizeRepoFullName(repoFullName); - return ( - purgeStoreByRepo(db, GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC, normalized) + - purgeStoreByRepo(db, GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC, normalized) - ); - }, - close() { - db.close(); - }, - }; - return state; + // BEGIN IMMEDIATE takes the write lock BEFORE `fn`'s read, so two processes on the same file (the loop daemon + // saving rate-limit/cap-usage state on every gated write, and an operator's `governor pause`/`resume` CLI + // invocation racing it) cannot interleave a stale read with each other's write and silently clobber the + // scalar-state column-group they don't own -- same fix shape as event-ledger.js's appendEvent (#7221). Shared + // by all three governor_scalar_state save methods below, since they all read-then-write across the same row. + function withTransaction(fn) { + db.exec("BEGIN IMMEDIATE"); + try { + const result = fn(); + db.exec("COMMIT"); + return result; + } + catch (error) { + db.exec("ROLLBACK"); + throw error; + } + } + const state = { + dbPath: resolvedPath, + loadRateLimitState() { + const row = getScalarStatement.get(); + return { + buckets: parseJsonColumn(row?.rate_limit_buckets_json, DEFAULT_RATE_LIMIT_BUCKETS), + backoffAttempts: parseJsonColumn(row?.rate_limit_backoff_json, DEFAULT_RATE_LIMIT_BACKOFF), + }; + }, + saveRateLimitState(rateLimitState) { + withTransaction(() => { + const row = getScalarStatement.get(); + upsertScalarStatement.run(JSON.stringify(rateLimitState?.buckets ?? DEFAULT_RATE_LIMIT_BUCKETS), JSON.stringify(rateLimitState?.backoffAttempts ?? DEFAULT_RATE_LIMIT_BACKOFF), row ? row.cap_usage_json : JSON.stringify(DEFAULT_CAP_USAGE), row ? row.paused : 0, row ? row.pause_reason : null, row ? row.paused_at : null, new Date().toISOString()); + }); + }, + loadCapUsage() { + const row = getScalarStatement.get(); + return parseJsonColumn(row?.cap_usage_json, DEFAULT_CAP_USAGE); + }, + saveCapUsage(capUsage) { + withTransaction(() => { + const row = getScalarStatement.get(); + upsertScalarStatement.run(row ? row.rate_limit_buckets_json : JSON.stringify(DEFAULT_RATE_LIMIT_BUCKETS), row ? row.rate_limit_backoff_json : JSON.stringify(DEFAULT_RATE_LIMIT_BACKOFF), JSON.stringify(capUsage ?? DEFAULT_CAP_USAGE), row ? row.paused : 0, row ? row.pause_reason : null, row ? row.paused_at : null, new Date().toISOString()); + }); + }, + // The governor pause/resume control surface (#4851): a real, persisted, operator/governor-writable flag the + // loop checks before each cycle -- distinct from governor-kill-switch.js (a read-only resolver over env/YAML + // inputs the miner does not itself write) and governor-run-halt.js (a one-way, run-scoped terminal breaker). + // `pausedAt` is stamped fresh on every transition INTO paused, and cleared on resume, so a status query can + // report how long a pause has been in effect without needing a separate history table. + loadPauseState() { + const row = getScalarStatement.get(); + return { + paused: row ? Boolean(row.paused) : false, + reason: row?.pause_reason ?? null, + pausedAt: row?.paused_at ?? null, + }; + }, + savePauseState(pauseState) { + const paused = Boolean(pauseState?.paused); + const reason = typeof pauseState?.reason === "string" && pauseState.reason.trim() ? pauseState.reason.trim() : null; + const pausedAt = paused ? new Date().toISOString() : null; + withTransaction(() => { + const row = getScalarStatement.get(); + upsertScalarStatement.run(row ? row.rate_limit_buckets_json : JSON.stringify(DEFAULT_RATE_LIMIT_BUCKETS), row ? row.rate_limit_backoff_json : JSON.stringify(DEFAULT_RATE_LIMIT_BACKOFF), row ? row.cap_usage_json : JSON.stringify(DEFAULT_CAP_USAGE), paused ? 1 : 0, reason, pausedAt, new Date().toISOString()); + }); + return { paused, reason, pausedAt }; + }, + loadReputationHistory(repoFullName, apiBaseUrl) { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const row = getReputationStatement.get(normalizedForge, normalizedRepo); + if (!row) + return { ...DEFAULT_REPUTATION_HISTORY }; + return { decided: row.decided, unfavorable: row.unfavorable }; + }, + saveReputationHistory(repoFullName, history, apiBaseUrl) { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const decided = Number.isInteger(history?.decided) ? history.decided : 0; + const unfavorable = Number.isInteger(history?.unfavorable) ? history.unfavorable : 0; + upsertReputationStatement.run(normalizedForge, normalizedRepo, decided, unfavorable, new Date().toISOString()); + return { decided, unfavorable }; + }, + recordOwnSubmission(record) { + const normalized = normalizeRepoFullName(record?.repoFullName); + if (typeof record?.fingerprint !== "string" || !record.fingerprint.trim()) { + throw new Error("invalid_fingerprint"); + } + const submittedAt = typeof record.submittedAt === "string" ? record.submittedAt : new Date().toISOString(); + const pullRequestNumber = Number.isInteger(record.pullRequestNumber) ? record.pullRequestNumber : null; + const issueNumber = Number.isInteger(record.issueNumber) ? record.issueNumber : null; + insertSubmissionStatement.run(normalized, record.fingerprint, submittedAt, pullRequestNumber, issueNumber); + return { repoFullName: normalized, fingerprint: record.fingerprint, submittedAt, pullRequestNumber, issueNumber }; + }, + listRecentOwnSubmissions(filter = {}) { + const limit = Number.isInteger(filter.limit) && filter.limit > 0 ? filter.limit : 200; + const rows = filter.repoFullName === undefined + ? listSubmissionsAllStatement.all(limit) + : listSubmissionsByRepoStatement.all(normalizeRepoFullName(filter.repoFullName), limit); + return rows.map((row) => rowToSubmission(row)); + }, + /** + * Delete every repo-scoped row for one repo across BOTH governor tables against this single open handle + * (#7091) — the right-to-be-forgotten path `loopover-miner purge` invokes. `governor_reputation_history` is + * purged on `repo_full_name` alone (its key is composite with `api_base_url`), so nothing survives on any + * forge host. `governor_scalar_state` is deliberately untouched — it has no repo dimension. Returns the + * total rows removed across both tables. + */ + purgeByRepo(repoFullName) { + const normalized = normalizeRepoFullName(repoFullName); + return (purgeStoreByRepo(db, GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC, normalized) + + purgeStoreByRepo(db, GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC, normalized)); + }, + close() { + db.close(); + }, + }; + return state; } - function getDefaultGovernorState() { - defaultGovernorState ??= openGovernorState(); - return defaultGovernorState; + defaultGovernorState ??= openGovernorState(); + return defaultGovernorState; } - export function loadRateLimitState() { - return getDefaultGovernorState().loadRateLimitState(); + return getDefaultGovernorState().loadRateLimitState(); } - export function saveRateLimitState(rateLimitState) { - return getDefaultGovernorState().saveRateLimitState(rateLimitState); + return getDefaultGovernorState().saveRateLimitState(rateLimitState); } - export function loadCapUsage() { - return getDefaultGovernorState().loadCapUsage(); + return getDefaultGovernorState().loadCapUsage(); } - export function saveCapUsage(capUsage) { - return getDefaultGovernorState().saveCapUsage(capUsage); + return getDefaultGovernorState().saveCapUsage(capUsage); } - export function loadPauseState() { - return getDefaultGovernorState().loadPauseState(); + return getDefaultGovernorState().loadPauseState(); } - export function savePauseState(pauseState) { - return getDefaultGovernorState().savePauseState(pauseState); + return getDefaultGovernorState().savePauseState(pauseState); } - export function loadReputationHistory(repoFullName, apiBaseUrl) { - return getDefaultGovernorState().loadReputationHistory(repoFullName, apiBaseUrl); + return getDefaultGovernorState().loadReputationHistory(repoFullName, apiBaseUrl); } - export function saveReputationHistory(repoFullName, history, apiBaseUrl) { - return getDefaultGovernorState().saveReputationHistory(repoFullName, history, apiBaseUrl); + return getDefaultGovernorState().saveReputationHistory(repoFullName, history, apiBaseUrl); } - export function recordOwnSubmission(record) { - return getDefaultGovernorState().recordOwnSubmission(record); + return getDefaultGovernorState().recordOwnSubmission(record); } - export function listRecentOwnSubmissions(filter) { - return getDefaultGovernorState().listRecentOwnSubmissions(filter); + return getDefaultGovernorState().listRecentOwnSubmissions(filter); } - export function closeDefaultGovernorState() { - if (!defaultGovernorState) return; - defaultGovernorState.close(); - defaultGovernorState = null; + if (!defaultGovernorState) + return; + defaultGovernorState.close(); + defaultGovernorState = null; } +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ292ZXJub3Itc3RhdGUuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJnb3Zlcm5vci1zdGF0ZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFRQSxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUN6RCxPQUFPLEVBQUUseUJBQXlCLEVBQUUsZ0JBQWdCLEVBQUUsdUJBQXVCLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUN4RyxPQUFPLEVBQ0wsbUNBQW1DLEVBQ25DLHNDQUFzQyxFQUN0QyxnQkFBZ0IsR0FDakIsTUFBTSx3QkFBd0IsQ0FBQztBQXdGaEMsTUFBTSxpQkFBaUIsR0FBRyx3QkFBd0IsQ0FBQztBQUNuRCxNQUFNLDBCQUEwQixHQUF3QyxNQUFNLENBQUMsTUFBTSxDQUFDLEVBQUUsTUFBTSxFQUFFLEVBQUUsRUFBRSxPQUFPLEVBQUUsRUFBRSxFQUFFLENBQUMsQ0FBQztBQUNuSCxNQUFNLDBCQUEwQixHQUF5QyxNQUFNLENBQUMsTUFBTSxDQUFDLEVBQUUsQ0FBQyxDQUFDO0FBQzNGLE1BQU0saUJBQWlCLEdBQStCLE1BQU0sQ0FBQyxNQUFNLENBQUMsRUFBRSxXQUFXLEVBQUUsQ0FBQyxFQUFFLFVBQVUsRUFBRSxDQUFDLEVBQUUsU0FBUyxFQUFFLENBQUMsRUFBRSxDQUFDLENBQUM7QUFDckgsTUFBTSwwQkFBMEIsR0FBaUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxFQUFFLE9BQU8sRUFBRSxDQUFDLEVBQUUsV0FBVyxFQUFFLENBQUMsRUFBRSxDQUFDLENBQUM7QUFDL0csSUFBSSxvQkFBb0IsR0FBeUIsSUFBSSxDQUFDO0FBRXRELE1BQU0sVUFBVSwwQkFBMEIsQ0FBQyxNQUEwQyxPQUFPLENBQUMsR0FBRztJQUM5RixPQUFPLHVCQUF1QixDQUFDLGlCQUFpQixFQUFFLGtDQUFrQyxFQUFFLEdBQUcsQ0FBQyxDQUFDO0FBQzdGLENBQUM7QUFFRCxTQUFTLGVBQWUsQ0FBQyxNQUFpQztJQUN4RCxPQUFPLHlCQUF5QixDQUFDLE1BQU0sRUFBRSwwQkFBMEIsRUFBRSxFQUFFLGdDQUFnQyxDQUFDLENBQUM7QUFDM0csQ0FBQztBQUVELFNBQVMscUJBQXFCLENBQUMsWUFBcUI7SUFDbEQsSUFBSSxPQUFPLFlBQVksS0FBSyxRQUFRO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyx3QkFBd0IsQ0FBQyxDQUFDO0lBQ2hGLE1BQU0sQ0FBQyxLQUFLLEVBQUUsSUFBSSxFQUFFLEtBQUssQ0FBQyxHQUFHLFlBQVksQ0FBQyxJQUFJLEVBQUUsQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDNUQsSUFBSSxDQUFDLEtBQUssSUFBSSxDQUFDLElBQUksSUFBSSxLQUFLLEtBQUssU0FBUztRQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsd0JBQXdCLENBQUMsQ0FBQztJQUN0RixPQUFPLEdBQUcsS0FBSyxJQUFJLElBQUksRUFBRSxDQUFDO0FBQzVCLENBQUM7QUFFRDt5R0FDeUc7QUFDekcsU0FBUyxtQkFBbUIsQ0FBQyxVQUFtQjtJQUM5QyxJQUFJLFVBQVUsS0FBSyxTQUFTLElBQUksVUFBVSxLQUFLLElBQUk7UUFBRSxPQUFPLG9CQUFvQixDQUFDLFVBQVUsQ0FBQztJQUM1RixJQUFJLE9BQU8sVUFBVSxLQUFLLFFBQVEsSUFBSSxDQUFDLFVBQVUsQ0FBQyxJQUFJLEVBQUU7UUFBRSxNQUFNLElBQUksS0FBSyxDQUFDLHNCQUFzQixDQUFDLENBQUM7SUFDbEcsT0FBTyxVQUFVLENBQUMsSUFBSSxFQUFFLENBQUM7QUFDM0IsQ0FBQztBQUVELFNBQVMsZUFBZSxDQUFtQixLQUFjLEVBQUUsUUFBVztJQUNwRSxJQUFJLE9BQU8sS0FBSyxLQUFLLFFBQVE7UUFBRSxPQUFPLFFBQVEsQ0FBQztJQUMvQyxJQUFJLENBQUM7UUFDSCxNQUFNLE1BQU0sR0FBWSxJQUFJLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxDQUFDO1FBQzFDLE9BQU8sTUFBTSxJQUFJLE9BQU8sTUFBTSxLQUFLLFFBQVEsQ0FBQyxDQUFDLENBQUUsTUFBWSxDQUFDLENBQUMsQ0FBQyxRQUFRLENBQUM7SUFDekUsQ0FBQztJQUFDLE1BQU0sQ0FBQztRQUNQLE9BQU8sUUFBUSxDQUFDO0lBQ2xCLENBQUM7QUFDSCxDQUFDO0FBRUQsNEdBQTRHO0FBQzVHLCtHQUErRztBQUMvRywyR0FBMkc7QUFDM0cseUdBQXlHO0FBQ3pHLDJEQUEyRDtBQUMzRCxTQUFTLGtCQUFrQixDQUFDLEVBQWdCO0lBQzFDLE1BQU0sZUFBZSxHQUFHLElBQUksR0FBRyxDQUM3QixFQUFFO1NBQ0MsT0FBTyxDQUFDLDBDQUEwQyxDQUFDO1NBQ25ELEdBQUcsRUFBRTtTQUNMLEdBQUcsQ0FBQyxDQUFDLE1BQU0sRUFBRSxFQUFFLENBQUUsTUFBdUIsQ0FBQyxJQUFJLENBQUMsQ0FDbEQsQ0FBQztJQUNGLElBQUksQ0FBQyxlQUFlLENBQUMsR0FBRyxDQUFDLFFBQVEsQ0FBQyxFQUFFLENBQUM7UUFDbkMsRUFBRSxDQUFDLElBQUksQ0FBQyxnRkFBZ0YsQ0FBQyxDQUFDO0lBQzVGLENBQUM7SUFDRCxJQUFJLENBQUMsZUFBZSxDQUFDLEdBQUcsQ0FBQyxjQUFjLENBQUMsRUFBRSxDQUFDO1FBQ3pDLEVBQUUsQ0FBQyxJQUFJLENBQUMsZ0VBQWdFLENBQUMsQ0FBQztJQUM1RSxDQUFDO0lBQ0QsSUFBSSxDQUFDLGVBQWUsQ0FBQyxHQUFHLENBQUMsV0FBVyxDQUFDLEVBQUUsQ0FBQztRQUN0QyxFQUFFLENBQUMsSUFBSSxDQUFDLDZEQUE2RCxDQUFDLENBQUM7SUFDekUsQ0FBQztBQUNILENBQUM7QUFFRCxnSEFBZ0g7QUFDaEgsMEdBQTBHO0FBQzFHLDJHQUEyRztBQUMzRywrR0FBK0c7QUFDL0csK0dBQStHO0FBQy9HLGdGQUFnRjtBQUNoRixTQUFTLGlDQUFpQyxDQUFDLEVBQWdCO0lBQ3pELE1BQU0sbUJBQW1CLEdBQUcsRUFBRTtTQUMzQixPQUFPLENBQUMsZ0RBQWdELENBQUM7U0FDekQsR0FBRyxFQUFFO1NBQ0wsSUFBSSxDQUFDLENBQUMsTUFBTSxFQUFFLEVBQUUsQ0FBRSxNQUF1QixDQUFDLElBQUksS0FBSyxjQUFjLENBQUMsQ0FBQztJQUN0RSxJQUFJLG1CQUFtQjtRQUFFLE9BQU87SUFDaEMsRUFBRSxDQUFDLElBQUksQ0FBQzs7Ozs7Ozs7O0dBU1AsQ0FBQyxDQUFDO0lBQ0gsdUdBQXVHO0lBQ3ZHLDRHQUE0RztJQUM1Ryx5Q0FBeUM7SUFDekMsRUFBRSxDQUFDLE9BQU8sQ0FDUjtpR0FDNkYsQ0FDOUYsQ0FBQyxHQUFHLENBQUMsb0JBQW9CLENBQUMsVUFBVSxDQUFDLENBQUM7SUFDdkMsRUFBRSxDQUFDLElBQUksQ0FBQyx3Q0FBd0MsQ0FBQyxDQUFDO0lBQ2xELEVBQUUsQ0FBQyxJQUFJLENBQUMsa0ZBQWtGLENBQUMsQ0FBQztBQUM5RixDQUFDO0FBRUQsMEVBQTBFO0FBQzFFLE1BQU0sVUFBVSxpQkFBaUIsQ0FBQyxTQUFpQiwwQkFBMEIsRUFBRTtJQUM3RSxNQUFNLFlBQVksR0FBRyxlQUFlLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDN0MsTUFBTSxFQUFFLEdBQUcsZ0JBQWdCLENBQUMsWUFBWSxDQUFDLENBQUM7SUFFMUMsNEdBQTRHO0lBQzVHLHFHQUFxRztJQUNyRyx1R0FBdUc7SUFDdkcsdUdBQXVHO0lBQ3ZHLEVBQUUsQ0FBQyxJQUFJLENBQUM7Ozs7Ozs7Ozs7O0dBV1AsQ0FBQyxDQUFDO0lBQ0gsa0JBQWtCLENBQUMsRUFBRSxDQUFDLENBQUM7SUFDdkIsRUFBRSxDQUFDLElBQUksQ0FBQzs7Ozs7OztHQU9QLENBQUMsQ0FBQztJQUNILGlDQUFpQyxDQUFDLEVBQUUsQ0FBQyxDQUFDO0lBQ3RDLEVBQUUsQ0FBQyxJQUFJLENBQUM7Ozs7Ozs7OztHQVNQLENBQUMsQ0FBQztJQUNILEVBQUUsQ0FBQyxJQUFJLENBQUMsK0dBQStHLENBQUMsQ0FBQztJQUV6SCxNQUFNLGtCQUFrQixHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUMsa0RBQWtELENBQUMsQ0FBQztJQUMxRixNQUFNLHFCQUFxQixHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUM7Ozs7Ozs7Ozs7OztHQVl4QyxDQUFDLENBQUM7SUFDSCxNQUFNLHNCQUFzQixHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQ3ZDLHlGQUF5RixDQUMxRixDQUFDO0lBQ0YsTUFBTSx5QkFBeUIsR0FBRyxFQUFFLENBQUMsT0FBTyxDQUFDOzs7Ozs7O0dBTzVDLENBQUMsQ0FBQztJQUNILE1BQU0seUJBQXlCLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FBQzs7O0dBRzVDLENBQUMsQ0FBQztJQUNILE1BQU0sMkJBQTJCLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FDNUMsaUVBQWlFLENBQ2xFLENBQUM7SUFDRixNQUFNLDhCQUE4QixHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQy9DLDBGQUEwRixDQUMzRixDQUFDO0lBRUYsU0FBUyxlQUFlLENBQUMsR0FBcUI7UUFDNUMsT0FBTztZQUNMLFlBQVksRUFBRSxHQUFHLENBQUMsY0FBYztZQUNoQyxXQUFXLEVBQUUsR0FBRyxDQUFDLFdBQVc7WUFDNUIsV0FBVyxFQUFFLEdBQUcsQ0FBQyxZQUFZO1lBQzdCLGlCQUFpQixFQUFFLEdBQUcsQ0FBQyxtQkFBbUI7WUFDMUMsV0FBVyxFQUFFLEdBQUcsQ0FBQyxZQUFZO1NBQzlCLENBQUM7SUFDSixDQUFDO0lBRUQsOEdBQThHO0lBQzlHLDBHQUEwRztJQUMxRyx3R0FBd0c7SUFDeEcsOEdBQThHO0lBQzlHLDZHQUE2RztJQUM3RyxTQUFTLGVBQWUsQ0FBSSxFQUFXO1FBQ3JDLEVBQUUsQ0FBQyxJQUFJLENBQUMsaUJBQWlCLENBQUMsQ0FBQztRQUMzQixJQUFJLENBQUM7WUFDSCxNQUFNLE1BQU0sR0FBRyxFQUFFLEVBQUUsQ0FBQztZQUNwQixFQUFFLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxDQUFDO1lBQ2xCLE9BQU8sTUFBTSxDQUFDO1FBQ2hCLENBQUM7UUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1lBQ2YsRUFBRSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztZQUNwQixNQUFNLEtBQUssQ0FBQztRQUNkLENBQUM7SUFDSCxDQUFDO0lBRUQsTUFBTSxLQUFLLEdBQWtCO1FBQzNCLE1BQU0sRUFBRSxZQUFZO1FBQ3BCLGtCQUFrQjtZQUNoQixNQUFNLEdBQUcsR0FBRyxrQkFBa0IsQ0FBQyxHQUFHLEVBQWdDLENBQUM7WUFDbkUsT0FBTztnQkFDTCxPQUFPLEVBQUUsZUFBZSxDQUFDLEdBQUcsRUFBRSx1QkFBdUIsRUFBRSwwQkFBMEIsQ0FBQztnQkFDbEYsZUFBZSxFQUFFLGVBQWUsQ0FBQyxHQUFHLEVBQUUsdUJBQXVCLEVBQUUsMEJBQTBCLENBQUM7YUFDM0YsQ0FBQztRQUNKLENBQUM7UUFDRCxrQkFBa0IsQ0FBQyxjQUFzQztZQUN2RCxlQUFlLENBQUMsR0FBRyxFQUFFO2dCQUNuQixNQUFNLEdBQUcsR0FBRyxrQkFBa0IsQ0FBQyxHQUFHLEVBQWdDLENBQUM7Z0JBQ25FLHFCQUFxQixDQUFDLEdBQUcsQ0FDdkIsSUFBSSxDQUFDLFNBQVMsQ0FBQyxjQUFjLEVBQUUsT0FBTyxJQUFJLDBCQUEwQixDQUFDLEVBQ3JFLElBQUksQ0FBQyxTQUFTLENBQUMsY0FBYyxFQUFFLGVBQWUsSUFBSSwwQkFBMEIsQ0FBQyxFQUM3RSxHQUFHLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxjQUFjLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsaUJBQWlCLENBQUMsRUFDNUQsR0FBRyxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxDQUFDLEVBQ3BCLEdBQUcsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsSUFBSSxFQUM3QixHQUFHLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLElBQUksRUFDMUIsSUFBSSxJQUFJLEVBQUUsQ0FBQyxXQUFXLEVBQUUsQ0FDekIsQ0FBQztZQUNKLENBQUMsQ0FBQyxDQUFDO1FBQ0wsQ0FBQztRQUNELFlBQVk7WUFDVixNQUFNLEdBQUcsR0FBRyxrQkFBa0IsQ0FBQyxHQUFHLEVBQWdDLENBQUM7WUFDbkUsT0FBTyxlQUFlLENBQUMsR0FBRyxFQUFFLGNBQWMsRUFBRSxpQkFBaUIsQ0FBQyxDQUFDO1FBQ2pFLENBQUM7UUFDRCxZQUFZLENBQUMsUUFBMEI7WUFDckMsZUFBZSxDQUFDLEdBQUcsRUFBRTtnQkFDbkIsTUFBTSxHQUFHLEdBQUcsa0JBQWtCLENBQUMsR0FBRyxFQUFnQyxDQUFDO2dCQUNuRSxxQkFBcUIsQ0FBQyxHQUFHLENBQ3ZCLEdBQUcsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLHVCQUF1QixDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLDBCQUEwQixDQUFDLEVBQzlFLEdBQUcsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLHVCQUF1QixDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLDBCQUEwQixDQUFDLEVBQzlFLElBQUksQ0FBQyxTQUFTLENBQUMsUUFBUSxJQUFJLGlCQUFpQixDQUFDLEVBQzdDLEdBQUcsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUNwQixHQUFHLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFDLElBQUksRUFDN0IsR0FBRyxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUMsU0FBUyxDQUFDLENBQUMsQ0FBQyxJQUFJLEVBQzFCLElBQUksSUFBSSxFQUFFLENBQUMsV0FBVyxFQUFFLENBQ3pCLENBQUM7WUFDSixDQUFDLENBQUMsQ0FBQztRQUNMLENBQUM7UUFDRCw0R0FBNEc7UUFDNUcsNkdBQTZHO1FBQzdHLDZHQUE2RztRQUM3Ryw0R0FBNEc7UUFDNUcsdUZBQXVGO1FBQ3ZGLGNBQWM7WUFDWixNQUFNLEdBQUcsR0FBRyxrQkFBa0IsQ0FBQyxHQUFHLEVBQWdDLENBQUM7WUFDbkUsT0FBTztnQkFDTCxNQUFNLEVBQUUsR0FBRyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLO2dCQUN6QyxNQUFNLEVBQUUsR0FBRyxFQUFFLFlBQVksSUFBSSxJQUFJO2dCQUNqQyxRQUFRLEVBQUUsR0FBRyxFQUFFLFNBQVMsSUFBSSxJQUFJO2FBQ2pDLENBQUM7UUFDSixDQUFDO1FBQ0QsY0FBYyxDQUFDLFVBQThCO1lBQzNDLE1BQU0sTUFBTSxHQUFHLE9BQU8sQ0FBQyxVQUFVLEVBQUUsTUFBTSxDQUFDLENBQUM7WUFDM0MsTUFBTSxNQUFNLEdBQ1YsT0FBTyxVQUFVLEVBQUUsTUFBTSxLQUFLLFFBQVEsSUFBSSxVQUFVLENBQUMsTUFBTSxDQUFDLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQyxVQUFVLENBQUMsTUFBTSxDQUFDLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUM7WUFDdkcsTUFBTSxRQUFRLEdBQUcsTUFBTSxDQUFDLENBQUMsQ0FBQyxJQUFJLElBQUksRUFBRSxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUM7WUFDMUQsZUFBZSxDQUFDLEdBQUcsRUFBRTtnQkFDbkIsTUFBTSxHQUFHLEdBQUcsa0JBQWtCLENBQUMsR0FBRyxFQUFnQyxDQUFDO2dCQUNuRSxxQkFBcUIsQ0FBQyxHQUFHLENBQ3ZCLEdBQUcsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLHVCQUF1QixDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLDBCQUEwQixDQUFDLEVBQzlFLEdBQUcsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLHVCQUF1QixDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsU0FBUyxDQUFDLDBCQUEwQixDQUFDLEVBQzlFLEdBQUcsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLGNBQWMsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxpQkFBaUIsQ0FBQyxFQUM1RCxNQUFNLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUNkLE1BQU0sRUFDTixRQUFRLEVBQ1IsSUFBSSxJQUFJLEVBQUUsQ0FBQyxXQUFXLEVBQUUsQ0FDekIsQ0FBQztZQUNKLENBQUMsQ0FBQyxDQUFDO1lBQ0gsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLEVBQUUsUUFBUSxFQUFFLENBQUM7UUFDdEMsQ0FBQztRQUNELHFCQUFxQixDQUFDLFlBQW9CLEVBQUUsVUFBbUI7WUFDN0QsTUFBTSxlQUFlLEdBQUcsbUJBQW1CLENBQUMsVUFBVSxDQUFDLENBQUM7WUFDeEQsTUFBTSxjQUFjLEdBQUcscUJBQXFCLENBQUMsWUFBWSxDQUFDLENBQUM7WUFDM0QsTUFBTSxHQUFHLEdBQUcsc0JBQXNCLENBQUMsR0FBRyxDQUFDLGVBQWUsRUFBRSxjQUFjLENBQXFDLENBQUM7WUFDNUcsSUFBSSxDQUFDLEdBQUc7Z0JBQUUsT0FBTyxFQUFFLEdBQUcsMEJBQTBCLEVBQUUsQ0FBQztZQUNuRCxPQUFPLEVBQUUsT0FBTyxFQUFFLEdBQUcsQ0FBQyxPQUFPLEVBQUUsV0FBVyxFQUFFLEdBQUcsQ0FBQyxXQUFXLEVBQUUsQ0FBQztRQUNoRSxDQUFDO1FBQ0QscUJBQXFCLENBQUMsWUFBb0IsRUFBRSxPQUEyQixFQUFFLFVBQW1CO1lBQzFGLE1BQU0sZUFBZSxHQUFHLG1CQUFtQixDQUFDLFVBQVUsQ0FBQyxDQUFDO1lBQ3hELE1BQU0sY0FBYyxHQUFHLHFCQUFxQixDQUFDLFlBQVksQ0FBQyxDQUFDO1lBQzNELE1BQU0sT0FBTyxHQUFHLE1BQU0sQ0FBQyxTQUFTLENBQUMsT0FBTyxFQUFFLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQyxPQUFPLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7WUFDekUsTUFBTSxXQUFXLEdBQUcsTUFBTSxDQUFDLFNBQVMsQ0FBQyxPQUFPLEVBQUUsV0FBVyxDQUFDLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQztZQUNyRix5QkFBeUIsQ0FBQyxHQUFHLENBQUMsZUFBZSxFQUFFLGNBQWMsRUFBRSxPQUFPLEVBQUUsV0FBVyxFQUFFLElBQUksSUFBSSxFQUFFLENBQUMsV0FBVyxFQUFFLENBQUMsQ0FBQztZQUMvRyxPQUFPLEVBQUUsT0FBTyxFQUFFLFdBQVcsRUFBRSxDQUFDO1FBQ2xDLENBQUM7UUFDRCxtQkFBbUIsQ0FBQyxNQUEyQjtZQUM3QyxNQUFNLFVBQVUsR0FBRyxxQkFBcUIsQ0FBQyxNQUFNLEVBQUUsWUFBWSxDQUFDLENBQUM7WUFDL0QsSUFBSSxPQUFPLE1BQU0sRUFBRSxXQUFXLEtBQUssUUFBUSxJQUFJLENBQUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxJQUFJLEVBQUUsRUFBRSxDQUFDO2dCQUMxRSxNQUFNLElBQUksS0FBSyxDQUFDLHFCQUFxQixDQUFDLENBQUM7WUFDekMsQ0FBQztZQUNELE1BQU0sV0FBVyxHQUFHLE9BQU8sTUFBTSxDQUFDLFdBQVcsS0FBSyxRQUFRLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxXQUFXLENBQUMsQ0FBQyxDQUFDLElBQUksSUFBSSxFQUFFLENBQUMsV0FBVyxFQUFFLENBQUM7WUFDM0csTUFBTSxpQkFBaUIsR0FBa0IsTUFBTSxDQUFDLFNBQVMsQ0FBQyxNQUFNLENBQUMsaUJBQWlCLENBQUMsQ0FBQyxDQUFDLENBQUUsTUFBTSxDQUFDLGlCQUE0QixDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUM7WUFDbEksTUFBTSxXQUFXLEdBQWtCLE1BQU0sQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsQ0FBRSxNQUFNLENBQUMsV0FBc0IsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDO1lBQ2hILHlCQUF5QixDQUFDLEdBQUcsQ0FBQyxVQUFVLEVBQUUsTUFBTSxDQUFDLFdBQVcsRUFBRSxXQUFXLEVBQUUsaUJBQWlCLEVBQUUsV0FBVyxDQUFDLENBQUM7WUFDM0csT0FBTyxFQUFFLFlBQVksRUFBRSxVQUFVLEVBQUUsV0FBVyxFQUFFLE1BQU0sQ0FBQyxXQUFXLEVBQUUsV0FBVyxFQUFFLGlCQUFpQixFQUFFLFdBQVcsRUFBRSxDQUFDO1FBQ3BILENBQUM7UUFDRCx3QkFBd0IsQ0FBQyxTQUF5QyxFQUFFO1lBQ2xFLE1BQU0sS0FBSyxHQUFHLE1BQU0sQ0FBQyxTQUFTLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxJQUFLLE1BQU0sQ0FBQyxLQUFnQixHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUUsTUFBTSxDQUFDLEtBQWdCLENBQUMsQ0FBQyxDQUFDLEdBQUcsQ0FBQztZQUM5RyxNQUFNLElBQUksR0FDUixNQUFNLENBQUMsWUFBWSxLQUFLLFNBQVM7Z0JBQy9CLENBQUMsQ0FBQywyQkFBMkIsQ0FBQyxHQUFHLENBQUMsS0FBSyxDQUFDO2dCQUN4QyxDQUFDLENBQUMsOEJBQThCLENBQUMsR0FBRyxDQUFDLHFCQUFxQixDQUFDLE1BQU0sQ0FBQyxZQUFZLENBQUMsRUFBRSxLQUFLLENBQUMsQ0FBQztZQUM1RixPQUFPLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQyxHQUFHLEVBQUUsRUFBRSxDQUFDLGVBQWUsQ0FBQyxHQUF1QixDQUFDLENBQUMsQ0FBQztRQUNyRSxDQUFDO1FBQ0Q7Ozs7OztXQU1HO1FBQ0gsV0FBVyxDQUFDLFlBQW9CO1lBQzlCLE1BQU0sVUFBVSxHQUFHLHFCQUFxQixDQUFDLFlBQVksQ0FBQyxDQUFDO1lBQ3ZELE9BQU8sQ0FDTCxnQkFBZ0IsQ0FBQyxFQUFFLEVBQUUsc0NBQXNDLEVBQUUsVUFBVSxDQUFDO2dCQUN4RSxnQkFBZ0IsQ0FBQyxFQUFFLEVBQUUsbUNBQW1DLEVBQUUsVUFBVSxDQUFDLENBQ3RFLENBQUM7UUFDSixDQUFDO1FBQ0QsS0FBSztZQUNILEVBQUUsQ0FBQyxLQUFLLEVBQUUsQ0FBQztRQUNiLENBQUM7S0FDRixDQUFDO0lBQ0YsT0FBTyxLQUFLLENBQUM7QUFDZixDQUFDO0FBRUQsU0FBUyx1QkFBdUI7SUFDOUIsb0JBQW9CLEtBQUssaUJBQWlCLEVBQUUsQ0FBQztJQUM3QyxPQUFPLG9CQUFvQixDQUFDO0FBQzlCLENBQUM7QUFFRCxNQUFNLFVBQVUsa0JBQWtCO0lBQ2hDLE9BQU8sdUJBQXVCLEVBQUUsQ0FBQyxrQkFBa0IsRUFBRSxDQUFDO0FBQ3hELENBQUM7QUFFRCxNQUFNLFVBQVUsa0JBQWtCLENBQUMsY0FBc0M7SUFDdkUsT0FBTyx1QkFBdUIsRUFBRSxDQUFDLGtCQUFrQixDQUFDLGNBQWMsQ0FBQyxDQUFDO0FBQ3RFLENBQUM7QUFFRCxNQUFNLFVBQVUsWUFBWTtJQUMxQixPQUFPLHVCQUF1QixFQUFFLENBQUMsWUFBWSxFQUFFLENBQUM7QUFDbEQsQ0FBQztBQUVELE1BQU0sVUFBVSxZQUFZLENBQUMsUUFBMEI7SUFDckQsT0FBTyx1QkFBdUIsRUFBRSxDQUFDLFlBQVksQ0FBQyxRQUFRLENBQUMsQ0FBQztBQUMxRCxDQUFDO0FBRUQsTUFBTSxVQUFVLGNBQWM7SUFDNUIsT0FBTyx1QkFBdUIsRUFBRSxDQUFDLGNBQWMsRUFBRSxDQUFDO0FBQ3BELENBQUM7QUFFRCxNQUFNLFVBQVUsY0FBYyxDQUFDLFVBQThCO0lBQzNELE9BQU8sdUJBQXVCLEVBQUUsQ0FBQyxjQUFjLENBQUMsVUFBVSxDQUFDLENBQUM7QUFDOUQsQ0FBQztBQUVELE1BQU0sVUFBVSxxQkFBcUIsQ0FBQyxZQUFvQixFQUFFLFVBQW1CO0lBQzdFLE9BQU8sdUJBQXVCLEVBQUUsQ0FBQyxxQkFBcUIsQ0FBQyxZQUFZLEVBQUUsVUFBVSxDQUFDLENBQUM7QUFDbkYsQ0FBQztBQUVELE1BQU0sVUFBVSxxQkFBcUIsQ0FBQyxZQUFvQixFQUFFLE9BQTJCLEVBQUUsVUFBbUI7SUFDMUcsT0FBTyx1QkFBdUIsRUFBRSxDQUFDLHFCQUFxQixDQUFDLFlBQVksRUFBRSxPQUFPLEVBQUUsVUFBVSxDQUFDLENBQUM7QUFDNUYsQ0FBQztBQUVELE1BQU0sVUFBVSxtQkFBbUIsQ0FBQyxNQUEyQjtJQUM3RCxPQUFPLHVCQUF1QixFQUFFLENBQUMsbUJBQW1CLENBQUMsTUFBTSxDQUFDLENBQUM7QUFDL0QsQ0FBQztBQUVELE1BQU0sVUFBVSx3QkFBd0IsQ0FBQyxNQUF1QztJQUM5RSxPQUFPLHVCQUF1QixFQUFFLENBQUMsd0JBQXdCLENBQUMsTUFBTSxDQUFDLENBQUM7QUFDcEUsQ0FBQztBQUVELE1BQU0sVUFBVSx5QkFBeUI7SUFDdkMsSUFBSSxDQUFDLG9CQUFvQjtRQUFFLE9BQU87SUFDbEMsb0JBQW9CLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDN0Isb0JBQW9CLEdBQUcsSUFBSSxDQUFDO0FBQzlCLENBQUMifQ== \ No newline at end of file diff --git a/packages/loopover-miner/lib/governor-state.ts b/packages/loopover-miner/lib/governor-state.ts new file mode 100644 index 0000000000..10b625ff04 --- /dev/null +++ b/packages/loopover-miner/lib/governor-state.ts @@ -0,0 +1,482 @@ +import type { + GovernorCapUsage, + OwnSubmissionRecord, + RepoOutcomeHistory, + WriteRateLimitBackoffStore, + WriteRateLimitBucketStore, +} from "@loopover/engine"; +import type { DatabaseSync } from "node:sqlite"; +import { DEFAULT_FORGE_CONFIG } from "./forge-config.js"; +import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js"; +import { + GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC, + GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC, + purgeStoreByRepo, +} from "./store-maintenance.js"; + +// Governor cross-attempt state persistence (#5134, Wave 3.5). Every governor-*.js wrapper +// (governor-chokepoint.js) is a pure in/out transform: it computes and RETURNS +// updated rate-limit buckets/backoff attempts, but nothing writes them to disk, so they reset to zero on +// every process start -- the mutable counters that should gate the NEXT decision never survive past one +// process. governor-ledger.js already persists the DECISION HISTORY (an append-only audit log); this module +// persists the DECISION INPUT state instead -- a second, distinct concern, not a duplicate of that log (see +// its own module doc for the ledger/state split this issue's acceptance criteria requires). +// +// This module does not alter evaluateGovernorChokepoint's precedence ladder or any pure calculator's logic -- +// it only gives their existing, already-optional input fields (rateLimitBuckets, rateLimitBackoffAttempts, +// capUsage, reputationHistory, recentOwnSubmissions) a real load-at-start/save-at-end home. Convergence input +// (packages/loopover-engine/src/portfolio/non-convergence.ts's PortfolioConvergenceInput) is NOT persisted +// here: that module's own doc comment says its counters belong on the portfolio-queue table (a pre-existing +// store this issue's boundaries don't touch) once that table grows attempt-history columns -- inventing a +// second, competing store for the same concept here would violate the same non-duplication principle the +// ledger/state split above is built on. + +export type GovernorRateLimitState = { + buckets: WriteRateLimitBucketStore; + backoffAttempts: WriteRateLimitBackoffStore; +}; + +export type ListRecentOwnSubmissionsFilter = { + repoFullName?: string; + limit?: number; +}; + +export type GovernorPauseState = { + paused: boolean; + reason: string | null; + pausedAt: string | null; +}; + +export type GovernorPauseInput = { + paused: boolean; + reason?: string | null; +}; + +export type GovernorState = { + dbPath: string; + loadRateLimitState(): GovernorRateLimitState; + saveRateLimitState(rateLimitState: GovernorRateLimitState): void; + loadCapUsage(): GovernorCapUsage; + saveCapUsage(capUsage: GovernorCapUsage): void; + loadPauseState(): GovernorPauseState; + savePauseState(pauseState: GovernorPauseInput): GovernorPauseState; + loadReputationHistory(repoFullName: string, apiBaseUrl?: string): RepoOutcomeHistory; + saveReputationHistory(repoFullName: string, history: RepoOutcomeHistory, apiBaseUrl?: string): RepoOutcomeHistory; + recordOwnSubmission(record: OwnSubmissionRecord): OwnSubmissionRecord; + listRecentOwnSubmissions(filter?: ListRecentOwnSubmissionsFilter): OwnSubmissionRecord[]; + /** Delete every repo-scoped row for one repo across both governor tables (#7091); returns total rows removed. */ + purgeByRepo(repoFullName: string): number; + close(): void; +}; + +/** SQLite `governor_scalar_state` row shape (StatementSync returns `Record`). */ +type ScalarStateRow = { + id: number; + rate_limit_buckets_json: string; + rate_limit_backoff_json: string; + cap_usage_json: string; + paused: number; + pause_reason: string | null; + paused_at: string | null; + updated_at: string; +}; + +type ReputationHistoryRow = { + api_base_url: string; + repo_full_name: string; + decided: number; + unfavorable: number; + updated_at: string; +}; + +type OwnSubmissionRow = { + id: number; + repo_full_name: string; + fingerprint: string; + submitted_at: string | null; + pull_request_number: number | null; + issue_number: number | null; +}; + +type TableInfoRow = { name: string }; + +const defaultDbFileName = "governor-state.sqlite3"; +const DEFAULT_RATE_LIMIT_BUCKETS: Readonly = Object.freeze({ global: {}, perRepo: {} }); +const DEFAULT_RATE_LIMIT_BACKOFF: Readonly = Object.freeze({}); +const DEFAULT_CAP_USAGE: Readonly = Object.freeze({ budgetSpent: 0, turnsTaken: 0, elapsedMs: 0 }); +const DEFAULT_REPUTATION_HISTORY: Readonly = Object.freeze({ decided: 0, unfavorable: 0 }); +let defaultGovernorState: GovernorState | null = null; + +export function resolveGovernorStateDbPath(env: Record = process.env): string { + return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_GOVERNOR_STATE_DB", env); +} + +function normalizeDbPath(dbPath: string | null | undefined): string { + return normalizeLocalStoreDbPath(dbPath, resolveGovernorStateDbPath(), "invalid_governor_state_db_path"); +} + +function normalizeRepoFullName(repoFullName: unknown): string { + if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name"); + const [owner, repo, extra] = repoFullName.trim().split("/"); + if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name"); + return `${owner}/${repo}`; +} + +/** Optional forge host, scoping rows so two hosts serving the same owner/repo name never collide (#5563). + * Omitted/nullish → the github.com default, so every pre-existing single-forge caller is unaffected. */ +function normalizeApiBaseUrl(apiBaseUrl: unknown): string { + if (apiBaseUrl === undefined || apiBaseUrl === null) return DEFAULT_FORGE_CONFIG.apiBaseUrl; + if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) throw new Error("invalid_api_base_url"); + return apiBaseUrl.trim(); +} + +function parseJsonColumn(value: unknown, fallback: T): T { + if (typeof value !== "string") return fallback; + try { + const parsed: unknown = JSON.parse(value); + return parsed && typeof parsed === "object" ? (parsed as T) : fallback; + } catch { + return fallback; + } +} + +// Add the pause/resume columns (#4851) to an on-disk file created before they existed. `CREATE TABLE IF NOT +// EXISTS` above is a no-op against an already-existing table, so a pre-#4851 file needs this explicit ALTER -- +// guarded by a per-column presence check (rather than a single `paused`-only check) so a file that somehow +// has `paused` but not `pause_reason`/`paused_at` still gets the columns it's missing, same technique as +// portfolio-queue.js's own post-creation column migration. +function ensurePauseColumns(db: DatabaseSync): void { + const existingColumns = new Set( + db + .prepare("PRAGMA table_info(governor_scalar_state)") + .all() + .map((column) => (column as TableInfoRow).name), + ); + if (!existingColumns.has("paused")) { + db.exec("ALTER TABLE governor_scalar_state ADD COLUMN paused INTEGER NOT NULL DEFAULT 0"); + } + if (!existingColumns.has("pause_reason")) { + db.exec("ALTER TABLE governor_scalar_state ADD COLUMN pause_reason TEXT"); + } + if (!existingColumns.has("paused_at")) { + db.exec("ALTER TABLE governor_scalar_state ADD COLUMN paused_at TEXT"); + } +} + +// Rebuild governor_reputation_history's bare `repo_full_name` PRIMARY KEY into a (api_base_url, repo_full_name) +// composite (#5563) -- two forge hosts serving a same-named owner/repo must not share one reputation row. +// SQLite cannot ALTER a PRIMARY KEY in place, so this rebuilds the table: create the new shape, copy every +// existing row with the pre-#4784 implicit single-forge default backfilled, drop the old table, rename the new +// one in. Guarded by a column-presence check (matching ensurePauseColumns' idempotence) so this only runs once +// per file, same technique as portfolio-queue.js's own post-creation migration. +function ensureReputationHistoryForgeScope(db: DatabaseSync): void { + const hasApiBaseUrlColumn = db + .prepare("PRAGMA table_info(governor_reputation_history)") + .all() + .some((column) => (column as TableInfoRow).name === "api_base_url"); + if (hasApiBaseUrlColumn) return; + db.exec(` + CREATE TABLE governor_reputation_history_v2 ( + api_base_url TEXT NOT NULL, + repo_full_name TEXT NOT NULL, + decided INTEGER NOT NULL, + unfavorable INTEGER NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (api_base_url, repo_full_name) + ) + `); + // OR IGNORE: a source row that somehow violates the rebuilt table's NOT NULL columns (a hand-edited or + // otherwise corrupted file) is skipped rather than aborting the whole migration -- same fail-closed posture + // as run-state.js's own #5563 migration. + db.prepare( + `INSERT OR IGNORE INTO governor_reputation_history_v2 (api_base_url, repo_full_name, decided, unfavorable, updated_at) + SELECT ?, repo_full_name, decided, unfavorable, updated_at FROM governor_reputation_history`, + ).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); + db.exec("DROP TABLE governor_reputation_history"); + db.exec("ALTER TABLE governor_reputation_history_v2 RENAME TO governor_reputation_history"); +} + +/** Opens the local governor-state store, creating tables on first use. */ +export function openGovernorState(dbPath: string = resolveGovernorStateDbPath()): GovernorState { + const resolvedPath = normalizeDbPath(dbPath); + const db = openLocalStoreDb(resolvedPath); + + // ONE row (id=1) holding the whole-run scalar state: rate-limit buckets/backoff and budget/turn/termination + // usage have no natural per-repo key of their own beyond what's already encoded inside the JSON blob + // (WriteRateLimitBucketStore.perRepo is itself keyed by `${actionClass}:${repoFullName}`), so a single + // UPSERTed row is simpler and more honest than inventing a relational key that doesn't exist upstream. + db.exec(` + CREATE TABLE IF NOT EXISTS governor_scalar_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + rate_limit_buckets_json TEXT NOT NULL, + rate_limit_backoff_json TEXT NOT NULL, + cap_usage_json TEXT NOT NULL, + paused INTEGER NOT NULL DEFAULT 0, + pause_reason TEXT, + paused_at TEXT, + updated_at TEXT NOT NULL + ) + `); + ensurePauseColumns(db); + db.exec(` + CREATE TABLE IF NOT EXISTS governor_reputation_history ( + repo_full_name TEXT PRIMARY KEY, + decided INTEGER NOT NULL, + unfavorable INTEGER NOT NULL, + updated_at TEXT NOT NULL + ) + `); + ensureReputationHistoryForgeScope(db); + db.exec(` + CREATE TABLE IF NOT EXISTS governor_own_submissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_full_name TEXT NOT NULL, + fingerprint TEXT NOT NULL, + submitted_at TEXT, + pull_request_number INTEGER, + issue_number INTEGER + ) + `); + db.exec("CREATE INDEX IF NOT EXISTS idx_governor_own_submissions_repo ON governor_own_submissions (repo_full_name, id)"); + + const getScalarStatement = db.prepare("SELECT * FROM governor_scalar_state WHERE id = 1"); + const upsertScalarStatement = db.prepare(` + INSERT INTO governor_scalar_state + (id, rate_limit_buckets_json, rate_limit_backoff_json, cap_usage_json, paused, pause_reason, paused_at, updated_at) + VALUES (1, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + rate_limit_buckets_json = excluded.rate_limit_buckets_json, + rate_limit_backoff_json = excluded.rate_limit_backoff_json, + cap_usage_json = excluded.cap_usage_json, + paused = excluded.paused, + pause_reason = excluded.pause_reason, + paused_at = excluded.paused_at, + updated_at = excluded.updated_at + `); + const getReputationStatement = db.prepare( + "SELECT * FROM governor_reputation_history WHERE api_base_url = ? AND repo_full_name = ?", + ); + const upsertReputationStatement = db.prepare(` + INSERT INTO governor_reputation_history (api_base_url, repo_full_name, decided, unfavorable, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(api_base_url, repo_full_name) DO UPDATE SET + decided = excluded.decided, + unfavorable = excluded.unfavorable, + updated_at = excluded.updated_at + `); + const insertSubmissionStatement = db.prepare(` + INSERT INTO governor_own_submissions (repo_full_name, fingerprint, submitted_at, pull_request_number, issue_number) + VALUES (?, ?, ?, ?, ?) + `); + const listSubmissionsAllStatement = db.prepare( + "SELECT * FROM governor_own_submissions ORDER BY id DESC LIMIT ?", + ); + const listSubmissionsByRepoStatement = db.prepare( + "SELECT * FROM governor_own_submissions WHERE repo_full_name = ? ORDER BY id DESC LIMIT ?", + ); + + function rowToSubmission(row: OwnSubmissionRow): OwnSubmissionRecord { + return { + repoFullName: row.repo_full_name, + fingerprint: row.fingerprint, + submittedAt: row.submitted_at, + pullRequestNumber: row.pull_request_number, + issueNumber: row.issue_number, + }; + } + + // BEGIN IMMEDIATE takes the write lock BEFORE `fn`'s read, so two processes on the same file (the loop daemon + // saving rate-limit/cap-usage state on every gated write, and an operator's `governor pause`/`resume` CLI + // invocation racing it) cannot interleave a stale read with each other's write and silently clobber the + // scalar-state column-group they don't own -- same fix shape as event-ledger.js's appendEvent (#7221). Shared + // by all three governor_scalar_state save methods below, since they all read-then-write across the same row. + function withTransaction(fn: () => T): T { + db.exec("BEGIN IMMEDIATE"); + try { + const result = fn(); + db.exec("COMMIT"); + return result; + } catch (error) { + db.exec("ROLLBACK"); + throw error; + } + } + + const state: GovernorState = { + dbPath: resolvedPath, + loadRateLimitState(): GovernorRateLimitState { + const row = getScalarStatement.get() as ScalarStateRow | undefined; + return { + buckets: parseJsonColumn(row?.rate_limit_buckets_json, DEFAULT_RATE_LIMIT_BUCKETS), + backoffAttempts: parseJsonColumn(row?.rate_limit_backoff_json, DEFAULT_RATE_LIMIT_BACKOFF), + }; + }, + saveRateLimitState(rateLimitState: GovernorRateLimitState): void { + withTransaction(() => { + const row = getScalarStatement.get() as ScalarStateRow | undefined; + upsertScalarStatement.run( + JSON.stringify(rateLimitState?.buckets ?? DEFAULT_RATE_LIMIT_BUCKETS), + JSON.stringify(rateLimitState?.backoffAttempts ?? DEFAULT_RATE_LIMIT_BACKOFF), + row ? row.cap_usage_json : JSON.stringify(DEFAULT_CAP_USAGE), + row ? row.paused : 0, + row ? row.pause_reason : null, + row ? row.paused_at : null, + new Date().toISOString(), + ); + }); + }, + loadCapUsage(): GovernorCapUsage { + const row = getScalarStatement.get() as ScalarStateRow | undefined; + return parseJsonColumn(row?.cap_usage_json, DEFAULT_CAP_USAGE); + }, + saveCapUsage(capUsage: GovernorCapUsage): void { + withTransaction(() => { + const row = getScalarStatement.get() as ScalarStateRow | undefined; + upsertScalarStatement.run( + row ? row.rate_limit_buckets_json : JSON.stringify(DEFAULT_RATE_LIMIT_BUCKETS), + row ? row.rate_limit_backoff_json : JSON.stringify(DEFAULT_RATE_LIMIT_BACKOFF), + JSON.stringify(capUsage ?? DEFAULT_CAP_USAGE), + row ? row.paused : 0, + row ? row.pause_reason : null, + row ? row.paused_at : null, + new Date().toISOString(), + ); + }); + }, + // The governor pause/resume control surface (#4851): a real, persisted, operator/governor-writable flag the + // loop checks before each cycle -- distinct from governor-kill-switch.js (a read-only resolver over env/YAML + // inputs the miner does not itself write) and governor-run-halt.js (a one-way, run-scoped terminal breaker). + // `pausedAt` is stamped fresh on every transition INTO paused, and cleared on resume, so a status query can + // report how long a pause has been in effect without needing a separate history table. + loadPauseState(): GovernorPauseState { + const row = getScalarStatement.get() as ScalarStateRow | undefined; + return { + paused: row ? Boolean(row.paused) : false, + reason: row?.pause_reason ?? null, + pausedAt: row?.paused_at ?? null, + }; + }, + savePauseState(pauseState: GovernorPauseInput): GovernorPauseState { + const paused = Boolean(pauseState?.paused); + const reason = + typeof pauseState?.reason === "string" && pauseState.reason.trim() ? pauseState.reason.trim() : null; + const pausedAt = paused ? new Date().toISOString() : null; + withTransaction(() => { + const row = getScalarStatement.get() as ScalarStateRow | undefined; + upsertScalarStatement.run( + row ? row.rate_limit_buckets_json : JSON.stringify(DEFAULT_RATE_LIMIT_BUCKETS), + row ? row.rate_limit_backoff_json : JSON.stringify(DEFAULT_RATE_LIMIT_BACKOFF), + row ? row.cap_usage_json : JSON.stringify(DEFAULT_CAP_USAGE), + paused ? 1 : 0, + reason, + pausedAt, + new Date().toISOString(), + ); + }); + return { paused, reason, pausedAt }; + }, + loadReputationHistory(repoFullName: string, apiBaseUrl?: string): RepoOutcomeHistory { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const row = getReputationStatement.get(normalizedForge, normalizedRepo) as ReputationHistoryRow | undefined; + if (!row) return { ...DEFAULT_REPUTATION_HISTORY }; + return { decided: row.decided, unfavorable: row.unfavorable }; + }, + saveReputationHistory(repoFullName: string, history: RepoOutcomeHistory, apiBaseUrl?: string): RepoOutcomeHistory { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const decided = Number.isInteger(history?.decided) ? history.decided : 0; + const unfavorable = Number.isInteger(history?.unfavorable) ? history.unfavorable : 0; + upsertReputationStatement.run(normalizedForge, normalizedRepo, decided, unfavorable, new Date().toISOString()); + return { decided, unfavorable }; + }, + recordOwnSubmission(record: OwnSubmissionRecord): OwnSubmissionRecord { + const normalized = normalizeRepoFullName(record?.repoFullName); + if (typeof record?.fingerprint !== "string" || !record.fingerprint.trim()) { + throw new Error("invalid_fingerprint"); + } + const submittedAt = typeof record.submittedAt === "string" ? record.submittedAt : new Date().toISOString(); + const pullRequestNumber: number | null = Number.isInteger(record.pullRequestNumber) ? (record.pullRequestNumber as number) : null; + const issueNumber: number | null = Number.isInteger(record.issueNumber) ? (record.issueNumber as number) : null; + insertSubmissionStatement.run(normalized, record.fingerprint, submittedAt, pullRequestNumber, issueNumber); + return { repoFullName: normalized, fingerprint: record.fingerprint, submittedAt, pullRequestNumber, issueNumber }; + }, + listRecentOwnSubmissions(filter: ListRecentOwnSubmissionsFilter = {}): OwnSubmissionRecord[] { + const limit = Number.isInteger(filter.limit) && (filter.limit as number) > 0 ? (filter.limit as number) : 200; + const rows = + filter.repoFullName === undefined + ? listSubmissionsAllStatement.all(limit) + : listSubmissionsByRepoStatement.all(normalizeRepoFullName(filter.repoFullName), limit); + return rows.map((row) => rowToSubmission(row as OwnSubmissionRow)); + }, + /** + * Delete every repo-scoped row for one repo across BOTH governor tables against this single open handle + * (#7091) — the right-to-be-forgotten path `loopover-miner purge` invokes. `governor_reputation_history` is + * purged on `repo_full_name` alone (its key is composite with `api_base_url`), so nothing survives on any + * forge host. `governor_scalar_state` is deliberately untouched — it has no repo dimension. Returns the + * total rows removed across both tables. + */ + purgeByRepo(repoFullName: string): number { + const normalized = normalizeRepoFullName(repoFullName); + return ( + purgeStoreByRepo(db, GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC, normalized) + + purgeStoreByRepo(db, GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC, normalized) + ); + }, + close(): void { + db.close(); + }, + }; + return state; +} + +function getDefaultGovernorState(): GovernorState { + defaultGovernorState ??= openGovernorState(); + return defaultGovernorState; +} + +export function loadRateLimitState(): GovernorRateLimitState { + return getDefaultGovernorState().loadRateLimitState(); +} + +export function saveRateLimitState(rateLimitState: GovernorRateLimitState): void { + return getDefaultGovernorState().saveRateLimitState(rateLimitState); +} + +export function loadCapUsage(): GovernorCapUsage { + return getDefaultGovernorState().loadCapUsage(); +} + +export function saveCapUsage(capUsage: GovernorCapUsage): void { + return getDefaultGovernorState().saveCapUsage(capUsage); +} + +export function loadPauseState(): GovernorPauseState { + return getDefaultGovernorState().loadPauseState(); +} + +export function savePauseState(pauseState: GovernorPauseInput): GovernorPauseState { + return getDefaultGovernorState().savePauseState(pauseState); +} + +export function loadReputationHistory(repoFullName: string, apiBaseUrl?: string): RepoOutcomeHistory { + return getDefaultGovernorState().loadReputationHistory(repoFullName, apiBaseUrl); +} + +export function saveReputationHistory(repoFullName: string, history: RepoOutcomeHistory, apiBaseUrl?: string): RepoOutcomeHistory { + return getDefaultGovernorState().saveReputationHistory(repoFullName, history, apiBaseUrl); +} + +export function recordOwnSubmission(record: OwnSubmissionRecord): OwnSubmissionRecord { + return getDefaultGovernorState().recordOwnSubmission(record); +} + +export function listRecentOwnSubmissions(filter?: ListRecentOwnSubmissionsFilter): OwnSubmissionRecord[] { + return getDefaultGovernorState().listRecentOwnSubmissions(filter); +} + +export function closeDefaultGovernorState(): void { + if (!defaultGovernorState) return; + defaultGovernorState.close(); + defaultGovernorState = null; +} diff --git a/packages/loopover-miner/lib/local-store.d.ts b/packages/loopover-miner/lib/local-store.d.ts index e391c2a20d..391a52e222 100644 --- a/packages/loopover-miner/lib/local-store.d.ts +++ b/packages/loopover-miner/lib/local-store.d.ts @@ -1,27 +1,32 @@ -import type { DatabaseSync } from "node:sqlite"; - -export function resolveLocalStoreDbPath( - defaultDbFileName: string, - explicitEnvVarName: string, - env?: Record, -): string; - -export function normalizeLocalStoreDbPath( - dbPath: string | null | undefined, - resolvedDefault: string, - invalidPathError: string, -): string; - -export function openLocalStoreDb( - resolvedPath: string, - options?: { busyTimeoutMs?: number }, -): DatabaseSync; - -export function openLocalStoreAdapter( - resolvedPath: string, - options?: { busyTimeoutMs?: number }, -): { - db: DatabaseSync; - driver: import("./store-db-adapter.js").SqliteDriver; - d1: import("./store-db-adapter.js").MinerD1Database; +import { DatabaseSync } from "node:sqlite"; +import { type MinerD1Database, type SqliteDriver } from "./store-db-adapter.js"; +/** + * Resolve a local store's DB path from, in order: an explicit env var, `LOOPOVER_MINER_CONFIG_DIR`, + * `XDG_CONFIG_HOME` (falling back to `~/.config`) — mirroring every store's prior hand-written resolver. + */ +export declare function resolveLocalStoreDbPath(defaultDbFileName: string, explicitEnvVarName: string, env?: Record): string; +/** Trim and validate a caller-supplied (or resolved-default) DB path, throwing `invalidPathError` if it is empty. */ +export declare function normalizeLocalStoreDbPath(dbPath: string | null | undefined, resolvedDefault: string, invalidPathError: string): string; +/** + * Open (creating parent dirs on first use) a local store's SQLite file with 0700/0600 permissions and a shared + * busy-timeout, so two instances of the same store on one file serialize writes instead of racing. Skips the + * mkdir/chmod steps for the special `:memory:` path, which has no on-disk file. `run-state.js` previously opened + * its DB with no busy-timeout at all (the one inconsistency among the four stores this issue found); folding it + * through this shared helper gives it the same wait-don't-fail behavior the other three already had. + */ +export declare function openLocalStoreDb(resolvedPath: string, options?: { + busyTimeoutMs?: number; +}): DatabaseSync; +/** + * Open a local store through the #7175 SqliteDriver / D1 adapter seam. + * Returns the underlying DatabaseSync (for schema migrations / purge helpers that still take it), + * the sync SqliteDriver (preferred for store CRUD until a store goes fully async), and the async D1 + * adapter (same surface ORB uses — ready for a later createPgAdapter swap). + */ +export declare function openLocalStoreAdapter(resolvedPath: string, options?: { + busyTimeoutMs?: number; +}): { + db: DatabaseSync; + driver: SqliteDriver; + d1: MinerD1Database; }; diff --git a/packages/loopover-miner/lib/local-store.js b/packages/loopover-miner/lib/local-store.js index 0f18210ff2..899b20ccb0 100644 --- a/packages/loopover-miner/lib/local-store.js +++ b/packages/loopover-miner/lib/local-store.js @@ -4,7 +4,6 @@ import { dirname, join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { registerCleanupResource } from "./process-lifecycle.js"; import { createD1Adapter, nodeSqliteDriver } from "./store-db-adapter.js"; - // Shared path-resolution + DB-open boilerplate for the package's local SQLite stores (#4272). This is a DRY pass // only, not a merge: run-state.js, claim-ledger.js, portfolio-queue.js, and event-ledger.js each keep their own // `.sqlite3` file, table, and env var — this module just extracts the ~15 lines each hand-duplicated @@ -12,33 +11,31 @@ import { createD1Adapter, nodeSqliteDriver } from "./store-db-adapter.js"; // // #7175 part 1 adds `openLocalStoreAdapter`: same open path, then wraps the handle as SqliteDriver + D1 adapter // so stores can migrate onto the shared seam without changing self-host's node:sqlite default. - /** * Resolve a local store's DB path from, in order: an explicit env var, `LOOPOVER_MINER_CONFIG_DIR`, * `XDG_CONFIG_HOME` (falling back to `~/.config`) — mirroring every store's prior hand-written resolver. */ export function resolveLocalStoreDbPath(defaultDbFileName, explicitEnvVarName, env = process.env) { - const explicitPath = typeof env[explicitEnvVarName] === "string" ? env[explicitEnvVarName].trim() : ""; - if (explicitPath) return explicitPath; - - const explicitConfigDir = typeof env.LOOPOVER_MINER_CONFIG_DIR === "string" - ? env.LOOPOVER_MINER_CONFIG_DIR.trim() - : ""; - if (explicitConfigDir) return join(explicitConfigDir, defaultDbFileName); - - const configHome = typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim() - ? env.XDG_CONFIG_HOME.trim() - : join(homedir(), ".config"); - return join(configHome, "loopover-miner", defaultDbFileName); + const explicitPath = typeof env[explicitEnvVarName] === "string" ? env[explicitEnvVarName].trim() : ""; + if (explicitPath) + return explicitPath; + const explicitConfigDir = typeof env.LOOPOVER_MINER_CONFIG_DIR === "string" + ? env.LOOPOVER_MINER_CONFIG_DIR.trim() + : ""; + if (explicitConfigDir) + return join(explicitConfigDir, defaultDbFileName); + const configHome = typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim() + ? env.XDG_CONFIG_HOME.trim() + : join(homedir(), ".config"); + return join(configHome, "loopover-miner", defaultDbFileName); } - /** Trim and validate a caller-supplied (or resolved-default) DB path, throwing `invalidPathError` if it is empty. */ export function normalizeLocalStoreDbPath(dbPath, resolvedDefault, invalidPathError) { - const raw = dbPath ?? resolvedDefault; - if (typeof raw !== "string" || !raw.trim()) throw new Error(invalidPathError); - return raw.trim(); + const raw = dbPath ?? resolvedDefault; + if (typeof raw !== "string" || !raw.trim()) + throw new Error(invalidPathError); + return raw.trim(); } - /** * Open (creating parent dirs on first use) a local store's SQLite file with 0700/0600 permissions and a shared * busy-timeout, so two instances of the same store on one file serialize writes instead of racing. Skips the @@ -47,37 +44,37 @@ export function normalizeLocalStoreDbPath(dbPath, resolvedDefault, invalidPathEr * through this shared helper gives it the same wait-don't-fail behavior the other three already had. */ export function openLocalStoreDb(resolvedPath, options = {}) { - const busyTimeoutMs = options.busyTimeoutMs ?? 5000; - const isMemory = resolvedPath === ":memory:"; - if (!isMemory) mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 }); - const db = new DatabaseSync(resolvedPath); - if (!isMemory) chmodSync(resolvedPath, 0o600); - db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`); - // Crash-safety (#4826): register every opened store so a SIGINT/SIGTERM/uncaught-exception handler can close it - // mid-run instead of leaving it half-written. The normal `close()` unregisters first, so the happy path never - // double-closes and a long-running `loop` doesn't accumulate stale references. - const unregister = registerCleanupResource(db); - const originalClose = db.close.bind(db); - db.close = () => { - unregister(); - return originalClose(); - }; - return db; + const busyTimeoutMs = options.busyTimeoutMs ?? 5000; + const isMemory = resolvedPath === ":memory:"; + if (!isMemory) + mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 }); + const db = new DatabaseSync(resolvedPath); + if (!isMemory) + chmodSync(resolvedPath, 0o600); + db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`); + // Crash-safety (#4826): register every opened store so a SIGINT/SIGTERM/uncaught-exception handler can close it + // mid-run instead of leaving it half-written. The normal `close()` unregisters first, so the happy path never + // double-closes and a long-running `loop` doesn't accumulate stale references. + const unregister = registerCleanupResource(db); + const originalClose = db.close.bind(db); + // Wrap close so cleanup registration is torn down with the handle. Assignment is intentional runtime behavior + // that the DatabaseSync type does not expose as writable. + db.close = () => { + unregister(); + return originalClose(); + }; + return db; } - /** * Open a local store through the #7175 SqliteDriver / D1 adapter seam. * Returns the underlying DatabaseSync (for schema migrations / purge helpers that still take it), * the sync SqliteDriver (preferred for store CRUD until a store goes fully async), and the async D1 * adapter (same surface ORB uses — ready for a later createPgAdapter swap). - * - * @param {string} resolvedPath - * @param {{ busyTimeoutMs?: number }} [options] - * @returns {{ db: import("node:sqlite").DatabaseSync, driver: import("./store-db-adapter.js").SqliteDriver, d1: ReturnType }} */ export function openLocalStoreAdapter(resolvedPath, options = {}) { - const db = openLocalStoreDb(resolvedPath, options); - const driver = nodeSqliteDriver(db); - const d1 = createD1Adapter(driver); - return { db, driver, d1 }; + const db = openLocalStoreDb(resolvedPath, options); + const driver = nodeSqliteDriver(db); + const d1 = createD1Adapter(driver); + return { db, driver, d1 }; } +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibG9jYWwtc3RvcmUuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJsb2NhbC1zdG9yZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsU0FBUyxFQUFFLFNBQVMsRUFBRSxNQUFNLFNBQVMsQ0FBQztBQUMvQyxPQUFPLEVBQUUsT0FBTyxFQUFFLE1BQU0sU0FBUyxDQUFDO0FBQ2xDLE9BQU8sRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLE1BQU0sV0FBVyxDQUFDO0FBQzFDLE9BQU8sRUFBRSxZQUFZLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDM0MsT0FBTyxFQUFFLHVCQUF1QixFQUFFLE1BQU0sd0JBQXdCLENBQUM7QUFDakUsT0FBTyxFQUFFLGVBQWUsRUFBRSxnQkFBZ0IsRUFBMkMsTUFBTSx1QkFBdUIsQ0FBQztBQUVuSCxpSEFBaUg7QUFDakgsZ0hBQWdIO0FBQ2hILHFHQUFxRztBQUNyRyw0R0FBNEc7QUFDNUcsRUFBRTtBQUNGLGdIQUFnSDtBQUNoSCwrRkFBK0Y7QUFFL0Y7OztHQUdHO0FBQ0gsTUFBTSxVQUFVLHVCQUF1QixDQUNyQyxpQkFBeUIsRUFDekIsa0JBQTBCLEVBQzFCLE1BQTBDLE9BQU8sQ0FBQyxHQUFHO0lBRXJELE1BQU0sWUFBWSxHQUFHLE9BQU8sR0FBRyxDQUFDLGtCQUFrQixDQUFDLEtBQUssUUFBUSxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUMsa0JBQWtCLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO0lBQ3ZHLElBQUksWUFBWTtRQUFFLE9BQU8sWUFBWSxDQUFDO0lBRXRDLE1BQU0saUJBQWlCLEdBQUcsT0FBTyxHQUFHLENBQUMseUJBQXlCLEtBQUssUUFBUTtRQUN6RSxDQUFDLENBQUMsR0FBRyxDQUFDLHlCQUF5QixDQUFDLElBQUksRUFBRTtRQUN0QyxDQUFDLENBQUMsRUFBRSxDQUFDO0lBQ1AsSUFBSSxpQkFBaUI7UUFBRSxPQUFPLElBQUksQ0FBQyxpQkFBaUIsRUFBRSxpQkFBaUIsQ0FBQyxDQUFDO0lBRXpFLE1BQU0sVUFBVSxHQUFHLE9BQU8sR0FBRyxDQUFDLGVBQWUsS0FBSyxRQUFRLElBQUksR0FBRyxDQUFDLGVBQWUsQ0FBQyxJQUFJLEVBQUU7UUFDdEYsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxlQUFlLENBQUMsSUFBSSxFQUFFO1FBQzVCLENBQUMsQ0FBQyxJQUFJLENBQUMsT0FBTyxFQUFFLEVBQUUsU0FBUyxDQUFDLENBQUM7SUFDL0IsT0FBTyxJQUFJLENBQUMsVUFBVSxFQUFFLGdCQUFnQixFQUFFLGlCQUFpQixDQUFDLENBQUM7QUFDL0QsQ0FBQztBQUVELHFIQUFxSDtBQUNySCxNQUFNLFVBQVUseUJBQXlCLENBQ3ZDLE1BQWlDLEVBQ2pDLGVBQXVCLEVBQ3ZCLGdCQUF3QjtJQUV4QixNQUFNLEdBQUcsR0FBRyxNQUFNLElBQUksZUFBZSxDQUFDO0lBQ3RDLElBQUksT0FBTyxHQUFHLEtBQUssUUFBUSxJQUFJLENBQUMsR0FBRyxDQUFDLElBQUksRUFBRTtRQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsZ0JBQWdCLENBQUMsQ0FBQztJQUM5RSxPQUFPLEdBQUcsQ0FBQyxJQUFJLEVBQUUsQ0FBQztBQUNwQixDQUFDO0FBRUQ7Ozs7OztHQU1HO0FBQ0gsTUFBTSxVQUFVLGdCQUFnQixDQUM5QixZQUFvQixFQUNwQixVQUFzQyxFQUFFO0lBRXhDLE1BQU0sYUFBYSxHQUFHLE9BQU8sQ0FBQyxhQUFhLElBQUksSUFBSSxDQUFDO0lBQ3BELE1BQU0sUUFBUSxHQUFHLFlBQVksS0FBSyxVQUFVLENBQUM7SUFDN0MsSUFBSSxDQUFDLFFBQVE7UUFBRSxTQUFTLENBQUMsT0FBTyxDQUFDLFlBQVksQ0FBQyxFQUFFLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsS0FBSyxFQUFFLENBQUMsQ0FBQztJQUNsRixNQUFNLEVBQUUsR0FBRyxJQUFJLFlBQVksQ0FBQyxZQUFZLENBQUMsQ0FBQztJQUMxQyxJQUFJLENBQUMsUUFBUTtRQUFFLFNBQVMsQ0FBQyxZQUFZLEVBQUUsS0FBSyxDQUFDLENBQUM7SUFDOUMsRUFBRSxDQUFDLElBQUksQ0FBQyx5QkFBeUIsYUFBYSxFQUFFLENBQUMsQ0FBQztJQUNsRCxnSEFBZ0g7SUFDaEgsOEdBQThHO0lBQzlHLCtFQUErRTtJQUMvRSxNQUFNLFVBQVUsR0FBRyx1QkFBdUIsQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUMvQyxNQUFNLGFBQWEsR0FBRyxFQUFFLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUN4Qyw4R0FBOEc7SUFDOUcsMERBQTBEO0lBQ3pELEVBQTRCLENBQUMsS0FBSyxHQUFHLEdBQUcsRUFBRTtRQUN6QyxVQUFVLEVBQUUsQ0FBQztRQUNiLE9BQU8sYUFBYSxFQUFFLENBQUM7SUFDekIsQ0FBQyxDQUFDO0lBQ0YsT0FBTyxFQUFFLENBQUM7QUFDWixDQUFDO0FBRUQ7Ozs7O0dBS0c7QUFDSCxNQUFNLFVBQVUscUJBQXFCLENBQ25DLFlBQW9CLEVBQ3BCLFVBQXNDLEVBQUU7SUFNeEMsTUFBTSxFQUFFLEdBQUcsZ0JBQWdCLENBQUMsWUFBWSxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBQ25ELE1BQU0sTUFBTSxHQUFHLGdCQUFnQixDQUFDLEVBQUUsQ0FBQyxDQUFDO0lBQ3BDLE1BQU0sRUFBRSxHQUFHLGVBQWUsQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUNuQyxPQUFPLEVBQUUsRUFBRSxFQUFFLE1BQU0sRUFBRSxFQUFFLEVBQUUsQ0FBQztBQUM1QixDQUFDIn0= \ No newline at end of file diff --git a/packages/loopover-miner/lib/local-store.ts b/packages/loopover-miner/lib/local-store.ts new file mode 100644 index 0000000000..06600cfb66 --- /dev/null +++ b/packages/loopover-miner/lib/local-store.ts @@ -0,0 +1,99 @@ +import { chmodSync, mkdirSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { registerCleanupResource } from "./process-lifecycle.js"; +import { createD1Adapter, nodeSqliteDriver, type MinerD1Database, type SqliteDriver } from "./store-db-adapter.js"; + +// Shared path-resolution + DB-open boilerplate for the package's local SQLite stores (#4272). This is a DRY pass +// only, not a merge: run-state.js, claim-ledger.js, portfolio-queue.js, and event-ledger.js each keep their own +// `.sqlite3` file, table, and env var — this module just extracts the ~15 lines each hand-duplicated +// (env-var/config-dir/XDG path resolution, mkdirSync(0o700) + chmodSync(0o600), and `PRAGMA busy_timeout`). +// +// #7175 part 1 adds `openLocalStoreAdapter`: same open path, then wraps the handle as SqliteDriver + D1 adapter +// so stores can migrate onto the shared seam without changing self-host's node:sqlite default. + +/** + * Resolve a local store's DB path from, in order: an explicit env var, `LOOPOVER_MINER_CONFIG_DIR`, + * `XDG_CONFIG_HOME` (falling back to `~/.config`) — mirroring every store's prior hand-written resolver. + */ +export function resolveLocalStoreDbPath( + defaultDbFileName: string, + explicitEnvVarName: string, + env: Record = process.env, +): string { + const explicitPath = typeof env[explicitEnvVarName] === "string" ? env[explicitEnvVarName].trim() : ""; + if (explicitPath) return explicitPath; + + const explicitConfigDir = typeof env.LOOPOVER_MINER_CONFIG_DIR === "string" + ? env.LOOPOVER_MINER_CONFIG_DIR.trim() + : ""; + if (explicitConfigDir) return join(explicitConfigDir, defaultDbFileName); + + const configHome = typeof env.XDG_CONFIG_HOME === "string" && env.XDG_CONFIG_HOME.trim() + ? env.XDG_CONFIG_HOME.trim() + : join(homedir(), ".config"); + return join(configHome, "loopover-miner", defaultDbFileName); +} + +/** Trim and validate a caller-supplied (or resolved-default) DB path, throwing `invalidPathError` if it is empty. */ +export function normalizeLocalStoreDbPath( + dbPath: string | null | undefined, + resolvedDefault: string, + invalidPathError: string, +): string { + const raw = dbPath ?? resolvedDefault; + if (typeof raw !== "string" || !raw.trim()) throw new Error(invalidPathError); + return raw.trim(); +} + +/** + * Open (creating parent dirs on first use) a local store's SQLite file with 0700/0600 permissions and a shared + * busy-timeout, so two instances of the same store on one file serialize writes instead of racing. Skips the + * mkdir/chmod steps for the special `:memory:` path, which has no on-disk file. `run-state.js` previously opened + * its DB with no busy-timeout at all (the one inconsistency among the four stores this issue found); folding it + * through this shared helper gives it the same wait-don't-fail behavior the other three already had. + */ +export function openLocalStoreDb( + resolvedPath: string, + options: { busyTimeoutMs?: number } = {}, +): DatabaseSync { + const busyTimeoutMs = options.busyTimeoutMs ?? 5000; + const isMemory = resolvedPath === ":memory:"; + if (!isMemory) mkdirSync(dirname(resolvedPath), { recursive: true, mode: 0o700 }); + const db = new DatabaseSync(resolvedPath); + if (!isMemory) chmodSync(resolvedPath, 0o600); + db.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}`); + // Crash-safety (#4826): register every opened store so a SIGINT/SIGTERM/uncaught-exception handler can close it + // mid-run instead of leaving it half-written. The normal `close()` unregisters first, so the happy path never + // double-closes and a long-running `loop` doesn't accumulate stale references. + const unregister = registerCleanupResource(db); + const originalClose = db.close.bind(db); + // Wrap close so cleanup registration is torn down with the handle. Assignment is intentional runtime behavior + // that the DatabaseSync type does not expose as writable. + (db as { close: () => void }).close = () => { + unregister(); + return originalClose(); + }; + return db; +} + +/** + * Open a local store through the #7175 SqliteDriver / D1 adapter seam. + * Returns the underlying DatabaseSync (for schema migrations / purge helpers that still take it), + * the sync SqliteDriver (preferred for store CRUD until a store goes fully async), and the async D1 + * adapter (same surface ORB uses — ready for a later createPgAdapter swap). + */ +export function openLocalStoreAdapter( + resolvedPath: string, + options: { busyTimeoutMs?: number } = {}, +): { + db: DatabaseSync; + driver: SqliteDriver; + d1: MinerD1Database; +} { + const db = openLocalStoreDb(resolvedPath, options); + const driver = nodeSqliteDriver(db); + const d1 = createD1Adapter(driver); + return { db, driver, d1 }; +} diff --git a/packages/loopover-miner/lib/prediction-ledger.d.ts b/packages/loopover-miner/lib/prediction-ledger.d.ts index c9c9174654..39db201346 100644 --- a/packages/loopover-miner/lib/prediction-ledger.d.ts +++ b/packages/loopover-miner/lib/prediction-ledger.d.ts @@ -1,47 +1,43 @@ export type PredictionLedgerEntry = { - id: number; - ts: string; - repoFullName: string; - targetId: number; - headSha: string | null; - conclusion: string; - pack: string; - readinessScore: number | null; - blockerCodes: string[]; - warningCodes: string[]; - engineVersion: string; + id: number; + ts: string; + repoFullName: string; + targetId: number; + headSha: string | null; + conclusion: string; + pack: string; + readinessScore: number | null; + blockerCodes: string[]; + warningCodes: string[]; + engineVersion: string; }; - export type AppendPredictionInput = { - repoFullName: string; - targetId: number; - headSha?: string | null; - conclusion: string; - pack: string; - readinessScore?: number | null; - blockerCodes?: string[]; - warningCodes?: string[]; - engineVersion: string; + repoFullName: string; + targetId: number; + headSha?: string | null; + conclusion: string; + pack: string; + readinessScore?: number | null; + blockerCodes?: string[]; + warningCodes?: string[]; + engineVersion: string; }; - export type ReadPredictionsFilter = { - repoFullName?: string | null; + repoFullName?: string | null; }; - export type PredictionLedger = { - dbPath: string; - appendPrediction(input: AppendPredictionInput): PredictionLedgerEntry; - readPredictions(filter?: ReadPredictionsFilter): PredictionLedgerEntry[]; - purgeByRepo(repoFullName: string): number; - close(): void; + dbPath: string; + appendPrediction(input: AppendPredictionInput): PredictionLedgerEntry; + readPredictions(filter?: ReadPredictionsFilter): PredictionLedgerEntry[]; + purgeByRepo(repoFullName: string): number; + close(): void; }; - -export function resolvePredictionLedgerDbPath(env?: Record): string; - -export function initPredictionLedger(dbPath?: string): PredictionLedger; - -export function appendPrediction(input: AppendPredictionInput): PredictionLedgerEntry; - -export function readPredictions(filter?: ReadPredictionsFilter): PredictionLedgerEntry[]; - -export function closeDefaultPredictionLedger(): void; +export declare function resolvePredictionLedgerDbPath(env?: Record): string; +/** + * Opens the append-only prediction ledger, creating the table on first use. Rows are returned in ascending `id` + * order (insertion order). (#4263) + */ +export declare function initPredictionLedger(dbPath?: string): PredictionLedger; +export declare function appendPrediction(input: AppendPredictionInput): PredictionLedgerEntry; +export declare function readPredictions(filter?: ReadPredictionsFilter): PredictionLedgerEntry[]; +export declare function closeDefaultPredictionLedger(): void; diff --git a/packages/loopover-miner/lib/prediction-ledger.js b/packages/loopover-miner/lib/prediction-ledger.js index cdba896920..f3e6575e19 100644 --- a/packages/loopover-miner/lib/prediction-ledger.js +++ b/packages/loopover-miner/lib/prediction-ledger.js @@ -1,138 +1,128 @@ import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js"; import { applySchemaMigrations } from "./schema-version.js"; -import { - PREDICTION_LEDGER_PURGE_SPEC, - PREDICTION_LEDGER_RETENTION_SPEC, - purgeStoreByRepo, - pruneLedgerByRetention, - resolveLedgerRetentionPolicy, -} from "./store-maintenance.js"; - -// Append-only prediction ledger (#4263): every predicted-gate verdict the miner computes for a target lands in -// a local SQLite table so a later self-improve pass can score the prediction against the realized pr_outcome. -// IMMUTABILITY INVARIANT: `appendPrediction`/`readPredictions` only ever issue INSERT and SELECT — never -// UPDATE/DELETE. Two documented exceptions, both separate maintenance operations rather than part of normal -// ledger operation: opt-in retention pruning (#4834, automatic) and `purgeByRepo` (#5564, always explicit and -// operator-invoked, never automatic). Rows are kept small and stable for later diffing: blocker/warning CODES -// only (no free-text detail), plus the ENGINE_VERSION that produced the call so a row self-reports which engine -// build made it. Mirrors governor-ledger.js's shape; normalization is local (like event-ledger.js) so the -// offline miner package pulls in no engine module. - +import { PREDICTION_LEDGER_PURGE_SPEC, PREDICTION_LEDGER_RETENTION_SPEC, purgeStoreByRepo, pruneLedgerByRetention, resolveLedgerRetentionPolicy, } from "./store-maintenance.js"; const defaultDbFileName = "prediction-ledger.sqlite3"; let defaultPredictionLedger = null; - export function resolvePredictionLedgerDbPath(env = process.env) { - return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_PREDICTION_LEDGER_DB", env); + return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_PREDICTION_LEDGER_DB", env); } - function normalizeDbPath(dbPath) { - return normalizeLocalStoreDbPath(dbPath, resolvePredictionLedgerDbPath(), "invalid_prediction_ledger_db_path"); + return normalizeLocalStoreDbPath(dbPath, resolvePredictionLedgerDbPath(), "invalid_prediction_ledger_db_path"); } - function normalizeRepoFullName(repoFullName) { - if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name"); - const [owner, repo, extra] = repoFullName.trim().split("/"); - if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name"); - return `${owner}/${repo}`; + if (typeof repoFullName !== "string") + throw new Error("invalid_repo_full_name"); + const [owner, repo, extra] = repoFullName.trim().split("/"); + if (!owner || !repo || extra !== undefined) + throw new Error("invalid_repo_full_name"); + return `${owner}/${repo}`; } - function normalizeOptionalRepoFullName(repoFullName) { - if (repoFullName === undefined || repoFullName === null) return undefined; - return normalizeRepoFullName(repoFullName); + if (repoFullName === undefined || repoFullName === null) + return undefined; + return normalizeRepoFullName(repoFullName); } - function requiredNonEmptyString(value, error) { - if (typeof value !== "string" || !value.trim()) throw new Error(error); - return value.trim(); + if (typeof value !== "string" || !value.trim()) + throw new Error(error); + return value.trim(); } - function optionalString(value) { - if (value === undefined || value === null) return null; - if (typeof value !== "string") throw new Error("invalid_head_sha"); - const trimmed = value.trim(); - return trimmed || null; + if (value === undefined || value === null) + return null; + if (typeof value !== "string") + throw new Error("invalid_head_sha"); + const trimmed = value.trim(); + return trimmed || null; } - // Codes are stored as a JSON array of the non-empty trimmed strings, in order — a stable, small projection of a // verdict's blockers/warnings that drops all free-text detail. function normalizeCodes(codes, error) { - if (codes === undefined || codes === null) return []; - if (!Array.isArray(codes)) throw new Error(error); - return codes.map((code) => { - if (typeof code !== "string" || !code.trim()) throw new Error(error); - return code.trim(); - }); -} - + if (codes === undefined || codes === null) + return []; + if (!Array.isArray(codes)) + throw new Error(error); + return codes.map((code) => { + if (typeof code !== "string" || !code.trim()) + throw new Error(error); + return code.trim(); + }); +} function normalizeReadinessScore(value) { - if (value === undefined || value === null) return null; - if (typeof value !== "number" || !Number.isFinite(value)) throw new Error("invalid_readiness_score"); - return value; + if (value === undefined || value === null) + return null; + if (typeof value !== "number" || !Number.isFinite(value)) + throw new Error("invalid_readiness_score"); + return value; } - /** Validate + normalize an append input, throwing on any invalid field (mirrors normalizeGovernorLedgerEvent). */ function normalizePredictionInput(input) { - if (!input || typeof input !== "object" || Array.isArray(input)) throw new Error("invalid_prediction_input"); - if (!Number.isInteger(input.targetId) || input.targetId <= 0) throw new Error("invalid_target_id"); - return { - repoFullName: normalizeRepoFullName(input.repoFullName), - targetId: input.targetId, - headSha: optionalString(input.headSha), - conclusion: requiredNonEmptyString(input.conclusion, "invalid_conclusion"), - pack: requiredNonEmptyString(input.pack, "invalid_pack"), - readinessScore: normalizeReadinessScore(input.readinessScore), - blockerCodes: normalizeCodes(input.blockerCodes, "invalid_blocker_codes"), - warningCodes: normalizeCodes(input.warningCodes, "invalid_warning_codes"), - engineVersion: requiredNonEmptyString(input.engineVersion, "invalid_engine_version"), - }; -} - + if (!input || typeof input !== "object" || Array.isArray(input)) + throw new Error("invalid_prediction_input"); + if (!Number.isInteger(input.targetId) || input.targetId <= 0) + throw new Error("invalid_target_id"); + return { + repoFullName: normalizeRepoFullName(input.repoFullName), + targetId: input.targetId, + headSha: optionalString(input.headSha), + conclusion: requiredNonEmptyString(input.conclusion, "invalid_conclusion"), + pack: requiredNonEmptyString(input.pack, "invalid_pack"), + readinessScore: normalizeReadinessScore(input.readinessScore), + blockerCodes: normalizeCodes(input.blockerCodes, "invalid_blocker_codes"), + warningCodes: normalizeCodes(input.warningCodes, "invalid_warning_codes"), + engineVersion: requiredNonEmptyString(input.engineVersion, "invalid_engine_version"), + }; +} function rowToEntry(row) { - let blockerCodes; - let warningCodes; - try { - blockerCodes = JSON.parse(row.blocker_codes_json); - warningCodes = JSON.parse(row.warning_codes_json); - if (!Array.isArray(blockerCodes) || !Array.isArray(warningCodes)) throw new Error("corrupted_prediction_row"); - } catch { - throw new Error("corrupted_prediction_row"); - } - return { - id: row.id, - ts: row.ts, - repoFullName: row.repo_full_name, - targetId: row.target_id, - headSha: row.head_sha, - conclusion: row.conclusion, - pack: row.pack, - readinessScore: row.readiness_score, - blockerCodes, - warningCodes, - engineVersion: row.engine_version, - }; -} - + let blockerCodes; + let warningCodes; + try { + blockerCodes = JSON.parse(row.blocker_codes_json); + warningCodes = JSON.parse(row.warning_codes_json); + if (!Array.isArray(blockerCodes) || !Array.isArray(warningCodes)) + throw new Error("corrupted_prediction_row"); + } + catch { + throw new Error("corrupted_prediction_row"); + } + return { + id: row.id, + ts: row.ts, + repoFullName: row.repo_full_name, + targetId: row.target_id, + headSha: row.head_sha, + conclusion: row.conclusion, + pack: row.pack, + readinessScore: row.readiness_score, + blockerCodes: blockerCodes, + warningCodes: warningCodes, + engineVersion: row.engine_version, + }; +} +function asPredictionDbRow(row) { + return row; +} // v1 -> v2 (#4939): additive tenant-scoping column, a prerequisite for any hosted, multi-tenant use of this // same store's logic. NULL for every row today -- self-host behavior is byte-identical, since nothing reads or // writes it yet (no consumer exists until a future hosted deployment populates it). Same defensive // column-presence guard as this file's sibling stores' own additive migrations (e.g. event-ledger.js's and // run-state.js's own tenant_id additions), so re-running it against an already-migrated file is a no-op. function addTenantIdColumn(db) { - const hasTenantIdColumn = db - .prepare("PRAGMA table_info(predictions)") - .all() - .some((column) => column.name === "tenant_id"); - if (!hasTenantIdColumn) db.exec("ALTER TABLE predictions ADD COLUMN tenant_id TEXT"); + const hasTenantIdColumn = db + .prepare("PRAGMA table_info(predictions)") + .all() + .some((column) => column.name === "tenant_id"); + if (!hasTenantIdColumn) + db.exec("ALTER TABLE predictions ADD COLUMN tenant_id TEXT"); } - /** * Opens the append-only prediction ledger, creating the table on first use. Rows are returned in ascending `id` * order (insertion order). (#4263) */ export function initPredictionLedger(dbPath = resolvePredictionLedgerDbPath()) { - const resolvedPath = normalizeDbPath(dbPath); - const db = openLocalStoreDb(resolvedPath); - db.exec(` + const resolvedPath = normalizeDbPath(dbPath); + const db = openLocalStoreDb(resolvedPath); + db.exec(` CREATE TABLE IF NOT EXISTS predictions ( id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT NOT NULL, @@ -147,71 +137,56 @@ export function initPredictionLedger(dbPath = resolvePredictionLedgerDbPath()) { engine_version TEXT NOT NULL ) `); - db.exec("CREATE INDEX IF NOT EXISTS idx_predictions_repo ON predictions (repo_full_name, id)"); - // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. - applySchemaMigrations(db, [addTenantIdColumn]); - // Opt-in retention (#4834): prune aged/excess rows when an operator has enabled it; a no-op by default. - pruneLedgerByRetention(db, PREDICTION_LEDGER_RETENTION_SPEC, resolveLedgerRetentionPolicy(), Date.now()); - - const appendStatement = db.prepare(` + db.exec("CREATE INDEX IF NOT EXISTS idx_predictions_repo ON predictions (repo_full_name, id)"); + // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. + applySchemaMigrations(db, [addTenantIdColumn]); + // Opt-in retention (#4834): prune aged/excess rows when an operator has enabled it; a no-op by default. + pruneLedgerByRetention(db, PREDICTION_LEDGER_RETENTION_SPEC, resolveLedgerRetentionPolicy(), Date.now()); + const appendStatement = db.prepare(` INSERT INTO predictions (ts, repo_full_name, target_id, head_sha, conclusion, pack, readiness_score, blocker_codes_json, warning_codes_json, engine_version) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); - const getByIdStatement = db.prepare("SELECT * FROM predictions WHERE id = ?"); - const readAllStatement = db.prepare("SELECT * FROM predictions ORDER BY id ASC"); - const readByRepoStatement = db.prepare("SELECT * FROM predictions WHERE repo_full_name = ? ORDER BY id ASC"); - - return { - dbPath: resolvedPath, - appendPrediction(input) { - const n = normalizePredictionInput(input); - const ts = new Date().toISOString(); - const result = appendStatement.run( - ts, - n.repoFullName, - n.targetId, - n.headSha, - n.conclusion, - n.pack, - n.readinessScore, - JSON.stringify(n.blockerCodes), - JSON.stringify(n.warningCodes), - n.engineVersion, - ); - return rowToEntry(getByIdStatement.get(Number(result.lastInsertRowid))); - }, - readPredictions(filter = {}) { - const repoFullName = normalizeOptionalRepoFullName(filter.repoFullName); - const rows = repoFullName === undefined ? readAllStatement.all() : readByRepoStatement.all(repoFullName); - return rows.map(rowToEntry); - }, - // Explicit, operator-invoked right-to-be-forgotten purge (#5564) — never runs automatically. See the - // IMMUTABILITY INVARIANT note above: this is a deliberate, separate exception, not a normal ledger write. - purgeByRepo(repoFullName) { - return purgeStoreByRepo(db, PREDICTION_LEDGER_PURGE_SPEC, normalizeRepoFullName(repoFullName)); - }, - close() { - db.close(); - }, - }; -} - + const getByIdStatement = db.prepare("SELECT * FROM predictions WHERE id = ?"); + const readAllStatement = db.prepare("SELECT * FROM predictions ORDER BY id ASC"); + const readByRepoStatement = db.prepare("SELECT * FROM predictions WHERE repo_full_name = ? ORDER BY id ASC"); + return { + dbPath: resolvedPath, + appendPrediction(input) { + const n = normalizePredictionInput(input); + const ts = new Date().toISOString(); + const result = appendStatement.run(ts, n.repoFullName, n.targetId, n.headSha, n.conclusion, n.pack, n.readinessScore, JSON.stringify(n.blockerCodes), JSON.stringify(n.warningCodes), n.engineVersion); + return rowToEntry(asPredictionDbRow(getByIdStatement.get(Number(result.lastInsertRowid)))); + }, + readPredictions(filter = {}) { + const repoFullName = normalizeOptionalRepoFullName(filter.repoFullName); + const rows = repoFullName === undefined ? readAllStatement.all() : readByRepoStatement.all(repoFullName); + return rows.map((row) => rowToEntry(asPredictionDbRow(row))); + }, + // Explicit, operator-invoked right-to-be-forgotten purge (#5564) — never runs automatically. See the + // IMMUTABILITY INVARIANT note above: this is a deliberate, separate exception, not a normal ledger write. + purgeByRepo(repoFullName) { + return purgeStoreByRepo(db, PREDICTION_LEDGER_PURGE_SPEC, normalizeRepoFullName(repoFullName)); + }, + close() { + db.close(); + }, + }; +} function getDefaultPredictionLedger() { - defaultPredictionLedger ??= initPredictionLedger(); - return defaultPredictionLedger; + defaultPredictionLedger ??= initPredictionLedger(); + return defaultPredictionLedger; } - export function appendPrediction(input) { - return getDefaultPredictionLedger().appendPrediction(input); + return getDefaultPredictionLedger().appendPrediction(input); } - export function readPredictions(filter) { - return getDefaultPredictionLedger().readPredictions(filter); + return getDefaultPredictionLedger().readPredictions(filter); } - export function closeDefaultPredictionLedger() { - if (!defaultPredictionLedger) return; - defaultPredictionLedger.close(); - defaultPredictionLedger = null; + if (!defaultPredictionLedger) + return; + defaultPredictionLedger.close(); + defaultPredictionLedger = null; } +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJlZGljdGlvbi1sZWRnZXIuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJwcmVkaWN0aW9uLWxlZGdlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFDQSxPQUFPLEVBQUUseUJBQXlCLEVBQUUsZ0JBQWdCLEVBQUUsdUJBQXVCLEVBQUUsTUFBTSxrQkFBa0IsQ0FBQztBQUN4RyxPQUFPLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxxQkFBcUIsQ0FBQztBQUM1RCxPQUFPLEVBQ0wsNEJBQTRCLEVBQzVCLGdDQUFnQyxFQUNoQyxnQkFBZ0IsRUFDaEIsc0JBQXNCLEVBQ3RCLDRCQUE0QixHQUM3QixNQUFNLHdCQUF3QixDQUFDO0FBaUVoQyxNQUFNLGlCQUFpQixHQUFHLDJCQUEyQixDQUFDO0FBQ3RELElBQUksdUJBQXVCLEdBQTRCLElBQUksQ0FBQztBQUU1RCxNQUFNLFVBQVUsNkJBQTZCLENBQUMsTUFBMEMsT0FBTyxDQUFDLEdBQUc7SUFDakcsT0FBTyx1QkFBdUIsQ0FBQyxpQkFBaUIsRUFBRSxxQ0FBcUMsRUFBRSxHQUFHLENBQUMsQ0FBQztBQUNoRyxDQUFDO0FBRUQsU0FBUyxlQUFlLENBQUMsTUFBYztJQUNyQyxPQUFPLHlCQUF5QixDQUFDLE1BQU0sRUFBRSw2QkFBNkIsRUFBRSxFQUFFLG1DQUFtQyxDQUFDLENBQUM7QUFDakgsQ0FBQztBQUVELFNBQVMscUJBQXFCLENBQUMsWUFBb0I7SUFDakQsSUFBSSxPQUFPLFlBQVksS0FBSyxRQUFRO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyx3QkFBd0IsQ0FBQyxDQUFDO0lBQ2hGLE1BQU0sQ0FBQyxLQUFLLEVBQUUsSUFBSSxFQUFFLEtBQUssQ0FBQyxHQUFHLFlBQVksQ0FBQyxJQUFJLEVBQUUsQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDNUQsSUFBSSxDQUFDLEtBQUssSUFBSSxDQUFDLElBQUksSUFBSSxLQUFLLEtBQUssU0FBUztRQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsd0JBQXdCLENBQUMsQ0FBQztJQUN0RixPQUFPLEdBQUcsS0FBSyxJQUFJLElBQUksRUFBRSxDQUFDO0FBQzVCLENBQUM7QUFFRCxTQUFTLDZCQUE2QixDQUFDLFlBQXVDO0lBQzVFLElBQUksWUFBWSxLQUFLLFNBQVMsSUFBSSxZQUFZLEtBQUssSUFBSTtRQUFFLE9BQU8sU0FBUyxDQUFDO0lBQzFFLE9BQU8scUJBQXFCLENBQUMsWUFBWSxDQUFDLENBQUM7QUFDN0MsQ0FBQztBQUVELFNBQVMsc0JBQXNCLENBQUMsS0FBYyxFQUFFLEtBQWE7SUFDM0QsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxFQUFFO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQztJQUN2RSxPQUFPLEtBQUssQ0FBQyxJQUFJLEVBQUUsQ0FBQztBQUN0QixDQUFDO0FBRUQsU0FBUyxjQUFjLENBQUMsS0FBZ0M7SUFDdEQsSUFBSSxLQUFLLEtBQUssU0FBUyxJQUFJLEtBQUssS0FBSyxJQUFJO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDdkQsSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyxrQkFBa0IsQ0FBQyxDQUFDO0lBQ25FLE1BQU0sT0FBTyxHQUFHLEtBQUssQ0FBQyxJQUFJLEVBQUUsQ0FBQztJQUM3QixPQUFPLE9BQU8sSUFBSSxJQUFJLENBQUM7QUFDekIsQ0FBQztBQUVELGdIQUFnSDtBQUNoSCwrREFBK0Q7QUFDL0QsU0FBUyxjQUFjLENBQUMsS0FBa0MsRUFBRSxLQUFhO0lBQ3ZFLElBQUksS0FBSyxLQUFLLFNBQVMsSUFBSSxLQUFLLEtBQUssSUFBSTtRQUFFLE9BQU8sRUFBRSxDQUFDO0lBQ3JELElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLEtBQUssQ0FBQztRQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsS0FBSyxDQUFDLENBQUM7SUFDbEQsT0FBTyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxFQUFFLEVBQUU7UUFDeEIsSUFBSSxPQUFPLElBQUksS0FBSyxRQUFRLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxFQUFFO1lBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyxLQUFLLENBQUMsQ0FBQztRQUNyRSxPQUFPLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztJQUNyQixDQUFDLENBQUMsQ0FBQztBQUNMLENBQUM7QUFFRCxTQUFTLHVCQUF1QixDQUFDLEtBQWdDO0lBQy9ELElBQUksS0FBSyxLQUFLLFNBQVMsSUFBSSxLQUFLLEtBQUssSUFBSTtRQUFFLE9BQU8sSUFBSSxDQUFDO0lBQ3ZELElBQUksT0FBTyxLQUFLLEtBQUssUUFBUSxJQUFJLENBQUMsTUFBTSxDQUFDLFFBQVEsQ0FBQyxLQUFLLENBQUM7UUFBRSxNQUFNLElBQUksS0FBSyxDQUFDLHlCQUF5QixDQUFDLENBQUM7SUFDckcsT0FBTyxLQUFLLENBQUM7QUFDZixDQUFDO0FBRUQsa0hBQWtIO0FBQ2xILFNBQVMsd0JBQXdCLENBQUMsS0FBNEI7SUFXNUQsSUFBSSxDQUFDLEtBQUssSUFBSSxPQUFPLEtBQUssS0FBSyxRQUFRLElBQUksS0FBSyxDQUFDLE9BQU8sQ0FBQyxLQUFLLENBQUM7UUFBRSxNQUFNLElBQUksS0FBSyxDQUFDLDBCQUEwQixDQUFDLENBQUM7SUFDN0csSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsS0FBSyxDQUFDLFFBQVEsQ0FBQyxJQUFJLEtBQUssQ0FBQyxRQUFRLElBQUksQ0FBQztRQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsbUJBQW1CLENBQUMsQ0FBQztJQUNuRyxPQUFPO1FBQ0wsWUFBWSxFQUFFLHFCQUFxQixDQUFDLEtBQUssQ0FBQyxZQUFZLENBQUM7UUFDdkQsUUFBUSxFQUFFLEtBQUssQ0FBQyxRQUFRO1FBQ3hCLE9BQU8sRUFBRSxjQUFjLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQztRQUN0QyxVQUFVLEVBQUUsc0JBQXNCLENBQUMsS0FBSyxDQUFDLFVBQVUsRUFBRSxvQkFBb0IsQ0FBQztRQUMxRSxJQUFJLEVBQUUsc0JBQXNCLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxjQUFjLENBQUM7UUFDeEQsY0FBYyxFQUFFLHVCQUF1QixDQUFDLEtBQUssQ0FBQyxjQUFjLENBQUM7UUFDN0QsWUFBWSxFQUFFLGNBQWMsQ0FBQyxLQUFLLENBQUMsWUFBWSxFQUFFLHVCQUF1QixDQUFDO1FBQ3pFLFlBQVksRUFBRSxjQUFjLENBQUMsS0FBSyxDQUFDLFlBQVksRUFBRSx1QkFBdUIsQ0FBQztRQUN6RSxhQUFhLEVBQUUsc0JBQXNCLENBQUMsS0FBSyxDQUFDLGFBQWEsRUFBRSx3QkFBd0IsQ0FBQztLQUNyRixDQUFDO0FBQ0osQ0FBQztBQUVELFNBQVMsVUFBVSxDQUFDLEdBQW9CO0lBQ3RDLElBQUksWUFBcUIsQ0FBQztJQUMxQixJQUFJLFlBQXFCLENBQUM7SUFDMUIsSUFBSSxDQUFDO1FBQ0gsWUFBWSxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLGtCQUFrQixDQUFDLENBQUM7UUFDbEQsWUFBWSxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDLGtCQUFrQixDQUFDLENBQUM7UUFDbEQsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsWUFBWSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDLFlBQVksQ0FBQztZQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsMEJBQTBCLENBQUMsQ0FBQztJQUNoSCxDQUFDO0lBQUMsTUFBTSxDQUFDO1FBQ1AsTUFBTSxJQUFJLEtBQUssQ0FBQywwQkFBMEIsQ0FBQyxDQUFDO0lBQzlDLENBQUM7SUFDRCxPQUFPO1FBQ0wsRUFBRSxFQUFFLEdBQUcsQ0FBQyxFQUFFO1FBQ1YsRUFBRSxFQUFFLEdBQUcsQ0FBQyxFQUFFO1FBQ1YsWUFBWSxFQUFFLEdBQUcsQ0FBQyxjQUFjO1FBQ2hDLFFBQVEsRUFBRSxHQUFHLENBQUMsU0FBUztRQUN2QixPQUFPLEVBQUUsR0FBRyxDQUFDLFFBQVE7UUFDckIsVUFBVSxFQUFFLEdBQUcsQ0FBQyxVQUFVO1FBQzFCLElBQUksRUFBRSxHQUFHLENBQUMsSUFBSTtRQUNkLGNBQWMsRUFBRSxHQUFHLENBQUMsZUFBZTtRQUNuQyxZQUFZLEVBQUUsWUFBd0I7UUFDdEMsWUFBWSxFQUFFLFlBQXdCO1FBQ3RDLGFBQWEsRUFBRSxHQUFHLENBQUMsY0FBYztLQUNsQyxDQUFDO0FBQ0osQ0FBQztBQUVELFNBQVMsaUJBQWlCLENBQUMsR0FBbUM7SUFDNUQsT0FBTyxHQUFpQyxDQUFDO0FBQzNDLENBQUM7QUFFRCw0R0FBNEc7QUFDNUcsK0dBQStHO0FBQy9HLG1HQUFtRztBQUNuRywyR0FBMkc7QUFDM0cseUdBQXlHO0FBQ3pHLFNBQVMsaUJBQWlCLENBQUMsRUFBZ0I7SUFDekMsTUFBTSxpQkFBaUIsR0FBRyxFQUFFO1NBQ3pCLE9BQU8sQ0FBQyxnQ0FBZ0MsQ0FBQztTQUN6QyxHQUFHLEVBQUU7U0FDTCxJQUFJLENBQUMsQ0FBQyxNQUFNLEVBQUUsRUFBRSxDQUFDLE1BQU0sQ0FBQyxJQUFJLEtBQUssV0FBVyxDQUFDLENBQUM7SUFDakQsSUFBSSxDQUFDLGlCQUFpQjtRQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsbURBQW1ELENBQUMsQ0FBQztBQUN2RixDQUFDO0FBRUQ7OztHQUdHO0FBQ0gsTUFBTSxVQUFVLG9CQUFvQixDQUFDLFNBQWlCLDZCQUE2QixFQUFFO0lBQ25GLE1BQU0sWUFBWSxHQUFHLGVBQWUsQ0FBQyxNQUFNLENBQUMsQ0FBQztJQUM3QyxNQUFNLEVBQUUsR0FBRyxnQkFBZ0IsQ0FBQyxZQUFZLENBQUMsQ0FBQztJQUMxQyxFQUFFLENBQUMsSUFBSSxDQUFDOzs7Ozs7Ozs7Ozs7OztHQWNQLENBQUMsQ0FBQztJQUNILEVBQUUsQ0FBQyxJQUFJLENBQUMscUZBQXFGLENBQUMsQ0FBQztJQUMvRiw4RkFBOEY7SUFDOUYscUJBQXFCLENBQUMsRUFBRSxFQUFFLENBQUMsaUJBQWlCLENBQUMsQ0FBQyxDQUFDO0lBQy9DLHdHQUF3RztJQUN4RyxzQkFBc0IsQ0FBQyxFQUFFLEVBQUUsZ0NBQWdDLEVBQUUsNEJBQTRCLEVBQUUsRUFBRSxJQUFJLENBQUMsR0FBRyxFQUFFLENBQUMsQ0FBQztJQUV6RyxNQUFNLGVBQWUsR0FBRyxFQUFFLENBQUMsT0FBTyxDQUFDOzs7O0dBSWxDLENBQUMsQ0FBQztJQUNILE1BQU0sZ0JBQWdCLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FBQyx3Q0FBd0MsQ0FBQyxDQUFDO0lBQzlFLE1BQU0sZ0JBQWdCLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FBQywyQ0FBMkMsQ0FBQyxDQUFDO0lBQ2pGLE1BQU0sbUJBQW1CLEdBQUcsRUFBRSxDQUFDLE9BQU8sQ0FBQyxvRUFBb0UsQ0FBQyxDQUFDO0lBRTdHLE9BQU87UUFDTCxNQUFNLEVBQUUsWUFBWTtRQUNwQixnQkFBZ0IsQ0FBQyxLQUFLO1lBQ3BCLE1BQU0sQ0FBQyxHQUFHLHdCQUF3QixDQUFDLEtBQUssQ0FBQyxDQUFDO1lBQzFDLE1BQU0sRUFBRSxHQUFHLElBQUksSUFBSSxFQUFFLENBQUMsV0FBVyxFQUFFLENBQUM7WUFDcEMsTUFBTSxNQUFNLEdBQUcsZUFBZSxDQUFDLEdBQUcsQ0FDaEMsRUFBRSxFQUNGLENBQUMsQ0FBQyxZQUFZLEVBQ2QsQ0FBQyxDQUFDLFFBQVEsRUFDVixDQUFDLENBQUMsT0FBTyxFQUNULENBQUMsQ0FBQyxVQUFVLEVBQ1osQ0FBQyxDQUFDLElBQUksRUFDTixDQUFDLENBQUMsY0FBYyxFQUNoQixJQUFJLENBQUMsU0FBUyxDQUFDLENBQUMsQ0FBQyxZQUFZLENBQUMsRUFDOUIsSUFBSSxDQUFDLFNBQVMsQ0FBQyxDQUFDLENBQUMsWUFBWSxDQUFDLEVBQzlCLENBQUMsQ0FBQyxhQUFhLENBQ2hCLENBQUM7WUFDRixPQUFPLFVBQVUsQ0FBQyxpQkFBaUIsQ0FBQyxnQkFBZ0IsQ0FBQyxHQUFHLENBQUMsTUFBTSxDQUFDLE1BQU0sQ0FBQyxlQUFlLENBQUMsQ0FBRSxDQUFDLENBQUMsQ0FBQztRQUM5RixDQUFDO1FBQ0QsZUFBZSxDQUFDLE1BQU0sR0FBRyxFQUFFO1lBQ3pCLE1BQU0sWUFBWSxHQUFHLDZCQUE2QixDQUFDLE1BQU0sQ0FBQyxZQUFZLENBQUMsQ0FBQztZQUN4RSxNQUFNLElBQUksR0FBRyxZQUFZLEtBQUssU0FBUyxDQUFDLENBQUMsQ0FBQyxnQkFBZ0IsQ0FBQyxHQUFHLEVBQUUsQ0FBQyxDQUFDLENBQUMsbUJBQW1CLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDO1lBQ3pHLE9BQU8sSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFLENBQUMsVUFBVSxDQUFDLGlCQUFpQixDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUMvRCxDQUFDO1FBQ0QscUdBQXFHO1FBQ3JHLDBHQUEwRztRQUMxRyxXQUFXLENBQUMsWUFBWTtZQUN0QixPQUFPLGdCQUFnQixDQUFDLEVBQUUsRUFBRSw0QkFBNEIsRUFBRSxxQkFBcUIsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFDO1FBQ2pHLENBQUM7UUFDRCxLQUFLO1lBQ0gsRUFBRSxDQUFDLEtBQUssRUFBRSxDQUFDO1FBQ2IsQ0FBQztLQUNGLENBQUM7QUFDSixDQUFDO0FBRUQsU0FBUywwQkFBMEI7SUFDakMsdUJBQXVCLEtBQUssb0JBQW9CLEVBQUUsQ0FBQztJQUNuRCxPQUFPLHVCQUF1QixDQUFDO0FBQ2pDLENBQUM7QUFFRCxNQUFNLFVBQVUsZ0JBQWdCLENBQUMsS0FBNEI7SUFDM0QsT0FBTywwQkFBMEIsRUFBRSxDQUFDLGdCQUFnQixDQUFDLEtBQUssQ0FBQyxDQUFDO0FBQzlELENBQUM7QUFFRCxNQUFNLFVBQVUsZUFBZSxDQUFDLE1BQThCO0lBQzVELE9BQU8sMEJBQTBCLEVBQUUsQ0FBQyxlQUFlLENBQUMsTUFBTSxDQUFDLENBQUM7QUFDOUQsQ0FBQztBQUVELE1BQU0sVUFBVSw0QkFBNEI7SUFDMUMsSUFBSSxDQUFDLHVCQUF1QjtRQUFFLE9BQU87SUFDckMsdUJBQXVCLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDaEMsdUJBQXVCLEdBQUcsSUFBSSxDQUFDO0FBQ2pDLENBQUMifQ== \ No newline at end of file diff --git a/packages/loopover-miner/lib/prediction-ledger.ts b/packages/loopover-miner/lib/prediction-ledger.ts new file mode 100644 index 0000000000..a7bff5674b --- /dev/null +++ b/packages/loopover-miner/lib/prediction-ledger.ts @@ -0,0 +1,285 @@ +import type { DatabaseSync, SQLOutputValue } from "node:sqlite"; +import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js"; +import { applySchemaMigrations } from "./schema-version.js"; +import { + PREDICTION_LEDGER_PURGE_SPEC, + PREDICTION_LEDGER_RETENTION_SPEC, + purgeStoreByRepo, + pruneLedgerByRetention, + resolveLedgerRetentionPolicy, +} from "./store-maintenance.js"; + +// Append-only prediction ledger (#4263): every predicted-gate verdict the miner computes for a target lands in +// a local SQLite table so a later self-improve pass can score the prediction against the realized pr_outcome. +// IMMUTABILITY INVARIANT: `appendPrediction`/`readPredictions` only ever issue INSERT and SELECT — never +// UPDATE/DELETE. Two documented exceptions, both separate maintenance operations rather than part of normal +// ledger operation: opt-in retention pruning (#4834, automatic) and `purgeByRepo` (#5564, always explicit and +// operator-invoked, never automatic). Rows are kept small and stable for later diffing: blocker/warning CODES +// only (no free-text detail), plus the ENGINE_VERSION that produced the call so a row self-reports which engine +// build made it. Mirrors governor-ledger.js's shape; normalization is local (like event-ledger.js) so the +// offline miner package pulls in no engine module. + +export type PredictionLedgerEntry = { + id: number; + ts: string; + repoFullName: string; + targetId: number; + headSha: string | null; + conclusion: string; + pack: string; + readinessScore: number | null; + blockerCodes: string[]; + warningCodes: string[]; + engineVersion: string; +}; + +export type AppendPredictionInput = { + repoFullName: string; + targetId: number; + headSha?: string | null; + conclusion: string; + pack: string; + readinessScore?: number | null; + blockerCodes?: string[]; + warningCodes?: string[]; + engineVersion: string; +}; + +export type ReadPredictionsFilter = { + repoFullName?: string | null; +}; + +export type PredictionLedger = { + dbPath: string; + appendPrediction(input: AppendPredictionInput): PredictionLedgerEntry; + readPredictions(filter?: ReadPredictionsFilter): PredictionLedgerEntry[]; + purgeByRepo(repoFullName: string): number; + close(): void; +}; + +/** Private shape of a `predictions` SELECT * row after casting off `Record`. */ +type PredictionDbRow = { + id: number; + ts: string; + repo_full_name: string; + target_id: number; + head_sha: string | null; + conclusion: string; + pack: string; + readiness_score: number | null; + blocker_codes_json: string; + warning_codes_json: string; + engine_version: string; +}; + +const defaultDbFileName = "prediction-ledger.sqlite3"; +let defaultPredictionLedger: PredictionLedger | null = null; + +export function resolvePredictionLedgerDbPath(env: Record = process.env): string { + return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_PREDICTION_LEDGER_DB", env); +} + +function normalizeDbPath(dbPath: string): string { + return normalizeLocalStoreDbPath(dbPath, resolvePredictionLedgerDbPath(), "invalid_prediction_ledger_db_path"); +} + +function normalizeRepoFullName(repoFullName: string): string { + if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name"); + const [owner, repo, extra] = repoFullName.trim().split("/"); + if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name"); + return `${owner}/${repo}`; +} + +function normalizeOptionalRepoFullName(repoFullName: string | null | undefined): string | undefined { + if (repoFullName === undefined || repoFullName === null) return undefined; + return normalizeRepoFullName(repoFullName); +} + +function requiredNonEmptyString(value: unknown, error: string): string { + if (typeof value !== "string" || !value.trim()) throw new Error(error); + return value.trim(); +} + +function optionalString(value: string | null | undefined): string | null { + if (value === undefined || value === null) return null; + if (typeof value !== "string") throw new Error("invalid_head_sha"); + const trimmed = value.trim(); + return trimmed || null; +} + +// Codes are stored as a JSON array of the non-empty trimmed strings, in order — a stable, small projection of a +// verdict's blockers/warnings that drops all free-text detail. +function normalizeCodes(codes: string[] | null | undefined, error: string): string[] { + if (codes === undefined || codes === null) return []; + if (!Array.isArray(codes)) throw new Error(error); + return codes.map((code) => { + if (typeof code !== "string" || !code.trim()) throw new Error(error); + return code.trim(); + }); +} + +function normalizeReadinessScore(value: number | null | undefined): number | null { + if (value === undefined || value === null) return null; + if (typeof value !== "number" || !Number.isFinite(value)) throw new Error("invalid_readiness_score"); + return value; +} + +/** Validate + normalize an append input, throwing on any invalid field (mirrors normalizeGovernorLedgerEvent). */ +function normalizePredictionInput(input: AppendPredictionInput): { + repoFullName: string; + targetId: number; + headSha: string | null; + conclusion: string; + pack: string; + readinessScore: number | null; + blockerCodes: string[]; + warningCodes: string[]; + engineVersion: string; +} { + if (!input || typeof input !== "object" || Array.isArray(input)) throw new Error("invalid_prediction_input"); + if (!Number.isInteger(input.targetId) || input.targetId <= 0) throw new Error("invalid_target_id"); + return { + repoFullName: normalizeRepoFullName(input.repoFullName), + targetId: input.targetId, + headSha: optionalString(input.headSha), + conclusion: requiredNonEmptyString(input.conclusion, "invalid_conclusion"), + pack: requiredNonEmptyString(input.pack, "invalid_pack"), + readinessScore: normalizeReadinessScore(input.readinessScore), + blockerCodes: normalizeCodes(input.blockerCodes, "invalid_blocker_codes"), + warningCodes: normalizeCodes(input.warningCodes, "invalid_warning_codes"), + engineVersion: requiredNonEmptyString(input.engineVersion, "invalid_engine_version"), + }; +} + +function rowToEntry(row: PredictionDbRow): PredictionLedgerEntry { + let blockerCodes: unknown; + let warningCodes: unknown; + try { + blockerCodes = JSON.parse(row.blocker_codes_json); + warningCodes = JSON.parse(row.warning_codes_json); + if (!Array.isArray(blockerCodes) || !Array.isArray(warningCodes)) throw new Error("corrupted_prediction_row"); + } catch { + throw new Error("corrupted_prediction_row"); + } + return { + id: row.id, + ts: row.ts, + repoFullName: row.repo_full_name, + targetId: row.target_id, + headSha: row.head_sha, + conclusion: row.conclusion, + pack: row.pack, + readinessScore: row.readiness_score, + blockerCodes: blockerCodes as string[], + warningCodes: warningCodes as string[], + engineVersion: row.engine_version, + }; +} + +function asPredictionDbRow(row: Record): PredictionDbRow { + return row as unknown as PredictionDbRow; +} + +// v1 -> v2 (#4939): additive tenant-scoping column, a prerequisite for any hosted, multi-tenant use of this +// same store's logic. NULL for every row today -- self-host behavior is byte-identical, since nothing reads or +// writes it yet (no consumer exists until a future hosted deployment populates it). Same defensive +// column-presence guard as this file's sibling stores' own additive migrations (e.g. event-ledger.js's and +// run-state.js's own tenant_id additions), so re-running it against an already-migrated file is a no-op. +function addTenantIdColumn(db: DatabaseSync): void { + const hasTenantIdColumn = db + .prepare("PRAGMA table_info(predictions)") + .all() + .some((column) => column.name === "tenant_id"); + if (!hasTenantIdColumn) db.exec("ALTER TABLE predictions ADD COLUMN tenant_id TEXT"); +} + +/** + * Opens the append-only prediction ledger, creating the table on first use. Rows are returned in ascending `id` + * order (insertion order). (#4263) + */ +export function initPredictionLedger(dbPath: string = resolvePredictionLedgerDbPath()): PredictionLedger { + const resolvedPath = normalizeDbPath(dbPath); + const db = openLocalStoreDb(resolvedPath); + db.exec(` + CREATE TABLE IF NOT EXISTS predictions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + repo_full_name TEXT NOT NULL, + target_id INTEGER NOT NULL, + head_sha TEXT, + conclusion TEXT NOT NULL, + pack TEXT NOT NULL, + readiness_score REAL, + blocker_codes_json TEXT NOT NULL, + warning_codes_json TEXT NOT NULL, + engine_version TEXT NOT NULL + ) + `); + db.exec("CREATE INDEX IF NOT EXISTS idx_predictions_repo ON predictions (repo_full_name, id)"); + // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. + applySchemaMigrations(db, [addTenantIdColumn]); + // Opt-in retention (#4834): prune aged/excess rows when an operator has enabled it; a no-op by default. + pruneLedgerByRetention(db, PREDICTION_LEDGER_RETENTION_SPEC, resolveLedgerRetentionPolicy(), Date.now()); + + const appendStatement = db.prepare(` + INSERT INTO predictions + (ts, repo_full_name, target_id, head_sha, conclusion, pack, readiness_score, blocker_codes_json, warning_codes_json, engine_version) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const getByIdStatement = db.prepare("SELECT * FROM predictions WHERE id = ?"); + const readAllStatement = db.prepare("SELECT * FROM predictions ORDER BY id ASC"); + const readByRepoStatement = db.prepare("SELECT * FROM predictions WHERE repo_full_name = ? ORDER BY id ASC"); + + return { + dbPath: resolvedPath, + appendPrediction(input) { + const n = normalizePredictionInput(input); + const ts = new Date().toISOString(); + const result = appendStatement.run( + ts, + n.repoFullName, + n.targetId, + n.headSha, + n.conclusion, + n.pack, + n.readinessScore, + JSON.stringify(n.blockerCodes), + JSON.stringify(n.warningCodes), + n.engineVersion, + ); + return rowToEntry(asPredictionDbRow(getByIdStatement.get(Number(result.lastInsertRowid))!)); + }, + readPredictions(filter = {}) { + const repoFullName = normalizeOptionalRepoFullName(filter.repoFullName); + const rows = repoFullName === undefined ? readAllStatement.all() : readByRepoStatement.all(repoFullName); + return rows.map((row) => rowToEntry(asPredictionDbRow(row))); + }, + // Explicit, operator-invoked right-to-be-forgotten purge (#5564) — never runs automatically. See the + // IMMUTABILITY INVARIANT note above: this is a deliberate, separate exception, not a normal ledger write. + purgeByRepo(repoFullName) { + return purgeStoreByRepo(db, PREDICTION_LEDGER_PURGE_SPEC, normalizeRepoFullName(repoFullName)); + }, + close() { + db.close(); + }, + }; +} + +function getDefaultPredictionLedger(): PredictionLedger { + defaultPredictionLedger ??= initPredictionLedger(); + return defaultPredictionLedger; +} + +export function appendPrediction(input: AppendPredictionInput): PredictionLedgerEntry { + return getDefaultPredictionLedger().appendPrediction(input); +} + +export function readPredictions(filter?: ReadPredictionsFilter): PredictionLedgerEntry[] { + return getDefaultPredictionLedger().readPredictions(filter); +} + +export function closeDefaultPredictionLedger(): void { + if (!defaultPredictionLedger) return; + defaultPredictionLedger.close(); + defaultPredictionLedger = null; +} diff --git a/packages/loopover-miner/lib/run-state.d.ts b/packages/loopover-miner/lib/run-state.d.ts index fb1bed014d..52eb2167e6 100644 --- a/packages/loopover-miner/lib/run-state.d.ts +++ b/packages/loopover-miner/lib/run-state.d.ts @@ -1,38 +1,36 @@ export type RunState = "idle" | "discovering" | "planning" | "preparing"; - export type RunStateWrite = { - apiBaseUrl: string; - repoFullName: string; - state: RunState; - updatedAt: string; + apiBaseUrl: string; + repoFullName: string; + state: RunState; + updatedAt: string; }; - export type RunStateRow = { - apiBaseUrl: string; - repoFullName: string; - state: RunState; - updatedAt: string; + apiBaseUrl: string; + repoFullName: string; + state: RunState; + updatedAt: string; }; - export type RunStateStore = { - dbPath: string; - getRunState(repoFullName: string, apiBaseUrl?: string): RunState | null; - setRunState(repoFullName: string, state: RunState, apiBaseUrl?: string): RunStateWrite; - listRunStates(): RunStateRow[]; - purgeByRepo(repoFullName: string): number; - close(): void; + dbPath: string; + getRunState(repoFullName: string, apiBaseUrl?: string): RunState | null; + setRunState(repoFullName: string, state: RunState, apiBaseUrl?: string): RunStateWrite; + listRunStates(): RunStateRow[]; + purgeByRepo(repoFullName: string): number; + close(): void; }; - -export const RUN_STATES: readonly RunState[]; - -export function resolveRunStateDbPath(env?: Record): string; - -export function initRunStateStore(dbPath?: string): RunStateStore; - -export function getRunState(repoFullName: string, apiBaseUrl?: string): RunState | null; - -export function setRunState(repoFullName: string, state: RunState, apiBaseUrl?: string): RunStateWrite; - -export function listRunStates(): RunStateRow[]; - -export function closeDefaultRunStateStore(): void; +export declare const RUN_STATES: readonly RunState[]; +export declare function resolveRunStateDbPath(env?: Record): string; +/** + * Opens the 100% local/client-side miner run-state store. The database only lives on this machine; + * this module never uploads, syncs, or phones home with its contents. (#2289, #5563) + * + * Opened through the #7175 SqliteDriver seam (`openLocalStoreAdapter`): CRUD goes through `driver.query`, + * while schema migrations / purge still use the underlying DatabaseSync until those helpers are migrated. + * Public API stays synchronous so loop/CLI/MCP callers need no async cascade in this part-1 slice. + */ +export declare function initRunStateStore(dbPath?: string): RunStateStore; +export declare function getRunState(repoFullName: string, apiBaseUrl?: string): RunState | null; +export declare function setRunState(repoFullName: string, state: RunState, apiBaseUrl?: string): RunStateWrite; +export declare function listRunStates(): RunStateRow[]; +export declare function closeDefaultRunStateStore(): void; diff --git a/packages/loopover-miner/lib/run-state.js b/packages/loopover-miner/lib/run-state.js index d2c9508984..b782a1412f 100644 --- a/packages/loopover-miner/lib/run-state.js +++ b/packages/loopover-miner/lib/run-state.js @@ -2,48 +2,53 @@ import { DEFAULT_FORGE_CONFIG } from "./forge-config.js"; import { normalizeLocalStoreDbPath, openLocalStoreAdapter, resolveLocalStoreDbPath } from "./local-store.js"; import { applySchemaMigrations } from "./schema-version.js"; import { RUN_STATE_PURGE_SPEC, purgeStoreByRepo } from "./store-maintenance.js"; - -export const RUN_STATES = Object.freeze(["idle", "discovering", "planning", "preparing"]); - +export const RUN_STATES = Object.freeze([ + "idle", + "discovering", + "planning", + "preparing", +]); const runStateSet = new Set(RUN_STATES); const defaultDbFileName = "run-state.sqlite3"; let defaultRunStateStore = null; - +function isRunState(value) { + return runStateSet.has(value); +} export function resolveRunStateDbPath(env = process.env) { - return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_RUN_STATE_DB", env); + return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_RUN_STATE_DB", env); } - function normalizeDbPath(dbPath) { - return normalizeLocalStoreDbPath(dbPath, resolveRunStateDbPath(), "invalid_run_state_db_path"); + return normalizeLocalStoreDbPath(dbPath, resolveRunStateDbPath(), "invalid_run_state_db_path"); } - function normalizeRepoFullName(repoFullName) { - if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name"); - const trimmed = repoFullName.trim(); - const [owner, repo, extra] = trimmed.split("/"); - if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name"); - return `${owner}/${repo}`; + if (typeof repoFullName !== "string") + throw new Error("invalid_repo_full_name"); + const trimmed = repoFullName.trim(); + const [owner, repo, extra] = trimmed.split("/"); + if (!owner || !repo || extra !== undefined) + throw new Error("invalid_repo_full_name"); + return `${owner}/${repo}`; } - function normalizeRunState(state) { - if (runStateSet.has(state)) return state; - throw new Error("invalid_run_state"); + if (runStateSet.has(state)) + return state; + throw new Error("invalid_run_state"); } - /** Optional forge host, scoping rows so two hosts serving the same owner/repo name never collide (#5563). * Omitted/nullish → the github.com default, so every pre-existing single-forge caller is unaffected. */ function normalizeApiBaseUrl(apiBaseUrl) { - if (apiBaseUrl === undefined || apiBaseUrl === null) return DEFAULT_FORGE_CONFIG.apiBaseUrl; - if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) throw new Error("invalid_api_base_url"); - return apiBaseUrl.trim(); + if (apiBaseUrl === undefined || apiBaseUrl === null) + return DEFAULT_FORGE_CONFIG.apiBaseUrl; + if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) + throw new Error("invalid_api_base_url"); + return apiBaseUrl.trim(); } - // v1 -> v2 (#5563): rebuild the bare `repo_full_name` PRIMARY KEY into a (api_base_url, repo_full_name) composite // -- two forge hosts serving a same-named owner/repo must not share one "current state" row. SQLite cannot ALTER // a PRIMARY KEY in place, so this rebuilds the table: create the new shape, copy every existing row with the // pre-#4784 implicit single-forge default backfilled, drop the old table, rename the new one in. function addApiBaseUrlScope(db) { - db.exec(` + db.exec(` CREATE TABLE miner_run_state_v2 ( api_base_url TEXT NOT NULL, repo_full_name TEXT NOT NULL, @@ -52,31 +57,28 @@ function addApiBaseUrlScope(db) { PRIMARY KEY (api_base_url, repo_full_name) ) `); - // OR IGNORE: a row this store's own read path already treats as unusable garbage (an unrecognized `state`, - // e.g. from a hand-edited or otherwise corrupted file -- getRunState/listRunStates fail closed on it too) - // would violate the CHECK constraint above and abort the whole migration. Skipping it here is consistent with - // that same fail-closed posture, rather than turning one bad row into a permanently unmigratable file. - db.prepare( - `INSERT OR IGNORE INTO miner_run_state_v2 (api_base_url, repo_full_name, state, updated_at) - SELECT ?, repo_full_name, state, updated_at FROM miner_run_state`, - ).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); - db.exec("DROP TABLE miner_run_state"); - db.exec("ALTER TABLE miner_run_state_v2 RENAME TO miner_run_state"); + // OR IGNORE: a row this store's own read path already treats as unusable garbage (an unrecognized `state`, + // e.g. from a hand-edited or otherwise corrupted file -- getRunState/listRunStates fail closed on it too) + // would violate the CHECK constraint above and abort the whole migration. Skipping it here is consistent with + // that same fail-closed posture, rather than turning one bad row into a permanently unmigratable file. + db.prepare(`INSERT OR IGNORE INTO miner_run_state_v2 (api_base_url, repo_full_name, state, updated_at) + SELECT ?, repo_full_name, state, updated_at FROM miner_run_state`).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); + db.exec("DROP TABLE miner_run_state"); + db.exec("ALTER TABLE miner_run_state_v2 RENAME TO miner_run_state"); } - // v2 -> v3 (#4939): additive tenant-scoping column, a prerequisite for any hosted, multi-tenant use of this // same store's logic. NULL for every row today -- self-host behavior is byte-identical, since nothing reads or // writes it yet (no consumer exists until a future hosted deployment populates it). Same defensive // column-presence guard as every other additive migration in this file's siblings (e.g. // portfolio-queue.js's v3->v4 attempts_count addition). function addTenantIdColumn(db) { - const hasTenantIdColumn = db - .prepare("PRAGMA table_info(miner_run_state)") - .all() - .some((column) => column.name === "tenant_id"); - if (!hasTenantIdColumn) db.exec("ALTER TABLE miner_run_state ADD COLUMN tenant_id TEXT"); + const hasTenantIdColumn = db + .prepare("PRAGMA table_info(miner_run_state)") + .all() + .some((column) => column.name === "tenant_id"); + if (!hasTenantIdColumn) + db.exec("ALTER TABLE miner_run_state ADD COLUMN tenant_id TEXT"); } - /** * Opens the 100% local/client-side miner run-state store. The database only lives on this machine; * this module never uploads, syncs, or phones home with its contents. (#2289, #5563) @@ -86,89 +88,84 @@ function addTenantIdColumn(db) { * Public API stays synchronous so loop/CLI/MCP callers need no async cascade in this part-1 slice. */ export function initRunStateStore(dbPath = resolveRunStateDbPath()) { - const resolvedPath = normalizeDbPath(dbPath); - const { db, driver } = openLocalStoreAdapter(resolvedPath); - db.exec(` + const resolvedPath = normalizeDbPath(dbPath); + const { db, driver } = openLocalStoreAdapter(resolvedPath); + db.exec(` CREATE TABLE IF NOT EXISTS miner_run_state ( repo_full_name TEXT PRIMARY KEY, state TEXT NOT NULL CHECK (state IN ('idle', 'discovering', 'planning', 'preparing')), updated_at TEXT NOT NULL ) `); - // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. - applySchemaMigrations(db, [addApiBaseUrlScope, addTenantIdColumn]); - - const getSql = "SELECT state FROM miner_run_state WHERE api_base_url = ? AND repo_full_name = ?"; - const setSql = ` + // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. + applySchemaMigrations(db, [addApiBaseUrlScope, addTenantIdColumn]); + const getSql = "SELECT state FROM miner_run_state WHERE api_base_url = ? AND repo_full_name = ?"; + const setSql = ` INSERT INTO miner_run_state (api_base_url, repo_full_name, state, updated_at) VALUES (?, ?, ?, ?) ON CONFLICT(api_base_url, repo_full_name) DO UPDATE SET state = excluded.state, updated_at = excluded.updated_at `; - const listSql = - "SELECT api_base_url, repo_full_name, state, updated_at FROM miner_run_state ORDER BY repo_full_name"; - - return { - dbPath: resolvedPath, - getRunState(repoFullName, apiBaseUrl) { - const { rows } = driver.query(getSql, [ - normalizeApiBaseUrl(apiBaseUrl), - normalizeRepoFullName(repoFullName), - ]); - const row = rows[0]; - return runStateSet.has(row?.state) ? row.state : null; - }, - setRunState(repoFullName, state, apiBaseUrl) { - const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); - const normalizedRepo = normalizeRepoFullName(repoFullName); - const normalizedState = normalizeRunState(state); - const updatedAt = new Date().toISOString(); - driver.query(setSql, [normalizedForge, normalizedRepo, normalizedState, updatedAt]); - return { apiBaseUrl: normalizedForge, repoFullName: normalizedRepo, state: normalizedState, updatedAt }; - }, - /** Every repo with a recorded run state, across the whole store — the per-repo discover/plan/prepare - * signal a "run portfolio" view folds alongside managed PR rows (#4279). */ - listRunStates() { - const { rows } = driver.query(listSql, []); - return rows - .filter((row) => runStateSet.has(row.state)) - .map((row) => ({ - apiBaseUrl: row.api_base_url, - repoFullName: row.repo_full_name, - state: row.state, - updatedAt: row.updated_at, - })); - }, - // Explicit, operator-invoked right-to-be-forgotten purge (#5564, #6599) — never runs automatically. - purgeByRepo(repoFullName) { - return purgeStoreByRepo(db, RUN_STATE_PURGE_SPEC, normalizeRepoFullName(repoFullName)); - }, - close() { - db.close(); - }, - }; + const listSql = "SELECT api_base_url, repo_full_name, state, updated_at FROM miner_run_state ORDER BY repo_full_name"; + return { + dbPath: resolvedPath, + getRunState(repoFullName, apiBaseUrl) { + const { rows } = driver.query(getSql, [ + normalizeApiBaseUrl(apiBaseUrl), + normalizeRepoFullName(repoFullName), + ]); + const row = rows[0]; + const state = row?.state; + return isRunState(state) ? state : null; + }, + setRunState(repoFullName, state, apiBaseUrl) { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const normalizedState = normalizeRunState(state); + const updatedAt = new Date().toISOString(); + driver.query(setSql, [normalizedForge, normalizedRepo, normalizedState, updatedAt]); + return { apiBaseUrl: normalizedForge, repoFullName: normalizedRepo, state: normalizedState, updatedAt }; + }, + /** Every repo with a recorded run state, across the whole store — the per-repo discover/plan/prepare + * signal a "run portfolio" view folds alongside managed PR rows (#4279). */ + listRunStates() { + const { rows } = driver.query(listSql, []); + return rows + .filter((row) => isRunState(row.state)) + .map((row) => ({ + apiBaseUrl: row.api_base_url, + repoFullName: row.repo_full_name, + state: row.state, + updatedAt: row.updated_at, + })); + }, + // Explicit, operator-invoked right-to-be-forgotten purge (#5564, #6599) — never runs automatically. + purgeByRepo(repoFullName) { + return purgeStoreByRepo(db, RUN_STATE_PURGE_SPEC, normalizeRepoFullName(repoFullName)); + }, + close() { + db.close(); + }, + }; } - function getDefaultRunStateStore() { - defaultRunStateStore ??= initRunStateStore(); - return defaultRunStateStore; + defaultRunStateStore ??= initRunStateStore(); + return defaultRunStateStore; } - export function getRunState(repoFullName, apiBaseUrl) { - return getDefaultRunStateStore().getRunState(repoFullName, apiBaseUrl); + return getDefaultRunStateStore().getRunState(repoFullName, apiBaseUrl); } - export function setRunState(repoFullName, state, apiBaseUrl) { - return getDefaultRunStateStore().setRunState(repoFullName, state, apiBaseUrl); + return getDefaultRunStateStore().setRunState(repoFullName, state, apiBaseUrl); } - export function listRunStates() { - return getDefaultRunStateStore().listRunStates(); + return getDefaultRunStateStore().listRunStates(); } - export function closeDefaultRunStateStore() { - if (!defaultRunStateStore) return; - defaultRunStateStore.close(); - defaultRunStateStore = null; + if (!defaultRunStateStore) + return; + defaultRunStateStore.close(); + defaultRunStateStore = null; } +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVuLXN0YXRlLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsicnVuLXN0YXRlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sRUFBRSxvQkFBb0IsRUFBRSxNQUFNLG1CQUFtQixDQUFDO0FBQ3pELE9BQU8sRUFBRSx5QkFBeUIsRUFBRSxxQkFBcUIsRUFBRSx1QkFBdUIsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBQzdHLE9BQU8sRUFBRSxxQkFBcUIsRUFBRSxNQUFNLHFCQUFxQixDQUFDO0FBQzVELE9BQU8sRUFBRSxvQkFBb0IsRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLHdCQUF3QixDQUFDO0FBMkJoRixNQUFNLENBQUMsTUFBTSxVQUFVLEdBQUcsTUFBTSxDQUFDLE1BQU0sQ0FBQztJQUN0QyxNQUFNO0lBQ04sYUFBYTtJQUNiLFVBQVU7SUFDVixXQUFXO0NBQ1osQ0FBd0IsQ0FBQztBQUUxQixNQUFNLFdBQVcsR0FBRyxJQUFJLEdBQUcsQ0FBUyxVQUFVLENBQUMsQ0FBQztBQUNoRCxNQUFNLGlCQUFpQixHQUFHLG1CQUFtQixDQUFDO0FBQzlDLElBQUksb0JBQW9CLEdBQXlCLElBQUksQ0FBQztBQUV0RCxTQUFTLFVBQVUsQ0FBQyxLQUFjO0lBQ2hDLE9BQU8sV0FBVyxDQUFDLEdBQUcsQ0FBQyxLQUFlLENBQUMsQ0FBQztBQUMxQyxDQUFDO0FBRUQsTUFBTSxVQUFVLHFCQUFxQixDQUFDLE1BQTBDLE9BQU8sQ0FBQyxHQUFHO0lBQ3pGLE9BQU8sdUJBQXVCLENBQUMsaUJBQWlCLEVBQUUsNkJBQTZCLEVBQUUsR0FBRyxDQUFDLENBQUM7QUFDeEYsQ0FBQztBQUVELFNBQVMsZUFBZSxDQUFDLE1BQWM7SUFDckMsT0FBTyx5QkFBeUIsQ0FBQyxNQUFNLEVBQUUscUJBQXFCLEVBQUUsRUFBRSwyQkFBMkIsQ0FBQyxDQUFDO0FBQ2pHLENBQUM7QUFFRCxTQUFTLHFCQUFxQixDQUFDLFlBQW9CO0lBQ2pELElBQUksT0FBTyxZQUFZLEtBQUssUUFBUTtRQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsd0JBQXdCLENBQUMsQ0FBQztJQUNoRixNQUFNLE9BQU8sR0FBRyxZQUFZLENBQUMsSUFBSSxFQUFFLENBQUM7SUFDcEMsTUFBTSxDQUFDLEtBQUssRUFBRSxJQUFJLEVBQUUsS0FBSyxDQUFDLEdBQUcsT0FBTyxDQUFDLEtBQUssQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUNoRCxJQUFJLENBQUMsS0FBSyxJQUFJLENBQUMsSUFBSSxJQUFJLEtBQUssS0FBSyxTQUFTO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyx3QkFBd0IsQ0FBQyxDQUFDO0lBQ3RGLE9BQU8sR0FBRyxLQUFLLElBQUksSUFBSSxFQUFFLENBQUM7QUFDNUIsQ0FBQztBQUVELFNBQVMsaUJBQWlCLENBQUMsS0FBYTtJQUN0QyxJQUFJLFdBQVcsQ0FBQyxHQUFHLENBQUMsS0FBSyxDQUFDO1FBQUUsT0FBTyxLQUFpQixDQUFDO0lBQ3JELE1BQU0sSUFBSSxLQUFLLENBQUMsbUJBQW1CLENBQUMsQ0FBQztBQUN2QyxDQUFDO0FBRUQ7eUdBQ3lHO0FBQ3pHLFNBQVMsbUJBQW1CLENBQUMsVUFBMEI7SUFDckQsSUFBSSxVQUFVLEtBQUssU0FBUyxJQUFJLFVBQVUsS0FBSyxJQUFJO1FBQUUsT0FBTyxvQkFBb0IsQ0FBQyxVQUFVLENBQUM7SUFDNUYsSUFBSSxPQUFPLFVBQVUsS0FBSyxRQUFRLElBQUksQ0FBQyxVQUFVLENBQUMsSUFBSSxFQUFFO1FBQUUsTUFBTSxJQUFJLEtBQUssQ0FBQyxzQkFBc0IsQ0FBQyxDQUFDO0lBQ2xHLE9BQU8sVUFBVSxDQUFDLElBQUksRUFBRSxDQUFDO0FBQzNCLENBQUM7QUFFRCxrSEFBa0g7QUFDbEgsaUhBQWlIO0FBQ2pILDZHQUE2RztBQUM3RyxpR0FBaUc7QUFDakcsU0FBUyxrQkFBa0IsQ0FBQyxFQUFnQjtJQUMxQyxFQUFFLENBQUMsSUFBSSxDQUFDOzs7Ozs7OztHQVFQLENBQUMsQ0FBQztJQUNILDJHQUEyRztJQUMzRywwR0FBMEc7SUFDMUcsOEdBQThHO0lBQzlHLHVHQUF1RztJQUN2RyxFQUFFLENBQUMsT0FBTyxDQUNSO3NFQUNrRSxDQUNuRSxDQUFDLEdBQUcsQ0FBQyxvQkFBb0IsQ0FBQyxVQUFVLENBQUMsQ0FBQztJQUN2QyxFQUFFLENBQUMsSUFBSSxDQUFDLDRCQUE0QixDQUFDLENBQUM7SUFDdEMsRUFBRSxDQUFDLElBQUksQ0FBQywwREFBMEQsQ0FBQyxDQUFDO0FBQ3RFLENBQUM7QUFFRCw0R0FBNEc7QUFDNUcsK0dBQStHO0FBQy9HLG1HQUFtRztBQUNuRyx3RkFBd0Y7QUFDeEYsd0RBQXdEO0FBQ3hELFNBQVMsaUJBQWlCLENBQUMsRUFBZ0I7SUFDekMsTUFBTSxpQkFBaUIsR0FBRyxFQUFFO1NBQ3pCLE9BQU8sQ0FBQyxvQ0FBb0MsQ0FBQztTQUM3QyxHQUFHLEVBQUU7U0FDTCxJQUFJLENBQUMsQ0FBQyxNQUFNLEVBQUUsRUFBRSxDQUFDLE1BQU0sQ0FBQyxJQUFJLEtBQUssV0FBVyxDQUFDLENBQUM7SUFDakQsSUFBSSxDQUFDLGlCQUFpQjtRQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsdURBQXVELENBQUMsQ0FBQztBQUMzRixDQUFDO0FBRUQ7Ozs7Ozs7R0FPRztBQUNILE1BQU0sVUFBVSxpQkFBaUIsQ0FBQyxTQUFpQixxQkFBcUIsRUFBRTtJQUN4RSxNQUFNLFlBQVksR0FBRyxlQUFlLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDN0MsTUFBTSxFQUFFLEVBQUUsRUFBRSxNQUFNLEVBQUUsR0FBRyxxQkFBcUIsQ0FBQyxZQUFZLENBQUMsQ0FBQztJQUMzRCxFQUFFLENBQUMsSUFBSSxDQUFDOzs7Ozs7R0FNUCxDQUFDLENBQUM7SUFDSCw4RkFBOEY7SUFDOUYscUJBQXFCLENBQUMsRUFBRSxFQUFFLENBQUMsa0JBQWtCLEVBQUUsaUJBQWlCLENBQUMsQ0FBQyxDQUFDO0lBRW5FLE1BQU0sTUFBTSxHQUFHLGlGQUFpRixDQUFDO0lBQ2pHLE1BQU0sTUFBTSxHQUFHOzs7Ozs7R0FNZCxDQUFDO0lBQ0YsTUFBTSxPQUFPLEdBQ1gscUdBQXFHLENBQUM7SUFFeEcsT0FBTztRQUNMLE1BQU0sRUFBRSxZQUFZO1FBQ3BCLFdBQVcsQ0FBQyxZQUFZLEVBQUUsVUFBVTtZQUNsQyxNQUFNLEVBQUUsSUFBSSxFQUFFLEdBQUcsTUFBTSxDQUFDLEtBQUssQ0FBQyxNQUFNLEVBQUU7Z0JBQ3BDLG1CQUFtQixDQUFDLFVBQVUsQ0FBQztnQkFDL0IscUJBQXFCLENBQUMsWUFBWSxDQUFDO2FBQ3BDLENBQUMsQ0FBQztZQUNILE1BQU0sR0FBRyxHQUFHLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQztZQUNwQixNQUFNLEtBQUssR0FBRyxHQUFHLEVBQUUsS0FBSyxDQUFDO1lBQ3pCLE9BQU8sVUFBVSxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQztRQUMxQyxDQUFDO1FBQ0QsV0FBVyxDQUFDLFlBQVksRUFBRSxLQUFLLEVBQUUsVUFBVTtZQUN6QyxNQUFNLGVBQWUsR0FBRyxtQkFBbUIsQ0FBQyxVQUFVLENBQUMsQ0FBQztZQUN4RCxNQUFNLGNBQWMsR0FBRyxxQkFBcUIsQ0FBQyxZQUFZLENBQUMsQ0FBQztZQUMzRCxNQUFNLGVBQWUsR0FBRyxpQkFBaUIsQ0FBQyxLQUFLLENBQUMsQ0FBQztZQUNqRCxNQUFNLFNBQVMsR0FBRyxJQUFJLElBQUksRUFBRSxDQUFDLFdBQVcsRUFBRSxDQUFDO1lBQzNDLE1BQU0sQ0FBQyxLQUFLLENBQUMsTUFBTSxFQUFFLENBQUMsZUFBZSxFQUFFLGNBQWMsRUFBRSxlQUFlLEVBQUUsU0FBUyxDQUFDLENBQUMsQ0FBQztZQUNwRixPQUFPLEVBQUUsVUFBVSxFQUFFLGVBQWUsRUFBRSxZQUFZLEVBQUUsY0FBYyxFQUFFLEtBQUssRUFBRSxlQUFlLEVBQUUsU0FBUyxFQUFFLENBQUM7UUFDMUcsQ0FBQztRQUNEO3FGQUM2RTtRQUM3RSxhQUFhO1lBQ1gsTUFBTSxFQUFFLElBQUksRUFBRSxHQUFHLE1BQU0sQ0FBQyxLQUFLLENBQUMsT0FBTyxFQUFFLEVBQUUsQ0FBQyxDQUFDO1lBQzNDLE9BQU8sSUFBSTtpQkFDUixNQUFNLENBQUMsQ0FBQyxHQUFHLEVBQXdELEVBQUUsQ0FBQyxVQUFVLENBQUMsR0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDO2lCQUM1RixHQUFHLENBQUMsQ0FBQyxHQUFHLEVBQUUsRUFBRSxDQUFDLENBQUM7Z0JBQ2IsVUFBVSxFQUFFLEdBQUcsQ0FBQyxZQUFzQjtnQkFDdEMsWUFBWSxFQUFFLEdBQUcsQ0FBQyxjQUF3QjtnQkFDMUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxLQUFLO2dCQUNoQixTQUFTLEVBQUUsR0FBRyxDQUFDLFVBQW9CO2FBQ3BDLENBQUMsQ0FBQyxDQUFDO1FBQ1IsQ0FBQztRQUNELG9HQUFvRztRQUNwRyxXQUFXLENBQUMsWUFBWTtZQUN0QixPQUFPLGdCQUFnQixDQUFDLEVBQUUsRUFBRSxvQkFBb0IsRUFBRSxxQkFBcUIsQ0FBQyxZQUFZLENBQUMsQ0FBQyxDQUFDO1FBQ3pGLENBQUM7UUFDRCxLQUFLO1lBQ0gsRUFBRSxDQUFDLEtBQUssRUFBRSxDQUFDO1FBQ2IsQ0FBQztLQUNGLENBQUM7QUFDSixDQUFDO0FBRUQsU0FBUyx1QkFBdUI7SUFDOUIsb0JBQW9CLEtBQUssaUJBQWlCLEVBQUUsQ0FBQztJQUM3QyxPQUFPLG9CQUFvQixDQUFDO0FBQzlCLENBQUM7QUFFRCxNQUFNLFVBQVUsV0FBVyxDQUFDLFlBQW9CLEVBQUUsVUFBbUI7SUFDbkUsT0FBTyx1QkFBdUIsRUFBRSxDQUFDLFdBQVcsQ0FBQyxZQUFZLEVBQUUsVUFBVSxDQUFDLENBQUM7QUFDekUsQ0FBQztBQUVELE1BQU0sVUFBVSxXQUFXLENBQUMsWUFBb0IsRUFBRSxLQUFlLEVBQUUsVUFBbUI7SUFDcEYsT0FBTyx1QkFBdUIsRUFBRSxDQUFDLFdBQVcsQ0FBQyxZQUFZLEVBQUUsS0FBSyxFQUFFLFVBQVUsQ0FBQyxDQUFDO0FBQ2hGLENBQUM7QUFFRCxNQUFNLFVBQVUsYUFBYTtJQUMzQixPQUFPLHVCQUF1QixFQUFFLENBQUMsYUFBYSxFQUFFLENBQUM7QUFDbkQsQ0FBQztBQUVELE1BQU0sVUFBVSx5QkFBeUI7SUFDdkMsSUFBSSxDQUFDLG9CQUFvQjtRQUFFLE9BQU87SUFDbEMsb0JBQW9CLENBQUMsS0FBSyxFQUFFLENBQUM7SUFDN0Isb0JBQW9CLEdBQUcsSUFBSSxDQUFDO0FBQzlCLENBQUMifQ== \ No newline at end of file diff --git a/packages/loopover-miner/lib/run-state.ts b/packages/loopover-miner/lib/run-state.ts new file mode 100644 index 0000000000..ec8de4d4a4 --- /dev/null +++ b/packages/loopover-miner/lib/run-state.ts @@ -0,0 +1,210 @@ +import type { DatabaseSync } from "node:sqlite"; +import { DEFAULT_FORGE_CONFIG } from "./forge-config.js"; +import { normalizeLocalStoreDbPath, openLocalStoreAdapter, resolveLocalStoreDbPath } from "./local-store.js"; +import { applySchemaMigrations } from "./schema-version.js"; +import { RUN_STATE_PURGE_SPEC, purgeStoreByRepo } from "./store-maintenance.js"; + +export type RunState = "idle" | "discovering" | "planning" | "preparing"; + +export type RunStateWrite = { + apiBaseUrl: string; + repoFullName: string; + state: RunState; + updatedAt: string; +}; + +export type RunStateRow = { + apiBaseUrl: string; + repoFullName: string; + state: RunState; + updatedAt: string; +}; + +export type RunStateStore = { + dbPath: string; + getRunState(repoFullName: string, apiBaseUrl?: string): RunState | null; + setRunState(repoFullName: string, state: RunState, apiBaseUrl?: string): RunStateWrite; + listRunStates(): RunStateRow[]; + purgeByRepo(repoFullName: string): number; + close(): void; +}; + +export const RUN_STATES = Object.freeze([ + "idle", + "discovering", + "planning", + "preparing", +]) as readonly RunState[]; + +const runStateSet = new Set(RUN_STATES); +const defaultDbFileName = "run-state.sqlite3"; +let defaultRunStateStore: RunStateStore | null = null; + +function isRunState(value: unknown): value is RunState { + return runStateSet.has(value as string); +} + +export function resolveRunStateDbPath(env: Record = process.env): string { + return resolveLocalStoreDbPath(defaultDbFileName, "LOOPOVER_MINER_RUN_STATE_DB", env); +} + +function normalizeDbPath(dbPath: string): string { + return normalizeLocalStoreDbPath(dbPath, resolveRunStateDbPath(), "invalid_run_state_db_path"); +} + +function normalizeRepoFullName(repoFullName: string): string { + if (typeof repoFullName !== "string") throw new Error("invalid_repo_full_name"); + const trimmed = repoFullName.trim(); + const [owner, repo, extra] = trimmed.split("/"); + if (!owner || !repo || extra !== undefined) throw new Error("invalid_repo_full_name"); + return `${owner}/${repo}`; +} + +function normalizeRunState(state: string): RunState { + if (runStateSet.has(state)) return state as RunState; + throw new Error("invalid_run_state"); +} + +/** Optional forge host, scoping rows so two hosts serving the same owner/repo name never collide (#5563). + * Omitted/nullish → the github.com default, so every pre-existing single-forge caller is unaffected. */ +function normalizeApiBaseUrl(apiBaseUrl?: string | null): string { + if (apiBaseUrl === undefined || apiBaseUrl === null) return DEFAULT_FORGE_CONFIG.apiBaseUrl; + if (typeof apiBaseUrl !== "string" || !apiBaseUrl.trim()) throw new Error("invalid_api_base_url"); + return apiBaseUrl.trim(); +} + +// v1 -> v2 (#5563): rebuild the bare `repo_full_name` PRIMARY KEY into a (api_base_url, repo_full_name) composite +// -- two forge hosts serving a same-named owner/repo must not share one "current state" row. SQLite cannot ALTER +// a PRIMARY KEY in place, so this rebuilds the table: create the new shape, copy every existing row with the +// pre-#4784 implicit single-forge default backfilled, drop the old table, rename the new one in. +function addApiBaseUrlScope(db: DatabaseSync): void { + db.exec(` + CREATE TABLE miner_run_state_v2 ( + api_base_url TEXT NOT NULL, + repo_full_name TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('idle', 'discovering', 'planning', 'preparing')), + updated_at TEXT NOT NULL, + PRIMARY KEY (api_base_url, repo_full_name) + ) + `); + // OR IGNORE: a row this store's own read path already treats as unusable garbage (an unrecognized `state`, + // e.g. from a hand-edited or otherwise corrupted file -- getRunState/listRunStates fail closed on it too) + // would violate the CHECK constraint above and abort the whole migration. Skipping it here is consistent with + // that same fail-closed posture, rather than turning one bad row into a permanently unmigratable file. + db.prepare( + `INSERT OR IGNORE INTO miner_run_state_v2 (api_base_url, repo_full_name, state, updated_at) + SELECT ?, repo_full_name, state, updated_at FROM miner_run_state`, + ).run(DEFAULT_FORGE_CONFIG.apiBaseUrl); + db.exec("DROP TABLE miner_run_state"); + db.exec("ALTER TABLE miner_run_state_v2 RENAME TO miner_run_state"); +} + +// v2 -> v3 (#4939): additive tenant-scoping column, a prerequisite for any hosted, multi-tenant use of this +// same store's logic. NULL for every row today -- self-host behavior is byte-identical, since nothing reads or +// writes it yet (no consumer exists until a future hosted deployment populates it). Same defensive +// column-presence guard as every other additive migration in this file's siblings (e.g. +// portfolio-queue.js's v3->v4 attempts_count addition). +function addTenantIdColumn(db: DatabaseSync): void { + const hasTenantIdColumn = db + .prepare("PRAGMA table_info(miner_run_state)") + .all() + .some((column) => column.name === "tenant_id"); + if (!hasTenantIdColumn) db.exec("ALTER TABLE miner_run_state ADD COLUMN tenant_id TEXT"); +} + +/** + * Opens the 100% local/client-side miner run-state store. The database only lives on this machine; + * this module never uploads, syncs, or phones home with its contents. (#2289, #5563) + * + * Opened through the #7175 SqliteDriver seam (`openLocalStoreAdapter`): CRUD goes through `driver.query`, + * while schema migrations / purge still use the underlying DatabaseSync until those helpers are migrated. + * Public API stays synchronous so loop/CLI/MCP callers need no async cascade in this part-1 slice. + */ +export function initRunStateStore(dbPath: string = resolveRunStateDbPath()): RunStateStore { + const resolvedPath = normalizeDbPath(dbPath); + const { db, driver } = openLocalStoreAdapter(resolvedPath); + db.exec(` + CREATE TABLE IF NOT EXISTS miner_run_state ( + repo_full_name TEXT PRIMARY KEY, + state TEXT NOT NULL CHECK (state IN ('idle', 'discovering', 'planning', 'preparing')), + updated_at TEXT NOT NULL + ) + `); + // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations. + applySchemaMigrations(db, [addApiBaseUrlScope, addTenantIdColumn]); + + const getSql = "SELECT state FROM miner_run_state WHERE api_base_url = ? AND repo_full_name = ?"; + const setSql = ` + INSERT INTO miner_run_state (api_base_url, repo_full_name, state, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(api_base_url, repo_full_name) DO UPDATE SET + state = excluded.state, + updated_at = excluded.updated_at + `; + const listSql = + "SELECT api_base_url, repo_full_name, state, updated_at FROM miner_run_state ORDER BY repo_full_name"; + + return { + dbPath: resolvedPath, + getRunState(repoFullName, apiBaseUrl) { + const { rows } = driver.query(getSql, [ + normalizeApiBaseUrl(apiBaseUrl), + normalizeRepoFullName(repoFullName), + ]); + const row = rows[0]; + const state = row?.state; + return isRunState(state) ? state : null; + }, + setRunState(repoFullName, state, apiBaseUrl) { + const normalizedForge = normalizeApiBaseUrl(apiBaseUrl); + const normalizedRepo = normalizeRepoFullName(repoFullName); + const normalizedState = normalizeRunState(state); + const updatedAt = new Date().toISOString(); + driver.query(setSql, [normalizedForge, normalizedRepo, normalizedState, updatedAt]); + return { apiBaseUrl: normalizedForge, repoFullName: normalizedRepo, state: normalizedState, updatedAt }; + }, + /** Every repo with a recorded run state, across the whole store — the per-repo discover/plan/prepare + * signal a "run portfolio" view folds alongside managed PR rows (#4279). */ + listRunStates() { + const { rows } = driver.query(listSql, []); + return rows + .filter((row): row is Record & { state: RunState } => isRunState(row.state)) + .map((row) => ({ + apiBaseUrl: row.api_base_url as string, + repoFullName: row.repo_full_name as string, + state: row.state, + updatedAt: row.updated_at as string, + })); + }, + // Explicit, operator-invoked right-to-be-forgotten purge (#5564, #6599) — never runs automatically. + purgeByRepo(repoFullName) { + return purgeStoreByRepo(db, RUN_STATE_PURGE_SPEC, normalizeRepoFullName(repoFullName)); + }, + close() { + db.close(); + }, + }; +} + +function getDefaultRunStateStore(): RunStateStore { + defaultRunStateStore ??= initRunStateStore(); + return defaultRunStateStore; +} + +export function getRunState(repoFullName: string, apiBaseUrl?: string): RunState | null { + return getDefaultRunStateStore().getRunState(repoFullName, apiBaseUrl); +} + +export function setRunState(repoFullName: string, state: RunState, apiBaseUrl?: string): RunStateWrite { + return getDefaultRunStateStore().setRunState(repoFullName, state, apiBaseUrl); +} + +export function listRunStates(): RunStateRow[] { + return getDefaultRunStateStore().listRunStates(); +} + +export function closeDefaultRunStateStore(): void { + if (!defaultRunStateStore) return; + defaultRunStateStore.close(); + defaultRunStateStore = null; +} diff --git a/packages/loopover-miner/lib/schema-version.d.ts b/packages/loopover-miner/lib/schema-version.d.ts index 5c52cd1c07..729e92da73 100644 --- a/packages/loopover-miner/lib/schema-version.d.ts +++ b/packages/loopover-miner/lib/schema-version.d.ts @@ -1,17 +1,18 @@ import type { DatabaseSync } from "node:sqlite"; - /** A single post-baseline schema migration: mutate the store in place to advance it exactly one version. */ export type SchemaMigration = (db: DatabaseSync) => void; - /** The bootstrap schema every store creates inline is, by convention, schema version 1. */ -export const BASELINE_SCHEMA_VERSION: number; - +export declare const BASELINE_SCHEMA_VERSION = 1; /** Read a store's current `PRAGMA user_version`, coercing any absent/invalid value to 0 (pre-versioning). */ -export function readSchemaVersion(db: DatabaseSync): number; - +export declare function readSchemaVersion(db: DatabaseSync): number; /** - * Run pending post-baseline migrations in order and stamp the resulting `user_version`. `migrations[i]` upgrades - * from version i+1 to i+2, so the target version is `BASELINE_SCHEMA_VERSION + migrations.length`. Returns the - * resulting version. + * Bring a store's on-disk schema up to date, then stamp its `user_version`. `migrations[i]` upgrades from + * version i+1 to i+2, so the target version is `BASELINE_SCHEMA_VERSION + migrations.length`. Every migration + * whose resulting version is above the file's current version runs, in order; a file already at (or past) the + * target runs none. Returns the resulting version. Never runs a migration twice (the stamped `user_version` + * gates re-runs on the next open) and never DOWNGRADES: a file written by newer code with more migrations is + * left at its higher version rather than stamped back down. Each migration and its version stamp are applied in + * one transaction, so a failure part-way through the sequence leaves the file at the last fully-applied version + * and re-opening resumes at the failed migration (a throwing migration rethrows after its changes roll back). */ -export function applySchemaMigrations(db: DatabaseSync, migrations?: SchemaMigration[]): number; +export declare function applySchemaMigrations(db: DatabaseSync, migrations?: SchemaMigration[]): number; diff --git a/packages/loopover-miner/lib/schema-version.js b/packages/loopover-miner/lib/schema-version.js index 6fcacfd6eb..1ac409cf5d 100644 --- a/packages/loopover-miner/lib/schema-version.js +++ b/packages/loopover-miner/lib/schema-version.js @@ -1,28 +1,11 @@ -// Lightweight schema-versioning convention shared across the miner's local SQLite stores (#4832). -// -// Every store bootstraps its tables with `CREATE TABLE IF NOT EXISTS ...` but, until now, carried no -// `user_version`/migration mechanism at all — so an older on-disk file was silently reused with a stale shape. -// This module adds the missing convention without the weight of the main product's `migrations/` runner: each -// store's bootstrap schema is treated as version 1 (BASELINE_SCHEMA_VERSION), and a store's `migrations` array -// describes ONLY the changes AFTER that baseline (`migrations[i]` upgrades the schema from version i+1 to i+2). -// `applySchemaMigrations` reads the file's current `PRAGMA user_version`, runs exactly the pending migrations in -// order, and stamps the new version — so opening an older-schema file runs its outstanding migrations instead of -// silently continuing on an incompatible shape. A pre-versioning file (user_version 0) already carries the -// baseline tables (the idempotent `CREATE TABLE IF NOT EXISTS` ran), so it is treated as the baseline before any -// post-baseline migration is applied. Pure control flow over an injected `DatabaseSync` handle: no IO of its own -// beyond the PRAGMA read/write and the caller-supplied migration functions, and deterministic given the same -// handle + migration list. - /** The bootstrap schema every store creates inline is, by convention, schema version 1. */ export const BASELINE_SCHEMA_VERSION = 1; - /** Read a store's current `PRAGMA user_version`, coercing any absent/invalid value to 0 (pre-versioning). */ export function readSchemaVersion(db) { - const row = db.prepare("PRAGMA user_version").get(); - const raw = row ? Number(row.user_version) : 0; - return Number.isInteger(raw) && raw >= 0 ? raw : 0; + const row = db.prepare("PRAGMA user_version").get(); + const raw = row ? Number(row.user_version) : 0; + return Number.isInteger(raw) && raw >= 0 ? raw : 0; } - /** * Bring a store's on-disk schema up to date, then stamp its `user_version`. `migrations[i]` upgrades from * version i+1 to i+2, so the target version is `BASELINE_SCHEMA_VERSION + migrations.length`. Every migration @@ -32,40 +15,40 @@ export function readSchemaVersion(db) { * left at its higher version rather than stamped back down. Each migration and its version stamp are applied in * one transaction, so a failure part-way through the sequence leaves the file at the last fully-applied version * and re-opening resumes at the failed migration (a throwing migration rethrows after its changes roll back). - * - * @param {import("node:sqlite").DatabaseSync} db - an open store handle whose baseline tables already exist. - * @param {Array<(db: import("node:sqlite").DatabaseSync) => void>} [migrations] - post-baseline migrations. - * @returns {number} the schema version after applying any pending migrations. */ export function applySchemaMigrations(db, migrations = []) { - const target = BASELINE_SCHEMA_VERSION + migrations.length; - const current = readSchemaVersion(db); - // A pre-versioning file (0) already holds the baseline schema, so advance from the baseline, not from 0. - const effective = current < BASELINE_SCHEMA_VERSION ? BASELINE_SCHEMA_VERSION : current; - // Stamp a pre-versioning file up to the baseline first, so a store with NO post-baseline migrations still - // records a version. Only ever stamp UPWARD: a file already at or past the baseline (including one written by - // newer code with more migrations) is never downgraded. `user_version` is an integer PRAGMA that cannot be - // parameterized; every stamped value here is a computed integer, never caller text, so interpolating is safe. - if (current < BASELINE_SCHEMA_VERSION) { - db.exec(`PRAGMA user_version = ${BASELINE_SCHEMA_VERSION}`); - } - for (let version = effective; version < target; version += 1) { - // Apply each migration AND stamp its resulting version in ONE transaction, so a failure part-way through the - // sequence leaves the file at the LAST fully-applied version: the next open resumes at the failed migration - // rather than re-running the ones that already succeeded (which, for a non-idempotent ALTER, would be a hard - // duplicate-column error). PRAGMA user_version is transactional in SQLite, so ROLLBACK undoes the migration's - // partial changes and its version stamp together. - db.exec("BEGIN"); - try { - migrations[version - BASELINE_SCHEMA_VERSION](db); - db.exec(`PRAGMA user_version = ${version + 1}`); - db.exec("COMMIT"); - } catch (error) { - db.exec("ROLLBACK"); - throw error; + const target = BASELINE_SCHEMA_VERSION + migrations.length; + const current = readSchemaVersion(db); + // A pre-versioning file (0) already holds the baseline schema, so advance from the baseline, not from 0. + const effective = current < BASELINE_SCHEMA_VERSION ? BASELINE_SCHEMA_VERSION : current; + // Stamp a pre-versioning file up to the baseline first, so a store with NO post-baseline migrations still + // records a version. Only ever stamp UPWARD: a file already at or past the baseline (including one written by + // newer code with more migrations) is never downgraded. `user_version` is an integer PRAGMA that cannot be + // parameterized; every stamped value here is a computed integer, never caller text, so interpolating is safe. + if (current < BASELINE_SCHEMA_VERSION) { + db.exec(`PRAGMA user_version = ${BASELINE_SCHEMA_VERSION}`); } - } - // The resulting on-disk version: `target` after an upgrade, or the file's own higher version when it was - // written by newer code (never downgraded). - return Math.max(current, target); + for (let version = effective; version < target; version += 1) { + // Apply each migration AND stamp its resulting version in ONE transaction, so a failure part-way through the + // sequence leaves the file at the LAST fully-applied version: the next open resumes at the failed migration + // rather than re-running the ones that already succeeded (which, for a non-idempotent ALTER, would be a hard + // duplicate-column error). PRAGMA user_version is transactional in SQLite, so ROLLBACK undoes the migration's + // partial changes and its version stamp together. + db.exec("BEGIN"); + try { + const migration = migrations[version - BASELINE_SCHEMA_VERSION]; + // Index is in-range by construction (`version < target` and `target = BASELINE + migrations.length`). + migration(db); + db.exec(`PRAGMA user_version = ${version + 1}`); + db.exec("COMMIT"); + } + catch (error) { + db.exec("ROLLBACK"); + throw error; + } + } + // The resulting on-disk version: `target` after an upgrade, or the file's own higher version when it was + // written by newer code (never downgraded). + return Math.max(current, target); } +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2NoZW1hLXZlcnNpb24uanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJzY2hlbWEtdmVyc2lvbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFvQkEsMkZBQTJGO0FBQzNGLE1BQU0sQ0FBQyxNQUFNLHVCQUF1QixHQUFHLENBQUMsQ0FBQztBQUV6Qyw2R0FBNkc7QUFDN0csTUFBTSxVQUFVLGlCQUFpQixDQUFDLEVBQWdCO0lBQ2hELE1BQU0sR0FBRyxHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUMscUJBQXFCLENBQUMsQ0FBQyxHQUFHLEVBQUUsQ0FBQztJQUNwRCxNQUFNLEdBQUcsR0FBRyxHQUFHLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUMvQyxPQUFPLE1BQU0sQ0FBQyxTQUFTLENBQUMsR0FBRyxDQUFDLElBQUksR0FBRyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUM7QUFDckQsQ0FBQztBQUVEOzs7Ozs7Ozs7R0FTRztBQUNILE1BQU0sVUFBVSxxQkFBcUIsQ0FBQyxFQUFnQixFQUFFLGFBQWdDLEVBQUU7SUFDeEYsTUFBTSxNQUFNLEdBQUcsdUJBQXVCLEdBQUcsVUFBVSxDQUFDLE1BQU0sQ0FBQztJQUMzRCxNQUFNLE9BQU8sR0FBRyxpQkFBaUIsQ0FBQyxFQUFFLENBQUMsQ0FBQztJQUN0Qyx5R0FBeUc7SUFDekcsTUFBTSxTQUFTLEdBQUcsT0FBTyxHQUFHLHVCQUF1QixDQUFDLENBQUMsQ0FBQyx1QkFBdUIsQ0FBQyxDQUFDLENBQUMsT0FBTyxDQUFDO0lBQ3hGLDBHQUEwRztJQUMxRyw4R0FBOEc7SUFDOUcsMkdBQTJHO0lBQzNHLDhHQUE4RztJQUM5RyxJQUFJLE9BQU8sR0FBRyx1QkFBdUIsRUFBRSxDQUFDO1FBQ3RDLEVBQUUsQ0FBQyxJQUFJLENBQUMseUJBQXlCLHVCQUF1QixFQUFFLENBQUMsQ0FBQztJQUM5RCxDQUFDO0lBQ0QsS0FBSyxJQUFJLE9BQU8sR0FBRyxTQUFTLEVBQUUsT0FBTyxHQUFHLE1BQU0sRUFBRSxPQUFPLElBQUksQ0FBQyxFQUFFLENBQUM7UUFDN0QsNkdBQTZHO1FBQzdHLDRHQUE0RztRQUM1Ryw2R0FBNkc7UUFDN0csOEdBQThHO1FBQzlHLGtEQUFrRDtRQUNsRCxFQUFFLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQ2pCLElBQUksQ0FBQztZQUNILE1BQU0sU0FBUyxHQUFHLFVBQVUsQ0FBQyxPQUFPLEdBQUcsdUJBQXVCLENBQUMsQ0FBQztZQUNoRSxzR0FBc0c7WUFDdEcsU0FBVSxDQUFDLEVBQUUsQ0FBQyxDQUFDO1lBQ2YsRUFBRSxDQUFDLElBQUksQ0FBQyx5QkFBeUIsT0FBTyxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUM7WUFDaEQsRUFBRSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsQ0FBQztRQUNwQixDQUFDO1FBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztZQUNmLEVBQUUsQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUM7WUFDcEIsTUFBTSxLQUFLLENBQUM7UUFDZCxDQUFDO0lBQ0gsQ0FBQztJQUNELHlHQUF5RztJQUN6Ryw0Q0FBNEM7SUFDNUMsT0FBTyxJQUFJLENBQUMsR0FBRyxDQUFDLE9BQU8sRUFBRSxNQUFNLENBQUMsQ0FBQztBQUNuQyxDQUFDIn0= \ No newline at end of file diff --git a/packages/loopover-miner/lib/schema-version.ts b/packages/loopover-miner/lib/schema-version.ts new file mode 100644 index 0000000000..7438970038 --- /dev/null +++ b/packages/loopover-miner/lib/schema-version.ts @@ -0,0 +1,74 @@ +import type { DatabaseSync } from "node:sqlite"; + +// Lightweight schema-versioning convention shared across the miner's local SQLite stores (#4832). +// +// Every store bootstraps its tables with `CREATE TABLE IF NOT EXISTS ...` but, until now, carried no +// `user_version`/migration mechanism at all — so an older on-disk file was silently reused with a stale shape. +// This module adds the missing convention without the weight of the main product's `migrations/` runner: each +// store's bootstrap schema is treated as version 1 (BASELINE_SCHEMA_VERSION), and a store's `migrations` array +// describes ONLY the changes AFTER that baseline (`migrations[i]` upgrades the schema from version i+1 to i+2). +// `applySchemaMigrations` reads the file's current `PRAGMA user_version`, runs exactly the pending migrations in +// order, and stamps the new version — so opening an older-schema file runs its outstanding migrations instead of +// silently continuing on an incompatible shape. A pre-versioning file (user_version 0) already carries the +// baseline tables (the idempotent `CREATE TABLE IF NOT EXISTS` ran), so it is treated as the baseline before any +// post-baseline migration is applied. Pure control flow over an injected `DatabaseSync` handle: no IO of its own +// beyond the PRAGMA read/write and the caller-supplied migration functions, and deterministic given the same +// handle + migration list. + +/** A single post-baseline schema migration: mutate the store in place to advance it exactly one version. */ +export type SchemaMigration = (db: DatabaseSync) => void; + +/** The bootstrap schema every store creates inline is, by convention, schema version 1. */ +export const BASELINE_SCHEMA_VERSION = 1; + +/** Read a store's current `PRAGMA user_version`, coercing any absent/invalid value to 0 (pre-versioning). */ +export function readSchemaVersion(db: DatabaseSync): number { + const row = db.prepare("PRAGMA user_version").get(); + const raw = row ? Number(row.user_version) : 0; + return Number.isInteger(raw) && raw >= 0 ? raw : 0; +} + +/** + * Bring a store's on-disk schema up to date, then stamp its `user_version`. `migrations[i]` upgrades from + * version i+1 to i+2, so the target version is `BASELINE_SCHEMA_VERSION + migrations.length`. Every migration + * whose resulting version is above the file's current version runs, in order; a file already at (or past) the + * target runs none. Returns the resulting version. Never runs a migration twice (the stamped `user_version` + * gates re-runs on the next open) and never DOWNGRADES: a file written by newer code with more migrations is + * left at its higher version rather than stamped back down. Each migration and its version stamp are applied in + * one transaction, so a failure part-way through the sequence leaves the file at the last fully-applied version + * and re-opening resumes at the failed migration (a throwing migration rethrows after its changes roll back). + */ +export function applySchemaMigrations(db: DatabaseSync, migrations: SchemaMigration[] = []): number { + const target = BASELINE_SCHEMA_VERSION + migrations.length; + const current = readSchemaVersion(db); + // A pre-versioning file (0) already holds the baseline schema, so advance from the baseline, not from 0. + const effective = current < BASELINE_SCHEMA_VERSION ? BASELINE_SCHEMA_VERSION : current; + // Stamp a pre-versioning file up to the baseline first, so a store with NO post-baseline migrations still + // records a version. Only ever stamp UPWARD: a file already at or past the baseline (including one written by + // newer code with more migrations) is never downgraded. `user_version` is an integer PRAGMA that cannot be + // parameterized; every stamped value here is a computed integer, never caller text, so interpolating is safe. + if (current < BASELINE_SCHEMA_VERSION) { + db.exec(`PRAGMA user_version = ${BASELINE_SCHEMA_VERSION}`); + } + for (let version = effective; version < target; version += 1) { + // Apply each migration AND stamp its resulting version in ONE transaction, so a failure part-way through the + // sequence leaves the file at the LAST fully-applied version: the next open resumes at the failed migration + // rather than re-running the ones that already succeeded (which, for a non-idempotent ALTER, would be a hard + // duplicate-column error). PRAGMA user_version is transactional in SQLite, so ROLLBACK undoes the migration's + // partial changes and its version stamp together. + db.exec("BEGIN"); + try { + const migration = migrations[version - BASELINE_SCHEMA_VERSION]; + // Index is in-range by construction (`version < target` and `target = BASELINE + migrations.length`). + migration!(db); + db.exec(`PRAGMA user_version = ${version + 1}`); + db.exec("COMMIT"); + } catch (error) { + db.exec("ROLLBACK"); + throw error; + } + } + // The resulting on-disk version: `target` after an upgrade, or the file's own higher version when it was + // written by newer code (never downgraded). + return Math.max(current, target); +} diff --git a/packages/loopover-miner/lib/store-maintenance.d.ts b/packages/loopover-miner/lib/store-maintenance.d.ts index ddc82cdce0..8b67a9ad0b 100644 --- a/packages/loopover-miner/lib/store-maintenance.d.ts +++ b/packages/loopover-miner/lib/store-maintenance.d.ts @@ -1,37 +1,100 @@ -import type { DatabaseSync } from "node:sqlite"; - -export const LEDGER_RETENTION_DAYS_ENV: string; -export const LEDGER_RETENTION_MAX_ROWS_ENV: string; - -export type LedgerRetentionSpec = { table: string; timestampColumn: string; orderColumn: string }; -export const EVENT_LEDGER_RETENTION_SPEC: LedgerRetentionSpec; -export const GOVERNOR_LEDGER_RETENTION_SPEC: LedgerRetentionSpec; -export const PREDICTION_LEDGER_RETENTION_SPEC: LedgerRetentionSpec; - -export type LedgerPurgeSpec = { table: string; repoColumn: string }; -export const CLAIM_LEDGER_PURGE_SPEC: LedgerPurgeSpec; -export const EVENT_LEDGER_PURGE_SPEC: LedgerPurgeSpec; -export const GOVERNOR_LEDGER_PURGE_SPEC: LedgerPurgeSpec; -export const PREDICTION_LEDGER_PURGE_SPEC: LedgerPurgeSpec; -export const PORTFOLIO_QUEUE_PURGE_SPEC: LedgerPurgeSpec; -export const RUN_STATE_PURGE_SPEC: LedgerPurgeSpec; -export const CONTRIBUTION_PROFILE_CACHE_PURGE_SPEC: LedgerPurgeSpec; -export const GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC: LedgerPurgeSpec; -export const GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC: LedgerPurgeSpec; -export const POLICY_VERDICT_CACHE_PURGE_SPEC: LedgerPurgeSpec; - -export type StoreIntegrityResult = { name: string; ok: boolean; detail: string }; -export type LedgerRetentionPolicy = { maxAgeMs?: number; maxRows?: number }; - -export function describeError(error: unknown): string; -export function classifyIntegrityRows(rows: Array<{ integrity_check?: unknown }>): { ok: boolean; note: string }; -export function checkStoreIntegrity(name: string, dbPath: string): StoreIntegrityResult; -export function resolveLedgerRetentionPolicy(env?: Record): LedgerRetentionPolicy | null; -export function pruneLedgerByRetention( - db: DatabaseSync, - spec: LedgerRetentionSpec, - policy: LedgerRetentionPolicy | null, - nowMs: number, -): number; -export function purgeStoreByRepo(db: DatabaseSync, spec: LedgerPurgeSpec, repoFullName: string): number; -export function countStoreByRepo(db: DatabaseSync, spec: LedgerPurgeSpec, repoFullName: string): number; +import { DatabaseSync } from "node:sqlite"; +/** Env opt-ins for ledger retention (unset ⇒ retention disabled). */ +export declare const LEDGER_RETENTION_DAYS_ENV = "LOOPOVER_MINER_LEDGER_RETENTION_DAYS"; +export declare const LEDGER_RETENTION_MAX_ROWS_ENV = "LOOPOVER_MINER_LEDGER_RETENTION_MAX_ROWS"; +export type LedgerRetentionSpec = { + table: string; + timestampColumn: string; + orderColumn: string; +}; +/** Fixed retention specs for the three append-only ledgers. These identifiers are INTERNAL constants — never + * caller/user text — and are validated as plain identifiers before interpolation as defence in depth. */ +export declare const EVENT_LEDGER_RETENTION_SPEC: LedgerRetentionSpec; +export declare const GOVERNOR_LEDGER_RETENTION_SPEC: LedgerRetentionSpec; +export declare const PREDICTION_LEDGER_RETENTION_SPEC: LedgerRetentionSpec; +export type LedgerPurgeSpec = { + table: string; + repoColumn: string; +}; +/** Fixed purge specs (#5564, #6599) for the six stores whose rows are directly scoped by a `repoColumn`. Same + * internal-constant-only discipline as the retention specs above. `attempt-log.js` is deliberately absent: its + * payload is a free-form `Record` with no dedicated repo column, so a precise per-repo purge + * isn't possible there without risking false matches — `purge-cli.js` reports it as not-purgeable instead. */ +export declare const CLAIM_LEDGER_PURGE_SPEC: LedgerPurgeSpec; +export declare const EVENT_LEDGER_PURGE_SPEC: LedgerPurgeSpec; +export declare const GOVERNOR_LEDGER_PURGE_SPEC: LedgerPurgeSpec; +export declare const PREDICTION_LEDGER_PURGE_SPEC: LedgerPurgeSpec; +export declare const PORTFOLIO_QUEUE_PURGE_SPEC: LedgerPurgeSpec; +export declare const RUN_STATE_PURGE_SPEC: LedgerPurgeSpec; +/** Three more repo-scoped stores the original six missed (#7091), same `repoColumn` shape and same internal- + * constant-only discipline. The contribution-profile-cache table name comes from its schema module's own + * `CONTRIBUTION_PROFILE_STORE_TABLE` constant so this spec can't drift from a second hardcoded literal. + * governor-state holds two genuinely repo-scoped tables (reputation history + own submissions); + * `governor_scalar_state` is intentionally excluded — it is a single whole-run scalar row with no repo + * dimension. `governor_reputation_history` is purged on `repo_full_name` alone (its key is composite with + * `api_base_url`), so a right-to-be-forgotten sweep clears the repo across every forge host it was recorded + * against, not just the default one. */ +export declare const CONTRIBUTION_PROFILE_CACHE_PURGE_SPEC: LedgerPurgeSpec; +export declare const GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC: LedgerPurgeSpec; +export declare const GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC: LedgerPurgeSpec; +/** policy-verdict-cache (#6987), another repo-scoped store the earlier sweeps missed. Its `repo_scope TEXT + * PRIMARY KEY` is the per-repo column (a tenant forge host + `owner/repo`), the same `repoColumn` shape and + * internal-constant-only discipline as the specs above. `policy-doc-cache.js` stays out (keyed by URL, no repo + * column, exactly like `attempt-log.js`). */ +export declare const POLICY_VERDICT_CACHE_PURGE_SPEC: LedgerPurgeSpec; +export type StoreIntegrityResult = { + name: string; + ok: boolean; + detail: string; +}; +export type LedgerRetentionPolicy = { + maxAgeMs?: number; + maxRows?: number; +}; +/** A readable message for a caught value, whether or not it is an Error. */ +export declare function describeError(error: unknown): string; +/** + * Classify raw `PRAGMA integrity_check` rows. A healthy database yields a single `"ok"` row; a corrupt one yields + * one row per problem. Pure — extracted so both the healthy and problem paths are testable without a genuinely + * corrupt file (which SQLite typically refuses to open at all, i.e. the catch path below). + */ +export declare function classifyIntegrityRows(rows: Array<{ + integrity_check?: unknown; +}>): { + ok: boolean; + note: string; +}; +/** + * Run `PRAGMA integrity_check` on a single store file. A store that does not exist yet is healthy by absence + * (nothing to corrupt). Never throws: a store that cannot be opened or read is reported as not-ok, so one bad + * store cannot abort the whole doctor sweep. Opens the connection driver-enforced read-only -- `readOnly` + * (camelCase) is the only option key node:sqlite recognizes for this; the lowercase `readonly` is silently + * ignored and opens read-write instead (the exact gotcha claim-ledger.js's own openClaimLedgerReadOnly already + * documents), which would defeat the read-only guarantee this function's own docs claim. + */ +export declare function checkStoreIntegrity(name: string, dbPath: string): StoreIntegrityResult; +/** + * Resolve the opt-in ledger retention policy from an env object. OFF by default: returns null unless at least + * one bound is set to a positive value. A zero/negative/non-numeric value is treated as unset. When set, returns + * `{ maxAgeMs? }` (from a day count) and/or `{ maxRows? }`. + */ +export declare function resolveLedgerRetentionPolicy(env?: Record): LedgerRetentionPolicy | null; +/** + * Prune one append-only ledger per a resolved retention policy: delete rows older than the age bound AND rows + * beyond the row-count bound (keeping the newest `maxRows` by `orderColumn`), atomically. A null policy is a + * no-op. `nowMs` is caller-supplied (no internal clock). Timestamp columns are UTC ISO-8601 strings, which sort + * lexicographically in chronological order, so a string comparison against the ISO cutoff selects older rows. + */ +export declare function pruneLedgerByRetention(db: DatabaseSync, spec: LedgerRetentionSpec, policy: LedgerRetentionPolicy | null, nowMs: number): number; +/** + * Delete every row for one repo from a store (#5564). Unlike `pruneLedgerByRetention`, this never runs + * automatically — it exists solely so `purge-cli.js` can give an operator a real right-to-be-forgotten path. + * `repoFullName` is caller-normalized (owner/repo) before reaching here; this function only guards the SQL + * identifiers, matching `pruneLedgerByRetention`'s own defence-in-depth discipline. + */ +export declare function purgeStoreByRepo(db: DatabaseSync, spec: LedgerPurgeSpec, repoFullName: string): number; +/** + * Count rows for one repo in a store without deleting anything (#5564) — the read-only counterpart to + * `purgeStoreByRepo`, used by `purge-cli.js --dry-run` to report what a real purge would remove. + */ +export declare function countStoreByRepo(db: DatabaseSync, spec: LedgerPurgeSpec, repoFullName: string): number; diff --git a/packages/loopover-miner/lib/store-maintenance.js b/packages/loopover-miner/lib/store-maintenance.js index 687e4ae017..860991d691 100644 --- a/packages/loopover-miner/lib/store-maintenance.js +++ b/packages/loopover-miner/lib/store-maintenance.js @@ -14,17 +14,14 @@ import { existsSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; import { CONTRIBUTION_PROFILE_STORE_TABLE } from "./contribution-profile.js"; - /** Env opt-ins for ledger retention (unset ⇒ retention disabled). */ export const LEDGER_RETENTION_DAYS_ENV = "LOOPOVER_MINER_LEDGER_RETENTION_DAYS"; export const LEDGER_RETENTION_MAX_ROWS_ENV = "LOOPOVER_MINER_LEDGER_RETENTION_MAX_ROWS"; - /** Fixed retention specs for the three append-only ledgers. These identifiers are INTERNAL constants — never * caller/user text — and are validated as plain identifiers before interpolation as defence in depth. */ export const EVENT_LEDGER_RETENTION_SPEC = { table: "miner_event_ledger", timestampColumn: "created_at", orderColumn: "id" }; export const GOVERNOR_LEDGER_RETENTION_SPEC = { table: "governor_events", timestampColumn: "ts", orderColumn: "id" }; export const PREDICTION_LEDGER_RETENTION_SPEC = { table: "predictions", timestampColumn: "ts", orderColumn: "id" }; - /** Fixed purge specs (#5564, #6599) for the six stores whose rows are directly scoped by a `repoColumn`. Same * internal-constant-only discipline as the retention specs above. `attempt-log.js` is deliberately absent: its * payload is a free-form `Record` with no dedicated repo column, so a precise per-repo purge @@ -35,7 +32,6 @@ export const GOVERNOR_LEDGER_PURGE_SPEC = { table: "governor_events", repoColumn export const PREDICTION_LEDGER_PURGE_SPEC = { table: "predictions", repoColumn: "repo_full_name" }; export const PORTFOLIO_QUEUE_PURGE_SPEC = { table: "miner_portfolio_queue", repoColumn: "repo_full_name" }; export const RUN_STATE_PURGE_SPEC = { table: "miner_run_state", repoColumn: "repo_full_name" }; - /** Three more repo-scoped stores the original six missed (#7091), same `repoColumn` shape and same internal- * constant-only discipline. The contribution-profile-cache table name comes from its schema module's own * `CONTRIBUTION_PROFILE_STORE_TABLE` constant so this spec can't drift from a second hardcoded literal. @@ -47,32 +43,25 @@ export const RUN_STATE_PURGE_SPEC = { table: "miner_run_state", repoColumn: "rep export const CONTRIBUTION_PROFILE_CACHE_PURGE_SPEC = { table: CONTRIBUTION_PROFILE_STORE_TABLE, repoColumn: "repo_full_name" }; export const GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC = { table: "governor_reputation_history", repoColumn: "repo_full_name" }; export const GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC = { table: "governor_own_submissions", repoColumn: "repo_full_name" }; - /** policy-verdict-cache (#6987), another repo-scoped store the earlier sweeps missed. Its `repo_scope TEXT * PRIMARY KEY` is the per-repo column (a tenant forge host + `owner/repo`), the same `repoColumn` shape and * internal-constant-only discipline as the specs above. `policy-doc-cache.js` stays out (keyed by URL, no repo * column, exactly like `attempt-log.js`). */ export const POLICY_VERDICT_CACHE_PURGE_SPEC = { table: "policy_verdict_cache", repoColumn: "repo_scope" }; - const SQL_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; - /** A readable message for a caught value, whether or not it is an Error. */ export function describeError(error) { - return error instanceof Error ? error.message : String(error); + return error instanceof Error ? error.message : String(error); } - /** * Classify raw `PRAGMA integrity_check` rows. A healthy database yields a single `"ok"` row; a corrupt one yields * one row per problem. Pure — extracted so both the healthy and problem paths are testable without a genuinely * corrupt file (which SQLite typically refuses to open at all, i.e. the catch path below). - * @param {Array<{ integrity_check?: unknown }>} rows - * @returns {{ ok: boolean, note: string }} */ export function classifyIntegrityRows(rows) { - const problems = rows.map((row) => String(row.integrity_check)).filter((value) => value !== "ok"); - return problems.length === 0 ? { ok: true, note: "ok" } : { ok: false, note: problems.join("; ") }; + const problems = rows.map((row) => String(row.integrity_check)).filter((value) => value !== "ok"); + return problems.length === 0 ? { ok: true, note: "ok" } : { ok: false, note: problems.join("; ") }; } - /** * Run `PRAGMA integrity_check` on a single store file. A store that does not exist yet is healthy by absence * (nothing to corrupt). Never throws: a store that cannot be opened or read is reported as not-ok, so one bad @@ -80,125 +69,112 @@ export function classifyIntegrityRows(rows) { * (camelCase) is the only option key node:sqlite recognizes for this; the lowercase `readonly` is silently * ignored and opens read-write instead (the exact gotcha claim-ledger.js's own openClaimLedgerReadOnly already * documents), which would defeat the read-only guarantee this function's own docs claim. - * @param {string} name - the check label (e.g. "event-ledger"). - * @param {string} dbPath - the store file path. - * @returns {{ name: string, ok: boolean, detail: string }} */ export function checkStoreIntegrity(name, dbPath) { - if (!existsSync(dbPath)) { - return { name, ok: true, detail: `${dbPath}: not created yet` }; - } - let db; - try { - db = new DatabaseSync(dbPath, { readOnly: true }); - const { ok, note } = classifyIntegrityRows(db.prepare("PRAGMA integrity_check").all()); - return { name, ok, detail: `${dbPath}: ${note}` }; - } catch (error) { - return { name, ok: false, detail: `${dbPath}: ${describeError(error)}` }; - } finally { - db?.close(); - } + if (!existsSync(dbPath)) { + return { name, ok: true, detail: `${dbPath}: not created yet` }; + } + let db; + try { + db = new DatabaseSync(dbPath, { readOnly: true }); + const { ok, note } = classifyIntegrityRows(db.prepare("PRAGMA integrity_check").all()); + return { name, ok, detail: `${dbPath}: ${note}` }; + } + catch (error) { + return { name, ok: false, detail: `${dbPath}: ${describeError(error)}` }; + } + finally { + db?.close(); + } } - /** Coerce an env value to a positive integer, or null (unset/blank/zero/negative/non-finite ⇒ null ⇒ disabled). * Floors BEFORE the positivity test, so a fractional value below 1 (e.g. "0.5") floors to 0 and disables the * bound rather than becoming a dangerous 0 that would prune the whole ledger. */ function positiveIntOrNull(raw) { - if (raw === undefined || raw === null || String(raw).trim() === "") return null; - const numeric = Math.floor(Number(raw)); - return Number.isFinite(numeric) && numeric > 0 ? numeric : null; + if (raw === undefined || raw === null || String(raw).trim() === "") + return null; + const numeric = Math.floor(Number(raw)); + return Number.isFinite(numeric) && numeric > 0 ? numeric : null; } - /** * Resolve the opt-in ledger retention policy from an env object. OFF by default: returns null unless at least * one bound is set to a positive value. A zero/negative/non-numeric value is treated as unset. When set, returns * `{ maxAgeMs? }` (from a day count) and/or `{ maxRows? }`. - * @param {NodeJS.ProcessEnv} [env] - * @returns {{ maxAgeMs?: number, maxRows?: number } | null} */ export function resolveLedgerRetentionPolicy(env = process.env) { - const maxAgeDays = positiveIntOrNull(env[LEDGER_RETENTION_DAYS_ENV]); - const maxRows = positiveIntOrNull(env[LEDGER_RETENTION_MAX_ROWS_ENV]); - if (maxAgeDays === null && maxRows === null) return null; - const policy = {}; - if (maxAgeDays !== null) policy.maxAgeMs = maxAgeDays * 24 * 60 * 60 * 1000; - if (maxRows !== null) policy.maxRows = maxRows; - return policy; + const maxAgeDays = positiveIntOrNull(env[LEDGER_RETENTION_DAYS_ENV]); + const maxRows = positiveIntOrNull(env[LEDGER_RETENTION_MAX_ROWS_ENV]); + if (maxAgeDays === null && maxRows === null) + return null; + const policy = {}; + if (maxAgeDays !== null) + policy.maxAgeMs = maxAgeDays * 24 * 60 * 60 * 1000; + if (maxRows !== null) + policy.maxRows = maxRows; + return policy; } - /** * Prune one append-only ledger per a resolved retention policy: delete rows older than the age bound AND rows * beyond the row-count bound (keeping the newest `maxRows` by `orderColumn`), atomically. A null policy is a * no-op. `nowMs` is caller-supplied (no internal clock). Timestamp columns are UTC ISO-8601 strings, which sort * lexicographically in chronological order, so a string comparison against the ISO cutoff selects older rows. - * @param {import("node:sqlite").DatabaseSync} db - * @param {{ table: string, timestampColumn: string, orderColumn: string }} spec - * @param {{ maxAgeMs?: number, maxRows?: number } | null} policy - * @param {number} nowMs - * @returns {number} rows deleted */ export function pruneLedgerByRetention(db, spec, policy, nowMs) { - if (!policy) return 0; - for (const identifier of [spec.table, spec.timestampColumn, spec.orderColumn]) { - if (!SQL_IDENTIFIER.test(identifier)) throw new Error(`unsafe SQL identifier: ${identifier}`); - } - let deleted = 0; - db.exec("BEGIN"); - try { - // Both bounds are guarded to be strictly positive as defence in depth: a 0 age would prune everything older - // than `now`, and a 0 row-cap makes `LIMIT 0` match no rows so `NOT IN (empty)` would delete the whole ledger. - if (policy.maxAgeMs !== undefined && policy.maxAgeMs > 0) { - const cutoff = new Date(nowMs - policy.maxAgeMs).toISOString(); - const info = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.timestampColumn} < ?`).run(cutoff); - deleted += Number(info.changes); + if (!policy) + return 0; + for (const identifier of [spec.table, spec.timestampColumn, spec.orderColumn]) { + if (!SQL_IDENTIFIER.test(identifier)) + throw new Error(`unsafe SQL identifier: ${identifier}`); } - if (policy.maxRows !== undefined && policy.maxRows >= 1) { - const info = db - .prepare( - `DELETE FROM ${spec.table} WHERE ${spec.orderColumn} NOT IN ` + - `(SELECT ${spec.orderColumn} FROM ${spec.table} ORDER BY ${spec.orderColumn} DESC LIMIT ?)`, - ) - .run(policy.maxRows); - deleted += Number(info.changes); + let deleted = 0; + db.exec("BEGIN"); + try { + // Both bounds are guarded to be strictly positive as defence in depth: a 0 age would prune everything older + // than `now`, and a 0 row-cap makes `LIMIT 0` match no rows so `NOT IN (empty)` would delete the whole ledger. + if (policy.maxAgeMs !== undefined && policy.maxAgeMs > 0) { + const cutoff = new Date(nowMs - policy.maxAgeMs).toISOString(); + const info = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.timestampColumn} < ?`).run(cutoff); + deleted += Number(info.changes); + } + if (policy.maxRows !== undefined && policy.maxRows >= 1) { + const info = db + .prepare(`DELETE FROM ${spec.table} WHERE ${spec.orderColumn} NOT IN ` + + `(SELECT ${spec.orderColumn} FROM ${spec.table} ORDER BY ${spec.orderColumn} DESC LIMIT ?)`) + .run(policy.maxRows); + deleted += Number(info.changes); + } + db.exec("COMMIT"); } - db.exec("COMMIT"); - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } - return deleted; + catch (error) { + db.exec("ROLLBACK"); + throw error; + } + return deleted; } - /** * Delete every row for one repo from a store (#5564). Unlike `pruneLedgerByRetention`, this never runs * automatically — it exists solely so `purge-cli.js` can give an operator a real right-to-be-forgotten path. * `repoFullName` is caller-normalized (owner/repo) before reaching here; this function only guards the SQL * identifiers, matching `pruneLedgerByRetention`'s own defence-in-depth discipline. - * @param {import("node:sqlite").DatabaseSync} db - * @param {{ table: string, repoColumn: string }} spec - * @param {string} repoFullName - * @returns {number} rows deleted */ export function purgeStoreByRepo(db, spec, repoFullName) { - for (const identifier of [spec.table, spec.repoColumn]) { - if (!SQL_IDENTIFIER.test(identifier)) throw new Error(`unsafe SQL identifier: ${identifier}`); - } - const info = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.repoColumn} = ?`).run(repoFullName); - return Number(info.changes); + for (const identifier of [spec.table, spec.repoColumn]) { + if (!SQL_IDENTIFIER.test(identifier)) + throw new Error(`unsafe SQL identifier: ${identifier}`); + } + const info = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.repoColumn} = ?`).run(repoFullName); + return Number(info.changes); } - /** * Count rows for one repo in a store without deleting anything (#5564) — the read-only counterpart to * `purgeStoreByRepo`, used by `purge-cli.js --dry-run` to report what a real purge would remove. - * @param {import("node:sqlite").DatabaseSync} db - * @param {{ table: string, repoColumn: string }} spec - * @param {string} repoFullName - * @returns {number} matching row count */ export function countStoreByRepo(db, spec, repoFullName) { - for (const identifier of [spec.table, spec.repoColumn]) { - if (!SQL_IDENTIFIER.test(identifier)) throw new Error(`unsafe SQL identifier: ${identifier}`); - } - const row = db.prepare(`SELECT COUNT(*) AS count FROM ${spec.table} WHERE ${spec.repoColumn} = ?`).get(repoFullName); - return Number(row.count); + for (const identifier of [spec.table, spec.repoColumn]) { + if (!SQL_IDENTIFIER.test(identifier)) + throw new Error(`unsafe SQL identifier: ${identifier}`); + } + const row = db.prepare(`SELECT COUNT(*) AS count FROM ${spec.table} WHERE ${spec.repoColumn} = ?`).get(repoFullName); + return Number(row?.count); } +//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3RvcmUtbWFpbnRlbmFuY2UuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJzdG9yZS1tYWludGVuYW5jZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSx5R0FBeUc7QUFDekcsRUFBRTtBQUNGLGtHQUFrRztBQUNsRywrR0FBK0c7QUFDL0csb0ZBQW9GO0FBQ3BGLDJHQUEyRztBQUMzRyxpSEFBaUg7QUFDakgsaUZBQWlGO0FBQ2pGLDhHQUE4RztBQUM5RywwR0FBMEc7QUFDMUcsK0ZBQStGO0FBQy9GLGlIQUFpSDtBQUNqSCx3RkFBd0Y7QUFDeEYsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLFNBQVMsQ0FBQztBQUNyQyxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sYUFBYSxDQUFDO0FBQzNDLE9BQU8sRUFBRSxnQ0FBZ0MsRUFBRSxNQUFNLDJCQUEyQixDQUFDO0FBRTdFLHFFQUFxRTtBQUNyRSxNQUFNLENBQUMsTUFBTSx5QkFBeUIsR0FBRyxzQ0FBc0MsQ0FBQztBQUNoRixNQUFNLENBQUMsTUFBTSw2QkFBNkIsR0FBRywwQ0FBMEMsQ0FBQztBQUl4RjswR0FDMEc7QUFDMUcsTUFBTSxDQUFDLE1BQU0sMkJBQTJCLEdBQXdCLEVBQUUsS0FBSyxFQUFFLG9CQUFvQixFQUFFLGVBQWUsRUFBRSxZQUFZLEVBQUUsV0FBVyxFQUFFLElBQUksRUFBRSxDQUFDO0FBQ2xKLE1BQU0sQ0FBQyxNQUFNLDhCQUE4QixHQUF3QixFQUFFLEtBQUssRUFBRSxpQkFBaUIsRUFBRSxlQUFlLEVBQUUsSUFBSSxFQUFFLFdBQVcsRUFBRSxJQUFJLEVBQUUsQ0FBQztBQUMxSSxNQUFNLENBQUMsTUFBTSxnQ0FBZ0MsR0FBd0IsRUFBRSxLQUFLLEVBQUUsYUFBYSxFQUFFLGVBQWUsRUFBRSxJQUFJLEVBQUUsV0FBVyxFQUFFLElBQUksRUFBRSxDQUFDO0FBSXhJOzs7K0dBRytHO0FBQy9HLE1BQU0sQ0FBQyxNQUFNLHVCQUF1QixHQUFvQixFQUFFLEtBQUssRUFBRSxjQUFjLEVBQUUsVUFBVSxFQUFFLGdCQUFnQixFQUFFLENBQUM7QUFDaEgsTUFBTSxDQUFDLE1BQU0sdUJBQXVCLEdBQW9CLEVBQUUsS0FBSyxFQUFFLG9CQUFvQixFQUFFLFVBQVUsRUFBRSxnQkFBZ0IsRUFBRSxDQUFDO0FBQ3RILE1BQU0sQ0FBQyxNQUFNLDBCQUEwQixHQUFvQixFQUFFLEtBQUssRUFBRSxpQkFBaUIsRUFBRSxVQUFVLEVBQUUsZ0JBQWdCLEVBQUUsQ0FBQztBQUN0SCxNQUFNLENBQUMsTUFBTSw0QkFBNEIsR0FBb0IsRUFBRSxLQUFLLEVBQUUsYUFBYSxFQUFFLFVBQVUsRUFBRSxnQkFBZ0IsRUFBRSxDQUFDO0FBQ3BILE1BQU0sQ0FBQyxNQUFNLDBCQUEwQixHQUFvQixFQUFFLEtBQUssRUFBRSx1QkFBdUIsRUFBRSxVQUFVLEVBQUUsZ0JBQWdCLEVBQUUsQ0FBQztBQUM1SCxNQUFNLENBQUMsTUFBTSxvQkFBb0IsR0FBb0IsRUFBRSxLQUFLLEVBQUUsaUJBQWlCLEVBQUUsVUFBVSxFQUFFLGdCQUFnQixFQUFFLENBQUM7QUFFaEg7Ozs7Ozs7eUNBT3lDO0FBQ3pDLE1BQU0sQ0FBQyxNQUFNLHFDQUFxQyxHQUFvQixFQUFFLEtBQUssRUFBRSxnQ0FBZ0MsRUFBRSxVQUFVLEVBQUUsZ0JBQWdCLEVBQUUsQ0FBQztBQUNoSixNQUFNLENBQUMsTUFBTSxzQ0FBc0MsR0FBb0IsRUFBRSxLQUFLLEVBQUUsNkJBQTZCLEVBQUUsVUFBVSxFQUFFLGdCQUFnQixFQUFFLENBQUM7QUFDOUksTUFBTSxDQUFDLE1BQU0sbUNBQW1DLEdBQW9CLEVBQUUsS0FBSyxFQUFFLDBCQUEwQixFQUFFLFVBQVUsRUFBRSxnQkFBZ0IsRUFBRSxDQUFDO0FBRXhJOzs7OENBRzhDO0FBQzlDLE1BQU0sQ0FBQyxNQUFNLCtCQUErQixHQUFvQixFQUFFLEtBQUssRUFBRSxzQkFBc0IsRUFBRSxVQUFVLEVBQUUsWUFBWSxFQUFFLENBQUM7QUFLNUgsTUFBTSxjQUFjLEdBQUcsMEJBQTBCLENBQUM7QUFFbEQsNEVBQTRFO0FBQzVFLE1BQU0sVUFBVSxhQUFhLENBQUMsS0FBYztJQUMxQyxPQUFPLEtBQUssWUFBWSxLQUFLLENBQUMsQ0FBQyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQztBQUNoRSxDQUFDO0FBRUQ7Ozs7R0FJRztBQUNILE1BQU0sVUFBVSxxQkFBcUIsQ0FBQyxJQUEwQztJQUM5RSxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsR0FBRyxFQUFFLEVBQUUsQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLGVBQWUsQ0FBQyxDQUFDLENBQUMsTUFBTSxDQUFDLENBQUMsS0FBSyxFQUFFLEVBQUUsQ0FBQyxLQUFLLEtBQUssSUFBSSxDQUFDLENBQUM7SUFDbEcsT0FBTyxRQUFRLENBQUMsTUFBTSxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUMsRUFBRSxFQUFFLEVBQUUsSUFBSSxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxFQUFFLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxRQUFRLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxFQUFFLENBQUM7QUFDckcsQ0FBQztBQUVEOzs7Ozs7O0dBT0c7QUFDSCxNQUFNLFVBQVUsbUJBQW1CLENBQUMsSUFBWSxFQUFFLE1BQWM7SUFDOUQsSUFBSSxDQUFDLFVBQVUsQ0FBQyxNQUFNLENBQUMsRUFBRSxDQUFDO1FBQ3hCLE9BQU8sRUFBRSxJQUFJLEVBQUUsRUFBRSxFQUFFLElBQUksRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLG1CQUFtQixFQUFFLENBQUM7SUFDbEUsQ0FBQztJQUNELElBQUksRUFBNEIsQ0FBQztJQUNqQyxJQUFJLENBQUM7UUFDSCxFQUFFLEdBQUcsSUFBSSxZQUFZLENBQUMsTUFBTSxFQUFFLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7UUFDbEQsTUFBTSxFQUFFLEVBQUUsRUFBRSxJQUFJLEVBQUUsR0FBRyxxQkFBcUIsQ0FBQyxFQUFFLENBQUMsT0FBTyxDQUFDLHdCQUF3QixDQUFDLENBQUMsR0FBRyxFQUFFLENBQUMsQ0FBQztRQUN2RixPQUFPLEVBQUUsSUFBSSxFQUFFLEVBQUUsRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEtBQUssSUFBSSxFQUFFLEVBQUUsQ0FBQztJQUNwRCxDQUFDO0lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztRQUNmLE9BQU8sRUFBRSxJQUFJLEVBQUUsRUFBRSxFQUFFLEtBQUssRUFBRSxNQUFNLEVBQUUsR0FBRyxNQUFNLEtBQUssYUFBYSxDQUFDLEtBQUssQ0FBQyxFQUFFLEVBQUUsQ0FBQztJQUMzRSxDQUFDO1lBQVMsQ0FBQztRQUNULEVBQUUsRUFBRSxLQUFLLEVBQUUsQ0FBQztJQUNkLENBQUM7QUFDSCxDQUFDO0FBRUQ7O2tGQUVrRjtBQUNsRixTQUFTLGlCQUFpQixDQUFDLEdBQXVCO0lBQ2hELElBQUksR0FBRyxLQUFLLFNBQVMsSUFBSSxHQUFHLEtBQUssSUFBSSxJQUFJLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQyxJQUFJLEVBQUUsS0FBSyxFQUFFO1FBQUUsT0FBTyxJQUFJLENBQUM7SUFDaEYsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLEtBQUssQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztJQUN4QyxPQUFPLE1BQU0sQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLElBQUksT0FBTyxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUM7QUFDbEUsQ0FBQztBQUVEOzs7O0dBSUc7QUFDSCxNQUFNLFVBQVUsNEJBQTRCLENBQzFDLE1BQTBDLE9BQU8sQ0FBQyxHQUFHO0lBRXJELE1BQU0sVUFBVSxHQUFHLGlCQUFpQixDQUFDLEdBQUcsQ0FBQyx5QkFBeUIsQ0FBQyxDQUFDLENBQUM7SUFDckUsTUFBTSxPQUFPLEdBQUcsaUJBQWlCLENBQUMsR0FBRyxDQUFDLDZCQUE2QixDQUFDLENBQUMsQ0FBQztJQUN0RSxJQUFJLFVBQVUsS0FBSyxJQUFJLElBQUksT0FBTyxLQUFLLElBQUk7UUFBRSxPQUFPLElBQUksQ0FBQztJQUN6RCxNQUFNLE1BQU0sR0FBMEIsRUFBRSxDQUFDO0lBQ3pDLElBQUksVUFBVSxLQUFLLElBQUk7UUFBRSxNQUFNLENBQUMsUUFBUSxHQUFHLFVBQVUsR0FBRyxFQUFFLEdBQUcsRUFBRSxHQUFHLEVBQUUsR0FBRyxJQUFJLENBQUM7SUFDNUUsSUFBSSxPQUFPLEtBQUssSUFBSTtRQUFFLE1BQU0sQ0FBQyxPQUFPLEdBQUcsT0FBTyxDQUFDO0lBQy9DLE9BQU8sTUFBTSxDQUFDO0FBQ2hCLENBQUM7QUFFRDs7Ozs7R0FLRztBQUNILE1BQU0sVUFBVSxzQkFBc0IsQ0FDcEMsRUFBZ0IsRUFDaEIsSUFBeUIsRUFDekIsTUFBb0MsRUFDcEMsS0FBYTtJQUViLElBQUksQ0FBQyxNQUFNO1FBQUUsT0FBTyxDQUFDLENBQUM7SUFDdEIsS0FBSyxNQUFNLFVBQVUsSUFBSSxDQUFDLElBQUksQ0FBQyxLQUFLLEVBQUUsSUFBSSxDQUFDLGVBQWUsRUFBRSxJQUFJLENBQUMsV0FBVyxDQUFDLEVBQUUsQ0FBQztRQUM5RSxJQUFJLENBQUMsY0FBYyxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUM7WUFBRSxNQUFNLElBQUksS0FBSyxDQUFDLDBCQUEwQixVQUFVLEVBQUUsQ0FBQyxDQUFDO0lBQ2hHLENBQUM7SUFDRCxJQUFJLE9BQU8sR0FBRyxDQUFDLENBQUM7SUFDaEIsRUFBRSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQztJQUNqQixJQUFJLENBQUM7UUFDSCw0R0FBNEc7UUFDNUcsK0dBQStHO1FBQy9HLElBQUksTUFBTSxDQUFDLFFBQVEsS0FBSyxTQUFTLElBQUksTUFBTSxDQUFDLFFBQVEsR0FBRyxDQUFDLEVBQUUsQ0FBQztZQUN6RCxNQUFNLE1BQU0sR0FBRyxJQUFJLElBQUksQ0FBQyxLQUFLLEdBQUcsTUFBTSxDQUFDLFFBQVEsQ0FBQyxDQUFDLFdBQVcsRUFBRSxDQUFDO1lBQy9ELE1BQU0sSUFBSSxHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUMsZUFBZSxJQUFJLENBQUMsS0FBSyxVQUFVLElBQUksQ0FBQyxlQUFlLE1BQU0sQ0FBQyxDQUFDLEdBQUcsQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUNuRyxPQUFPLElBQUksTUFBTSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQztRQUNsQyxDQUFDO1FBQ0QsSUFBSSxNQUFNLENBQUMsT0FBTyxLQUFLLFNBQVMsSUFBSSxNQUFNLENBQUMsT0FBTyxJQUFJLENBQUMsRUFBRSxDQUFDO1lBQ3hELE1BQU0sSUFBSSxHQUFHLEVBQUU7aUJBQ1osT0FBTyxDQUNOLGVBQWUsSUFBSSxDQUFDLEtBQUssVUFBVSxJQUFJLENBQUMsV0FBVyxVQUFVO2dCQUMzRCxXQUFXLElBQUksQ0FBQyxXQUFXLFNBQVMsSUFBSSxDQUFDLEtBQUssYUFBYSxJQUFJLENBQUMsV0FBVyxnQkFBZ0IsQ0FDOUY7aUJBQ0EsR0FBRyxDQUFDLE1BQU0sQ0FBQyxPQUFPLENBQUMsQ0FBQztZQUN2QixPQUFPLElBQUksTUFBTSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQztRQUNsQyxDQUFDO1FBQ0QsRUFBRSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsQ0FBQztJQUNwQixDQUFDO0lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztRQUNmLEVBQUUsQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUM7UUFDcEIsTUFBTSxLQUFLLENBQUM7SUFDZCxDQUFDO0lBQ0QsT0FBTyxPQUFPLENBQUM7QUFDakIsQ0FBQztBQUVEOzs7OztHQUtHO0FBQ0gsTUFBTSxVQUFVLGdCQUFnQixDQUFDLEVBQWdCLEVBQUUsSUFBcUIsRUFBRSxZQUFvQjtJQUM1RixLQUFLLE1BQU0sVUFBVSxJQUFJLENBQUMsSUFBSSxDQUFDLEtBQUssRUFBRSxJQUFJLENBQUMsVUFBVSxDQUFDLEVBQUUsQ0FBQztRQUN2RCxJQUFJLENBQUMsY0FBYyxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUM7WUFBRSxNQUFNLElBQUksS0FBSyxDQUFDLDBCQUEwQixVQUFVLEVBQUUsQ0FBQyxDQUFDO0lBQ2hHLENBQUM7SUFDRCxNQUFNLElBQUksR0FBRyxFQUFFLENBQUMsT0FBTyxDQUFDLGVBQWUsSUFBSSxDQUFDLEtBQUssVUFBVSxJQUFJLENBQUMsVUFBVSxNQUFNLENBQUMsQ0FBQyxHQUFHLENBQUMsWUFBWSxDQUFDLENBQUM7SUFDcEcsT0FBTyxNQUFNLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFDO0FBQzlCLENBQUM7QUFFRDs7O0dBR0c7QUFDSCxNQUFNLFVBQVUsZ0JBQWdCLENBQUMsRUFBZ0IsRUFBRSxJQUFxQixFQUFFLFlBQW9CO0lBQzVGLEtBQUssTUFBTSxVQUFVLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLElBQUksQ0FBQyxVQUFVLENBQUMsRUFBRSxDQUFDO1FBQ3ZELElBQUksQ0FBQyxjQUFjLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQztZQUFFLE1BQU0sSUFBSSxLQUFLLENBQUMsMEJBQTBCLFVBQVUsRUFBRSxDQUFDLENBQUM7SUFDaEcsQ0FBQztJQUNELE1BQU0sR0FBRyxHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUMsaUNBQWlDLElBQUksQ0FBQyxLQUFLLFVBQVUsSUFBSSxDQUFDLFVBQVUsTUFBTSxDQUFDLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDO0lBQ3JILE9BQU8sTUFBTSxDQUFDLEdBQUcsRUFBRSxLQUFLLENBQUMsQ0FBQztBQUM1QixDQUFDIn0= \ No newline at end of file diff --git a/packages/loopover-miner/lib/store-maintenance.ts b/packages/loopover-miner/lib/store-maintenance.ts new file mode 100644 index 0000000000..72fe9fe89a --- /dev/null +++ b/packages/loopover-miner/lib/store-maintenance.ts @@ -0,0 +1,198 @@ +// Local-store maintenance for the miner (#4834): SQLite integrity checks + append-only ledger retention. +// +// Three independent, side-effect-light helpers used by `doctor`, the ledgers, and `purge-cli.js`: +// 1. checkStoreIntegrity — run `PRAGMA integrity_check` on one store file and report health, so `doctor` can +// flag a corrupted store instead of only probing a single one with `SELECT 1`. +// 2. resolveLedgerRetentionPolicy / pruneLedgerByRetention — an opt-in, age- and/or size-based retention +// policy for the unbounded append-only ledgers (event, governor, prediction), which otherwise grow forever. +// OFF by default: retention only runs when an operator sets the env opt-in. +// 3. purgeStoreByRepo — an explicit, operator-invoked delete of every row for one repo (#5564, right-to-be- +// forgotten). Distinct from retention pruning: never runs automatically, always caller-initiated via +// `purge-cli.js`, and always reports how many rows it removed so a purge is never silent. +// Pure control flow over injected inputs (a DB handle, an env object, a caller-supplied clock) — no network, and +// no internal clock read in the prune path so it stays deterministic and unit-testable. +import { existsSync } from "node:fs"; +import { DatabaseSync } from "node:sqlite"; +import { CONTRIBUTION_PROFILE_STORE_TABLE } from "./contribution-profile.js"; + +/** Env opt-ins for ledger retention (unset ⇒ retention disabled). */ +export const LEDGER_RETENTION_DAYS_ENV = "LOOPOVER_MINER_LEDGER_RETENTION_DAYS"; +export const LEDGER_RETENTION_MAX_ROWS_ENV = "LOOPOVER_MINER_LEDGER_RETENTION_MAX_ROWS"; + +export type LedgerRetentionSpec = { table: string; timestampColumn: string; orderColumn: string }; + +/** Fixed retention specs for the three append-only ledgers. These identifiers are INTERNAL constants — never + * caller/user text — and are validated as plain identifiers before interpolation as defence in depth. */ +export const EVENT_LEDGER_RETENTION_SPEC: LedgerRetentionSpec = { table: "miner_event_ledger", timestampColumn: "created_at", orderColumn: "id" }; +export const GOVERNOR_LEDGER_RETENTION_SPEC: LedgerRetentionSpec = { table: "governor_events", timestampColumn: "ts", orderColumn: "id" }; +export const PREDICTION_LEDGER_RETENTION_SPEC: LedgerRetentionSpec = { table: "predictions", timestampColumn: "ts", orderColumn: "id" }; + +export type LedgerPurgeSpec = { table: string; repoColumn: string }; + +/** Fixed purge specs (#5564, #6599) for the six stores whose rows are directly scoped by a `repoColumn`. Same + * internal-constant-only discipline as the retention specs above. `attempt-log.js` is deliberately absent: its + * payload is a free-form `Record` with no dedicated repo column, so a precise per-repo purge + * isn't possible there without risking false matches — `purge-cli.js` reports it as not-purgeable instead. */ +export const CLAIM_LEDGER_PURGE_SPEC: LedgerPurgeSpec = { table: "miner_claims", repoColumn: "repo_full_name" }; +export const EVENT_LEDGER_PURGE_SPEC: LedgerPurgeSpec = { table: "miner_event_ledger", repoColumn: "repo_full_name" }; +export const GOVERNOR_LEDGER_PURGE_SPEC: LedgerPurgeSpec = { table: "governor_events", repoColumn: "repo_full_name" }; +export const PREDICTION_LEDGER_PURGE_SPEC: LedgerPurgeSpec = { table: "predictions", repoColumn: "repo_full_name" }; +export const PORTFOLIO_QUEUE_PURGE_SPEC: LedgerPurgeSpec = { table: "miner_portfolio_queue", repoColumn: "repo_full_name" }; +export const RUN_STATE_PURGE_SPEC: LedgerPurgeSpec = { table: "miner_run_state", repoColumn: "repo_full_name" }; + +/** Three more repo-scoped stores the original six missed (#7091), same `repoColumn` shape and same internal- + * constant-only discipline. The contribution-profile-cache table name comes from its schema module's own + * `CONTRIBUTION_PROFILE_STORE_TABLE` constant so this spec can't drift from a second hardcoded literal. + * governor-state holds two genuinely repo-scoped tables (reputation history + own submissions); + * `governor_scalar_state` is intentionally excluded — it is a single whole-run scalar row with no repo + * dimension. `governor_reputation_history` is purged on `repo_full_name` alone (its key is composite with + * `api_base_url`), so a right-to-be-forgotten sweep clears the repo across every forge host it was recorded + * against, not just the default one. */ +export const CONTRIBUTION_PROFILE_CACHE_PURGE_SPEC: LedgerPurgeSpec = { table: CONTRIBUTION_PROFILE_STORE_TABLE, repoColumn: "repo_full_name" }; +export const GOVERNOR_REPUTATION_HISTORY_PURGE_SPEC: LedgerPurgeSpec = { table: "governor_reputation_history", repoColumn: "repo_full_name" }; +export const GOVERNOR_OWN_SUBMISSIONS_PURGE_SPEC: LedgerPurgeSpec = { table: "governor_own_submissions", repoColumn: "repo_full_name" }; + +/** policy-verdict-cache (#6987), another repo-scoped store the earlier sweeps missed. Its `repo_scope TEXT + * PRIMARY KEY` is the per-repo column (a tenant forge host + `owner/repo`), the same `repoColumn` shape and + * internal-constant-only discipline as the specs above. `policy-doc-cache.js` stays out (keyed by URL, no repo + * column, exactly like `attempt-log.js`). */ +export const POLICY_VERDICT_CACHE_PURGE_SPEC: LedgerPurgeSpec = { table: "policy_verdict_cache", repoColumn: "repo_scope" }; + +export type StoreIntegrityResult = { name: string; ok: boolean; detail: string }; +export type LedgerRetentionPolicy = { maxAgeMs?: number; maxRows?: number }; + +const SQL_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; + +/** A readable message for a caught value, whether or not it is an Error. */ +export function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Classify raw `PRAGMA integrity_check` rows. A healthy database yields a single `"ok"` row; a corrupt one yields + * one row per problem. Pure — extracted so both the healthy and problem paths are testable without a genuinely + * corrupt file (which SQLite typically refuses to open at all, i.e. the catch path below). + */ +export function classifyIntegrityRows(rows: Array<{ integrity_check?: unknown }>): { ok: boolean; note: string } { + const problems = rows.map((row) => String(row.integrity_check)).filter((value) => value !== "ok"); + return problems.length === 0 ? { ok: true, note: "ok" } : { ok: false, note: problems.join("; ") }; +} + +/** + * Run `PRAGMA integrity_check` on a single store file. A store that does not exist yet is healthy by absence + * (nothing to corrupt). Never throws: a store that cannot be opened or read is reported as not-ok, so one bad + * store cannot abort the whole doctor sweep. Opens the connection driver-enforced read-only -- `readOnly` + * (camelCase) is the only option key node:sqlite recognizes for this; the lowercase `readonly` is silently + * ignored and opens read-write instead (the exact gotcha claim-ledger.js's own openClaimLedgerReadOnly already + * documents), which would defeat the read-only guarantee this function's own docs claim. + */ +export function checkStoreIntegrity(name: string, dbPath: string): StoreIntegrityResult { + if (!existsSync(dbPath)) { + return { name, ok: true, detail: `${dbPath}: not created yet` }; + } + let db: DatabaseSync | undefined; + try { + db = new DatabaseSync(dbPath, { readOnly: true }); + const { ok, note } = classifyIntegrityRows(db.prepare("PRAGMA integrity_check").all()); + return { name, ok, detail: `${dbPath}: ${note}` }; + } catch (error) { + return { name, ok: false, detail: `${dbPath}: ${describeError(error)}` }; + } finally { + db?.close(); + } +} + +/** Coerce an env value to a positive integer, or null (unset/blank/zero/negative/non-finite ⇒ null ⇒ disabled). + * Floors BEFORE the positivity test, so a fractional value below 1 (e.g. "0.5") floors to 0 and disables the + * bound rather than becoming a dangerous 0 that would prune the whole ledger. */ +function positiveIntOrNull(raw: string | undefined): number | null { + if (raw === undefined || raw === null || String(raw).trim() === "") return null; + const numeric = Math.floor(Number(raw)); + return Number.isFinite(numeric) && numeric > 0 ? numeric : null; +} + +/** + * Resolve the opt-in ledger retention policy from an env object. OFF by default: returns null unless at least + * one bound is set to a positive value. A zero/negative/non-numeric value is treated as unset. When set, returns + * `{ maxAgeMs? }` (from a day count) and/or `{ maxRows? }`. + */ +export function resolveLedgerRetentionPolicy( + env: Record = process.env, +): LedgerRetentionPolicy | null { + const maxAgeDays = positiveIntOrNull(env[LEDGER_RETENTION_DAYS_ENV]); + const maxRows = positiveIntOrNull(env[LEDGER_RETENTION_MAX_ROWS_ENV]); + if (maxAgeDays === null && maxRows === null) return null; + const policy: LedgerRetentionPolicy = {}; + if (maxAgeDays !== null) policy.maxAgeMs = maxAgeDays * 24 * 60 * 60 * 1000; + if (maxRows !== null) policy.maxRows = maxRows; + return policy; +} + +/** + * Prune one append-only ledger per a resolved retention policy: delete rows older than the age bound AND rows + * beyond the row-count bound (keeping the newest `maxRows` by `orderColumn`), atomically. A null policy is a + * no-op. `nowMs` is caller-supplied (no internal clock). Timestamp columns are UTC ISO-8601 strings, which sort + * lexicographically in chronological order, so a string comparison against the ISO cutoff selects older rows. + */ +export function pruneLedgerByRetention( + db: DatabaseSync, + spec: LedgerRetentionSpec, + policy: LedgerRetentionPolicy | null, + nowMs: number, +): number { + if (!policy) return 0; + for (const identifier of [spec.table, spec.timestampColumn, spec.orderColumn]) { + if (!SQL_IDENTIFIER.test(identifier)) throw new Error(`unsafe SQL identifier: ${identifier}`); + } + let deleted = 0; + db.exec("BEGIN"); + try { + // Both bounds are guarded to be strictly positive as defence in depth: a 0 age would prune everything older + // than `now`, and a 0 row-cap makes `LIMIT 0` match no rows so `NOT IN (empty)` would delete the whole ledger. + if (policy.maxAgeMs !== undefined && policy.maxAgeMs > 0) { + const cutoff = new Date(nowMs - policy.maxAgeMs).toISOString(); + const info = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.timestampColumn} < ?`).run(cutoff); + deleted += Number(info.changes); + } + if (policy.maxRows !== undefined && policy.maxRows >= 1) { + const info = db + .prepare( + `DELETE FROM ${spec.table} WHERE ${spec.orderColumn} NOT IN ` + + `(SELECT ${spec.orderColumn} FROM ${spec.table} ORDER BY ${spec.orderColumn} DESC LIMIT ?)`, + ) + .run(policy.maxRows); + deleted += Number(info.changes); + } + db.exec("COMMIT"); + } catch (error) { + db.exec("ROLLBACK"); + throw error; + } + return deleted; +} + +/** + * Delete every row for one repo from a store (#5564). Unlike `pruneLedgerByRetention`, this never runs + * automatically — it exists solely so `purge-cli.js` can give an operator a real right-to-be-forgotten path. + * `repoFullName` is caller-normalized (owner/repo) before reaching here; this function only guards the SQL + * identifiers, matching `pruneLedgerByRetention`'s own defence-in-depth discipline. + */ +export function purgeStoreByRepo(db: DatabaseSync, spec: LedgerPurgeSpec, repoFullName: string): number { + for (const identifier of [spec.table, spec.repoColumn]) { + if (!SQL_IDENTIFIER.test(identifier)) throw new Error(`unsafe SQL identifier: ${identifier}`); + } + const info = db.prepare(`DELETE FROM ${spec.table} WHERE ${spec.repoColumn} = ?`).run(repoFullName); + return Number(info.changes); +} + +/** + * Count rows for one repo in a store without deleting anything (#5564) — the read-only counterpart to + * `purgeStoreByRepo`, used by `purge-cli.js --dry-run` to report what a real purge would remove. + */ +export function countStoreByRepo(db: DatabaseSync, spec: LedgerPurgeSpec, repoFullName: string): number { + for (const identifier of [spec.table, spec.repoColumn]) { + if (!SQL_IDENTIFIER.test(identifier)) throw new Error(`unsafe SQL identifier: ${identifier}`); + } + const row = db.prepare(`SELECT COUNT(*) AS count FROM ${spec.table} WHERE ${spec.repoColumn} = ?`).get(repoFullName); + return Number(row?.count); +}