diff --git a/packages/gittensory-miner/README.md b/packages/gittensory-miner/README.md index 8307bb47f7..666a75b16a 100644 --- a/packages/gittensory-miner/README.md +++ b/packages/gittensory-miner/README.md @@ -67,7 +67,7 @@ Phase 6 of the same roadmap tracker and hasn't been scaffolded yet. (#4279) ## Local storage -Four independent local SQLite stores back the commands above. Each keeps its own file, its own table, and its own +Five independent local SQLite stores back the commands above. Each keeps its own file, its own table, and its own env-var override — this is a DRY pass over their shared path-resolution/open boilerplate (`local-store.js`), not a merge into one database. (#4272) @@ -77,6 +77,13 @@ merge into one database. (#4272) | Claim ledger | `claim-ledger.sqlite3` | `miner_claims` | `claim-ledger.js` | `GITTENSORY_MINER_CLAIM_LEDGER_DB` | | Portfolio queue | `portfolio-queue.sqlite3` | `miner_portfolio_queue` | `portfolio-queue.js` | `GITTENSORY_MINER_PORTFOLIO_QUEUE_DB` | | Event ledger | `event-ledger.sqlite3` | `miner_event_ledger` | `event-ledger.js` | `GITTENSORY_MINER_EVENT_LEDGER_DB` | +| Policy-doc cache | `policy-doc-cache.sqlite3` | `policy_doc_cache` | `policy-doc-cache.js` | `GITTENSORY_MINER_POLICY_DOC_CACHE_DB` | + +The policy-doc cache is the only one of the five that holds no miner state of its own: it caches the last-known +ETag + body of each target repo's fetched policy docs (AI-USAGE.md/CONTRIBUTING.md) so a repeated `discover` +revalidates them with a conditional GET (`If-None-Match`) instead of re-downloading static content, spending no +extra rate-limit budget when GitHub answers `304 Not Modified`. It is pure optimization — deleting the file only +forces the next run to refetch in full (#4842). Every store resolves its file the same way: the store-specific env var above, else `GITTENSORY_MINER_CONFIG_DIR`, else `XDG_CONFIG_HOME` (falling back to `~/.config`), joined with `gittensory-miner/`. Every store also opens diff --git a/packages/gittensory-miner/docs/env-reference.md b/packages/gittensory-miner/docs/env-reference.md index e5e9d47277..6997d7a87c 100644 --- a/packages/gittensory-miner/docs/env-reference.md +++ b/packages/gittensory-miner/docs/env-reference.md @@ -15,6 +15,7 @@ Generated by `npm run miner:env-reference`. Do not edit manually. | `GITTENSORY_MINER_NO_UPDATE_CHECK` | `lib/update-check.js` | `""` | | `GITTENSORY_MINER_ORB_EXPORT_DB` | `lib/orb-export.js` | `""` | | `GITTENSORY_MINER_PLAN_STORE_DB` | `lib/plan-store.js` | `""` | +| `GITTENSORY_MINER_POLICY_DOC_CACHE_DB` | `lib/policy-doc-cache.js` | (none) | | `GITTENSORY_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.js` | (none) | | `GITTENSORY_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.js` | `""` | | `GITTENSORY_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.js` | (none) | diff --git a/packages/gittensory-miner/lib/discover-cli.d.ts b/packages/gittensory-miner/lib/discover-cli.d.ts index 370b143935..8292681a6e 100644 --- a/packages/gittensory-miner/lib/discover-cli.d.ts +++ b/packages/gittensory-miner/lib/discover-cli.d.ts @@ -4,6 +4,7 @@ import type { RawCandidateIssue, } from "./opportunity-fanout.js"; import type { RankedCandidateIssue, RankedCandidateSummary } from "./opportunity-ranker.js"; +import type { PolicyDocCache, PolicyDocCacheStore } from "./policy-doc-cache.js"; import type { EnqueueRankedDiscoverySummary } from "./portfolio-discovery.js"; import type { PortfolioQueueStore } from "./portfolio-queue.js"; @@ -41,15 +42,16 @@ export type RunDiscoverOptions = { apiBaseUrl?: string; nowMs?: number; initPortfolioQueue?: () => PortfolioQueueStore; + initPolicyDocCache?: () => PolicyDocCacheStore; fetchCandidateIssuesWithSummary?: ( targets: FanoutTarget[], githubToken: string, - options?: { apiBaseUrl?: string }, + options?: { apiBaseUrl?: string; policyDocCache?: PolicyDocCache | null }, ) => Promise; searchCandidateIssuesWithSummary?: ( searchQuery: string, githubToken: string, - options?: { apiBaseUrl?: string }, + options?: { apiBaseUrl?: string; policyDocCache?: PolicyDocCache | null }, ) => Promise; rankCandidateIssuesWithSummary?: ( candidates: RawCandidateIssue[], diff --git a/packages/gittensory-miner/lib/discover-cli.js b/packages/gittensory-miner/lib/discover-cli.js index d392afb58b..3ea680369b 100644 --- a/packages/gittensory-miner/lib/discover-cli.js +++ b/packages/gittensory-miner/lib/discover-cli.js @@ -5,6 +5,7 @@ import { searchCandidateIssuesWithSummary, } from "./opportunity-fanout.js"; import { rankCandidateIssuesWithSummary } from "./opportunity-ranker.js"; +import { initPolicyDocCacheStore } from "./policy-doc-cache.js"; import { enqueueRankedDiscovery } from "./portfolio-discovery.js"; import { initPortfolioQueueStore } from "./portfolio-queue.js"; @@ -118,11 +119,18 @@ export async function runDiscover(args, options = {}) { const ownsPortfolioQueue = options.initPortfolioQueue === undefined; const portfolioQueue = (options.initPortfolioQueue ?? initPortfolioQueueStore)(); + // Local ETag cache so a repeated discover revalidates each repo's policy docs with a conditional GET instead of + // re-downloading them (#4842). Owned + closed here exactly like the portfolio queue above; an injected factory + // lets tests supply a temp/in-memory store instead of the real on-disk one. + const ownsPolicyDocCache = options.initPolicyDocCache === undefined; + const policyDocCache = (options.initPolicyDocCache ?? initPolicyDocCacheStore)(); + try { + const fanOutOptions = { apiBaseUrl: options.apiBaseUrl, policyDocCache }; const fanOut = parsed.search !== null - ? await searchTargets(parsed.search, githubToken, { apiBaseUrl: options.apiBaseUrl }) - : await fetchTargets(parsed.targets, githubToken, { apiBaseUrl: options.apiBaseUrl }); + ? await searchTargets(parsed.search, githubToken, fanOutOptions) + : await fetchTargets(parsed.targets, githubToken, fanOutOptions); const rankedSummary = rankIssues(fanOut.issues, { nowMs: options.nowMs }); const enqueueSummary = enqueue(rankedSummary.issues, { queueStore: portfolioQueue }); @@ -147,5 +155,6 @@ export async function runDiscover(args, options = {}) { return 2; } finally { if (ownsPortfolioQueue) portfolioQueue.close(); + if (ownsPolicyDocCache) policyDocCache.close(); } } diff --git a/packages/gittensory-miner/lib/opportunity-fanout.d.ts b/packages/gittensory-miner/lib/opportunity-fanout.d.ts index d179b9da14..b89b7996c2 100644 --- a/packages/gittensory-miner/lib/opportunity-fanout.d.ts +++ b/packages/gittensory-miner/lib/opportunity-fanout.d.ts @@ -1,8 +1,20 @@ +import type { PolicyDocCache } from "./policy-doc-cache.js"; + export type FanoutTarget = { owner: string; repo: string; }; +/** Options common to every fan-out/search entry point. `policyDocCache`, when supplied, lets discovery revalidate + * each repo's policy docs with a conditional GET instead of a full refetch (#4842). */ +export type FanoutOptions = { + apiBaseUrl?: string; + concurrency?: number; + perPage?: number; + sleepFn?: (ms: number) => Promise; + policyDocCache?: PolicyDocCache | null; +}; + export type RawCandidateIssue = { owner: string; repo: string; @@ -42,51 +54,23 @@ export function mapWithConcurrency( export function fetchCandidateIssuesWithSummary( targets: FanoutTarget[], githubToken: string, - options?: { - apiBaseUrl?: string; - concurrency?: number; - rateLimitLowWaterMark?: number; - rateLimitHighWaterMark?: number; - perPage?: number; - sleepFn?: (ms: number) => Promise; - }, + options?: FanoutOptions, ): Promise; export function fetchCandidateIssues( targets: FanoutTarget[], githubToken: string, - options?: { - apiBaseUrl?: string; - concurrency?: number; - rateLimitLowWaterMark?: number; - rateLimitHighWaterMark?: number; - perPage?: number; - sleepFn?: (ms: number) => Promise; - }, + options?: FanoutOptions, ): Promise; export function searchCandidateIssuesWithSummary( searchQuery: string, githubToken: string, - options?: { - apiBaseUrl?: string; - concurrency?: number; - rateLimitLowWaterMark?: number; - rateLimitHighWaterMark?: number; - perPage?: number; - sleepFn?: (ms: number) => Promise; - }, + options?: FanoutOptions, ): Promise; export function searchCandidateIssues( searchQuery: string, githubToken: string, - options?: { - apiBaseUrl?: string; - concurrency?: number; - rateLimitLowWaterMark?: number; - rateLimitHighWaterMark?: number; - perPage?: number; - sleepFn?: (ms: number) => Promise; - }, + options?: FanoutOptions, ): Promise; diff --git a/packages/gittensory-miner/lib/opportunity-fanout.js b/packages/gittensory-miner/lib/opportunity-fanout.js index 348aa87ded..4ec0301e24 100644 --- a/packages/gittensory-miner/lib/opportunity-fanout.js +++ b/packages/gittensory-miner/lib/opportunity-fanout.js @@ -112,13 +112,14 @@ function recordRateLimit(summary, response) { } } -async function githubGetJson(url, githubToken, summary, options) { +async function githubGetJson(url, githubToken, summary, options, extraHeaders = {}) { // Retry a transient 5xx from GitHub before dropping this target's results for the whole run (#4830) — the same // discipline as the CI/gate-verdict pollers. A thrown network error still propagates to each caller's try/catch. + // `extraHeaders` carries per-call additions (e.g. a policy-doc If-None-Match, #4842) on top of the base auth set. const response = await fetchWithRetry( fetch, url, - { method: "GET", headers: githubHeaders(githubToken) }, + { method: "GET", headers: { ...githubHeaders(githubToken), ...extraHeaders } }, { sleepFn: options?.sleepFn }, ); recordRateLimit(summary, response); @@ -139,19 +140,52 @@ function warning(target, stage, message) { return { repoFullName: target.repoFullName, stage, message }; } +// Read a URL's prior ETag so an unchanged doc can be revalidated with a conditional GET (#4842). A cache that is +// absent, or whose read throws (corrupt/locked file), is treated as a plain miss: the caller does a full fetch, +// per the "never risk a stale policy" rule — the cache only ever makes discovery cheaper, never less correct. +function readCachedPolicyDoc(cache, url) { + if (!cache) return null; + try { + return cache.get(url); + } catch { + return null; + } +} + +// Persist the fresh ETag + body so the NEXT discover run can revalidate instead of re-downloading. Only a real +// ETag paired with decoded content is stored, and a write that throws must never fail discovery (same stale-safe +// rule) — it degrades to "not cached", so the next run simply refetches in full. +function writeCachedPolicyDoc(cache, url, response, content) { + if (!cache || content === null) return; + const etag = response.headers.get("etag"); + if (typeof etag !== "string" || !etag.trim()) return; + try { + cache.put(url, etag, content); + } catch { + // Leave this URL uncached; the next run refetches fully rather than serving anything stale. + } +} + async function fetchRepoDoc(target, path, githubToken, options, summary, warnings) { const url = apiUrl( options.apiBaseUrl, repoPath(target, `/contents/${encodeURIComponent(path)}`), ); + const cached = readCachedPolicyDoc(options.policyDocCache, url); + const conditionalHeaders = cached ? { "if-none-match": cached.etag } : {}; try { - const { response, payload } = await githubGetJson(url, githubToken, summary, options); + const { response, payload } = await githubGetJson(url, githubToken, summary, options, conditionalHeaders); + // A 304 only ever follows the If-None-Match we send above, which we only send when `cached` exists — so the + // cached body is the GitHub-confirmed current content, served with no extra rate-limit spend. + if (response.status === 304) return cached.content; if (response.status === 404) return null; if (!response.ok) { warnings.push(warning(target, `policy:${path}`, `GitHub returned ${response.status}`)); return null; } - return decodeContentPayload(payload); + const content = decodeContentPayload(payload); + writeCachedPolicyDoc(options.policyDocCache, url, response, content); + return content; } catch (error) { warnings.push( warning(target, `policy:${path}`, error instanceof Error ? error.message : "policy fetch failed"), @@ -376,6 +410,9 @@ function normalizeOptions(options = {}) { maxPages: normalizeLimit(options.maxPages, defaultMaxPages, 1, 100), // Passed through to the per-fetch retry so tests can inject an instant sleep; undefined uses the real backoff. sleepFn: typeof options.sleepFn === "function" ? options.sleepFn : undefined, + // Optional local ETag cache for policy-doc revalidation (#4842). Absent (null) => every policy doc is fetched + // in full, exactly as before; discover-cli.js supplies the real on-disk store for a live run. + policyDocCache: options.policyDocCache ?? null, }; } diff --git a/packages/gittensory-miner/lib/policy-doc-cache.d.ts b/packages/gittensory-miner/lib/policy-doc-cache.d.ts new file mode 100644 index 0000000000..4cd5a5e5c6 --- /dev/null +++ b/packages/gittensory-miner/lib/policy-doc-cache.d.ts @@ -0,0 +1,25 @@ +export type PolicyDocCacheEntry = { + etag: string; + content: string; +}; + +export type PolicyDocCacheWrite = { + url: string; + etag: string; + content: string; + updatedAt: string; +}; + +export type PolicyDocCacheStore = { + dbPath: string; + get(url: string): PolicyDocCacheEntry | null; + put(url: string, etag: string, content: string): PolicyDocCacheWrite; + close(): void; +}; + +/** The read/write surface opportunity-fanout.js needs to inject a cache without depending on the SQLite store. */ +export type PolicyDocCache = Pick; + +export function resolvePolicyDocCacheDbPath(env?: Record): string; + +export function initPolicyDocCacheStore(dbPath?: string): PolicyDocCacheStore; diff --git a/packages/gittensory-miner/lib/policy-doc-cache.js b/packages/gittensory-miner/lib/policy-doc-cache.js new file mode 100644 index 0000000000..910eab45a4 --- /dev/null +++ b/packages/gittensory-miner/lib/policy-doc-cache.js @@ -0,0 +1,79 @@ +import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js"; +import { applySchemaMigrations } from "./schema-version.js"; + +// Local ETag cache for discovery's small policy-doc fetches (#4842). `discover` refetches each target repo's +// AI-USAGE.md/CONTRIBUTING.md on every run even though they rarely change, spending rate-limit budget on static +// content; this store lets opportunity-fanout.js revalidate with a conditional GET (If-None-Match) instead, and +// GitHub answers an unchanged doc with a 304 that costs no primary rate-limit budget. A 304 is a GitHub-confirmed +// unchanged body -- the cached content is only ever served AFTER a same-run revalidation, never blindly -- so this +// can never surface a stale policy that would wrongly permit autonomous work on an opted-out repo. Same 100% +// local/client-side discipline (mirrors run-state.js and the other stores this package owns via local-store.js): +// the file lives only on this machine and is never uploaded, synced, or phoned home with. + +const defaultDbFileName = "policy-doc-cache.sqlite3"; + +export function resolvePolicyDocCacheDbPath(env = process.env) { + return resolveLocalStoreDbPath(defaultDbFileName, "GITTENSORY_MINER_POLICY_DOC_CACHE_DB", env); +} + +function normalizeDbPath(dbPath) { + return normalizeLocalStoreDbPath(dbPath, resolvePolicyDocCacheDbPath(), "invalid_policy_doc_cache_db_path"); +} + +function normalizeUrl(url) { + if (typeof url !== "string") throw new Error("invalid_policy_doc_url"); + const trimmed = url.trim(); + if (!trimmed) throw new Error("invalid_policy_doc_url"); + return trimmed; +} + +/** + * Opens the 100% local/client-side miner policy-doc ETag cache. The database only lives on this machine; this + * module never uploads, syncs, or phones home with its contents. (#4842) + */ +export function initPolicyDocCacheStore(dbPath = resolvePolicyDocCacheDbPath()) { + const resolvedPath = normalizeDbPath(dbPath); + const db = openLocalStoreDb(resolvedPath); + db.exec(` + CREATE TABLE IF NOT EXISTS policy_doc_cache ( + url TEXT PRIMARY KEY, + etag TEXT NOT NULL, + content TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + `); + // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations (none yet). + applySchemaMigrations(db, []); + + const getStatement = db.prepare("SELECT etag, content FROM policy_doc_cache WHERE url = ?"); + const putStatement = db.prepare(` + INSERT INTO policy_doc_cache (url, etag, content, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(url) DO UPDATE SET + etag = excluded.etag, + content = excluded.content, + updated_at = excluded.updated_at + `); + + return { + dbPath: resolvedPath, + /** The last-known `{ etag, content }` for a policy-doc URL, or null when it has never been cached. Both columns + * are `TEXT NOT NULL`, so a present row always carries string values. */ + get(url) { + const row = getStatement.get(normalizeUrl(url)); + return row ? { etag: row.etag, content: row.content } : null; + }, + /** Record the fresh ETag + body so the next run can revalidate it with a conditional GET. */ + put(url, etag, content) { + const normalizedUrl = normalizeUrl(url); + if (typeof etag !== "string" || !etag.trim()) throw new Error("invalid_policy_doc_etag"); + if (typeof content !== "string") throw new Error("invalid_policy_doc_content"); + const updatedAt = new Date().toISOString(); + putStatement.run(normalizedUrl, etag, content, updatedAt); + return { url: normalizedUrl, etag, content, updatedAt }; + }, + close() { + db.close(); + }, + }; +} diff --git a/packages/gittensory-miner/package.json b/packages/gittensory-miner/package.json index b868ac8a6a..b3587d6b99 100644 --- a/packages/gittensory-miner/package.json +++ b/packages/gittensory-miner/package.json @@ -33,7 +33,7 @@ "expected-engine.version" ], "scripts": { - "build": "node --check bin/gittensory-miner.js && node --check bin/gittensory-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-run.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/gate-verdict-poller.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-open-pr.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/metrics-cli.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" + "build": "node --check bin/gittensory-miner.js && node --check bin/gittensory-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/gate-verdict-poller.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-open-pr.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/policy-doc-cache.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" }, "dependencies": { "@jsonbored/gittensory-engine": "*", diff --git a/test/unit/miner-discover-cli.test.ts b/test/unit/miner-discover-cli.test.ts index 5b35ac8aa1..3974829170 100644 --- a/test/unit/miner-discover-cli.test.ts +++ b/test/unit/miner-discover-cli.test.ts @@ -1,7 +1,8 @@ -import { mkdtempSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { initPolicyDocCacheStore } from "../../packages/gittensory-miner/lib/policy-doc-cache.js"; import { closeDefaultPortfolioQueueStore, initPortfolioQueueStore, @@ -27,6 +28,17 @@ function tempQueueStore() { return store; } +// An injected policy-doc cache keeps runDiscover from opening the real on-disk cache in ~/.config for every test +// that only cares about the fan-out/rank/enqueue path (#4842). runDiscover doesn't own an injected store, so the +// afterEach hook below closes it. +function tempPolicyDocCacheStore() { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-discover-cli-pdc-")); + roots.push(root); + const store = initPolicyDocCacheStore(join(root, "policy-doc-cache.sqlite3")); + stores.push(store); + return store; +} + function fanOutIssue(overrides: Record = {}) { return { owner: "acme", @@ -232,6 +244,7 @@ describe("runDiscover (#4247)", () => { const exitCode = await runDiscover(["acme/widgets", "--json"], { nowMs: NOW, initPortfolioQueue: () => portfolioQueue, + initPolicyDocCache: () => tempPolicyDocCacheStore(), fetchCandidateIssuesWithSummary, searchCandidateIssuesWithSummary, }); @@ -272,6 +285,7 @@ describe("runDiscover (#4247)", () => { const exitCode = await runDiscover(["--search", "label:bug"], { nowMs: NOW, initPortfolioQueue: () => portfolioQueue, + initPolicyDocCache: () => tempPolicyDocCacheStore(), fetchCandidateIssuesWithSummary, searchCandidateIssuesWithSummary, }); @@ -295,6 +309,7 @@ describe("runDiscover (#4247)", () => { const exitCode = await runDiscover(["acme/widgets"], { nowMs: NOW, initPortfolioQueue: () => portfolioQueue, + initPolicyDocCache: () => tempPolicyDocCacheStore(), fetchCandidateIssuesWithSummary, }); @@ -326,6 +341,7 @@ describe("runDiscover (#4247)", () => { const exitCode = await runDiscover(["acme/widgets"], { initPortfolioQueue: () => portfolioQueue, + initPolicyDocCache: () => tempPolicyDocCacheStore(), fetchCandidateIssuesWithSummary, }); @@ -348,7 +364,11 @@ describe("runDiscover (#4247)", () => { })); vi.spyOn(console, "log").mockImplementation(() => undefined); - const exitCode = await runDiscover(["acme/widgets"], { nowMs: NOW, fetchCandidateIssuesWithSummary }); + const exitCode = await runDiscover(["acme/widgets"], { + nowMs: NOW, + fetchCandidateIssuesWithSummary, + initPolicyDocCache: () => tempPolicyDocCacheStore(), + }); expect(exitCode).toBe(0); // runDiscover owned and closed this store itself (no initPortfolioQueue override was passed); reopening @@ -361,6 +381,36 @@ describe("runDiscover (#4247)", () => { else process.env.GITTENSORY_MINER_PORTFOLIO_QUEUE_DB = previousDbPath; } }); + + it("opens and closes the default on-disk policy-doc cache when no override is supplied", async () => { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-discover-cli-pdc-default-")); + roots.push(root); + const cacheDbPath = join(root, "policy-doc-cache.sqlite3"); + const previousCacheDbPath = process.env.GITTENSORY_MINER_POLICY_DOC_CACHE_DB; + process.env.GITTENSORY_MINER_POLICY_DOC_CACHE_DB = cacheDbPath; + try { + const portfolioQueue = tempQueueStore(); + const fetchCandidateIssuesWithSummary = vi.fn(async () => ({ issues: [fanOutIssue()], warnings: [] })); + vi.spyOn(console, "log").mockImplementation(() => undefined); + + // No initPolicyDocCache override: runDiscover opens the default on-disk cache at the env path and closes it + // in its finally block. Reopening the same file confirms the default code path created a usable store. + const exitCode = await runDiscover(["acme/widgets"], { + nowMs: NOW, + fetchCandidateIssuesWithSummary, + initPortfolioQueue: () => portfolioQueue, + }); + expect(exitCode).toBe(0); + expect(existsSync(cacheDbPath)).toBe(true); + + const reopened = initPolicyDocCacheStore(cacheDbPath); + stores.push(reopened); + expect(reopened.get("https://api.github.com/repos/acme/widgets/contents/AI-USAGE.md")).toBeNull(); + } finally { + if (previousCacheDbPath === undefined) delete process.env.GITTENSORY_MINER_POLICY_DOC_CACHE_DB; + else process.env.GITTENSORY_MINER_POLICY_DOC_CACHE_DB = previousCacheDbPath; + } + }); }); describe("gittensory-miner discover CLI entrypoint (#4247)", () => { diff --git a/test/unit/miner-local-store-readme.test.ts b/test/unit/miner-local-store-readme.test.ts index 05c3780277..37d1ec85c2 100644 --- a/test/unit/miner-local-store-readme.test.ts +++ b/test/unit/miner-local-store-readme.test.ts @@ -5,7 +5,7 @@ import { describe, expect, it } from "vitest"; const readmePath = join(process.cwd(), "packages/gittensory-miner/README.md"); describe("gittensory-miner local storage README (#4272)", () => { - it("documents all four local stores together with their file/table/module/env-var", () => { + it("documents all five local stores together with their file/table/module/env-var", () => { const readme = readFileSync(readmePath, "utf8"); expect(readme).toContain("## Local storage"); expect(readme).toContain("run-state.sqlite3"); @@ -16,10 +16,13 @@ describe("gittensory-miner local storage README (#4272)", () => { expect(readme).toContain("miner_portfolio_queue"); expect(readme).toContain("event-ledger.sqlite3"); expect(readme).toContain("miner_event_ledger"); + expect(readme).toContain("policy-doc-cache.sqlite3"); + expect(readme).toContain("policy_doc_cache"); expect(readme).toContain("GITTENSORY_MINER_RUN_STATE_DB"); expect(readme).toContain("GITTENSORY_MINER_CLAIM_LEDGER_DB"); expect(readme).toContain("GITTENSORY_MINER_PORTFOLIO_QUEUE_DB"); expect(readme).toContain("GITTENSORY_MINER_EVENT_LEDGER_DB"); + expect(readme).toContain("GITTENSORY_MINER_POLICY_DOC_CACHE_DB"); }); it("documents the PR-portfolio read-time-join decision", () => { diff --git a/test/unit/miner-policy-doc-cache.test.ts b/test/unit/miner-policy-doc-cache.test.ts new file mode 100644 index 0000000000..16975a2f1a --- /dev/null +++ b/test/unit/miner-policy-doc-cache.test.ts @@ -0,0 +1,109 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + initPolicyDocCacheStore, + resolvePolicyDocCacheDbPath, +} from "../../packages/gittensory-miner/lib/policy-doc-cache.js"; + +const roots: string[] = []; +const stores: Array<{ close(): void }> = []; + +function tempDbPath(): string { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-policy-doc-cache-")); + roots.push(root); + return join(root, "policy-doc-cache.sqlite3"); +} + +function openStore(dbPath = ":memory:") { + const store = initPolicyDocCacheStore(dbPath); + stores.push(store); + return store; +} + +const URL = "https://api.github.com/repos/acme/widgets/contents/AI-USAGE.md"; + +afterEach(() => { + for (const store of stores.splice(0)) store.close(); + vi.unstubAllEnvs(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("resolvePolicyDocCacheDbPath (#4842)", () => { + it("prefers the store-specific env var, then the config dir, then XDG/~config", () => { + expect(resolvePolicyDocCacheDbPath({ GITTENSORY_MINER_POLICY_DOC_CACHE_DB: "/custom/pdc.sqlite3" })).toBe( + "/custom/pdc.sqlite3", + ); + expect(resolvePolicyDocCacheDbPath({ GITTENSORY_MINER_CONFIG_DIR: "/cfg" })).toBe( + join("/cfg", "policy-doc-cache.sqlite3"), + ); + expect(resolvePolicyDocCacheDbPath({ XDG_CONFIG_HOME: "/xdg" })).toBe( + join("/xdg", "gittensory-miner", "policy-doc-cache.sqlite3"), + ); + }); +}); + +describe("gittensory-miner policy-doc cache store (#4842)", () => { + it("returns null for a URL that has never been cached", () => { + expect(openStore().get(URL)).toBeNull(); + }); + + it("stores and reads back an ETag + content, and reports its db path", () => { + const store = openStore(); + const write = store.put(URL, '"v1"', "# AI usage\nwelcome"); + expect(write).toMatchObject({ url: URL, etag: '"v1"', content: "# AI usage\nwelcome" }); + expect(typeof write.updatedAt).toBe("string"); + expect(store.get(URL)).toEqual({ etag: '"v1"', content: "# AI usage\nwelcome" }); + expect(store.dbPath).toBe(":memory:"); + }); + + it("overwrites the prior entry for the same URL (ON CONFLICT upsert)", () => { + const store = openStore(); + store.put(URL, '"v1"', "old"); + store.put(URL, '"v2"', "new"); + expect(store.get(URL)).toEqual({ etag: '"v2"', content: "new" }); + }); + + it("rejects a non-string or empty URL on both get and put", () => { + const store = openStore(); + expect(() => store.get("")).toThrow("invalid_policy_doc_url"); + expect(() => store.get(" ")).toThrow("invalid_policy_doc_url"); + // @ts-expect-error deliberately passing a non-string to exercise the guard. + expect(() => store.get(42)).toThrow("invalid_policy_doc_url"); + expect(() => store.put("", '"v1"', "x")).toThrow("invalid_policy_doc_url"); + }); + + it("rejects a missing/blank ETag or a non-string content", () => { + const store = openStore(); + // @ts-expect-error deliberately passing a non-string etag. + expect(() => store.put(URL, null, "x")).toThrow("invalid_policy_doc_etag"); + expect(() => store.put(URL, " ", "x")).toThrow("invalid_policy_doc_etag"); + // @ts-expect-error deliberately passing a non-string content. + expect(() => store.put(URL, '"v1"', 123)).toThrow("invalid_policy_doc_content"); + }); + + it("persists entries across a close + reopen of the same on-disk file", () => { + const dbPath = tempDbPath(); + const store = openStore(dbPath); + store.put(URL, '"v1"', "persisted"); + store.close(); + stores.splice(stores.indexOf(store), 1); + + const reopened = openStore(dbPath); + expect(reopened.get(URL)).toEqual({ etag: '"v1"', content: "persisted" }); + }); + + it("resolves its default path from the env when no path is passed", () => { + const dbPath = tempDbPath(); + vi.stubEnv("GITTENSORY_MINER_POLICY_DOC_CACHE_DB", dbPath); + // Call with no argument so the default parameter resolves the path from the env. + const store = initPolicyDocCacheStore(); + stores.push(store); + expect(store.dbPath).toBe(dbPath); + }); + + it("throws on an empty explicit db path", () => { + expect(() => initPolicyDocCacheStore("")).toThrow("invalid_policy_doc_cache_db_path"); + }); +}); diff --git a/test/unit/opportunity-fanout-policy-doc-cache.test.ts b/test/unit/opportunity-fanout-policy-doc-cache.test.ts new file mode 100644 index 0000000000..7401c99b49 --- /dev/null +++ b/test/unit/opportunity-fanout-policy-doc-cache.test.ts @@ -0,0 +1,278 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +// Route the miner's bare "@jsonbored/gittensory-engine" import at the engine source (mirrors +// opportunity-fanout-ai-policy.test.ts) so the fan-out uses the real resolveAiPolicyVerdict. +vi.mock("@jsonbored/gittensory-engine", async () => { + return import("../../packages/gittensory-engine/src/index"); +}); + +import { fetchCandidateIssuesWithSummary } from "../../packages/gittensory-miner/lib/opportunity-fanout.js"; +import { initPolicyDocCacheStore } from "../../packages/gittensory-miner/lib/policy-doc-cache.js"; + +const API = "https://api.test"; +const AI_USAGE_URL = `${API}/repos/acme/widgets/contents/AI-USAGE.md`; + +const fixtureDir = join(dirname(fileURLToPath(import.meta.url)), "../fixtures/ai-policy"); +const ALLOWED_AI_USAGE = readFileSync(join(fixtureDir, "allowed-encourages-ai.md"), "utf8"); + +type FetchCall = { url: string; headers: Record }; + +function headerRecord(init?: RequestInit): Record { + const headers = (init?.headers ?? {}) as Record; + return headers; +} + +function jsonResponse(body: unknown, init: ResponseInit = {}) { + return Response.json(body, { + ...init, + headers: { "x-ratelimit-remaining": "42", "x-ratelimit-reset": "1800000000", ...(init.headers ?? {}) }, + }); +} + +function contentResponse(content: string, etag?: string) { + const headers: Record = etag === undefined ? {} : { etag }; + return jsonResponse( + { type: "file", encoding: "base64", content: Buffer.from(content, "utf8").toString("base64") }, + { headers }, + ); +} + +function notModifiedResponse() { + return new Response(null, { + status: 304, + headers: { "x-ratelimit-remaining": "42", "x-ratelimit-reset": "1800000000" }, + }); +} + +const issue = (number: number) => ({ + number, + title: `Issue ${number}`, + labels: ["help wanted"], + comments: 1, + created_at: "2026-07-01T00:00:00Z", + updated_at: "2026-07-01T01:00:00Z", + html_url: `https://github.com/acme/widgets/issues/${number}`, +}); + +/** A minimal in-memory PolicyDocCache that records its writes, so a test can assert exactly what got cached. */ +function fakeCache(overrides: { getImpl?: (url: string) => unknown; putImpl?: () => void } = {}) { + const store = new Map(); + const puts: Array<{ url: string; etag: string; content: string }> = []; + return { + store, + puts, + get(url: string) { + if (overrides.getImpl) return overrides.getImpl(url); + return store.get(url) ?? null; + }, + put(url: string, etag: string, content: string) { + if (overrides.putImpl) overrides.putImpl(); + store.set(url, { etag, content }); + puts.push({ url, etag, content }); + return { url, etag, content, updatedAt: "t" }; + }, + }; +} + +/** Stub global fetch, recording each call's URL + headers; policy doc served per `policy`, issues always one. */ +function stubFetch(policy: (call: FetchCall) => Response | Promise) { + const calls: FetchCall[] = []; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + const call: FetchCall = { url, headers: headerRecord(init) }; + calls.push(call); + if (url.includes("/contents/AI-USAGE.md")) return policy(call); + if (url.includes("/repos/acme/widgets/issues?")) return jsonResponse([issue(1)]); + return jsonResponse({}, { status: 404 }); + }); + return calls; +} + +async function discover(policyDocCache: unknown) { + return fetchCandidateIssuesWithSummary([{ owner: "acme", repo: "widgets" }], "token", { + apiBaseUrl: API, + // biome-ignore lint/suspicious/noExplicitAny: the injected fake satisfies the structural PolicyDocCache surface. + policyDocCache: policyDocCache as any, + }); +} + +const roots: string[] = []; +const stores: Array<{ close(): void }> = []; + +afterEach(() => { + for (const store of stores.splice(0)) store.close(); + vi.unstubAllGlobals(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("opportunity fan-out policy-doc conditional-GET cache (#4842)", () => { + it("sends no conditional header on a cold cache and stores the fetched ETag", async () => { + const cache = fakeCache(); + const calls = stubFetch(() => contentResponse(ALLOWED_AI_USAGE, '"v1"')); + + const result = await discover(cache); + + expect(result.issues.map((entry) => entry.issueNumber)).toEqual([1]); + const policyCall = calls.find((call) => call.url === AI_USAGE_URL); + expect(policyCall?.headers["if-none-match"]).toBeUndefined(); + expect(cache.puts).toEqual([{ url: AI_USAGE_URL, etag: '"v1"', content: ALLOWED_AI_USAGE }]); + }); + + it("revalidates a cached doc with If-None-Match and serves the 304 body without re-downloading", async () => { + const cache = fakeCache(); + cache.store.set(AI_USAGE_URL, { etag: '"v1"', content: ALLOWED_AI_USAGE }); + const calls = stubFetch(() => notModifiedResponse()); + + const result = await discover(cache); + + // The policy resolved from the cached body (issue survives) even though the 304 carried no doc. + expect(result.issues.map((entry) => entry.issueNumber)).toEqual([1]); + const policyCall = calls.find((call) => call.url === AI_USAGE_URL); + expect(policyCall?.headers["if-none-match"]).toBe('"v1"'); + // A 304 revalidation never re-caches: the stored entry is untouched. + expect(cache.puts).toEqual([]); + expect(result.warnings).toEqual([]); + }); + + it("does not cache a 200 response that carries no ETag header", async () => { + const cache = fakeCache(); + stubFetch(() => contentResponse(ALLOWED_AI_USAGE)); + + const result = await discover(cache); + + expect(result.issues.map((entry) => entry.issueNumber)).toEqual([1]); + expect(cache.puts).toEqual([]); + }); + + it("does not cache a blank ETag", async () => { + const cache = fakeCache(); + stubFetch(() => contentResponse(ALLOWED_AI_USAGE, " ")); + + const result = await discover(cache); + + expect(result.issues.map((entry) => entry.issueNumber)).toEqual([1]); + expect(cache.puts).toEqual([]); + }); + + it("does not cache when the response body has no decodable content", async () => { + const cache = fakeCache(); + // AI-USAGE.md 200 with no `content` field decodes to null; policy falls through to a (404) CONTRIBUTING.md, + // so both docs resolve to a silent allow and nothing is cached. + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("/contents/AI-USAGE.md")) return jsonResponse({ type: "file" }, { headers: { etag: '"v1"' } }); + if (url.includes("/contents/CONTRIBUTING.md")) return jsonResponse({}, { status: 404 }); + if (url.includes("/repos/acme/widgets/issues?")) return jsonResponse([issue(1)]); + return jsonResponse({}, { status: 404 }); + }); + + const result = await discover(cache); + + expect(result.issues.map((entry) => entry.issueNumber)).toEqual([1]); + expect(cache.puts).toEqual([]); + }); + + it("treats a cache read failure as a miss and still fetches the doc in full", async () => { + const cache = fakeCache({ + getImpl: () => { + throw new Error("corrupt cache"); + }, + }); + const calls = stubFetch(() => contentResponse(ALLOWED_AI_USAGE, '"v1"')); + + const result = await discover(cache); + + expect(result.issues.map((entry) => entry.issueNumber)).toEqual([1]); + const policyCall = calls.find((call) => call.url === AI_USAGE_URL); + expect(policyCall?.headers["if-none-match"]).toBeUndefined(); + // The read failed but the write path still runs — a later run can revalidate. + expect(cache.puts).toHaveLength(1); + }); + + it("never fails discovery when the cache write throws", async () => { + const cache = fakeCache({ + putImpl: () => { + throw new Error("disk full"); + }, + }); + stubFetch(() => contentResponse(ALLOWED_AI_USAGE, '"v1"')); + + const result = await discover(cache); + + expect(result.issues.map((entry) => entry.issueNumber)).toEqual([1]); + expect(result.warnings).toEqual([]); + }); + + it("warns and caches nothing when the policy fetch returns a non-404 error", async () => { + const cache = fakeCache(); + // A 403 is neither 304, 404, nor ok: fetchRepoDoc records a warning and returns null, so nothing is cached. + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("/contents/")) return jsonResponse({ message: "forbidden" }, { status: 403 }); + if (url.includes("/repos/acme/widgets/issues?")) return jsonResponse([issue(1)]); + return jsonResponse({}, { status: 404 }); + }); + + const result = await discover(cache); + + expect(result.warnings.some((warning) => warning.stage.startsWith("policy:"))).toBe(true); + expect(cache.puts).toEqual([]); + }); + + it("records a warning and caches nothing when the policy fetch itself throws", async () => { + const cache = fakeCache(); + // A thrown fetch (network-level failure) is not retried; fetchRepoDoc catches it, warns, and returns null. + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("/contents/")) throw new Error("socket hang up"); + if (url.includes("/repos/acme/widgets/issues?")) return jsonResponse([issue(1)]); + return jsonResponse({}, { status: 404 }); + }); + + const result = await discover(cache); + + expect(result.warnings.some((warning) => warning.message === "socket hang up")).toBe(true); + expect(cache.puts).toEqual([]); + }); + + it("fetches normally when no cache is supplied (feature is inert without one)", async () => { + const calls = stubFetch(() => contentResponse(ALLOWED_AI_USAGE, '"v1"')); + + const result = await fetchCandidateIssuesWithSummary([{ owner: "acme", repo: "widgets" }], "token", { + apiBaseUrl: API, + }); + + expect(result.issues.map((entry) => entry.issueNumber)).toEqual([1]); + const policyCall = calls.find((call) => call.url === AI_USAGE_URL); + expect(policyCall?.headers["if-none-match"]).toBeUndefined(); + }); + + it("persists the ETag across two runs with the real on-disk store, then serves a 304", async () => { + const root = mkdtempSync(join(tmpdir(), "gittensory-miner-policy-doc-cache-fanout-")); + roots.push(root); + const dbPath = join(root, "policy-doc-cache.sqlite3"); + const store = initPolicyDocCacheStore(dbPath); + stores.push(store); + + const firstCalls = stubFetch(() => contentResponse(ALLOWED_AI_USAGE, '"v1"')); + const first = await discover(store); + expect(first.issues.map((entry) => entry.issueNumber)).toEqual([1]); + expect(firstCalls.find((call) => call.url === AI_USAGE_URL)?.headers["if-none-match"]).toBeUndefined(); + expect(store.get(AI_USAGE_URL)).toEqual({ etag: '"v1"', content: ALLOWED_AI_USAGE }); + + vi.unstubAllGlobals(); + const secondCalls = stubFetch(() => notModifiedResponse()); + const second = await discover(store); + expect(second.issues.map((entry) => entry.issueNumber)).toEqual([1]); + expect(secondCalls.find((call) => call.url === AI_USAGE_URL)?.headers["if-none-match"]).toBe('"v1"'); + + // The ETag really landed on disk: a freshly reopened handle still has it. + const reopened = initPolicyDocCacheStore(dbPath); + stores.push(reopened); + expect(reopened.get(AI_USAGE_URL)).toEqual({ etag: '"v1"', content: ALLOWED_AI_USAGE }); + }); +});