feat(miner): cache policy docs with conditional-GET revalidation

packages/gittensory-miner/README.md

@@ -106,6 +106,13 @@ See [`docs/env-reference.md`](docs/env-reference.md) for the full `GITTENSORY_MI
 | Deny-hook synthesis | `deny-hook-synthesis.sqlite3` | `deny_rule_proposals` | `deny-hook-synthesis.js` | `GITTENSORY_MINER_DENY_HOOK_SYNTHESIS_DB` |
 | Worktree allocator | `worktree-allocator.sqlite3` | `worktree_slots` | `worktree-allocator.js` | `GITTENSORY_MINER_WORKTREE_ALLOCATOR_DB` |
 | Orb export | `orb-export.sqlite3` | `orb_export_meta` | `orb-export.js` | `GITTENSORY_MINER_ORB_EXPORT_DB` |
+| Policy-doc cache | `policy-doc-cache.sqlite3` | `policy_doc_cache` | `policy-doc-cache.js` | `GITTENSORY_MINER_POLICY_DOC_CACHE_DB` |
+
+The policy-doc cache is the only store above that holds no miner state of its own: it caches the last-known ETag +
+body of each target repo's fetched policy docs (AI-USAGE.md/CONTRIBUTING.md) so a repeated `discover` revalidates
+them with a conditional GET (`If-None-Match`) instead of re-downloading static content, spending no extra
+rate-limit budget when GitHub answers `304 Not Modified`. It is pure optimization — deleting the file only forces
+the next run to refetch in full (#4842).
 
 Every store resolves its file the same way: the store-specific env var above, else `GITTENSORY_MINER_CONFIG_DIR`,
 else `XDG_CONFIG_HOME` (falling back to `~/.config`), joined with `gittensory-miner/<file>`. Every store also opens

packages/gittensory-miner/docs/env-reference.md

@@ -12,9 +12,11 @@ Generated by `npm run miner:env-reference`. Do not edit manually.
 | `GITTENSORY_MINER_EVENT_LEDGER_DB` | `lib/event-ledger.js` | (none) |
 | `GITTENSORY_MINER_GOVERNOR_LEDGER_DB` | `lib/governor-ledger.js` | `""` |
 | `GITTENSORY_MINER_GOVERNOR_STATE_DB` | `lib/governor-state.js` | (none) |
+| `GITTENSORY_MINER_KILL_SWITCH` | `lib/config-precedence.js` | `""` |
 | `GITTENSORY_MINER_NO_UPDATE_CHECK` | `lib/update-check.js` | `""` |
 | `GITTENSORY_MINER_ORB_EXPORT_DB` | `lib/orb-export.js` | `""` |
 | `GITTENSORY_MINER_PLAN_STORE_DB` | `lib/plan-store.js` | `""` |
+| `GITTENSORY_MINER_POLICY_DOC_CACHE_DB` | `lib/policy-doc-cache.js` | (none) |
 | `GITTENSORY_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.js` | (none) |
 | `GITTENSORY_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.js` | `""` |
 | `GITTENSORY_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.js` | (none) |

packages/gittensory-miner/lib/discover-cli.d.ts

@@ -10,6 +10,7 @@ import type {
   RankedCandidateIssue,
   RankedCandidateSummary,
 } from "./opportunity-ranker.js";
+import type { PolicyDocCacheStore } from "./policy-doc-cache.js";
 import type { EnqueueRankedDiscoverySummary } from "./portfolio-discovery.js";
 import type { PortfolioQueueStore } from "./portfolio-queue.js";
 
@@ -60,6 +61,7 @@ export type RunDiscoverOptions = {
   goalSpecsByRepo?: RankCandidateIssuesOptions["goalSpecsByRepo"];
   goalSpecContentByRepo?: RankCandidateIssuesOptions["goalSpecContentByRepo"];
   initPortfolioQueue?: () => PortfolioQueueStore;
+  initPolicyDocCache?: () => PolicyDocCacheStore;
   fetchCandidateIssuesWithSummary?: (
     targets: FanoutTarget[],
     githubToken: string,

packages/gittensory-miner/lib/discover-cli.js

@@ -6,6 +6,7 @@ import {
   searchCandidateIssuesWithSummary,
 } from "./opportunity-fanout.js";
 import { rankCandidateIssuesWithSummary } from "./opportunity-ranker.js";
+import { initPolicyDocCacheStore } from "./policy-doc-cache.js";
 import { enqueueRankedDiscovery } from "./portfolio-discovery.js";
 import { initPortfolioQueueStore } from "./portfolio-queue.js";
 
@@ -149,7 +150,6 @@ export async function runDiscover(args, options = {}) {
   // A `--api-base-url` flag (or `options.apiBaseUrl`) surfaces the fan-out's existing forge-host override at the CLI
   // (#4784); `options.forge` carries any remaining per-tenant forge knobs for a programmatic caller.
   const apiBaseUrl = parsed.apiBaseUrl ?? options.apiBaseUrl;
-  const fanOutOptions = { apiBaseUrl, forge: options.forge };
   const fetchTargets = options.fetchCandidateIssuesWithSummary ?? fetchCandidateIssuesWithSummary;
   const searchTargets = options.searchCandidateIssuesWithSummary ?? searchCandidateIssuesWithSummary;
   const rankIssues = options.rankCandidateIssuesWithSummary ?? rankCandidateIssuesWithSummary;
@@ -158,6 +158,23 @@ export async function runDiscover(args, options = {}) {
   const ownsPortfolioQueue = options.initPortfolioQueue === undefined;
   const portfolioQueue = (options.initPortfolioQueue ?? initPortfolioQueueStore)();
 
+  // Local ETag cache so a repeated discover revalidates each repo's policy docs with a conditional GET instead of
+  // re-downloading them (#4842). Opened inside its OWN try/catch, separate from the portfolio queue above: the
+  // queue is required infrastructure (discovery genuinely cannot enqueue anything without it, so a real open
+  // failure should abort the run), but the policy-doc cache is a pure performance optimization -- a corrupt or
+  // unwritable cache DB must degrade to "no cache" (every doc fetched in full, exactly as before #4842) rather
+  // than fail discovery outright.
+  let policyDocCache = null;
+  let ownsPolicyDocCache = false;
+  try {
+    ownsPolicyDocCache = options.initPolicyDocCache === undefined;
+    policyDocCache = (options.initPolicyDocCache ?? initPolicyDocCacheStore)();
+  } catch {
+    policyDocCache = null;
+    ownsPolicyDocCache = false;
+  }
+  const fanOutOptions = { apiBaseUrl, forge: options.forge, policyDocCache };
+
   try {
     const fanOut =
       parsed.search !== null
@@ -195,5 +212,6 @@ export async function runDiscover(args, options = {}) {
     return 2;
   } finally {
     if (ownsPortfolioQueue) portfolioQueue.close();
+    if (ownsPolicyDocCache && policyDocCache) policyDocCache.close();
   }
 }

packages/gittensory-miner/lib/opportunity-fanout.d.ts

@@ -1,12 +1,15 @@
 import type { ForgeConfig } from "./forge-config.js";
+import type { PolicyDocCache } from "./policy-doc-cache.js";
 
 export type FanoutTarget = {
   owner: string;
   repo: string;
 };
 
 /** Options shared by every fan-out entry point. `apiBaseUrl` is the legacy top-level forge-host override (it still
- * wins over `forge.apiBaseUrl`); `forge` (#4784) carries the rest of the per-tenant forge knobs. */
+ * wins over `forge.apiBaseUrl`); `forge` (#4784) carries the rest of the per-tenant forge knobs. `policyDocCache`,
+ * when supplied, lets discovery revalidate each repo's policy docs with a conditional GET instead of a full
+ * refetch (#4842). */
 export type FanoutOptions = {
   apiBaseUrl?: string;
   forge?: Partial<ForgeConfig>;
@@ -16,6 +19,7 @@ export type FanoutOptions = {
   perPage?: number;
   maxPages?: number;
   sleepFn?: (ms: number) => Promise<unknown>;
+  policyDocCache?: PolicyDocCache | null;
 };
 
 export type RawCandidateIssue = {

packages/gittensory-miner/lib/opportunity-fanout.js

@@ -119,13 +119,14 @@ function recordRateLimit(summary, response) {
   }
 }
 
-async function githubGetJson(url, githubToken, summary, options) {
+async function githubGetJson(url, githubToken, summary, options, extraHeaders = {}) {
   // Retry a transient 5xx from GitHub before dropping this target's results for the whole run (#4830) — the same
   // discipline as the CI/gate-verdict pollers. A thrown network error still propagates to each caller's try/catch.
+  // `extraHeaders` carries per-call additions (e.g. a policy-doc If-None-Match, #4842) on top of the base auth set.
   const response = await fetchWithRetry(
     fetch,
     url,
-    { method: "GET", headers: githubHeaders(githubToken, options.forge) },
+    { method: "GET", headers: { ...githubHeaders(githubToken, options.forge), ...extraHeaders } },
     { sleepFn: options?.sleepFn },
   );
   recordRateLimit(summary, response);
@@ -146,19 +147,52 @@ function warning(target, stage, message) {
   return { repoFullName: target.repoFullName, stage, message };
 }
 
+// Read a URL's prior ETag so an unchanged doc can be revalidated with a conditional GET (#4842). A cache that is
+// absent, or whose read throws (corrupt/locked file), is treated as a plain miss: the caller does a full fetch,
+// per the "never risk a stale policy" rule — the cache only ever makes discovery cheaper, never less correct.
+function readCachedPolicyDoc(cache, url) {
+  if (!cache) return null;
+  try {
+    return cache.get(url);
+  } catch {
+    return null;
+  }
+}
+
+// Persist the fresh ETag + body so the NEXT discover run can revalidate instead of re-downloading. Only a real
+// ETag paired with decoded content is stored, and a write that throws must never fail discovery (same stale-safe
+// rule) — it degrades to "not cached", so the next run simply refetches in full.
+function writeCachedPolicyDoc(cache, url, response, content) {
+  if (!cache || content === null) return;
+  const etag = response.headers.get("etag");
+  if (typeof etag !== "string" || !etag.trim()) return;
+  try {
+    cache.put(url, etag, content);
+  } catch {
+    // Leave this URL uncached; the next run refetches fully rather than serving anything stale.
+  }
+}
+
 async function fetchRepoDoc(target, path, githubToken, options, summary, warnings) {
   const url = apiUrl(
     options.apiBaseUrl,
     repoPath(options.forge, target, `/contents/${encodeURIComponent(path)}`),
   );
+  const cached = readCachedPolicyDoc(options.policyDocCache, url);
+  const conditionalHeaders = cached ? { "if-none-match": cached.etag } : {};
   try {
-    const { response, payload } = await githubGetJson(url, githubToken, summary, options);
+    const { response, payload } = await githubGetJson(url, githubToken, summary, options, conditionalHeaders);
+    // A 304 only ever follows the If-None-Match we send above, which we only send when `cached` exists — so the
+    // cached body is the GitHub-confirmed current content, served with no extra rate-limit spend.
+    if (response.status === 304) return cached.content;
     if (response.status === 404) return null;
     if (!response.ok) {
       warnings.push(warning(target, `policy:${path}`, `GitHub returned ${response.status}`));
       return null;
     }
-    return decodeContentPayload(payload);
+    const content = decodeContentPayload(payload);
+    writeCachedPolicyDoc(options.policyDocCache, url, response, content);
+    return content;
   } catch (error) {
     warnings.push(
       warning(target, `policy:${path}`, error instanceof Error ? error.message : "policy fetch failed"),
@@ -405,6 +439,9 @@ function normalizeOptions(options = {}) {
     maxPages: normalizeLimit(options.maxPages, defaultMaxPages, 1, 100),
     // Passed through to the per-fetch retry so tests can inject an instant sleep; undefined uses the real backoff.
     sleepFn: typeof options.sleepFn === "function" ? options.sleepFn : undefined,
+    // Optional local ETag cache for policy-doc revalidation (#4842). Absent (null) => every policy doc is fetched
+    // in full, exactly as before; discover-cli.js supplies the real on-disk store for a live run.
+    policyDocCache: options.policyDocCache ?? null,
   };
 }
 

packages/gittensory-miner/lib/policy-doc-cache.d.ts

@@ -0,0 +1,25 @@
+export type PolicyDocCacheEntry = {
+  etag: string;
+  content: string;
+};
+
+export type PolicyDocCacheWrite = {
+  url: string;
+  etag: string;
+  content: string;
+  updatedAt: string;
+};
+
+export type PolicyDocCacheStore = {
+  dbPath: string;
+  get(url: string): PolicyDocCacheEntry | null;
+  put(url: string, etag: string, content: string): PolicyDocCacheWrite;
+  close(): void;
+};
+
+/** The read/write surface opportunity-fanout.js needs to inject a cache without depending on the SQLite store. */
+export type PolicyDocCache = Pick<PolicyDocCacheStore, "get" | "put">;
+
+export function resolvePolicyDocCacheDbPath(env?: Record<string, string | undefined>): string;
+
+export function initPolicyDocCacheStore(dbPath?: string): PolicyDocCacheStore;

packages/gittensory-miner/lib/policy-doc-cache.js

@@ -0,0 +1,79 @@
+import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js";
+import { applySchemaMigrations } from "./schema-version.js";
+
+// Local ETag cache for discovery's small policy-doc fetches (#4842). `discover` refetches each target repo's
+// AI-USAGE.md/CONTRIBUTING.md on every run even though they rarely change, spending rate-limit budget on static
+// content; this store lets opportunity-fanout.js revalidate with a conditional GET (If-None-Match) instead, and
+// GitHub answers an unchanged doc with a 304 that costs no primary rate-limit budget. A 304 is a GitHub-confirmed
+// unchanged body -- the cached content is only ever served AFTER a same-run revalidation, never blindly -- so this
+// can never surface a stale policy that would wrongly permit autonomous work on an opted-out repo. Same 100%
+// local/client-side discipline (mirrors run-state.js and the other stores this package owns via local-store.js):
+// the file lives only on this machine and is never uploaded, synced, or phoned home with.
+
+const defaultDbFileName = "policy-doc-cache.sqlite3";
+
+export function resolvePolicyDocCacheDbPath(env = process.env) {
+  return resolveLocalStoreDbPath(defaultDbFileName, "GITTENSORY_MINER_POLICY_DOC_CACHE_DB", env);
+}
+
+function normalizeDbPath(dbPath) {
+  return normalizeLocalStoreDbPath(dbPath, resolvePolicyDocCacheDbPath(), "invalid_policy_doc_cache_db_path");
+}
+
+function normalizeUrl(url) {
+  if (typeof url !== "string") throw new Error("invalid_policy_doc_url");
+  const trimmed = url.trim();
+  if (!trimmed) throw new Error("invalid_policy_doc_url");
+  return trimmed;
+}
+
+/**
+ * Opens the 100% local/client-side miner policy-doc ETag cache. The database only lives on this machine; this
+ * module never uploads, syncs, or phones home with its contents. (#4842)
+ */
+export function initPolicyDocCacheStore(dbPath = resolvePolicyDocCacheDbPath()) {
+  const resolvedPath = normalizeDbPath(dbPath);
+  const db = openLocalStoreDb(resolvedPath);
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS policy_doc_cache (
+      url TEXT PRIMARY KEY,
+      etag TEXT NOT NULL,
+      content TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    )
+  `);
+  // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations (none yet).
+  applySchemaMigrations(db, []);
+
+  const getStatement = db.prepare("SELECT etag, content FROM policy_doc_cache WHERE url = ?");
+  const putStatement = db.prepare(`
+    INSERT INTO policy_doc_cache (url, etag, content, updated_at)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(url) DO UPDATE SET
+      etag = excluded.etag,
+      content = excluded.content,
+      updated_at = excluded.updated_at
+  `);
+
+  return {
+    dbPath: resolvedPath,
+    /** The last-known `{ etag, content }` for a policy-doc URL, or null when it has never been cached. Both columns
+     *  are `TEXT NOT NULL`, so a present row always carries string values. */
+    get(url) {
+      const row = getStatement.get(normalizeUrl(url));
+      return row ? { etag: row.etag, content: row.content } : null;
+    },
+    /** Record the fresh ETag + body so the next run can revalidate it with a conditional GET. */
+    put(url, etag, content) {
+      const normalizedUrl = normalizeUrl(url);
+      if (typeof etag !== "string" || !etag.trim()) throw new Error("invalid_policy_doc_etag");
+      if (typeof content !== "string") throw new Error("invalid_policy_doc_content");
+      const updatedAt = new Date().toISOString();
+      putStatement.run(normalizedUrl, etag, content, updatedAt);
+      return { url: normalizedUrl, etag, content, updatedAt };
+    },
+    close() {
+      db.close();
+    },
+  };
+}