feat(miner): cache policy docs with conditional-GET ETags in discovery (#4842)

packages/gittensory-miner/README.md

@@ -67,7 +67,7 @@ Phase 6 of the same roadmap tracker and hasn't been scaffolded yet. (#4279)
 
 ## Local storage
 
-Four independent local SQLite stores back the commands above. Each keeps its own file, its own table, and its own
+Five independent local SQLite stores back the commands above. Each keeps its own file, its own table, and its own
 env-var override — this is a DRY pass over their shared path-resolution/open boilerplate (`local-store.js`), not a
 merge into one database. (#4272)
 
@@ -77,6 +77,13 @@ merge into one database. (#4272)
 | Claim ledger | `claim-ledger.sqlite3` | `miner_claims` | `claim-ledger.js` | `GITTENSORY_MINER_CLAIM_LEDGER_DB` |
 | Portfolio queue | `portfolio-queue.sqlite3` | `miner_portfolio_queue` | `portfolio-queue.js` | `GITTENSORY_MINER_PORTFOLIO_QUEUE_DB` |
 | Event ledger | `event-ledger.sqlite3` | `miner_event_ledger` | `event-ledger.js` | `GITTENSORY_MINER_EVENT_LEDGER_DB` |
+| Policy-doc cache | `policy-doc-cache.sqlite3` | `policy_doc_cache` | `policy-doc-cache.js` | `GITTENSORY_MINER_POLICY_DOC_CACHE_DB` |
+
+The policy-doc cache is the only one of the five that holds no miner state of its own: it caches the last-known
+ETag + body of each target repo's fetched policy docs (AI-USAGE.md/CONTRIBUTING.md) so a repeated `discover`
+revalidates them with a conditional GET (`If-None-Match`) instead of re-downloading static content, spending no
+extra rate-limit budget when GitHub answers `304 Not Modified`. It is pure optimization — deleting the file only
+forces the next run to refetch in full (#4842).
 
 Every store resolves its file the same way: the store-specific env var above, else `GITTENSORY_MINER_CONFIG_DIR`,
 else `XDG_CONFIG_HOME` (falling back to `~/.config`), joined with `gittensory-miner/<file>`. Every store also opens

packages/gittensory-miner/docs/env-reference.md

@@ -15,6 +15,7 @@ Generated by `npm run miner:env-reference`. Do not edit manually.
 | `GITTENSORY_MINER_NO_UPDATE_CHECK` | `lib/update-check.js` | `""` |
 | `GITTENSORY_MINER_ORB_EXPORT_DB` | `lib/orb-export.js` | `""` |
 | `GITTENSORY_MINER_PLAN_STORE_DB` | `lib/plan-store.js` | `""` |
+| `GITTENSORY_MINER_POLICY_DOC_CACHE_DB` | `lib/policy-doc-cache.js` | (none) |
 | `GITTENSORY_MINER_PORTFOLIO_QUEUE_DB` | `lib/portfolio-queue.js` | (none) |
 | `GITTENSORY_MINER_PREDICTION_LEDGER_DB` | `lib/prediction-ledger.js` | `""` |
 | `GITTENSORY_MINER_REPLAY_SNAPSHOT_DB` | `lib/replay-snapshot.js` | (none) |

packages/gittensory-miner/lib/discover-cli.d.ts

@@ -4,6 +4,7 @@ import type {
   RawCandidateIssue,
 } from "./opportunity-fanout.js";
 import type { RankedCandidateIssue, RankedCandidateSummary } from "./opportunity-ranker.js";
+import type { PolicyDocCache, PolicyDocCacheStore } from "./policy-doc-cache.js";
 import type { EnqueueRankedDiscoverySummary } from "./portfolio-discovery.js";
 import type { PortfolioQueueStore } from "./portfolio-queue.js";
 
@@ -41,15 +42,16 @@ export type RunDiscoverOptions = {
   apiBaseUrl?: string;
   nowMs?: number;
   initPortfolioQueue?: () => PortfolioQueueStore;
+  initPolicyDocCache?: () => PolicyDocCacheStore;
   fetchCandidateIssuesWithSummary?: (
     targets: FanoutTarget[],
     githubToken: string,
-    options?: { apiBaseUrl?: string },
+    options?: { apiBaseUrl?: string; policyDocCache?: PolicyDocCache | null },
   ) => Promise<DiscoverFanOutSummary>;
   searchCandidateIssuesWithSummary?: (
     searchQuery: string,
     githubToken: string,
-    options?: { apiBaseUrl?: string },
+    options?: { apiBaseUrl?: string; policyDocCache?: PolicyDocCache | null },
   ) => Promise<DiscoverFanOutSummary>;
   rankCandidateIssuesWithSummary?: (
     candidates: RawCandidateIssue[],

packages/gittensory-miner/lib/discover-cli.js

@@ -5,6 +5,7 @@ import {
   searchCandidateIssuesWithSummary,
 } from "./opportunity-fanout.js";
 import { rankCandidateIssuesWithSummary } from "./opportunity-ranker.js";
+import { initPolicyDocCacheStore } from "./policy-doc-cache.js";
 import { enqueueRankedDiscovery } from "./portfolio-discovery.js";
 import { initPortfolioQueueStore } from "./portfolio-queue.js";
 
@@ -118,11 +119,18 @@ export async function runDiscover(args, options = {}) {
   const ownsPortfolioQueue = options.initPortfolioQueue === undefined;
   const portfolioQueue = (options.initPortfolioQueue ?? initPortfolioQueueStore)();
 
+  // Local ETag cache so a repeated discover revalidates each repo's policy docs with a conditional GET instead of
+  // re-downloading them (#4842). Owned + closed here exactly like the portfolio queue above; an injected factory
+  // lets tests supply a temp/in-memory store instead of the real on-disk one.
+  const ownsPolicyDocCache = options.initPolicyDocCache === undefined;
+  const policyDocCache = (options.initPolicyDocCache ?? initPolicyDocCacheStore)();
+
   try {
+    const fanOutOptions = { apiBaseUrl: options.apiBaseUrl, policyDocCache };
     const fanOut =
       parsed.search !== null
-        ? await searchTargets(parsed.search, githubToken, { apiBaseUrl: options.apiBaseUrl })
-        : await fetchTargets(parsed.targets, githubToken, { apiBaseUrl: options.apiBaseUrl });
+        ? await searchTargets(parsed.search, githubToken, fanOutOptions)
+        : await fetchTargets(parsed.targets, githubToken, fanOutOptions);
 
     const rankedSummary = rankIssues(fanOut.issues, { nowMs: options.nowMs });
     const enqueueSummary = enqueue(rankedSummary.issues, { queueStore: portfolioQueue });
@@ -147,5 +155,6 @@ export async function runDiscover(args, options = {}) {
     return 2;
   } finally {
     if (ownsPortfolioQueue) portfolioQueue.close();
+    if (ownsPolicyDocCache) policyDocCache.close();
   }
 }

packages/gittensory-miner/lib/opportunity-fanout.d.ts

@@ -1,8 +1,20 @@
+import type { PolicyDocCache } from "./policy-doc-cache.js";
+
 export type FanoutTarget = {
   owner: string;
   repo: string;
 };
 
+/** Options common to every fan-out/search entry point. `policyDocCache`, when supplied, lets discovery revalidate
+ * each repo's policy docs with a conditional GET instead of a full refetch (#4842). */
+export type FanoutOptions = {
+  apiBaseUrl?: string;
+  concurrency?: number;
+  perPage?: number;
+  sleepFn?: (ms: number) => Promise<unknown>;
+  policyDocCache?: PolicyDocCache | null;
+};
+
 export type RawCandidateIssue = {
   owner: string;
   repo: string;
@@ -42,51 +54,23 @@ export function mapWithConcurrency<T, R>(
 export function fetchCandidateIssuesWithSummary(
   targets: FanoutTarget[],
   githubToken: string,
-  options?: {
-    apiBaseUrl?: string;
-    concurrency?: number;
-    rateLimitLowWaterMark?: number;
-    rateLimitHighWaterMark?: number;
-    perPage?: number;
-    sleepFn?: (ms: number) => Promise<unknown>;
-  },
+  options?: FanoutOptions,
 ): Promise<CandidateIssueSummary>;
 
 export function fetchCandidateIssues(
   targets: FanoutTarget[],
   githubToken: string,
-  options?: {
-    apiBaseUrl?: string;
-    concurrency?: number;
-    rateLimitLowWaterMark?: number;
-    rateLimitHighWaterMark?: number;
-    perPage?: number;
-    sleepFn?: (ms: number) => Promise<unknown>;
-  },
+  options?: FanoutOptions,
 ): Promise<RawCandidateIssue[]>;
 
 export function searchCandidateIssuesWithSummary(
   searchQuery: string,
   githubToken: string,
-  options?: {
-    apiBaseUrl?: string;
-    concurrency?: number;
-    rateLimitLowWaterMark?: number;
-    rateLimitHighWaterMark?: number;
-    perPage?: number;
-    sleepFn?: (ms: number) => Promise<unknown>;
-  },
+  options?: FanoutOptions,
 ): Promise<CandidateIssueSummary>;
 
 export function searchCandidateIssues(
   searchQuery: string,
   githubToken: string,
-  options?: {
-    apiBaseUrl?: string;
-    concurrency?: number;
-    rateLimitLowWaterMark?: number;
-    rateLimitHighWaterMark?: number;
-    perPage?: number;
-    sleepFn?: (ms: number) => Promise<unknown>;
-  },
+  options?: FanoutOptions,
 ): Promise<RawCandidateIssue[]>;

packages/gittensory-miner/lib/opportunity-fanout.js

@@ -112,13 +112,14 @@ function recordRateLimit(summary, response) {
   }
 }
 
-async function githubGetJson(url, githubToken, summary, options) {
+async function githubGetJson(url, githubToken, summary, options, extraHeaders = {}) {
   // Retry a transient 5xx from GitHub before dropping this target's results for the whole run (#4830) — the same
   // discipline as the CI/gate-verdict pollers. A thrown network error still propagates to each caller's try/catch.
+  // `extraHeaders` carries per-call additions (e.g. a policy-doc If-None-Match, #4842) on top of the base auth set.
   const response = await fetchWithRetry(
     fetch,
     url,
-    { method: "GET", headers: githubHeaders(githubToken) },
+    { method: "GET", headers: { ...githubHeaders(githubToken), ...extraHeaders } },
     { sleepFn: options?.sleepFn },
   );
   recordRateLimit(summary, response);
@@ -139,19 +140,52 @@ function warning(target, stage, message) {
   return { repoFullName: target.repoFullName, stage, message };
 }
 
+// Read a URL's prior ETag so an unchanged doc can be revalidated with a conditional GET (#4842). A cache that is
+// absent, or whose read throws (corrupt/locked file), is treated as a plain miss: the caller does a full fetch,
+// per the "never risk a stale policy" rule — the cache only ever makes discovery cheaper, never less correct.
+function readCachedPolicyDoc(cache, url) {
+  if (!cache) return null;
+  try {
+    return cache.get(url);
+  } catch {
+    return null;
+  }
+}
+
+// Persist the fresh ETag + body so the NEXT discover run can revalidate instead of re-downloading. Only a real
+// ETag paired with decoded content is stored, and a write that throws must never fail discovery (same stale-safe
+// rule) — it degrades to "not cached", so the next run simply refetches in full.
+function writeCachedPolicyDoc(cache, url, response, content) {
+  if (!cache || content === null) return;
+  const etag = response.headers.get("etag");
+  if (typeof etag !== "string" || !etag.trim()) return;
+  try {
+    cache.put(url, etag, content);
+  } catch {
+    // Leave this URL uncached; the next run refetches fully rather than serving anything stale.
+  }
+}
+
 async function fetchRepoDoc(target, path, githubToken, options, summary, warnings) {
   const url = apiUrl(
     options.apiBaseUrl,
     repoPath(target, `/contents/${encodeURIComponent(path)}`),
   );
+  const cached = readCachedPolicyDoc(options.policyDocCache, url);
+  const conditionalHeaders = cached ? { "if-none-match": cached.etag } : {};
   try {
-    const { response, payload } = await githubGetJson(url, githubToken, summary, options);
+    const { response, payload } = await githubGetJson(url, githubToken, summary, options, conditionalHeaders);
+    // A 304 only ever follows the If-None-Match we send above, which we only send when `cached` exists — so the
+    // cached body is the GitHub-confirmed current content, served with no extra rate-limit spend.
+    if (response.status === 304) return cached.content;
     if (response.status === 404) return null;
     if (!response.ok) {
       warnings.push(warning(target, `policy:${path}`, `GitHub returned ${response.status}`));
       return null;
     }
-    return decodeContentPayload(payload);
+    const content = decodeContentPayload(payload);
+    writeCachedPolicyDoc(options.policyDocCache, url, response, content);
+    return content;
   } catch (error) {
     warnings.push(
       warning(target, `policy:${path}`, error instanceof Error ? error.message : "policy fetch failed"),
@@ -376,6 +410,9 @@ function normalizeOptions(options = {}) {
     maxPages: normalizeLimit(options.maxPages, defaultMaxPages, 1, 100),
     // Passed through to the per-fetch retry so tests can inject an instant sleep; undefined uses the real backoff.
     sleepFn: typeof options.sleepFn === "function" ? options.sleepFn : undefined,
+    // Optional local ETag cache for policy-doc revalidation (#4842). Absent (null) => every policy doc is fetched
+    // in full, exactly as before; discover-cli.js supplies the real on-disk store for a live run.
+    policyDocCache: options.policyDocCache ?? null,
   };
 }
 

packages/gittensory-miner/lib/policy-doc-cache.d.ts

@@ -0,0 +1,25 @@
+export type PolicyDocCacheEntry = {
+  etag: string;
+  content: string;
+};
+
+export type PolicyDocCacheWrite = {
+  url: string;
+  etag: string;
+  content: string;
+  updatedAt: string;
+};
+
+export type PolicyDocCacheStore = {
+  dbPath: string;
+  get(url: string): PolicyDocCacheEntry | null;
+  put(url: string, etag: string, content: string): PolicyDocCacheWrite;
+  close(): void;
+};
+
+/** The read/write surface opportunity-fanout.js needs to inject a cache without depending on the SQLite store. */
+export type PolicyDocCache = Pick<PolicyDocCacheStore, "get" | "put">;
+
+export function resolvePolicyDocCacheDbPath(env?: Record<string, string | undefined>): string;
+
+export function initPolicyDocCacheStore(dbPath?: string): PolicyDocCacheStore;

packages/gittensory-miner/lib/policy-doc-cache.js

@@ -0,0 +1,79 @@
+import { normalizeLocalStoreDbPath, openLocalStoreDb, resolveLocalStoreDbPath } from "./local-store.js";
+import { applySchemaMigrations } from "./schema-version.js";
+
+// Local ETag cache for discovery's small policy-doc fetches (#4842). `discover` refetches each target repo's
+// AI-USAGE.md/CONTRIBUTING.md on every run even though they rarely change, spending rate-limit budget on static
+// content; this store lets opportunity-fanout.js revalidate with a conditional GET (If-None-Match) instead, and
+// GitHub answers an unchanged doc with a 304 that costs no primary rate-limit budget. A 304 is a GitHub-confirmed
+// unchanged body -- the cached content is only ever served AFTER a same-run revalidation, never blindly -- so this
+// can never surface a stale policy that would wrongly permit autonomous work on an opted-out repo. Same 100%
+// local/client-side discipline (mirrors run-state.js and the other stores this package owns via local-store.js):
+// the file lives only on this machine and is never uploaded, synced, or phoned home with.
+
+const defaultDbFileName = "policy-doc-cache.sqlite3";
+
+export function resolvePolicyDocCacheDbPath(env = process.env) {
+  return resolveLocalStoreDbPath(defaultDbFileName, "GITTENSORY_MINER_POLICY_DOC_CACHE_DB", env);
+}
+
+function normalizeDbPath(dbPath) {
+  return normalizeLocalStoreDbPath(dbPath, resolvePolicyDocCacheDbPath(), "invalid_policy_doc_cache_db_path");
+}
+
+function normalizeUrl(url) {
+  if (typeof url !== "string") throw new Error("invalid_policy_doc_url");
+  const trimmed = url.trim();
+  if (!trimmed) throw new Error("invalid_policy_doc_url");
+  return trimmed;
+}
+
+/**
+ * Opens the 100% local/client-side miner policy-doc ETag cache. The database only lives on this machine; this
+ * module never uploads, syncs, or phones home with its contents. (#4842)
+ */
+export function initPolicyDocCacheStore(dbPath = resolvePolicyDocCacheDbPath()) {
+  const resolvedPath = normalizeDbPath(dbPath);
+  const db = openLocalStoreDb(resolvedPath);
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS policy_doc_cache (
+      url TEXT PRIMARY KEY,
+      etag TEXT NOT NULL,
+      content TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    )
+  `);
+  // Schema-version convention (#4832): stamp the baseline and run any post-baseline migrations (none yet).
+  applySchemaMigrations(db, []);
+
+  const getStatement = db.prepare("SELECT etag, content FROM policy_doc_cache WHERE url = ?");
+  const putStatement = db.prepare(`
+    INSERT INTO policy_doc_cache (url, etag, content, updated_at)
+    VALUES (?, ?, ?, ?)
+    ON CONFLICT(url) DO UPDATE SET
+      etag = excluded.etag,
+      content = excluded.content,
+      updated_at = excluded.updated_at
+  `);
+
+  return {
+    dbPath: resolvedPath,
+    /** The last-known `{ etag, content }` for a policy-doc URL, or null when it has never been cached. Both columns
+     *  are `TEXT NOT NULL`, so a present row always carries string values. */
+    get(url) {
+      const row = getStatement.get(normalizeUrl(url));
+      return row ? { etag: row.etag, content: row.content } : null;
+    },
+    /** Record the fresh ETag + body so the next run can revalidate it with a conditional GET. */
+    put(url, etag, content) {
+      const normalizedUrl = normalizeUrl(url);
+      if (typeof etag !== "string" || !etag.trim()) throw new Error("invalid_policy_doc_etag");
+      if (typeof content !== "string") throw new Error("invalid_policy_doc_content");
+      const updatedAt = new Date().toISOString();
+      putStatement.run(normalizedUrl, etag, content, updatedAt);
+      return { url: normalizedUrl, etag, content, updatedAt };
+    },
+    close() {
+      db.close();
+    },
+  };
+}