feat(miner): add cross-repo evaluation harness (#4788)

package.json

@@ -20,6 +20,7 @@
     "miner:env-reference": "node packages/loopover-miner/scripts/generate-env-reference.mjs",
     "miner:env-reference:check": "node packages/loopover-miner/scripts/generate-env-reference.mjs --check",
     "benchmark:miner": "node packages/loopover-miner/scripts/benchmark.mjs",
+    "cross-repo-eval:miner": "node packages/loopover-miner/scripts/cross-repo-evaluation.mjs",
     "loadtest:iterate-loop": "npm run build --workspace @loopover/engine && node packages/loopover-engine/scripts/load-test-iterate-loop.mjs",
     "command-reference": "node scripts/gen-command-reference.mjs",
     "command-reference:check": "node scripts/gen-command-reference.mjs --check",

packages/loopover-miner/README.md

@@ -22,6 +22,10 @@ Config precedence (`.loopover-miner.yml` vs operator env vs CLI flags) is docume
 A committed micro-benchmark for the discovery-ranking and local-store read/write paths lives at
 [`BENCHMARKS.md`](BENCHMARKS.md) — run it with `npm run benchmark:miner` from the repo root.
 
+The cross-repo evaluation harness (#4788) lives at
+[`docs/cross-repo-evaluation.md`](docs/cross-repo-evaluation.md) — run it with
+`npm run cross-repo-eval --workspace @loopover/miner` after cloning the benchmark repos.
+
 Real miner commands land in follow-up issues.
 
 The package also includes the first metadata-only discovery primitive: `fetchCandidateIssues` lists open issue

packages/loopover-miner/benchmarks/cross-repo/manifest.json

@@ -0,0 +1,34 @@
+{
+  "description": "Cross-repo evaluation benchmark set (#4788). Diverse public repos exercised by `node packages/loopover-miner/scripts/cross-repo-evaluation.mjs` after cloning into LOOPOVER_MINER_REPO_CLONE_DIR.",
+  "repos": [
+    {
+      "repoFullName": "sindresorhus/is",
+      "stackHint": "nodejs"
+    },
+    {
+      "repoFullName": "expressjs/express",
+      "stackHint": "nodejs",
+      "requireTestCommand": true
+    },
+    {
+      "repoFullName": "pallets/flask",
+      "stackHint": "python",
+      "requireTestCommand": true
+    },
+    {
+      "repoFullName": "BurntSushi/ripgrep",
+      "stackHint": "rust",
+      "requireTestCommand": true
+    },
+    {
+      "repoFullName": "gin-gonic/gin",
+      "stackHint": "go",
+      "requireTestCommand": true
+    },
+    {
+      "repoFullName": "lodash/lodash",
+      "stackHint": "nodejs",
+      "requireTestCommand": true
+    }
+  ]
+}

packages/loopover-miner/docs/cross-repo-evaluation.md

@@ -0,0 +1,78 @@
+# Cross-repo evaluation harness
+
+The **cross-repo evaluation harness** (#4788) is a repeatable readiness check that asks whether the miner can
+approach a diverse benchmark repo set **without gittensory-specific target-repo configuration** (no
+`.loopover-miner.yml` required in the benchmark repos). It exercises the same offline path a real attempt uses
+before the coding agent runs:
+
+1. **Clone setup** — the repo exists under `LOOPOVER_MINER_REPO_CLONE_DIR`
+2. **Stack auto-detection** (`detectRepoStack`, #4785)
+3. **Coding-task spec composition** (`buildCodingTaskSpec`, #4786) including validation guidance derived from the
+   detected stack
+4. **Assumption scan** — agent instructions must not positively mandate LoopOver/gittensory CI conventions
+
+Each benchmark repo receives a **pass/fail** line. Failures are categorized:
+
+| Category | Meaning |
+| --- | --- |
+| `stack_detection_gap` | No recognized manifest / stack could not be inferred |
+| `execution_gap` | Stack detected but the coding-task path is not ready (e.g. missing inferred test command when required) |
+| `gittensory_assumption` | Agent instructions leak gittensory-specific CI assumptions |
+| `clone_setup` | The repo has not been cloned to the expected cache path |
+| `other` | Unexpected errors |
+
+The run also reports whether a **strict majority** of repos passed and how many succeeded **without** a per-target
+`.loopover-miner.yml` (the default goal spec is acceptable).
+
+## Benchmark manifest
+
+Committed at [`benchmarks/cross-repo/manifest.json`](../benchmarks/cross-repo/manifest.json). Each entry is either a
+bare `"owner/repo"` string or an object:
+
+- **`repoFullName`** — canonical `owner/repo`
+- **`stackHint`** — documentation only (not used by the evaluator)
+- **`requireTestCommand`** — when `true`, stack detection must infer a test command or the repo fails with
+  `execution_gap`
+
+Malformed manifest fields degrade to documented defaults with warnings (same tolerant-parser convention as the
+fleet run-manifest).
+
+## Running locally
+
+1. Clone the benchmark repos into the miner clone cache (once per machine):
+
+   ```bash
+   export LOOPOVER_MINER_REPO_CLONE_DIR="${LOOPOVER_MINER_REPO_CLONE_DIR:-$HOME/.config/loopover-miner/repos}"
+   mkdir -p "$LOOPOVER_MINER_REPO_CLONE_DIR"
+   # Example for one entry — repeat for each repo in the manifest
+   git clone --depth 1 https://github.com/sindresorhus/is.git "$LOOPOVER_MINER_REPO_CLONE_DIR/sindresorhus/is"
+   ```
+
+2. Run the harness from the repo root:
+
+   ```bash
+   node packages/loopover-miner/scripts/cross-repo-evaluation.mjs
+   ```
+
+   Useful flags:
+
+   - `--json` — machine-readable `{ warnings, results, summary }` payload
+   - `--repo owner/repo` — evaluate a single manifest entry
+   - `--manifest path/to/manifest.json` — alternate benchmark set (e.g. a fixture manifest in tests)
+   - `--require-majority` — exit `1` unless a strict majority of repos pass (for CI-style gating)
+
+## Library API
+
+Pure functions live in [`lib/cross-repo-evaluation.js`](../lib/cross-repo-evaluation.js):
+
+- `parseCrossRepoEvaluationManifest(content)`
+- `evaluateRepoReadiness(entry, options)` — inject `existsSync`, `detectRepoStack`, etc. for unit tests
+- `runCrossRepoEvaluation(parsed, options)`
+- `summarizeCrossRepoEvaluation(results)`
+- `formatCrossRepoEvaluationReport(results, summary)`
+
+## Wiring
+
+This harness is **readiness-only**: it does not run the coding agent, open PRs, or call forge APIs. A green report
+means the miner’s repo-agnostic stack-detection and coding-task-spec path is prepared for the benchmark repo; a live
+attempt still needs credentials, governor policy, and queue state as documented in [`DEPLOYMENT.md`](../DEPLOYMENT.md).

packages/loopover-miner/lib/cross-repo-evaluation.d.ts

@@ -0,0 +1,90 @@
+import type { RepoStackResult } from "./stack-detection.js";
+
+export const CROSS_REPO_FAILURE_CATEGORY: Readonly<{
+  STACK_DETECTION: "stack_detection_gap";
+  EXECUTION: "execution_gap";
+  GITTENSOR_ASSUMPTION: "gittensory_assumption";
+  CLONE_SETUP: "clone_setup";
+  OTHER: "other";
+}>;
+
+export const GITTENSOR_POSITIVE_ASSUMPTION_CHECKS: ReadonlyArray<{
+  id: string;
+  pattern: RegExp;
+}>;
+
+export const DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH: string;
+export const MAX_CROSS_REPO_MANIFEST_BYTES: number;
+export const MAX_CROSS_REPO_MANIFEST_REPOS: number;
+
+export type CrossRepoEvaluationManifestRepo = {
+  repoFullName: string;
+  stackHint?: string;
+  requireTestCommand?: boolean;
+  fixturePath?: string;
+};
+
+export type ParsedCrossRepoEvaluationManifest = {
+  present: boolean;
+  manifest: { repos: CrossRepoEvaluationManifestRepo[] };
+  warnings: string[];
+};
+
+export type CrossRepoEvaluationResult = {
+  repoFullName: string;
+  passed: boolean;
+  failureCategory: string | null;
+  reason: string | null;
+  stackDetected: boolean;
+  usedDefaultGoalSpec: boolean | null;
+  assumptionFindings: Array<{ id: string; line: string }>;
+  stack?: RepoStackResult;
+};
+
+export type CrossRepoEvaluationSummary = {
+  total: number;
+  passed: number;
+  failed: number;
+  majorityPassed: boolean;
+  withoutGittensoryConfig: number;
+  failuresByCategory: Record<string, number>;
+};
+
+export function normalizeCrossRepoFullName(value: unknown): string | null;
+
+export function parseCrossRepoEvaluationManifest(
+  content: string | null | undefined,
+): ParsedCrossRepoEvaluationManifest;
+
+export function scanPositiveGittensoryAssumptions(text: string): Array<{ id: string; line: string }>;
+
+export function evaluateRepoReadiness(
+  entry: CrossRepoEvaluationManifestRepo,
+  options?: {
+    repoPath?: string;
+    resolveRepoPath?: (entry: { repoFullName: string }) => string;
+    env?: NodeJS.ProcessEnv;
+    existsSync?: (path: string) => boolean;
+    detectRepoStack?: (repoPath: string) => RepoStackResult;
+    resolveMinerGoalSpec?: (repoPath: string) => { present: boolean };
+    buildCodingTaskSpec?: (input: Record<string, unknown>) => {
+      ready: boolean;
+      verdict?: string;
+      instructions?: string;
+    };
+  },
+): CrossRepoEvaluationResult;
+
+export function runCrossRepoEvaluation(
+  parsed: ParsedCrossRepoEvaluationManifest,
+  options?: {
+    repoFilter?: string;
+  } & Parameters<typeof evaluateRepoReadiness>[1],
+): CrossRepoEvaluationResult[];
+
+export function summarizeCrossRepoEvaluation(results: CrossRepoEvaluationResult[]): CrossRepoEvaluationSummary;
+
+export function formatCrossRepoEvaluationReport(
+  results: CrossRepoEvaluationResult[],
+  summary?: CrossRepoEvaluationSummary,
+): string;