From 827e84bf8f9c11c2123ad0b2a1b487ce67c2ed1b Mon Sep 17 00:00:00 2001 From: Andriy Polanski Date: Tue, 14 Jul 2026 15:56:43 +0000 Subject: [PATCH 1/2] feat(miner): add cross-repo evaluation harness (#4788) --- packages/loopover-miner/README.md | 4 + .../benchmarks/cross-repo/manifest.json | 34 ++ .../docs/cross-repo-evaluation.md | 78 ++++ .../lib/cross-repo-evaluation.d.ts | 90 ++++ .../lib/cross-repo-evaluation.js | 397 +++++++++++++++++ packages/loopover-miner/package.json | 3 +- .../scripts/cross-repo-evaluation.d.mts | 28 ++ .../scripts/cross-repo-evaluation.mjs | 116 +++++ test/unit/miner-cross-repo-evaluation.test.ts | 413 ++++++++++++++++++ 9 files changed, 1162 insertions(+), 1 deletion(-) create mode 100644 packages/loopover-miner/benchmarks/cross-repo/manifest.json create mode 100644 packages/loopover-miner/docs/cross-repo-evaluation.md create mode 100644 packages/loopover-miner/lib/cross-repo-evaluation.d.ts create mode 100644 packages/loopover-miner/lib/cross-repo-evaluation.js create mode 100644 packages/loopover-miner/scripts/cross-repo-evaluation.d.mts create mode 100644 packages/loopover-miner/scripts/cross-repo-evaluation.mjs create mode 100644 test/unit/miner-cross-repo-evaluation.test.ts diff --git a/packages/loopover-miner/README.md b/packages/loopover-miner/README.md index d5ba8e2764..18e19693a6 100644 --- a/packages/loopover-miner/README.md +++ b/packages/loopover-miner/README.md @@ -22,6 +22,10 @@ Config precedence (`.loopover-miner.yml` vs operator env vs CLI flags) is docume A committed micro-benchmark for the discovery-ranking and local-store read/write paths lives at [`BENCHMARKS.md`](BENCHMARKS.md) — run it with `npm run benchmark:miner` from the repo root. +The cross-repo evaluation harness (#4788) lives at +[`docs/cross-repo-evaluation.md`](docs/cross-repo-evaluation.md) — run it with +`npm run cross-repo-eval --workspace @loopover/miner` after cloning the benchmark repos. + Real miner commands land in follow-up issues. The package also includes the first metadata-only discovery primitive: `fetchCandidateIssues` lists open issue diff --git a/packages/loopover-miner/benchmarks/cross-repo/manifest.json b/packages/loopover-miner/benchmarks/cross-repo/manifest.json new file mode 100644 index 0000000000..a6d1bc1b35 --- /dev/null +++ b/packages/loopover-miner/benchmarks/cross-repo/manifest.json @@ -0,0 +1,34 @@ +{ + "description": "Cross-repo evaluation benchmark set (#4788). Diverse public repos exercised by `node packages/loopover-miner/scripts/cross-repo-evaluation.mjs` after cloning into LOOPOVER_MINER_REPO_CLONE_DIR.", + "repos": [ + { + "repoFullName": "sindresorhus/is", + "stackHint": "nodejs" + }, + { + "repoFullName": "expressjs/express", + "stackHint": "nodejs", + "requireTestCommand": true + }, + { + "repoFullName": "pallets/flask", + "stackHint": "python", + "requireTestCommand": true + }, + { + "repoFullName": "BurntSushi/ripgrep", + "stackHint": "rust", + "requireTestCommand": true + }, + { + "repoFullName": "gin-gonic/gin", + "stackHint": "go", + "requireTestCommand": true + }, + { + "repoFullName": "lodash/lodash", + "stackHint": "nodejs", + "requireTestCommand": true + } + ] +} diff --git a/packages/loopover-miner/docs/cross-repo-evaluation.md b/packages/loopover-miner/docs/cross-repo-evaluation.md new file mode 100644 index 0000000000..ab80fe80a6 --- /dev/null +++ b/packages/loopover-miner/docs/cross-repo-evaluation.md @@ -0,0 +1,78 @@ +# Cross-repo evaluation harness + +The **cross-repo evaluation harness** (#4788) is a repeatable readiness check that asks whether the miner can +approach a diverse benchmark repo set **without gittensory-specific target-repo configuration** (no +`.loopover-miner.yml` required in the benchmark repos). It exercises the same offline path a real attempt uses +before the coding agent runs: + +1. **Clone setup** — the repo exists under `LOOPOVER_MINER_REPO_CLONE_DIR` +2. **Stack auto-detection** (`detectRepoStack`, #4785) +3. **Coding-task spec composition** (`buildCodingTaskSpec`, #4786) including validation guidance derived from the + detected stack +4. **Assumption scan** — agent instructions must not positively mandate LoopOver/gittensory CI conventions + +Each benchmark repo receives a **pass/fail** line. Failures are categorized: + +| Category | Meaning | +| --- | --- | +| `stack_detection_gap` | No recognized manifest / stack could not be inferred | +| `execution_gap` | Stack detected but the coding-task path is not ready (e.g. missing inferred test command when required) | +| `gittensory_assumption` | Agent instructions leak gittensory-specific CI assumptions | +| `clone_setup` | The repo has not been cloned to the expected cache path | +| `other` | Unexpected errors | + +The run also reports whether a **strict majority** of repos passed and how many succeeded **without** a per-target +`.loopover-miner.yml` (the default goal spec is acceptable). + +## Benchmark manifest + +Committed at [`benchmarks/cross-repo/manifest.json`](../benchmarks/cross-repo/manifest.json). Each entry is either a +bare `"owner/repo"` string or an object: + +- **`repoFullName`** — canonical `owner/repo` +- **`stackHint`** — documentation only (not used by the evaluator) +- **`requireTestCommand`** — when `true`, stack detection must infer a test command or the repo fails with + `execution_gap` + +Malformed manifest fields degrade to documented defaults with warnings (same tolerant-parser convention as the +fleet run-manifest). + +## Running locally + +1. Clone the benchmark repos into the miner clone cache (once per machine): + + ```bash + export LOOPOVER_MINER_REPO_CLONE_DIR="${LOOPOVER_MINER_REPO_CLONE_DIR:-$HOME/.config/loopover-miner/repos}" + mkdir -p "$LOOPOVER_MINER_REPO_CLONE_DIR" + # Example for one entry — repeat for each repo in the manifest + git clone --depth 1 https://github.com/sindresorhus/is.git "$LOOPOVER_MINER_REPO_CLONE_DIR/sindresorhus/is" + ``` + +2. Run the harness from the repo root: + + ```bash + node packages/loopover-miner/scripts/cross-repo-evaluation.mjs + ``` + + Useful flags: + + - `--json` — machine-readable `{ warnings, results, summary }` payload + - `--repo owner/repo` — evaluate a single manifest entry + - `--manifest path/to/manifest.json` — alternate benchmark set (e.g. a fixture manifest in tests) + - `--require-majority` — exit `1` unless a strict majority of repos pass (for CI-style gating) + +## Library API + +Pure functions live in [`lib/cross-repo-evaluation.js`](../lib/cross-repo-evaluation.js): + +- `parseCrossRepoEvaluationManifest(content)` +- `evaluateRepoReadiness(entry, options)` — inject `existsSync`, `detectRepoStack`, etc. for unit tests +- `runCrossRepoEvaluation(parsed, options)` +- `summarizeCrossRepoEvaluation(results)` +- `formatCrossRepoEvaluationReport(results, summary)` + +## Wiring + +This harness is **readiness-only**: it does not run the coding agent, open PRs, or call forge APIs. A green report +means the miner’s repo-agnostic stack-detection and coding-task-spec path is prepared for the benchmark repo; a live +attempt still needs credentials, governor policy, and queue state as documented in [`DEPLOYMENT.md`](../DEPLOYMENT.md). diff --git a/packages/loopover-miner/lib/cross-repo-evaluation.d.ts b/packages/loopover-miner/lib/cross-repo-evaluation.d.ts new file mode 100644 index 0000000000..84e617a3bd --- /dev/null +++ b/packages/loopover-miner/lib/cross-repo-evaluation.d.ts @@ -0,0 +1,90 @@ +import type { RepoStackResult } from "./stack-detection.js"; + +export const CROSS_REPO_FAILURE_CATEGORY: Readonly<{ + STACK_DETECTION: "stack_detection_gap"; + EXECUTION: "execution_gap"; + GITTENSOR_ASSUMPTION: "gittensory_assumption"; + CLONE_SETUP: "clone_setup"; + OTHER: "other"; +}>; + +export const GITTENSOR_POSITIVE_ASSUMPTION_CHECKS: ReadonlyArray<{ + id: string; + pattern: RegExp; +}>; + +export const DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH: string; +export const MAX_CROSS_REPO_MANIFEST_BYTES: number; +export const MAX_CROSS_REPO_MANIFEST_REPOS: number; + +export type CrossRepoEvaluationManifestRepo = { + repoFullName: string; + stackHint?: string; + requireTestCommand?: boolean; + fixturePath?: string; +}; + +export type ParsedCrossRepoEvaluationManifest = { + present: boolean; + manifest: { repos: CrossRepoEvaluationManifestRepo[] }; + warnings: string[]; +}; + +export type CrossRepoEvaluationResult = { + repoFullName: string; + passed: boolean; + failureCategory: string | null; + reason: string | null; + stackDetected: boolean; + usedDefaultGoalSpec: boolean | null; + assumptionFindings: Array<{ id: string; line: string }>; + stack?: RepoStackResult; +}; + +export type CrossRepoEvaluationSummary = { + total: number; + passed: number; + failed: number; + majorityPassed: boolean; + withoutGittensoryConfig: number; + failuresByCategory: Record; +}; + +export function normalizeCrossRepoFullName(value: unknown): string | null; + +export function parseCrossRepoEvaluationManifest( + content: string | null | undefined, +): ParsedCrossRepoEvaluationManifest; + +export function scanPositiveGittensoryAssumptions(text: string): Array<{ id: string; line: string }>; + +export function evaluateRepoReadiness( + entry: CrossRepoEvaluationManifestRepo, + options?: { + repoPath?: string; + resolveRepoPath?: (entry: { repoFullName: string }) => string; + env?: NodeJS.ProcessEnv; + existsSync?: (path: string) => boolean; + detectRepoStack?: (repoPath: string) => RepoStackResult; + resolveMinerGoalSpec?: (repoPath: string) => { present: boolean }; + buildCodingTaskSpec?: (input: Record) => { + ready: boolean; + verdict?: string; + instructions?: string; + }; + }, +): CrossRepoEvaluationResult; + +export function runCrossRepoEvaluation( + parsed: ParsedCrossRepoEvaluationManifest, + options?: { + repoFilter?: string; + } & Parameters[1], +): CrossRepoEvaluationResult[]; + +export function summarizeCrossRepoEvaluation(results: CrossRepoEvaluationResult[]): CrossRepoEvaluationSummary; + +export function formatCrossRepoEvaluationReport( + results: CrossRepoEvaluationResult[], + summary?: CrossRepoEvaluationSummary, +): string; diff --git a/packages/loopover-miner/lib/cross-repo-evaluation.js b/packages/loopover-miner/lib/cross-repo-evaluation.js new file mode 100644 index 0000000000..e3495585b3 --- /dev/null +++ b/packages/loopover-miner/lib/cross-repo-evaluation.js @@ -0,0 +1,397 @@ +// Cross-repo evaluation harness (#4788): a repeatable, offline-first readiness check that asks whether the miner +// can approach a diverse benchmark repo set without gittensory-specific target-repo configuration. Each repo is +// evaluated through the same stack-detection + coding-task-spec path a real attempt uses (detectRepoStack, +// resolveMinerGoalSpec, buildCodingTaskSpec) and failures are categorized as stack-detection gaps, execution +// readiness gaps, leaked gittensory assumptions in agent instructions, clone/setup problems, or other. + +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { buildCodingTaskSpec } from "./coding-task-spec.js"; +import { resolveMinerGoalSpec } from "./miner-goal-spec.js"; +import { resolveRepoCloneDir } from "./repo-clone.js"; +import { detectRepoStack } from "./stack-detection.js"; + +/** Failure taxonomy surfaced in per-repo reports (#4788). */ +export const CROSS_REPO_FAILURE_CATEGORY = Object.freeze({ + STACK_DETECTION: "stack_detection_gap", + EXECUTION: "execution_gap", + GITTENSOR_ASSUMPTION: "gittensory_assumption", + CLONE_SETUP: "clone_setup", + OTHER: "other", +}); + +/** Instruction substrings that indicate a POSITIVE gittensory/LoopOver CI assumption leaked into the agent prompt. + * Lines that explicitly tell the agent *not* to assume these are filtered out before scanning. */ +export const GITTENSOR_POSITIVE_ASSUMPTION_CHECKS = Object.freeze([ + { id: "test_ci_script", pattern: /npm run test:ci/i }, + { id: "codecov_patch", pattern: /codecov\/patch/i }, + { id: "gittensor_label", pattern: /gittensor:(?:bug|feature|priority)/i }, + { id: "loopover_gate", pattern: /loopover gate/i }, +]); + +export const DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH = "benchmarks/cross-repo/manifest.json"; +export const MAX_CROSS_REPO_MANIFEST_BYTES = 65_536; +export const MAX_CROSS_REPO_MANIFEST_REPOS = 100; + +const REPO_SEGMENT_PATTERN = /^[A-Za-z0-9._-]+$/; + +function cloneEmptyManifest(warnings = []) { + return { present: false, manifest: { repos: [] }, warnings }; +} + +function isPathTraversalSegment(segment) { + return segment === "." || segment === ".."; +} + +/** Canonical `owner/repo` with exactly one slash and safe segments; anything else → null. */ +export function normalizeCrossRepoFullName(value) { + if (typeof value !== "string") return null; + const [owner, repo, extra] = value.trim().split("/"); + if (!owner || !repo || extra !== undefined) return null; + if (!REPO_SEGMENT_PATTERN.test(owner) || !REPO_SEGMENT_PATTERN.test(repo)) return null; + if (isPathTraversalSegment(owner) || isPathTraversalSegment(repo)) return null; + return `${owner}/${repo}`; +} + +function normalizeBoolean(value, field, fallback, warnings) { + if (value === undefined || value === null) return fallback; + if (typeof value === "boolean") return value; + warnings.push(`CrossRepoEvaluationManifest field "${field}" must be a boolean; falling back to ${fallback}.`); + return fallback; +} + +function normalizeOptionalString(value, field, warnings) { + if (value === undefined || value === null) return null; + if (typeof value !== "string") { + warnings.push(`CrossRepoEvaluationManifest field "${field}" must be a string; ignoring the value.`); + return null; + } + const trimmed = value.trim(); + return trimmed || null; +} + +function normalizeRepoList(value, warnings) { + if (value === undefined || value === null) return []; + if (!Array.isArray(value)) { + warnings.push(`CrossRepoEvaluationManifest field "repos" must be a list; ignoring a ${typeof value} value.`); + return []; + } + const result = []; + const seen = new Set(); + for (const [index, entry] of value.entries()) { + if (index >= MAX_CROSS_REPO_MANIFEST_REPOS) { + warnings.push( + `CrossRepoEvaluationManifest field "repos" exceeded ${MAX_CROSS_REPO_MANIFEST_REPOS} entries; extra entries ignored.`, + ); + break; + } + let repoFullName = null; + let stackHint = null; + let requireTestCommand = false; + let fixturePath = null; + if (typeof entry === "string") { + repoFullName = normalizeCrossRepoFullName(entry); + } else if (entry && typeof entry === "object" && !Array.isArray(entry)) { + const record = entry; + repoFullName = normalizeCrossRepoFullName(record.repoFullName); + stackHint = normalizeOptionalString(record.stackHint, "stackHint", warnings); + requireTestCommand = normalizeBoolean(record.requireTestCommand, "requireTestCommand", false, warnings); + fixturePath = normalizeOptionalString(record.fixturePath, "fixturePath", warnings); + } else { + warnings.push(`CrossRepoEvaluationManifest "repos" skipped a non-string, non-mapping entry.`); + continue; + } + if (repoFullName === null) { + warnings.push(`CrossRepoEvaluationManifest "repos" skipped an entry with an invalid "owner/repo" name.`); + continue; + } + if (seen.has(repoFullName)) { + warnings.push(`CrossRepoEvaluationManifest "repos" skipped a duplicate entry for ${repoFullName}.`); + continue; + } + seen.add(repoFullName); + const normalized = { repoFullName, requireTestCommand }; + if (stackHint) normalized.stackHint = stackHint; + if (fixturePath) normalized.fixturePath = fixturePath; + result.push(normalized); + } + return result; +} + +/** + * Tolerant JSON manifest parser (#4788). Malformed input degrades to an empty repo list with warnings rather than + * throwing, mirroring the fleet-run-manifest / miner-goal-spec convention. + * + * @param {string | null | undefined} content + * @returns {{ present: boolean, manifest: { repos: Array<{ repoFullName: string, stackHint?: string, requireTestCommand?: boolean, fixturePath?: string }> }, warnings: string[] }} + */ +export function parseCrossRepoEvaluationManifest(content) { + if (content === undefined || content === null) return cloneEmptyManifest(); + if (typeof content !== "string") { + return cloneEmptyManifest([`CrossRepoEvaluationManifest content must be a string; got ${typeof content}.`]); + } + const trimmed = content.trim(); + if (!trimmed) return cloneEmptyManifest(); + if (trimmed.length > MAX_CROSS_REPO_MANIFEST_BYTES) { + return cloneEmptyManifest([ + `CrossRepoEvaluationManifest exceeded ${MAX_CROSS_REPO_MANIFEST_BYTES} bytes; ignoring the file.`, + ]); + } + let raw; + try { + raw = JSON.parse(trimmed); + } catch { + return cloneEmptyManifest(["CrossRepoEvaluationManifest is not valid JSON."]); + } + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + return cloneEmptyManifest(["CrossRepoEvaluationManifest root must be a JSON object."]); + } + const warnings = []; + const repos = normalizeRepoList(raw.repos, warnings); + return { present: true, manifest: { repos }, warnings }; +} + +/** + * Scan agent instructions for positive gittensory/LoopOver assumptions (#4788). Lines that already tell the agent + * *not* to assume LoopOver conventions (the negative guidance from buildValidationGuidance) are skipped. + * + * @param {string} text + * @returns {Array<{ id: string, line: string }>} + */ +export function scanPositiveGittensoryAssumptions(text) { + if (typeof text !== "string") return []; + const findings = []; + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed || /do not assume/i.test(trimmed)) continue; + for (const check of GITTENSOR_POSITIVE_ASSUMPTION_CHECKS) { + if (check.pattern.test(line)) findings.push({ id: check.id, line: trimmed }); + } + } + return findings; +} + +function buildFailure(repoFullName, category, reason, extra = {}) { + return { + repoFullName, + passed: false, + failureCategory: category, + reason, + stackDetected: false, + usedDefaultGoalSpec: null, + assumptionFindings: [], + ...extra, + }; +} + +function buildPass(repoFullName, extra = {}) { + return { + repoFullName, + passed: true, + failureCategory: null, + reason: null, + stackDetected: true, + usedDefaultGoalSpec: true, + assumptionFindings: [], + ...extra, + }; +} + +function resolveEvaluationRepoPath(entry, options = {}) { + if (entry.fixturePath && typeof entry.fixturePath === "string") return entry.fixturePath; + if (typeof options.repoPath === "string" && options.repoPath.trim()) return options.repoPath.trim(); + if (typeof options.resolveRepoPath === "function") return options.resolveRepoPath(entry); + return resolveRepoCloneDir(entry.repoFullName, options.env ?? process.env); +} + +function defaultClaimLedger(repoFullName) { + return { listClaims: () => [] }; +} + +/** + * Evaluate one benchmark repo's miner readiness without running a live coding agent (#4788). + * + * @param {{ repoFullName: string, stackHint?: string, requireTestCommand?: boolean, fixturePath?: string }} entry + * @param {{ + * repoPath?: string, + * resolveRepoPath?: (entry: { repoFullName: string }) => string, + * env?: NodeJS.ProcessEnv, + * existsSync?: (path: string) => boolean, + * detectRepoStack?: typeof detectRepoStack, + * resolveMinerGoalSpec?: typeof resolveMinerGoalSpec, + * buildCodingTaskSpec?: typeof buildCodingTaskSpec, + * }} [options] + */ +export function evaluateRepoReadiness(entry, options = {}) { + const repoFullName = entry?.repoFullName; + if (typeof repoFullName !== "string" || !normalizeCrossRepoFullName(repoFullName)) { + return buildFailure( + typeof repoFullName === "string" ? repoFullName : "(invalid)", + CROSS_REPO_FAILURE_CATEGORY.OTHER, + "Benchmark entry is missing a valid owner/repo name.", + ); + } + + const existsImpl = options.existsSync ?? existsSync; + const detectImpl = options.detectRepoStack ?? detectRepoStack; + const goalSpecImpl = options.resolveMinerGoalSpec ?? resolveMinerGoalSpec; + const buildSpecImpl = options.buildCodingTaskSpec ?? buildCodingTaskSpec; + const repoPath = resolveEvaluationRepoPath(entry, options); + + if (!existsImpl(repoPath)) { + return buildFailure( + repoFullName, + CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP, + `Repository path does not exist: ${repoPath}. Clone the repo or set LOOPOVER_MINER_REPO_CLONE_DIR.`, + ); + } + + const goalSpec = goalSpecImpl(repoPath); + const usedDefaultGoalSpec = goalSpec?.present !== true; + + const stack = detectImpl(repoPath); + if (stack?.detected !== true) { + return buildFailure( + repoFullName, + CROSS_REPO_FAILURE_CATEGORY.STACK_DETECTION, + stack?.reason ?? "Stack auto-detection did not recognize this repository.", + { stackDetected: false, usedDefaultGoalSpec }, + ); + } + + if (entry.requireTestCommand === true && !stack.testCommand) { + return buildFailure( + repoFullName, + CROSS_REPO_FAILURE_CATEGORY.EXECUTION, + "Stack detection succeeded but no test command was inferred while requireTestCommand is set.", + { stackDetected: true, usedDefaultGoalSpec, stack }, + ); + } + + let specResult; + try { + specResult = buildSpecImpl({ + repoFullName, + issue: { + number: 1, + title: "Cross-repo evaluation harness smoke issue", + body: "Synthetic issue used only by the cross-repo evaluation harness.", + labels: ["bug"], + }, + context: { issues: [{ number: 1 }], pullRequests: [] }, + claimLedger: defaultClaimLedger(repoFullName), + workingDirectory: repoPath, + detectRepoStack: detectImpl, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return buildFailure(repoFullName, CROSS_REPO_FAILURE_CATEGORY.OTHER, message, { + stackDetected: true, + usedDefaultGoalSpec, + stack, + }); + } + + if (specResult?.ready !== true) { + return buildFailure( + repoFullName, + CROSS_REPO_FAILURE_CATEGORY.EXECUTION, + `Coding task spec is not ready (verdict: ${specResult?.verdict ?? "unknown"}).`, + { stackDetected: true, usedDefaultGoalSpec, stack }, + ); + } + + const assumptionFindings = scanPositiveGittensoryAssumptions(specResult.instructions ?? ""); + if (assumptionFindings.length > 0) { + return buildFailure( + repoFullName, + CROSS_REPO_FAILURE_CATEGORY.GITTENSOR_ASSUMPTION, + `Agent instructions leak gittensory-specific assumptions (${assumptionFindings.map((f) => f.id).join(", ")}).`, + { stackDetected: true, usedDefaultGoalSpec, stack, assumptionFindings }, + ); + } + + return buildPass(repoFullName, { usedDefaultGoalSpec, stack }); +} + +/** + * Run the harness across every repo in a parsed manifest (#4788). + * + * @param {ReturnType} parsed + * @param {{ repoFilter?: string } & Parameters[1]} [options] + * @returns {ReturnType[]} + */ +export function runCrossRepoEvaluation(parsed, options = {}) { + const repos = parsed?.manifest?.repos ?? []; + const results = []; + for (const entry of repos) { + if (options.repoFilter && entry.repoFullName !== options.repoFilter) continue; + results.push(evaluateRepoReadiness(entry, options)); + } + return results; +} + +/** + * Reduce per-repo results to pass/fail counts and whether a strict majority passed (#4788). + * + * @param {ReturnType[]} results + */ +export function summarizeCrossRepoEvaluation(results) { + const list = Array.isArray(results) ? results : []; + let passed = 0; + let failed = 0; + const failuresByCategory = {}; + for (const result of list) { + if (result?.passed === true) { + passed += 1; + continue; + } + failed += 1; + const category = result?.failureCategory ?? CROSS_REPO_FAILURE_CATEGORY.OTHER; + failuresByCategory[category] = (failuresByCategory[category] ?? 0) + 1; + } + const total = passed + failed; + const majorityPassed = total > 0 ? passed > failed : false; + const withoutGittensoryConfig = list.filter((r) => r?.usedDefaultGoalSpec !== false).length; + return { + total, + passed, + failed, + majorityPassed, + withoutGittensoryConfig, + failuresByCategory, + }; +} + +/** + * Human-readable pass/fail report for one evaluation run (#4788). + * + * @param {ReturnType[]} results + * @param {ReturnType} [summary] + */ +export function formatCrossRepoEvaluationReport(results, summary = summarizeCrossRepoEvaluation(results)) { + const lines = ["loopover-miner cross-repo evaluation", ""]; + for (const result of results) { + if (result.passed) { + lines.push(`PASS ${result.repoFullName}`); + continue; + } + lines.push(`FAIL ${result.repoFullName} [${result.failureCategory}] ${result.reason}`); + } + lines.push( + "", + `summary: ${summary.passed}/${summary.total} passed` + + (summary.majorityPassed ? " (majority passed)" : " (majority failed)"), + ); + if (summary.total > 0) { + lines.push(`without gittensory-specific target config: ${summary.withoutGittensoryConfig}/${summary.total}`); + } + const categories = Object.entries(summary.failuresByCategory).sort(([a], [b]) => a.localeCompare(b)); + if (categories.length > 0) { + lines.push("", "failures by category:"); + for (const [category, count] of categories) { + lines.push(`- ${category}: ${count}`); + } + } + return lines.join("\n"); +} diff --git a/packages/loopover-miner/package.json b/packages/loopover-miner/package.json index de8e287347..fe81c41249 100644 --- a/packages/loopover-miner/package.json +++ b/packages/loopover-miner/package.json @@ -38,7 +38,8 @@ ], "scripts": { "benchmark": "node scripts/benchmark.mjs", - "build": "node --check bin/loopover-miner.js && node --check bin/loopover-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-run.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-conflict-resolver.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-metrics-cli.js && node --check lib/governor-open-pr.js && node --check lib/governor-pause-cli.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/init-wizard.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/logger.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/metrics-cli.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/policy-doc-cache.js && node --check lib/policy-verdict-cache.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-number-parse.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/purge-cli.js && node --check lib/ranked-candidates.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/stack-detection.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" + "cross-repo-eval": "node scripts/cross-repo-evaluation.mjs", + "build": "node --check bin/loopover-miner.js && node --check bin/loopover-miner-mcp.js && node --check lib/ams-policy.js && node --check lib/attempt-cli.js && node --check lib/attempt-input-builder.js && node --check lib/attempt-log.js && node --check lib/attempt-runner.js && node --check lib/attempt-worktree.js && node --check lib/calibration-run.js && node --check lib/calibration-types.js && node --check lib/calibration.js && node --check lib/ci-poller.js && node --check lib/claim-adjudication.js && node --check lib/claim-conflict-resolver.js && node --check lib/claim-ledger-cli.js && node --check lib/claim-ledger-expiry.js && node --check lib/claim-ledger.js && node --check lib/cli.js && node --check lib/coding-agent-construction.js && node --check lib/coding-agent-house-rules.js && node --check lib/coding-task-spec.js && node --check lib/cross-repo-evaluation.js && node --check lib/deny-check.js && node --check lib/deny-hook-synthesis.js && node --check lib/deny-hooks.js && node --check lib/deployment-docs-audit.js && node --check lib/discover-cli.js && node --check lib/event-ledger-cli.js && node --check lib/event-ledger.js && node --check lib/execute-local-write.js && node --check lib/feasibility-cli.js && node --check lib/governor-action-mode.js && node --check lib/governor-chokepoint-persisted.js && node --check lib/governor-chokepoint.js && node --check lib/governor-kill-switch.js && node --check lib/governor-ledger-cli.js && node --check lib/governor-ledger.js && node --check lib/governor-metrics-cli.js && node --check lib/governor-open-pr.js && node --check lib/governor-pause-cli.js && node --check lib/governor-run-halt.js && node --check lib/governor-state.js && node --check lib/governor-write-rate-limit.js && node --check lib/harness-submission-trigger.js && node --check lib/init-wizard.js && node --check lib/laptop-init.js && node --check lib/live-issue-snapshot.js && node --check lib/local-store.js && node --check lib/logger.js && node --check lib/loop-cli.js && node --check lib/loop-closure.js && node --check lib/loop-reentry.js && node --check lib/manage-poll.js && node --check lib/manage-status.js && node --check lib/metrics-cli.js && node --check lib/miner-goal-spec.js && node --check lib/opportunity-fanout.js && node --check lib/opportunity-ranker.js && node --check lib/orb-export.js && node --check lib/plan-store-cli.js && node --check lib/plan-store.js && node --check lib/policy-doc-cache.js && node --check lib/policy-verdict-cache.js && node --check lib/portfolio-dashboard.js && node --check lib/portfolio-discovery.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-queue-manager.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-expiry.js && node --check lib/pr-disposition-poller.js && node --check lib/pr-number-parse.js && node --check lib/pr-outcome.js && node --check lib/prediction-ledger.js && node --check lib/pretooluse-hook.js && node --check lib/purge-cli.js && node --check lib/ranked-candidates.js && node --check lib/rejection-signal.js && node --check lib/rejection-state-machine.js && node --check lib/rejection-templates.js && node --check lib/replay-objective-anchor.js && node --check lib/replay-snapshot.js && node --check lib/replay-task-generation.js && node --check lib/repo-clone.js && node --check lib/run-state-cli.js && node --check lib/run-state.js && node --check lib/self-review-context.js && node --check lib/slop-assessment.js && node --check lib/stack-detection.js && node --check lib/status.js && node --check lib/submission-freshness-check.js && node --check lib/update-check.js && node --check lib/version.js && node --check lib/worktree-allocator.js" }, "dependencies": { "@loopover/engine": "^3.0.0", diff --git a/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts b/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts new file mode 100644 index 0000000000..c263aa2347 --- /dev/null +++ b/packages/loopover-miner/scripts/cross-repo-evaluation.d.mts @@ -0,0 +1,28 @@ +import type { + CrossRepoEvaluationResult, + CrossRepoEvaluationSummary, + ParsedCrossRepoEvaluationManifest, +} from "../lib/cross-repo-evaluation.js"; + +export type CrossRepoEvaluationCliArgs = + | { manifestPath: string; json: boolean; repoFilter: string | null; requireMajority: boolean } + | { error: string } + | { help: true }; + +export type CrossRepoEvaluationCliOptions = { + parsed?: ParsedCrossRepoEvaluationManifest; + manifestPath?: string; + repoFilter?: string | null; +}; + +export declare function resolveDefaultManifestPath(): string; + +export declare function parseCrossRepoEvaluationArgs(argv?: readonly string[]): CrossRepoEvaluationCliArgs; + +export declare function loadCrossRepoEvaluationManifest(manifestPath: string): ParsedCrossRepoEvaluationManifest; + +export declare function runCrossRepoEvaluationCli(options?: CrossRepoEvaluationCliOptions): { + parsed: ParsedCrossRepoEvaluationManifest; + results: CrossRepoEvaluationResult[]; + summary: CrossRepoEvaluationSummary; +}; diff --git a/packages/loopover-miner/scripts/cross-repo-evaluation.mjs b/packages/loopover-miner/scripts/cross-repo-evaluation.mjs new file mode 100644 index 0000000000..04de1d9fa4 --- /dev/null +++ b/packages/loopover-miner/scripts/cross-repo-evaluation.mjs @@ -0,0 +1,116 @@ +#!/usr/bin/env node +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { + DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH, + formatCrossRepoEvaluationReport, + parseCrossRepoEvaluationManifest, + runCrossRepoEvaluation, + summarizeCrossRepoEvaluation, +} from "../lib/cross-repo-evaluation.js"; + +const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); + +export function resolveDefaultManifestPath() { + return join(PACKAGE_ROOT, DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH); +} + +export function parseCrossRepoEvaluationArgs(argv) { + const args = argv ?? process.argv.slice(2); + let manifestPath = resolveDefaultManifestPath(); + let json = false; + let repoFilter = null; + let requireMajority = false; + for (let i = 0; i < args.length; i += 1) { + const token = args[i]; + if (token === "--json") { + json = true; + continue; + } + if (token === "--require-majority") { + requireMajority = true; + continue; + } + if (token === "--manifest") { + const value = args[i + 1]; + if (!value) return { error: "Missing value for --manifest." }; + manifestPath = value; + i += 1; + continue; + } + if (token === "--repo") { + const value = args[i + 1]; + if (!value) return { error: "Missing value for --repo." }; + repoFilter = value; + i += 1; + continue; + } + if (token === "--help" || token === "-h") { + return { help: true }; + } + return { error: `Unknown argument: ${token}` }; + } + return { manifestPath, json, repoFilter, requireMajority }; +} + +export function loadCrossRepoEvaluationManifest(manifestPath) { + const content = readFileSync(manifestPath, "utf8"); + return parseCrossRepoEvaluationManifest(content); +} + +export function runCrossRepoEvaluationCli(options = {}) { + const parsed = options.parsed ?? loadCrossRepoEvaluationManifest(options.manifestPath ?? resolveDefaultManifestPath()); + const results = runCrossRepoEvaluation(parsed, { repoFilter: options.repoFilter ?? null }); + const summary = summarizeCrossRepoEvaluation(results); + return { parsed, results, summary }; +} + +function printHelp() { + console.log( + [ + "loopover-miner cross-repo evaluation (#4788)", + "", + "Usage:", + " node packages/loopover-miner/scripts/cross-repo-evaluation.mjs [options]", + "", + "Options:", + " --manifest Benchmark manifest (default: benchmarks/cross-repo/manifest.json)", + " --repo Evaluate a single benchmark entry", + " --json Emit machine-readable JSON on stdout", + " --require-majority Exit 1 unless a strict majority of repos pass", + " -h, --help Show this help", + "", + "Prerequisite: clone benchmark repos into LOOPOVER_MINER_REPO_CLONE_DIR (see docs/cross-repo-evaluation.md).", + ].join("\n"), + ); +} + +function main() { + const parsedArgs = parseCrossRepoEvaluationArgs(); + if (parsedArgs.help) { + printHelp(); + return 0; + } + if (parsedArgs.error) { + console.error(parsedArgs.error); + return 2; + } + + const { parsed, results, summary } = runCrossRepoEvaluationCli(parsedArgs); + if (parsedArgs.json) { + console.log(JSON.stringify({ warnings: parsed.warnings, results, summary }, null, 2)); + } else { + if (parsed.warnings.length > 0) { + console.error(`manifest warnings:\n- ${parsed.warnings.join("\n- ")}`); + } + console.log(formatCrossRepoEvaluationReport(results, summary)); + } + + if (parsedArgs.requireMajority && !summary.majorityPassed) return 1; + return 0; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + process.exitCode = main(); +} diff --git a/test/unit/miner-cross-repo-evaluation.test.ts b/test/unit/miner-cross-repo-evaluation.test.ts new file mode 100644 index 0000000000..026f7fccb2 --- /dev/null +++ b/test/unit/miner-cross-repo-evaluation.test.ts @@ -0,0 +1,413 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@loopover/engine", async () => { + return import("../../packages/loopover-engine/src/index"); +}); + +import { + CROSS_REPO_FAILURE_CATEGORY, + DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH, + MAX_CROSS_REPO_MANIFEST_BYTES, + formatCrossRepoEvaluationReport, + evaluateRepoReadiness, + normalizeCrossRepoFullName, + parseCrossRepoEvaluationManifest, + runCrossRepoEvaluation, + scanPositiveGittensoryAssumptions, + summarizeCrossRepoEvaluation, +} from "../../packages/loopover-miner/lib/cross-repo-evaluation.js"; +import { + loadCrossRepoEvaluationManifest, + parseCrossRepoEvaluationArgs, + resolveDefaultManifestPath, + runCrossRepoEvaluationCli, +} from "../../packages/loopover-miner/scripts/cross-repo-evaluation.mjs"; + +const roots: string[] = []; + +afterEach(() => { + vi.restoreAllMocks(); + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +function tempRepo(files: Record = {}) { + const root = mkdtempSync(join(tmpdir(), "gittensory-cross-repo-eval-")); + roots.push(root); + for (const [rel, content] of Object.entries(files)) { + writeFileSync(join(root, rel), content, "utf8"); + } + return root; +} + +const pkg = (value: Record) => JSON.stringify(value); + +describe("cross-repo evaluation harness (#4788)", () => { + describe("normalizeCrossRepoFullName", () => { + it("accepts canonical owner/repo names and rejects unsafe values", () => { + expect(normalizeCrossRepoFullName("acme/widgets")).toBe("acme/widgets"); + expect(normalizeCrossRepoFullName(" acme/widgets ")).toBe("acme/widgets"); + expect(normalizeCrossRepoFullName("acme")).toBeNull(); + expect(normalizeCrossRepoFullName("acme/widgets/extra")).toBeNull(); + expect(normalizeCrossRepoFullName("../evil/repo")).toBeNull(); + expect(normalizeCrossRepoFullName(12)).toBeNull(); + }); + }); + + describe("parseCrossRepoEvaluationManifest", () => { + it("degrades missing or invalid content to an empty repo list with warnings", () => { + expect(parseCrossRepoEvaluationManifest(null)).toEqual({ + present: false, + manifest: { repos: [] }, + warnings: [], + }); + expect(parseCrossRepoEvaluationManifest(42 as never).warnings[0]).toContain("string"); + expect(parseCrossRepoEvaluationManifest(" ").present).toBe(false); + expect(parseCrossRepoEvaluationManifest("{").warnings[0]).toContain("valid JSON"); + expect(parseCrossRepoEvaluationManifest("[]").warnings[0]).toContain("JSON object"); + }); + + it("rejects oversize manifests", () => { + const parsed = parseCrossRepoEvaluationManifest(`{"repos":${" ".repeat(MAX_CROSS_REPO_MANIFEST_BYTES)}}`); + expect(parsed.present).toBe(false); + expect(parsed.warnings[0]).toContain("exceeded"); + }); + + it("normalizes string and object repo entries and skips invalid duplicates", () => { + const parsed = parseCrossRepoEvaluationManifest( + JSON.stringify({ + repos: [ + "acme/alpha", + { repoFullName: "acme/beta", stackHint: "nodejs", requireTestCommand: true }, + "acme/alpha", + { repoFullName: "bad", requireTestCommand: "yes" }, + 7, + ], + }), + ); + expect(parsed.present).toBe(true); + expect(parsed.manifest.repos).toEqual([ + { repoFullName: "acme/alpha", requireTestCommand: false }, + { repoFullName: "acme/beta", stackHint: "nodejs", requireTestCommand: true }, + ]); + expect(parsed.warnings.some((w) => w.includes("duplicate"))).toBe(true); + expect(parsed.warnings.some((w) => w.includes("invalid"))).toBe(true); + expect(parsed.warnings.some((w) => w.includes("boolean"))).toBe(true); + expect(parsed.warnings.some((w) => w.includes("non-string"))).toBe(true); + }); + + it("truncates manifests with more than the documented repo cap", () => { + const repos = Array.from({ length: 105 }, (_, i) => `acme/repo-${i}`); + const parsed = parseCrossRepoEvaluationManifest(JSON.stringify({ repos })); + expect(parsed.manifest.repos).toHaveLength(100); + expect(parsed.warnings.some((w) => w.includes("exceeded"))).toBe(true); + }); + + it("ignores non-string stackHint values with a warning", () => { + const parsed = parseCrossRepoEvaluationManifest( + JSON.stringify({ repos: [{ repoFullName: "acme/hint", stackHint: 42 }] }), + ); + expect(parsed.manifest.repos[0]?.stackHint).toBeUndefined(); + expect(parsed.warnings.some((w) => w.includes("stackHint"))).toBe(true); + }); + it("treats a non-array repos field as empty", () => { + const parsed = parseCrossRepoEvaluationManifest(JSON.stringify({ repos: "nope" })); + expect(parsed.manifest.repos).toEqual([]); + expect(parsed.warnings[0]).toContain("must be a list"); + }); + }); + + describe("scanPositiveGittensoryAssumptions", () => { + it("ignores non-strings and negative guidance lines", () => { + expect(scanPositiveGittensoryAssumptions(null as never)).toEqual([]); + const text = [ + "Do not assume LoopOver/gittensory CI conventions or `npm run test:ci`.", + "Run npm run test:ci before finishing.", + ].join("\n"); + expect(scanPositiveGittensoryAssumptions(text)).toEqual([ + { id: "test_ci_script", line: "Run npm run test:ci before finishing." }, + ]); + }); + + it("detects other positive assumption markers", () => { + const findings = scanPositiveGittensoryAssumptions( + ["Ensure codecov/patch is green.", "Label with gittensor:bug.", "Wait for the loopover gate."].join("\n"), + ); + expect(findings.map((f) => f.id).sort()).toEqual(["codecov_patch", "gittensor_label", "loopover_gate"]); + }); + }); + + describe("evaluateRepoReadiness", () => { + it("fails clone_setup when the repo path is absent", () => { + const result = evaluateRepoReadiness( + { repoFullName: "acme/missing", requireTestCommand: false }, + { repoPath: "/tmp/definitely-missing-repo-path", existsSync: () => false }, + ); + expect(result.passed).toBe(false); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP); + }); + + it("fails stack_detection_gap when no manifest is recognized", () => { + const repoPath = tempRepo({ "README.md": "# hello" }); + const result = evaluateRepoReadiness( + { repoFullName: "acme/plain", requireTestCommand: false }, + { repoPath, existsSync: () => true }, + ); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.STACK_DETECTION); + expect(result.stackDetected).toBe(false); + }); + + it("fails execution_gap when requireTestCommand is set but no test command is inferred", () => { + const repoPath = tempRepo({ "package.json": pkg({}) }); + const result = evaluateRepoReadiness( + { repoFullName: "acme/no-test", requireTestCommand: true }, + { repoPath, existsSync: () => true }, + ); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.EXECUTION); + expect(result.stackDetected).toBe(true); + }); + + it("fails gittensory_assumption when injected instructions leak LoopOver CI defaults", () => { + const repoPath = tempRepo({ + "package.json": pkg({ scripts: { test: "node --test" } }), + }); + const result = evaluateRepoReadiness( + { repoFullName: "acme/leaky", requireTestCommand: false }, + { + repoPath, + existsSync: () => true, + buildCodingTaskSpec: () => ({ + ready: true, + instructions: "Please run npm run test:ci and satisfy codecov/patch.", + }), + }, + ); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.GITTENSOR_ASSUMPTION); + expect(result.assumptionFindings.length).toBeGreaterThan(0); + }); + + it("fails execution_gap when the coding-task spec is not ready", () => { + const repoPath = tempRepo({ + "package.json": pkg({ scripts: { test: "node --test" } }), + }); + const result = evaluateRepoReadiness( + { repoFullName: "acme/not-ready", requireTestCommand: false }, + { + repoPath, + existsSync: () => true, + buildCodingTaskSpec: () => ({ ready: false, verdict: "avoid" }), + }, + ); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.EXECUTION); + expect(result.reason).toContain("avoid"); + }); + + it("fails other when buildCodingTaskSpec throws", () => { + const repoPath = tempRepo({ + "package.json": pkg({ scripts: { test: "node --test" } }), + }); + const result = evaluateRepoReadiness( + { repoFullName: "acme/throws", requireTestCommand: false }, + { + repoPath, + existsSync: () => true, + buildCodingTaskSpec: () => { + throw new Error("boom"); + }, + }, + ); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.OTHER); + expect(result.reason).toBe("boom"); + }); + + it("passes end-to-end for a plain Node repo without gittensory-specific target config", () => { + const repoPath = tempRepo({ + "package.json": pkg({ scripts: { test: "node --test" } }), + }); + const result = evaluateRepoReadiness( + { repoFullName: "acme/pass", requireTestCommand: true }, + { repoPath, existsSync: () => true }, + ); + expect(result.passed).toBe(true); + expect(result.usedDefaultGoalSpec).toBe(true); + expect(result.assumptionFindings).toEqual([]); + }); + + it("honors fixturePath and resolveRepoPath overrides", () => { + const fixtureRepo = tempRepo({ "package.json": pkg({ scripts: { test: "node --test" } }) }); + const resolverRepo = tempRepo({ "package.json": pkg({ scripts: { test: "node --test" } }) }); + const viaFixture = evaluateRepoReadiness( + { repoFullName: "acme/fixture", fixturePath: fixtureRepo, requireTestCommand: false }, + { existsSync: (path) => path === fixtureRepo }, + ); + expect(viaFixture.passed).toBe(true); + + const viaResolver = evaluateRepoReadiness( + { repoFullName: "acme/resolver", requireTestCommand: false }, + { existsSync: (path) => path === resolverRepo, resolveRepoPath: () => resolverRepo }, + ); + expect(viaResolver.passed).toBe(true); + }); + + it("uses options.repoPath when no fixturePath is present", () => { + const repoPath = tempRepo({ "package.json": pkg({ scripts: { test: "node --test" } }) }); + const result = evaluateRepoReadiness( + { repoFullName: "acme/direct", requireTestCommand: false }, + { repoPath, existsSync: (path) => path === repoPath }, + ); + expect(result.passed).toBe(true); + }); + + it("falls back to a generic stack-detection reason when the detector omits one", () => { + const repoPath = tempRepo({ "package.json": pkg({}) }); + const result = evaluateRepoReadiness( + { repoFullName: "acme/no-reason", requireTestCommand: false }, + { + repoPath, + existsSync: () => true, + detectRepoStack: () => ({ detected: false, reason: "no manifest" }), + }, + ); + expect(result.reason).toContain("did not recognize"); + }); + + it("rejects benchmark entries with invalid repo names", () => { + const result = evaluateRepoReadiness({ repoFullName: "not-a-repo", requireTestCommand: false }); + expect(result.failureCategory).toBe(CROSS_REPO_FAILURE_CATEGORY.OTHER); + }); + }); + + describe("runCrossRepoEvaluation + summarizeCrossRepoEvaluation", () => { + it("filters to a single repo and computes majority + category counts", () => { + const parsed = parseCrossRepoEvaluationManifest( + JSON.stringify({ repos: ["acme/a", "acme/b", "acme/c"] }), + ); + const results = runCrossRepoEvaluation(parsed, { + repoFilter: "acme/b", + existsSync: () => false, + }); + expect(results).toHaveLength(1); + expect(results[0]?.repoFullName).toBe("acme/b"); + + const summary = summarizeCrossRepoEvaluation([ + { passed: true }, + { passed: false, failureCategory: CROSS_REPO_FAILURE_CATEGORY.STACK_DETECTION }, + { passed: false, failureCategory: CROSS_REPO_FAILURE_CATEGORY.EXECUTION }, + { passed: true, usedDefaultGoalSpec: true }, + ] as never); + expect(summary.total).toBe(4); + expect(summary.passed).toBe(2); + expect(summary.majorityPassed).toBe(false); + expect(summary.withoutGittensoryConfig).toBe(4); + expect(summary.failuresByCategory[CROSS_REPO_FAILURE_CATEGORY.STACK_DETECTION]).toBe(1); + expect(summary.failuresByCategory[CROSS_REPO_FAILURE_CATEGORY.EXECUTION]).toBe(1); + }); + + it("reports majority passed and renders a stable text report", () => { + const results = [ + { + repoFullName: "acme/ok", + passed: true, + failureCategory: null, + reason: null, + }, + { + repoFullName: "acme/bad", + passed: false, + failureCategory: CROSS_REPO_FAILURE_CATEGORY.CLONE_SETUP, + reason: "missing clone", + }, + ] as never; + const summary = summarizeCrossRepoEvaluation(results); + expect(summary.majorityPassed).toBe(false); + expect(formatCrossRepoEvaluationReport(results, summary)).toBe( + [ + "loopover-miner cross-repo evaluation", + "", + "PASS acme/ok", + "FAIL acme/bad [clone_setup] missing clone", + "", + "summary: 1/2 passed (majority failed)", + "without gittensory-specific target config: 2/2", + "", + "failures by category:", + "- clone_setup: 1", + ].join("\n"), + ); + }); + + it("treats an empty result set as no majority", () => { + const summary = summarizeCrossRepoEvaluation([]); + expect(summary.majorityPassed).toBe(false); + expect(summary.total).toBe(0); + }); + + it("reports a strict majority when more than half the repos pass", () => { + const summary = summarizeCrossRepoEvaluation([ + { passed: true, usedDefaultGoalSpec: true }, + { passed: true, usedDefaultGoalSpec: true }, + { passed: false, failureCategory: null }, + ] as never); + expect(summary.majorityPassed).toBe(true); + expect(summary.failuresByCategory.other).toBe(1); + }); + }); + + describe("committed benchmark manifest + CLI", () => { + it("parses the shipped cross-repo manifest", () => { + const manifestPath = join(process.cwd(), "packages/loopover-miner", DEFAULT_CROSS_REPO_MANIFEST_RELATIVE_PATH); + const parsed = loadCrossRepoEvaluationManifest(manifestPath); + expect(parsed.present).toBe(true); + expect(parsed.manifest.repos.length).toBeGreaterThanOrEqual(5); + expect(parsed.warnings).toEqual([]); + }); + + it("parses CLI flags and resolves the default manifest path", () => { + expect(parseCrossRepoEvaluationArgs(["--json", "--require-majority", "--repo", "acme/widgets"])).toEqual({ + manifestPath: resolveDefaultManifestPath(), + json: true, + repoFilter: "acme/widgets", + requireMajority: true, + }); + expect(parseCrossRepoEvaluationArgs(["--manifest"])).toEqual({ error: "Missing value for --manifest." }); + expect(parseCrossRepoEvaluationArgs(["--nope"])).toEqual({ error: "Unknown argument: --nope" }); + expect(parseCrossRepoEvaluationArgs(["--help"])).toEqual({ help: true }); + }); + + it("runs the harness driver against a fixture manifest", () => { + const repoPath = tempRepo({ + "package.json": pkg({ scripts: { test: "node --test" } }), + }); + const manifestPath = tempRepo(); + writeFileSync( + join(manifestPath, "manifest.json"), + JSON.stringify({ + repos: [{ repoFullName: "acme/fixture", fixturePath: repoPath, requireTestCommand: true }], + }), + "utf8", + ); + + const { parsed, results, summary } = runCrossRepoEvaluationCli({ + manifestPath: join(manifestPath, "manifest.json"), + }); + expect(parsed.warnings).toEqual([]); + expect(results[0]?.passed).toBe(true); + expect(summary.passed).toBe(1); + expect(formatCrossRepoEvaluationReport(results, summary)).toContain("PASS acme/fixture"); + }); + + it("parseCrossRepoEvaluationArgs treats a missing --repo value as an error", () => { + expect(parseCrossRepoEvaluationArgs(["--repo"])).toEqual({ error: "Missing value for --repo." }); + }); + }); + + it("documents the harness in packages/loopover-miner/docs/cross-repo-evaluation.md", () => { + const doc = readFileSync(join(process.cwd(), "packages/loopover-miner/docs/cross-repo-evaluation.md"), "utf8"); + expect(doc).toContain("#4788"); + expect(doc).toContain("stack_detection_gap"); + expect(doc).toContain("cross-repo-evaluation.mjs"); + expect(doc).toContain("benchmarks/cross-repo/manifest.json"); + }); +}); From 2f8ed92bc465e731ecf2476cddce3798b3a7ef9e Mon Sep 17 00:00:00 2001 From: Andriy Polanski Date: Tue, 14 Jul 2026 16:05:27 +0000 Subject: [PATCH 2/2] fix --- package.json | 1 + test/unit/miner-cross-repo-evaluation.test.ts | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 0b842df771..c254e7888c 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "miner:env-reference": "node packages/loopover-miner/scripts/generate-env-reference.mjs", "miner:env-reference:check": "node packages/loopover-miner/scripts/generate-env-reference.mjs --check", "benchmark:miner": "node packages/loopover-miner/scripts/benchmark.mjs", + "cross-repo-eval:miner": "node packages/loopover-miner/scripts/cross-repo-evaluation.mjs", "loadtest:iterate-loop": "npm run build --workspace @loopover/engine && node packages/loopover-engine/scripts/load-test-iterate-loop.mjs", "command-reference": "node scripts/gen-command-reference.mjs", "command-reference:check": "node scripts/gen-command-reference.mjs --check", diff --git a/test/unit/miner-cross-repo-evaluation.test.ts b/test/unit/miner-cross-repo-evaluation.test.ts index 026f7fccb2..99bd70e2dc 100644 --- a/test/unit/miner-cross-repo-evaluation.test.ts +++ b/test/unit/miner-cross-repo-evaluation.test.ts @@ -19,6 +19,7 @@ import { scanPositiveGittensoryAssumptions, summarizeCrossRepoEvaluation, } from "../../packages/loopover-miner/lib/cross-repo-evaluation.js"; +import type { RepoStackResult } from "../../packages/loopover-miner/lib/stack-detection.js"; import { loadCrossRepoEvaluationManifest, parseCrossRepoEvaluationArgs, @@ -267,7 +268,8 @@ describe("cross-repo evaluation harness (#4788)", () => { { repoPath, existsSync: () => true, - detectRepoStack: () => ({ detected: false, reason: "no manifest" }), + // Simulate a legacy detector that omits `reason` at runtime; evaluateRepoReadiness must fall back. + detectRepoStack: () => ({ detected: false }) as RepoStackResult, }, ); expect(result.reason).toContain("did not recognize");