diff --git a/packages/gittensory-miner/lib/replay-objective-anchor.d.ts b/packages/gittensory-miner/lib/replay-objective-anchor.d.ts new file mode 100644 index 0000000000..b47ebe14da --- /dev/null +++ b/packages/gittensory-miner/lib/replay-objective-anchor.d.ts @@ -0,0 +1,79 @@ +export type ChangeKind = + | "feature" + | "fix" + | "refactor" + | "docs" + | "test" + | "chore" + | "perf" + | "build" + | "ci" + | "style" + | "other"; + +export const CHANGE_KINDS: readonly ChangeKind[]; +export const MODULE_OVERLAP_WEIGHT: number; +export const CHANGE_KIND_WEIGHT: number; + +export type ReplayPlanInput = { + pathsTouched?: unknown; + changeKind?: unknown; + title?: unknown; +}; + +export type RevealedHistoryEntry = { + pathsTouched?: unknown; + changeKind?: unknown; + title?: unknown; +}; + +export type ReplayTargetFeatures = { + modules: string[]; + changeKind: ChangeKind; +}; + +export type RevealedFeatures = { + modules: string[]; + changeKinds: ChangeKind[]; +}; + +export type ObjectiveAnchorBreakdown = { + score: number; + moduleOverlap: number; + changeKindMatch: 0 | 1; + replayChangeKind: ChangeKind; + revealedChangeKinds: ChangeKind[]; + sharedModules: string[]; + replayOnlyModules: string[]; + revealedOnlyModules: string[]; +}; + +export type ObjectiveAnchorResult = ObjectiveAnchorBreakdown & { + replayFeatures: ReplayTargetFeatures; + revealedFeatures: RevealedFeatures; +}; + +export function classifyChangeKind(value: unknown): ChangeKind; + +export function extractReplayTargetFeatures( + plan: ReplayPlanInput | null | undefined, +): ReplayTargetFeatures; + +export function extractRevealedFeatures( + history: readonly unknown[] | RevealedHistoryEntry | null | undefined, +): RevealedFeatures; + +export function scoreObjectiveAnchor( + replayFeatures: { modules?: unknown; changeKind?: unknown } | null | undefined, + revealedFeatures: { modules?: unknown; changeKinds?: unknown } | null | undefined, +): ObjectiveAnchorBreakdown; + +export function computeObjectiveAnchor( + input: + | { + replayPlan?: ReplayPlanInput | null; + revealedHistory?: RevealedHistoryEntry[] | RevealedHistoryEntry | null; + } + | null + | undefined, +): ObjectiveAnchorResult; diff --git a/packages/gittensory-miner/lib/replay-objective-anchor.js b/packages/gittensory-miner/lib/replay-objective-anchor.js new file mode 100644 index 0000000000..26556c2cd6 --- /dev/null +++ b/packages/gittensory-miner/lib/replay-objective-anchor.js @@ -0,0 +1,179 @@ +// Deterministic structural "objective-anchor" score for the historical-replay calibration harness (#3012). +// +// Once a replay run produces a plan/PR against a frozen snapshot, half of the calibration score is meant to +// come from a deterministic, auditable structural comparison rather than an LLM judgment. This module is that +// structural half: it compares what the miner's replayed output *targeted* (modules touched + change kind) +// against what the revealed post-T history *actually* changed, and returns a reproducible `[0, 1]` score plus +// a full audit breakdown. There is no model call in this path — given the same two feature sets it is +// byte-for-byte reproducible. + +// Fixed change-kind vocabulary. Conventional-Commit types collapse onto these buckets; anything unrecognized +// degrades to "other" so a novel prefix lowers the signal instead of throwing. +export const CHANGE_KINDS = Object.freeze([ + "feature", + "fix", + "refactor", + "docs", + "test", + "chore", + "perf", + "build", + "ci", + "style", + "other", +]); + +const CONVENTIONAL_TYPE_TO_KIND = new Map([ + ["feat", "feature"], + ["feature", "feature"], + ["fix", "fix"], + ["bugfix", "fix"], + ["refactor", "refactor"], + ["docs", "docs"], + ["doc", "docs"], + ["test", "test"], + ["tests", "test"], + ["chore", "chore"], + ["perf", "perf"], + ["build", "build"], + ["ci", "ci"], + ["style", "style"], +]); + +// Fixed weights for the two structural components. They sum to 1 so the composed score stays in [0, 1]. +export const MODULE_OVERLAP_WEIGHT = 0.7; +export const CHANGE_KIND_WEIGHT = 0.3; + +const SCORE_PRECISION = 1e4; + +function roundScore(value) { + return Math.round(value * SCORE_PRECISION) / SCORE_PRECISION; +} + +// A path's "module" is its directory (everything before the final slash); a bare filename is its own module. +// Grouping by directory is what makes two different files in one directory a *partial* overlap, not a miss. +function pathToModule(path) { + const trimmed = path.trim().replace(/^(?:\.\/)+/, "").replace(/\/+$/, ""); + if (!trimmed) return null; + const slash = trimmed.lastIndexOf("/"); + return slash === -1 ? trimmed : trimmed.slice(0, slash); +} + +function normalizeModules(pathsTouched) { + if (!Array.isArray(pathsTouched)) return []; + const modules = new Set(); + for (const entry of pathsTouched) { + if (typeof entry !== "string") continue; + const module = pathToModule(entry); + if (module) modules.add(module); + } + return [...modules].sort(); +} + +function normalizeKindList(value) { + if (!Array.isArray(value)) return []; + const kinds = new Set(); + for (const entry of value) { + if (typeof entry === "string" && CHANGE_KINDS.includes(entry)) kinds.add(entry); + } + return [...kinds].sort(); +} + +function normalizeModuleList(value) { + if (!Array.isArray(value)) return []; + const modules = new Set(); + for (const entry of value) { + if (typeof entry === "string" && entry) modules.add(entry); + } + return [...modules].sort(); +} + +// Deterministically map a Conventional-Commit-style subject (`feat(scope)!: …`) to a change-kind bucket. +// Missing prefix, unknown type, or non-string input all resolve to "other" rather than throwing. +export function classifyChangeKind(value) { + if (typeof value !== "string") return "other"; + const match = /^\s*([A-Za-z]+)\s*(?:\([^)]*\))?\s*!?\s*:/.exec(value); + if (!match) return "other"; + return CONVENTIONAL_TYPE_TO_KIND.get(match[1].toLowerCase()) ?? "other"; +} + +function resolveChangeKind(entry) { + if (entry && typeof entry.changeKind === "string") { + const explicit = entry.changeKind.trim().toLowerCase(); + if (CHANGE_KINDS.includes(explicit)) return explicit; + } + return classifyChangeKind(entry?.title); +} + +// Structural features of the miner's replayed plan/PR: the sorted, de-duplicated set of modules it targeted +// and its single change kind (explicit `changeKind` wins; otherwise classified from `title`). +export function extractReplayTargetFeatures(plan) { + return { + modules: normalizeModules(plan?.pathsTouched), + changeKind: resolveChangeKind(plan), + }; +} + +// Structural features of the revealed post-T history. The history is a list of commits/PRs (a single object +// is tolerated as a one-element list); modules are unioned and change kinds collected into a set, since the +// revealed side legitimately spans several changes. +export function extractRevealedFeatures(history) { + const entries = Array.isArray(history) ? history : history ? [history] : []; + const modules = new Set(); + const changeKinds = new Set(); + for (const entry of entries) { + if (!entry || typeof entry !== "object") continue; + for (const module of normalizeModules(entry.pathsTouched)) modules.add(module); + changeKinds.add(resolveChangeKind(entry)); + } + return { + modules: [...modules].sort(), + changeKinds: [...changeKinds].sort(), + }; +} + +// Deterministic objective-anchor score from two already-extracted feature sets. No LLM, no clock, no +// randomness — identical inputs always yield an identical breakdown. A zero-overlap comparison (disjoint +// modules and a change kind the revealed side never shows) resolves to the score floor `0`, never an error. +export function scoreObjectiveAnchor(replayFeatures, revealedFeatures) { + const replayModules = normalizeModuleList(replayFeatures?.modules); + const revealedModules = normalizeModuleList(revealedFeatures?.modules); + const replayChangeKind = + typeof replayFeatures?.changeKind === "string" && CHANGE_KINDS.includes(replayFeatures.changeKind) + ? replayFeatures.changeKind + : "other"; + const revealedChangeKinds = normalizeKindList(revealedFeatures?.changeKinds); + + const replaySet = new Set(replayModules); + const revealedSet = new Set(revealedModules); + const sharedModules = replayModules.filter((module) => revealedSet.has(module)); + const replayOnlyModules = replayModules.filter((module) => !revealedSet.has(module)); + const revealedOnlyModules = revealedModules.filter((module) => !replaySet.has(module)); + + const unionSize = replayModules.length + revealedModules.length - sharedModules.length; + const moduleOverlap = unionSize === 0 ? 0 : sharedModules.length / unionSize; + const changeKindMatch = revealedChangeKinds.includes(replayChangeKind) ? 1 : 0; + + return { + score: roundScore(MODULE_OVERLAP_WEIGHT * moduleOverlap + CHANGE_KIND_WEIGHT * changeKindMatch), + moduleOverlap: roundScore(moduleOverlap), + changeKindMatch, + replayChangeKind, + revealedChangeKinds, + sharedModules, + replayOnlyModules, + revealedOnlyModules, + }; +} + +// One-shot entry point: extract both sides, score them, and return the score together with the extracted +// feature sets so a low score is auditable after the fact without re-running the extraction. +export function computeObjectiveAnchor(input) { + const replayFeatures = extractReplayTargetFeatures(input?.replayPlan); + const revealedFeatures = extractRevealedFeatures(input?.revealedHistory); + return { + ...scoreObjectiveAnchor(replayFeatures, revealedFeatures), + replayFeatures, + revealedFeatures, + }; +} diff --git a/packages/gittensory-miner/package.json b/packages/gittensory-miner/package.json index 8d5dea5338..5d107ef464 100644 --- a/packages/gittensory-miner/package.json +++ b/packages/gittensory-miner/package.json @@ -31,7 +31,7 @@ "lib" ], "scripts": { - "build": "node --check bin/gittensory-miner.js && node --check lib/cli.js && node --check lib/deny-check.js && node --check lib/run-state-cli.js && node --check lib/update-check.js && node --check lib/opportunity-fanout.js && node --check lib/ci-poller.js && node --check lib/run-state.js && node --check lib/deny-hooks.js && node --check lib/event-ledger.js && node --check lib/event-ledger-cli.js && node --check lib/claim-ledger.js && node --check lib/claim-ledger-expiry.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-discovery.js && node --check lib/opportunity-ranker.js && node --check lib/plan-store.js && node --check lib/plan-store-cli.js && node --check lib/rejection-templates.js && node --check lib/governor-ledger.js && node --check lib/governor-ledger-cli.js && node --check lib/manage-status.js && node --check lib/manage-poll.js && node --check lib/status.js" + "build": "node --check bin/gittensory-miner.js && node --check lib/cli.js && node --check lib/deny-check.js && node --check lib/run-state-cli.js && node --check lib/update-check.js && node --check lib/opportunity-fanout.js && node --check lib/ci-poller.js && node --check lib/run-state.js && node --check lib/deny-hooks.js && node --check lib/event-ledger.js && node --check lib/event-ledger-cli.js && node --check lib/claim-ledger.js && node --check lib/claim-ledger-expiry.js && node --check lib/portfolio-queue.js && node --check lib/portfolio-queue-cli.js && node --check lib/portfolio-discovery.js && node --check lib/opportunity-ranker.js && node --check lib/plan-store.js && node --check lib/plan-store-cli.js && node --check lib/rejection-templates.js && node --check lib/governor-ledger.js && node --check lib/governor-ledger-cli.js && node --check lib/manage-status.js && node --check lib/manage-poll.js && node --check lib/status.js && node --check lib/replay-objective-anchor.js" }, "dependencies": { "@jsonbored/gittensory-engine": "0.1.0" diff --git a/test/unit/miner-replay-objective-anchor.test.ts b/test/unit/miner-replay-objective-anchor.test.ts new file mode 100644 index 0000000000..bca72dc00c --- /dev/null +++ b/test/unit/miner-replay-objective-anchor.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from "vitest"; +import { + CHANGE_KINDS, + CHANGE_KIND_WEIGHT, + MODULE_OVERLAP_WEIGHT, + classifyChangeKind, + computeObjectiveAnchor, + extractReplayTargetFeatures, + extractRevealedFeatures, + scoreObjectiveAnchor, +} from "../../packages/gittensory-miner/lib/replay-objective-anchor.js"; + +describe("gittensory-miner replay objective-anchor scoring (#3012)", () => { + it("exposes a frozen change-kind vocabulary and weights that sum to 1", () => { + expect(Object.isFrozen(CHANGE_KINDS)).toBe(true); + expect(CHANGE_KINDS).toContain("feature"); + expect(CHANGE_KINDS).toContain("other"); + expect(MODULE_OVERLAP_WEIGHT + CHANGE_KIND_WEIGHT).toBe(1); + }); + + describe("classifyChangeKind", () => { + it("maps Conventional-Commit types (with scope/bang) onto the vocabulary", () => { + expect(classifyChangeKind("feat(miner): add anchor")).toBe("feature"); + expect(classifyChangeKind("fix: correct overlap")).toBe("fix"); + expect(classifyChangeKind("refactor(engine)!: reshape")).toBe("refactor"); + expect(classifyChangeKind("docs: update readme")).toBe("docs"); + expect(classifyChangeKind("test(unit): more cases")).toBe("test"); + expect(classifyChangeKind("chore!: bump")).toBe("chore"); + expect(classifyChangeKind("perf: speed up")).toBe("perf"); + }); + + it("resolves an unknown prefix, a prefix-less subject, and non-strings to 'other'", () => { + expect(classifyChangeKind("wip: scratch")).toBe("other"); + expect(classifyChangeKind("just a plain title")).toBe("other"); + expect(classifyChangeKind(undefined)).toBe("other"); + expect(classifyChangeKind(42)).toBe("other"); + }); + }); + + describe("extractReplayTargetFeatures", () => { + it("groups paths into sorted, de-duplicated modules and classifies from title", () => { + const features = extractReplayTargetFeatures({ + pathsTouched: ["src/a/y.ts", "src/a/x.ts", "./src/b/z.ts", "src/a/y.ts"], + title: "feat(x): thing", + }); + expect(features.modules).toEqual(["src/a", "src/b"]); + expect(features.changeKind).toBe("feature"); + }); + + it("treats a bare filename as its own module and honors an explicit changeKind over the title", () => { + const features = extractReplayTargetFeatures({ + pathsTouched: ["README.md"], + changeKind: "Docs", + title: "feat: mislabeled", + }); + expect(features.modules).toEqual(["README.md"]); + expect(features.changeKind).toBe("docs"); // explicit (case-insensitive) wins over the title + }); + + it("ignores an out-of-vocabulary explicit changeKind and falls back to the title", () => { + const features = extractReplayTargetFeatures({ + pathsTouched: ["src/a/x.ts"], + changeKind: "banana", + title: "fix: real kind", + }); + expect(features.changeKind).toBe("fix"); + }); + + it("degrades junk and missing input to empty modules and 'other'", () => { + expect(extractReplayTargetFeatures(null)).toEqual({ modules: [], changeKind: "other" }); + expect( + extractReplayTargetFeatures({ pathsTouched: [" ", 7, null, "src/a/x.ts"] }), + ).toEqual({ modules: ["src/a"], changeKind: "other" }); + expect(extractReplayTargetFeatures({ pathsTouched: "src/a/x.ts" }).modules).toEqual([]); + }); + }); + + describe("extractRevealedFeatures", () => { + it("unions modules and collects the set of change kinds across many entries", () => { + const features = extractRevealedFeatures([ + { pathsTouched: ["src/a/x.ts"], title: "feat: one" }, + { pathsTouched: ["src/b/y.ts", "src/a/z.ts"], title: "fix: two" }, + ]); + expect(features.modules).toEqual(["src/a", "src/b"]); + expect(features.changeKinds).toEqual(["feature", "fix"]); + }); + + it("tolerates a single object, and skips null/non-object entries", () => { + expect(extractRevealedFeatures({ pathsTouched: ["src/a/x.ts"], title: "docs: y" })).toEqual({ + modules: ["src/a"], + changeKinds: ["docs"], + }); + const features = extractRevealedFeatures([null, 3, { pathsTouched: ["src/a/x.ts"] }]); + expect(features.modules).toEqual(["src/a"]); + expect(features.changeKinds).toEqual(["other"]); + }); + + it("returns empty feature sets for empty or nullish history", () => { + expect(extractRevealedFeatures([])).toEqual({ modules: [], changeKinds: [] }); + expect(extractRevealedFeatures(null)).toEqual({ modules: [], changeKinds: [] }); + }); + }); + + describe("scoreObjectiveAnchor", () => { + it("scores full module + change-kind overlap as 1 with an empty divergence set", () => { + const result = scoreObjectiveAnchor( + { modules: ["src/a"], changeKind: "feature" }, + { modules: ["src/a"], changeKinds: ["feature"] }, + ); + expect(result.score).toBe(1); + expect(result.moduleOverlap).toBe(1); + expect(result.changeKindMatch).toBe(1); + expect(result.sharedModules).toEqual(["src/a"]); + expect(result.replayOnlyModules).toEqual([]); + expect(result.revealedOnlyModules).toEqual([]); + }); + + it("floors zero overlap (disjoint modules + unmatched kind) at 0 without throwing", () => { + const result = scoreObjectiveAnchor( + { modules: ["src/a"], changeKind: "feature" }, + { modules: ["src/b"], changeKinds: ["fix"] }, + ); + expect(result.score).toBe(0); + expect(result.moduleOverlap).toBe(0); + expect(result.changeKindMatch).toBe(0); + expect(result.sharedModules).toEqual([]); + expect(result.replayOnlyModules).toEqual(["src/a"]); + expect(result.revealedOnlyModules).toEqual(["src/b"]); + }); + + it("computes partial module overlap as a Jaccard ratio, weighted with a matched kind", () => { + const result = scoreObjectiveAnchor( + { modules: ["src/a", "src/b"], changeKind: "feature" }, + { modules: ["src/a", "src/c"], changeKinds: ["feature"] }, + ); + // shared {src/a}, union {src/a, src/b, src/c} → overlap 1/3; kind matches → 0.7*(1/3) + 0.3 + expect(result.moduleOverlap).toBe(0.3333); + expect(result.score).toBe(0.5333); + expect(result.sharedModules).toEqual(["src/a"]); + expect(result.replayOnlyModules).toEqual(["src/b"]); + expect(result.revealedOnlyModules).toEqual(["src/c"]); + }); + + it("separates the module and change-kind contributions (overlap with a mismatched kind)", () => { + const result = scoreObjectiveAnchor( + { modules: ["src/a"], changeKind: "feature" }, + { modules: ["src/a"], changeKinds: ["fix"] }, + ); + expect(result.moduleOverlap).toBe(1); + expect(result.changeKindMatch).toBe(0); + expect(result.score).toBe(MODULE_OVERLAP_WEIGHT); // 0.7 from modules only + }); + + it("floors both-empty feature sets at 0 rather than dividing by zero", () => { + const result = scoreObjectiveAnchor({ modules: [] }, { modules: [], changeKinds: [] }); + expect(result.moduleOverlap).toBe(0); + expect(result.score).toBe(0); + expect(result.replayChangeKind).toBe("other"); + }); + + it("normalizes malformed feature inputs (non-array modules, out-of-vocab kinds) defensively", () => { + const result = scoreObjectiveAnchor( + { modules: "src/a", changeKind: "banana" }, + { modules: [7, "src/a"], changeKinds: ["banana", "feature"] }, + ); + expect(result.replayChangeKind).toBe("other"); + expect(result.revealedChangeKinds).toEqual(["feature"]); // "banana" dropped as out-of-vocab + expect(result.replayOnlyModules).toEqual([]); // replay modules coerced to [] + expect(result.revealedOnlyModules).toEqual(["src/a"]); + }); + }); + + describe("computeObjectiveAnchor", () => { + it("extracts both sides, scores them, and logs the extracted features for audit", () => { + const result = computeObjectiveAnchor({ + replayPlan: { pathsTouched: ["src/a/x.ts"], title: "feat: build the thing" }, + revealedHistory: [ + { pathsTouched: ["src/a/y.ts"], title: "feat: shipped the thing" }, + { pathsTouched: ["src/b/z.ts"], title: "docs: mention it" }, + ], + }); + expect(result.replayFeatures).toEqual({ modules: ["src/a"], changeKind: "feature" }); + expect(result.revealedFeatures).toEqual({ + modules: ["src/a", "src/b"], + changeKinds: ["docs", "feature"], + }); + // shared {src/a}, union {src/a, src/b} → 1/2; kind "feature" present → 0.7*0.5 + 0.3 + expect(result.score).toBe(0.65); + }); + + it("floors a replay that targets modules the revealed history never touches", () => { + const result = computeObjectiveAnchor({ + replayPlan: { pathsTouched: ["src/ghost/x.ts"], title: "feat: guess" }, + revealedHistory: [{ pathsTouched: ["src/real/y.ts"], title: "fix: actual" }], + }); + expect(result.score).toBe(0); + expect(result.sharedModules).toEqual([]); + }); + + it("is byte-for-byte reproducible across repeated runs on the same inputs", () => { + const input = { + replayPlan: { pathsTouched: ["src/a/x.ts", "src/b/y.ts"], changeKind: "refactor" }, + revealedHistory: [{ pathsTouched: ["src/a/z.ts"], changeKind: "refactor" }], + }; + expect(computeObjectiveAnchor(input)).toEqual(computeObjectiveAnchor(input)); + }); + + it("floors a fully empty run at 0 without error", () => { + expect(computeObjectiveAnchor(null).score).toBe(0); + expect(computeObjectiveAnchor({}).score).toBe(0); + }); + }); +});