diff --git a/src/services/recommendation-snapshots.ts b/src/services/recommendation-snapshots.ts index 3e3d0e3828..18c045adda 100644 --- a/src/services/recommendation-snapshots.ts +++ b/src/services/recommendation-snapshots.ts @@ -1,5 +1,44 @@ import type { AgentActionRecord, AgentContextSnapshotRecord, AgentActionType, JsonValue } from "../types"; +export type SnapshotProvenanceConfidence = "high" | "medium" | "low"; +export type SnapshotProvenanceFreshness = + | "fresh" + | "stale" + | "rebuilding" + | "missing" + | "degraded" + | "possibly_stale" + | "unknown"; + +/** + * Public-safe provenance for a single evidence source. Only structured + * identifiers and metadata are exposed — never the raw human-readable summary, + * which can carry private repo/login/scoreability context. + */ +export type RecommendationSnapshotSourceProvenance = { + name: string; + freshness: SnapshotProvenanceFreshness; + generatedAt: string | null; +}; + +/** + * Public-safe provenance attached to a recommendation snapshot. This is the + * only provenance shape serialized into public GitHub text: it surfaces which + * evidence was used, how fresh it was, the model confidence, and any known + * gaps, without leaking private/authenticated evidence detail. Advisory + * metadata only — never public scoring or reward prediction. + */ +export type RecommendationSnapshotProvenance = { + confidence: SnapshotProvenanceConfidence; + freshness: SnapshotProvenanceFreshness; + generatedAt: string | null; + scoringModelId: string | null; + repoSignalSnapshotIds: string[]; + sources: RecommendationSnapshotSourceProvenance[]; + evidenceGaps: string[]; + evidenceComplete: boolean; +}; + export type RecommendationSnapshotEnvelope = { kind: "recommendation_snapshot"; version: 1; @@ -15,8 +54,109 @@ export type RecommendationSnapshotEnvelope = { pullNumber?: number; issueNumber?: number; }; + provenance: RecommendationSnapshotProvenance; }; +const CONFIDENCE_VALUES: ReadonlySet = new Set(["high", "medium", "low"]); +const FRESHNESS_VALUES: ReadonlySet = new Set([ + "fresh", + "stale", + "rebuilding", + "missing", + "degraded", + "possibly_stale", + "unknown", +]); + +function narrowConfidence(value: JsonValue | undefined): SnapshotProvenanceConfidence { + return typeof value === "string" && CONFIDENCE_VALUES.has(value as SnapshotProvenanceConfidence) + ? (value as SnapshotProvenanceConfidence) + : "low"; +} + +function narrowFreshness(value: JsonValue | undefined): SnapshotProvenanceFreshness { + return typeof value === "string" && FRESHNESS_VALUES.has(value as SnapshotProvenanceFreshness) + ? (value as SnapshotProvenanceFreshness) + : "unknown"; +} + +function isJsonRecord(value: JsonValue | undefined): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Defensively read public-safe provenance inputs from the private + * `recommendationEvidence` blob carried on an action payload. Fails closed: + * missing or malformed evidence yields low-confidence, unknown-freshness + * provenance with the gap recorded explicitly rather than silently omitted. + */ +function readEvidenceProvenance(raw: JsonValue | undefined): { + confidence: SnapshotProvenanceConfidence; + freshness: SnapshotProvenanceFreshness; + sources: RecommendationSnapshotSourceProvenance[]; + hasEvidence: boolean; +} { + if (!isJsonRecord(raw)) { + return { confidence: "low", freshness: "unknown", sources: [], hasEvidence: false }; + } + const rawSources = Array.isArray(raw.sources) ? raw.sources : []; + const sources: RecommendationSnapshotSourceProvenance[] = []; + for (const entry of rawSources) { + if (!isJsonRecord(entry)) continue; + const name = typeof entry.name === "string" ? entry.name.trim() : ""; + if (!name) continue; + sources.push({ + name, + freshness: narrowFreshness(entry.freshness), + generatedAt: typeof entry.generatedAt === "string" ? entry.generatedAt : null, + }); + } + return { + confidence: narrowConfidence(raw.confidence), + freshness: narrowFreshness(raw.freshness), + sources, + hasEvidence: true, + }; +} + +function snapshotGeneratedAt(context: AgentContextSnapshotRecord): string | null { + return context.createdAt ?? context.decisionPackVersion ?? null; +} + +/** + * Build the public-safe provenance for a recommendation snapshot from the + * action's evidence and the durable context snapshot. Stale and missing + * evidence are represented explicitly via `evidenceGaps`/`evidenceComplete`. + */ +export function recommendationSnapshotProvenance( + action: AgentActionRecord, + context: AgentContextSnapshotRecord, +): RecommendationSnapshotProvenance { + const { confidence, freshness, sources, hasEvidence } = readEvidenceProvenance(action.payload.recommendationEvidence); + + const evidenceGaps: string[] = []; + if (!hasEvidence) { + evidenceGaps.push("evidence: missing"); + } else if (sources.length === 0) { + evidenceGaps.push("evidence_sources: missing"); + } else { + for (const source of sources) { + if (source.freshness !== "fresh") evidenceGaps.push(`${source.name}: ${source.freshness}`); + } + } + + return { + confidence, + freshness, + generatedAt: snapshotGeneratedAt(context), + scoringModelId: context.scoringModelId ?? null, + repoSignalSnapshotIds: [...context.repoSignalSnapshotIds], + sources, + evidenceGaps, + evidenceComplete: hasEvidence && sources.length > 0 && evidenceGaps.length === 0, + }; +} + export function recommendationSnapshotId(contextSnapshotId: string, actionId: string): string { return `recommendation:${contextSnapshotId}:${actionId}`; } @@ -37,9 +177,10 @@ export function recommendationSnapshotEnvelope( actionId: action.id, runId: action.runId, actionType: action.actionType, - generatedAt: context.createdAt ?? context.decisionPackVersion ?? null, + generatedAt: snapshotGeneratedAt(context), publicSafe: true, target, + provenance: recommendationSnapshotProvenance(action, context), }; } diff --git a/test/unit/recommendation-snapshots.test.ts b/test/unit/recommendation-snapshots.test.ts index 05456b9f56..485bf79b16 100644 --- a/test/unit/recommendation-snapshots.test.ts +++ b/test/unit/recommendation-snapshots.test.ts @@ -4,8 +4,9 @@ import { attachRecommendationSnapshots, recommendationSnapshotEnvelope, recommendationSnapshotId, + recommendationSnapshotProvenance, } from "../../src/services/recommendation-snapshots"; -import type { AgentActionRecord, AgentContextSnapshotRecord } from "../../src/types"; +import type { AgentActionRecord, AgentContextSnapshotRecord, JsonValue } from "../../src/types"; describe("recommendation snapshot envelopes", () => { it("creates stable ids from the durable context snapshot and action ids", () => { @@ -30,6 +31,16 @@ describe("recommendation snapshot envelopes", () => { repoFullName: "JSONbored/gittensory", pullNumber: 12, }, + provenance: { + confidence: "low", + freshness: "unknown", + generatedAt: "2026-06-01T00:00:00.000Z", + scoringModelId: "scoring-1", + repoSignalSnapshotIds: [], + sources: [], + evidenceGaps: ["evidence: missing"], + evidenceComplete: false, + }, }); expect(JSON.stringify(envelope)).not.toMatch( /wallet|hotkey|coldkey|raw trust|private reviewability|private scoreability|reward estimate|payload|recommendationEvidence/i, @@ -46,6 +57,7 @@ describe("recommendation snapshot envelopes", () => { expect(attached.payload.recommendationSnapshot).toMatchObject({ snapshotId: "recommendation:context-123:run-1:00:choose_next_work", publicSafe: true, + provenance: { evidenceComplete: false }, }); }); @@ -81,6 +93,139 @@ describe("recommendation snapshot envelopes", () => { }); }); +describe("recommendation snapshot provenance", () => { + it("derives confidence, freshness, and complete evidence from fresh sources", () => { + const provenance = recommendationSnapshotProvenance( + actionWithEvidence({ + confidence: "high", + freshness: "fresh", + sources: [ + { name: "contributor_decision_pack", freshness: "fresh", generatedAt: "2026-06-01T00:00:00.000Z" }, + { name: "repo_decision", freshness: "fresh" }, + ], + }), + context({ repoSignalSnapshotIds: ["sig-1", "sig-2"] }), + ); + expect(provenance).toEqual({ + confidence: "high", + freshness: "fresh", + generatedAt: "2026-06-01T00:00:00.000Z", + scoringModelId: "scoring-1", + repoSignalSnapshotIds: ["sig-1", "sig-2"], + sources: [ + { name: "contributor_decision_pack", freshness: "fresh", generatedAt: "2026-06-01T00:00:00.000Z" }, + { name: "repo_decision", freshness: "fresh", generatedAt: null }, + ], + evidenceGaps: [], + evidenceComplete: true, + }); + }); + + it("records stale and missing sources as explicit gaps instead of omitting them", () => { + const provenance = recommendationSnapshotProvenance( + actionWithEvidence({ + confidence: "medium", + freshness: "stale", + sources: [ + { name: "contributor_decision_pack", freshness: "fresh" }, + { name: "official_contributor_stats", freshness: "missing" }, + { name: "repo_outcome_patterns", freshness: "possibly_stale" }, + ], + }), + context(), + ); + expect(provenance.freshness).toBe("stale"); + expect(provenance.evidenceGaps).toEqual([ + "official_contributor_stats: missing", + "repo_outcome_patterns: possibly_stale", + ]); + expect(provenance.evidenceComplete).toBe(false); + }); + + it("fails closed when the action carries no evidence", () => { + const provenance = recommendationSnapshotProvenance(action({ payload: {} }), context()); + expect(provenance).toMatchObject({ + confidence: "low", + freshness: "unknown", + sources: [], + evidenceGaps: ["evidence: missing"], + evidenceComplete: false, + }); + }); + + it("flags evidence that exists but exposes no sources", () => { + const provenance = recommendationSnapshotProvenance( + actionWithEvidence({ confidence: "medium", freshness: "degraded" }), + context(), + ); + expect(provenance.evidenceGaps).toEqual(["evidence_sources: missing"]); + expect(provenance.evidenceComplete).toBe(false); + }); + + it("narrows unknown confidence and freshness values and skips malformed sources", () => { + const provenance = recommendationSnapshotProvenance( + actionWithEvidence({ + confidence: "superb", + freshness: "ancient", + sources: [ + "not-an-object", + {}, + { name: " " }, + { name: "repo_decision", freshness: "weird" }, + ], + } as unknown as Record), + context(), + ); + expect(provenance.confidence).toBe("low"); + expect(provenance.freshness).toBe("unknown"); + expect(provenance.sources).toEqual([{ name: "repo_decision", freshness: "unknown", generatedAt: null }]); + expect(provenance.evidenceGaps).toEqual(["repo_decision: unknown"]); + }); + + it("uses createdAt over decisionPackVersion and null when neither exists", () => { + expect( + recommendationSnapshotProvenance(action({ payload: {} }), context({ createdAt: "2026-06-03T00:00:00.000Z" })).generatedAt, + ).toBe("2026-06-03T00:00:00.000Z"); + expect( + recommendationSnapshotProvenance(action({ payload: {} }), { + ...context(), + decisionPackVersion: null, + scoringModelId: null, + }).generatedAt, + ).toBeNull(); + }); + + it("never serializes private evidence summaries, assumptions, or warnings into the envelope", () => { + const envelope = recommendationSnapshotEnvelope( + actionWithEvidence({ + confidence: "high", + freshness: "fresh", + sourceSummary: "private reviewability and raw trust score detail", + assumptions: ["wallet hotkey coldkey assumption"], + warnings: ["reward estimate leak"], + sources: [ + { + name: "contributor_decision_pack", + freshness: "fresh", + generatedAt: "2026-06-01T00:00:00.000Z", + source: "cache", + summary: "jsonbored reward estimate private reviewability", + }, + ], + }), + context(), + ); + expect(envelope.provenance.sources[0]).toEqual({ + name: "contributor_decision_pack", + freshness: "fresh", + generatedAt: "2026-06-01T00:00:00.000Z", + }); + expect(JSON.stringify(envelope)).not.toMatch( + /wallet|hotkey|coldkey|raw trust|private reviewability|reward estimate|summary/i, + ); + }); +}); + function action(overrides: Partial = {}): AgentActionRecord { return { id: "run-1:00:choose_next_work", @@ -101,6 +246,13 @@ function action(overrides: Partial = {}): AgentActionRecord { }; } +function actionWithEvidence( + evidence: Record, + overrides: Partial = {}, +): AgentActionRecord { + return action({ payload: { recommendationEvidence: evidence as unknown as JsonValue }, ...overrides }); +} + function context(overrides: Partial = {}): AgentContextSnapshotRecord { return { id: "context-123",