chore: organize dirty work from codex/backlog-diagnostics-reconciliation

.github/workflows/eval-canary.yml

@@ -173,7 +173,7 @@ jobs:
           fi
           mkdir -p .local/eval-canary
           set -o pipefail
-          npm run eval:quality -- --rag-only --limit "$ANSWER_CASE_LIMIT" --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log
+          npm run eval:quality -- --rag-only --retrieval-results .local/eval-canary/golden-retrieval.json --limit "$ANSWER_CASE_LIMIT" --output-dir .local/eval-canary/quality-reports --fail-on-threshold 2>&1 | tee .local/eval-canary/answer-quality.log
 
       # Phase E baseline/paired-run instrument: the five answer-quality metrics (relevance,
       # readability, artifact leaks, intent coverage, fail-closed) + per-intent structural

scripts/eval-quality.ts

@@ -38,6 +38,7 @@ export type EvalQualityProviderMode = "openai" | "offline";
 
 type EvalQualityArgs = {
   fixture: string;
+  retrievalResults?: string;
   ownerEmail?: string;
   ownerId?: string;
   limit?: number;
@@ -293,6 +294,7 @@ function parseArgs(argv: string[]): EvalQualityArgs {
     index += 1;
 
     if (token === "--fixture") args.fixture = value;
+    if (token === "--retrieval-results") args.retrievalResults = value;
     if (token === "--owner-email") args.ownerEmail = value;
     if (token === "--owner-id") args.ownerId = value;
     if (token === "--limit") args.limit = Number.parseInt(value, 10);
@@ -309,13 +311,43 @@ function parseArgs(argv: string[]): EvalQualityArgs {
   }
 
   if (args.retrievalOnly && args.ragOnly) throw new Error("Use only one of --retrieval-only or --rag-only.");
+  if (args.retrievalResults && !args.ragOnly) {
+    throw new Error("--retrieval-results is only supported with --rag-only.");
+  }
   if (args.limit !== undefined && (!Number.isInteger(args.limit) || args.limit <= 0)) {
     throw new Error("--limit must be a positive integer.");
   }
 
   return args;
 }
 
+export function goldenRetrievalResultsFromArtifact(payload: unknown): GoldenRetrievalResult[] {
+  if (!payload || typeof payload !== "object" || !("results" in payload) || !Array.isArray(payload.results)) {
+    throw new Error("Golden retrieval artifact must contain a results array.");
+  }
+  if (
+    payload.results.some(
+      (result) =>
+        !result || typeof result !== "object" || !("topResults" in result) || !Array.isArray(result.topResults),
+    )
+  ) {
+    throw new Error("Every golden retrieval result must contain a topResults array.");
+  }
+  return payload.results as GoldenRetrievalResult[];
+}
+
+async function loadGoldenRetrievalResultsArtifact(path: string) {
+  let payload: unknown;
+  try {
+    payload = JSON.parse(await readFile(path, "utf8"));
+  } catch (error) {
+    throw new Error(
+      `Unable to read golden retrieval artifact ${path}: ${error instanceof Error ? error.message : String(error)}`,
+    );
+  }
+  return goldenRetrievalResultsFromArtifact(payload);
+}
+
 export function configureEvalProviderEnvironment(providerMode: EvalQualityProviderMode) {
   process.env.RAG_PROVIDER_MODE = providerMode;
   if (providerMode !== "offline") return;
@@ -597,14 +629,15 @@ function summarizeRagQualityResults(results: RagQualityResult[], providerMode: E
 export function buildEvalQualityReport(args: {
   generatedAt?: string;
   retrievalResults: GoldenRetrievalResult[];
+  sourceGovernanceResults?: GoldenRetrievalResult[];
   ragResults: RagQualityResult[];
   sourceMetadataDebtAcceptance?: SourceMetadataDebtAcceptance;
   providerMode?: EvalQualityProviderMode;
 }) {
   const providerMode = args.providerMode ?? "openai";
   const retrievalSummary = summarizeGoldenRetrievalResults(args.retrievalResults);
   const ragSummary = summarizeRagQualityResults(args.ragResults, providerMode);
-  const governance = topResultGovernanceCounts(args.retrievalResults);
+  const governance = topResultGovernanceCounts(args.sourceGovernanceResults ?? args.retrievalResults);
   const thresholdFailures: string[] = [];
   const providerEvidence = {
     mode: providerMode,
@@ -1258,10 +1291,14 @@ async function main() {
     : undefined;
 
   const ownerId = await resolveEvalOwnerId(supabase, args);
+  const sourceGovernanceResults = args.retrievalResults
+    ? await loadGoldenRetrievalResultsArtifact(args.retrievalResults)
+    : undefined;
   const retrievalResults = args.ragOnly ? [] : await runRetrievalQualityCases({ ...args, ownerId, supabase });
   const ragResults = args.retrievalOnly ? [] : await runRagQualityCases({ ...args, ownerId, supabase });
   const report = buildEvalQualityReport({
     retrievalResults,
+    sourceGovernanceResults,
     ragResults,
     sourceMetadataDebtAcceptance,
     providerMode: args.providerMode,

src/lib/eval-document-matching.ts

@@ -116,12 +116,32 @@ export function expectedFileCoverage(
   sources: Array<Pick<SearchResult, "file_name" | "title">>,
   limit = 3,
 ): ExpectedFileCoverage {
-  const topFiles = sources.slice(0, limit).map(resultDocumentText);
-  const matchedFiles = expectedFiles.filter((expected) =>
-    documentExpectationAlternatives(expected).some((alternative) =>
-      topFiles.some((file) => file.includes(alternative)),
-    ),
-  );
+  const topFiles = Array.from(new Set(sources.slice(0, limit).map(resultDocumentText)));
+  const alternatives = expectedFiles.map(documentExpectationAlternatives);
+  const expectedForSource = Array<number>(topFiles.length).fill(-1);
+  const sourceForExpected = Array<number>(expectedFiles.length).fill(-1);
+
+  const assignExpected = (expectedIndex: number, visitedSources: Set<number>): boolean => {
+    for (let sourceIndex = 0; sourceIndex < topFiles.length; sourceIndex += 1) {
+      if (visitedSources.has(sourceIndex)) continue;
+      if (!alternatives[expectedIndex].some((alternative) => topFiles[sourceIndex].includes(alternative))) continue;
+
+      visitedSources.add(sourceIndex);
+      const currentExpected = expectedForSource[sourceIndex];
+      if (currentExpected >= 0 && !assignExpected(currentExpected, visitedSources)) continue;
+
+      expectedForSource[sourceIndex] = expectedIndex;
+      sourceForExpected[expectedIndex] = sourceIndex;
+      return true;
+    }
+    return false;
+  };
+
+  for (let expectedIndex = 0; expectedIndex < expectedFiles.length; expectedIndex += 1) {
+    assignExpected(expectedIndex, new Set());
+  }
+
+  const matchedFiles = expectedFiles.filter((_, index) => sourceForExpected[index] >= 0);
 
   return {
     expectedFiles,

tests/eval-canary-workflow.test.ts

@@ -37,6 +37,7 @@ describe("eval canary workflow input", () => {
     expect(workflow).toContain("tee .local/eval-canary/golden-retrieval.log");
     expect(workflow).toContain("tee .local/eval-canary/answer-quality.log");
     expect(workflow).toContain("--output-dir .local/eval-canary/quality-reports");
+    expect(workflow).toContain("--retrieval-results .local/eval-canary/golden-retrieval.json");
     expect(workflow).toContain(
       "await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/scripts/productivity-core.mjs`).href)",
     );

tests/eval-quality.test.ts

@@ -5,6 +5,7 @@ import {
   configureEvalProviderEnvironment,
   deliveredGroundedAfterSourceGovernancePolicy,
   evalQualityRunContext,
+  goldenRetrievalResultsFromArtifact,
   qualityFailureCategory,
   ragAnswerTimingDiagnostics,
   renderEvalQualityMarkdown,
@@ -658,6 +659,39 @@ describe("eval quality reporting", () => {
     expect(markdown).toContain("Policy: unknown, unverified");
   });
 
+  it("renders governance counts from the preceding golden retrieval artifact", () => {
+    const retrievalResults = goldenRetrievalResultsFromArtifact({
+      results: [
+        retrievalResult({
+          topResults: [
+            {
+              ...retrievalResult().topResults[0],
+              document_status: "review_due",
+              clinical_validation_status: "unverified",
+              extraction_quality: "poor",
+            },
+          ],
+        }),
+      ],
+    });
+    const report = buildEvalQualityReport({
+      retrievalResults: [],
+      sourceGovernanceResults: retrievalResults,
+      ragResults: [ragResult()],
+    });
+    const markdown = renderEvalQualityMarkdown(report);
+
+    expect(markdown).toContain("| Top results | 1 |");
+    expect(markdown).toContain("| Review-due top results | 1 |");
+    expect(markdown).toContain("| Unverified top results | 1 |");
+    expect(markdown).toContain("| Poor-extraction top results | 1 |");
+    expect(markdown).toContain("| Review-required top results | 1 |");
+    expect(report.retrieval.summary.case_count).toBe(0);
+    expect(report.threshold_failures).not.toEqual(
+      expect.arrayContaining([expect.stringContaining("top-result review_required_rate")]),
+    );
+  });
+
   it("hard-fails any answer case that exceeds its route ceiling", () => {
     const report = buildEvalQualityReport({
       generatedAt: "2026-07-13T00:00:00.000Z",

tests/eval-search.test.ts

@@ -72,6 +72,44 @@ describe("search eval thresholds", () => {
     expect(partial.missingFiles).toEqual(["MHSP.AdmissionCommunityPts.pdf"]);
   });
 
+  it("does not let one aliased source satisfy two expected document slots", () => {
+    const coverage = expectedFileCoverage(
+      ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+      [
+        {
+          title: "Admission to Discharge for Mental Health Inpatients",
+          file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf",
+        },
+      ],
+      5,
+    );
+
+    expect(coverage.matchedFiles).toHaveLength(1);
+    expect(coverage.missingFiles).toHaveLength(1);
+    expect(coverage.allHit).toBe(false);
+  });
+
+  it("assigns overlapping aliases to distinct sources when both expected documents exist", () => {
+    const coverage = expectedFileCoverage(
+      ["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"],
+      [
+        {
+          title: "Admission to Discharge for Mental Health Inpatients",
+          file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf",
+        },
+        {
+          title: "Admission of Community Patients",
+          file_name: "Admission of Community Patients (AKG).pdf",
+        },
+      ],
+      5,
+    );
+
+    expect(coverage.matchedFiles).toEqual(["MHSP.AdmissionCommunityPts.pdf", "MHSP.Discharge.pdf"]);
+    expect(coverage.missingFiles).toEqual([]);
+    expect(coverage.allHit).toBe(true);
+  });
+
   it("matches legacy eval expectations to current clinical source filenames", () => {
     expect(
       expectedFileHit(