diff --git a/docs/branch-review-records/ce864242309b87ec27f7947bddac2350e40dcd2241dfeea7fdcafbb6733823f3.record.md b/docs/branch-review-records/ce864242309b87ec27f7947bddac2350e40dcd2241dfeea7fdcafbb6733823f3.record.md new file mode 100644 index 0000000000..c851c77a59 --- /dev/null +++ b/docs/branch-review-records/ce864242309b87ec27f7947bddac2350e40dcd2241dfeea7fdcafbb6733823f3.record.md @@ -0,0 +1 @@ +| 2026-08-17 | claude/s1d-final-gate-gap-recovery-dxgrn2 | 42134f42b9fe8676af99cfb7dbe377ac1306c9e6 | S1d final-gate gap recovery: finalizeRagAnswerQualityCore extractive recovery for fast strong_routine_retrieval gap-like answers (rag-extractive-answer.ts + tests + behaviour-map) | PR #2054 open; behaviour change, post-merge canary pair owed (baseline 32039841070) | verify:pr-local heavy scope green (lint, typecheck, test, build, eval:rag:offline); focused vitest 227/227 + 91/91; check:rag:fixtures 36 golden; check:maintainability-budgets green; discriminating-fixture proof (2 fail without diff) | diff --git a/docs/outstanding-issues-inbox/776405e0-c2d9-4dec-b688-e26c22143f04.json b/docs/outstanding-issues-inbox/776405e0-c2d9-4dec-b688-e26c22143f04.json new file mode 100644 index 0000000000..a7be3f93e4 --- /dev/null +++ b/docs/outstanding-issues-inbox/776405e0-c2d9-4dec-b688-e26c22143f04.json @@ -0,0 +1,10 @@ +{ + "version": 2, + "id": "776405e0-c2d9-4dec-b688-e26c22143f04", + "createdOn": "2026-08-17", + "action": "cancel", + "payload": { + "requestId": "e8a5480e-0b0e-4ab1-b7bd-6090699dd7c1", + "reason": "S1d implemented: finalizeRagAnswerQualityCore's gap conversion now attempts the same source-backed extractive recovery as the in-loop and outer-catch paths for fast strong_routine_retrieval answers over non-empty results (branch claude/s1d-final-gate-gap-recovery-dxgrn2, HANDOVER §2 S1d row updated in the same PR). No open work remains for this row. If this cancellation loses the race to an earlier reconcile that applied the add, close the materialized row with issues:done citing the same PR." + } +} diff --git a/docs/rag-behaviour/behaviour-map.md b/docs/rag-behaviour/behaviour-map.md index 80e616a837..4f62470819 100644 --- a/docs/rag-behaviour/behaviour-map.md +++ b/docs/rag-behaviour/behaviour-map.md @@ -96,6 +96,12 @@ free real estate for ordering keys (see §2's critical property). the post-finalize source-safe gates in `rag.ts` (`claim_support_high_risk_gap`, `material_source_governance_gap`, `numeric_band_coherence_gap`, `numeric_faithfulness_gap`) and the `finalizeRagAnswerQualityCore` gates. +- Since 2026-08-17 (S1d) the finalizer's gap conversion first attempts the same + source-backed extractive recovery the loop and outer catch use + (`recoverFinalGateGapExtractively`): a fast + `strong_routine_retrieval` gap-like answer + over non-empty results rebuilds extractively (marker + `final_quality_gate_source_backed_recovery:`); empty-retrieval, strong-route, and + comparison/dose/threshold gaps stay terminal as `final_quality_gate:`. - Since 2026-08-12 the quality-gate throw sites raise `GenerationQualityError` (`src/lib/rag/rag-generation-quality-diagnostics.ts`) carrying `{stage, gateReason, answerShape}` where `answerShape` is provider-safe counts/lengths only — never prose. The diff --git a/docs/rag-improvement/HANDOVER.md b/docs/rag-improvement/HANDOVER.md index 59f6cb8a71..d9d9f32967 100644 --- a/docs/rag-improvement/HANDOVER.md +++ b/docs/rag-improvement/HANDOVER.md @@ -67,25 +67,25 @@ generation-quality verdict on fallback`), merged 2026-08-13 — structured ## 2. Status table — update in every programme PR -| Packet | Scope | Branch | PR | State | Canary / evidence refs | -| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | --------------------- | ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Guide | Programme guide | `claude/rag-plan-review-guide-vhrls9` | #1895 | Merged 2026-08-13 | docs-only | -| Handover | Multi-session handover + coordination | `claude/rag-plan-review-guide-vhrls9` | #1908 / #2024 | Merged 2026-08-13; coordination layer PR #2024 | docs-only | -| S0 | A1 phase 1: structured fallback diagnostics | `claude/lithium-generation-quality-debug-ji1vce` | #1899 | Merged 2026-08-13 | offline 93/93 focused | -| S1 | A1 phase 2: rung-1 verification-faithfulness fixes | `claude/s1-rag-mitigation-231-86c182` | #2022 | Merged 2026-08-17 (squash `2bd146eed`, landed by content) | 8 pre-fix + 5 post-fix live probes 2026-08-17; offline 583/583; canary pair run 31964560921 (baseline `8f8d111ab`) -> run 32025082010 (`2bd146eed`): recall 1.0/1.0, zero per-case rr regressions, answer gate 44/44 (denominator reconciled by S5; see baseline-record §3); rung-2 measurement in `docs/audit/live-drift-forensics-2026-08.md` §5 | -| S1b | A1 rung 3 (R1): pre-deadline strong routing for dosing class | `claude/s1b-rag-dosing-routing-6u1mik` | #2035 | Merged 2026-08-17 (PR #2035, merge `92f7618`) | canary pair pending: baseline run 32025082010 (`2bd146eed`) -> post-merge dispatch (owner-approved); offline 586/586 + verify:pr-local heavy scope green | -| S1c | A1 residuals R2 + R3: claim-support strictness | `claude/s1c-residuals-r2-r3-4pb1at` | #2052 | PR open 2026-08-17 | offline 613/613 + verify:pr-local heavy scope green; R3 measured before loosening: 87->78 sole-overlap rejections (46->42 unique), zero protective flips; canary pair planned post-merge: baseline run 32039841070 (`92f7618`) -> post-merge dispatch (owner-approved) | -| S1d | A1 final-gate gap recovery: hedged cited low-confidence fast answers must recover extractively, not collapse to a citation-free `provider_source_gap` | `claude/rag-a1-final-gate-gap-recovery-` | — | Ready — dispatch now; parallel-safe with S1c (different file); lands before S2 | needs canary pair; evidence: canary runs 32038751592 (red) vs 32039841070 (green), 3/3 live probes on the extractive branch | -| G1 | Governance: provenance tag for document-summary rows (Option B) | `claude/rag-g1-document-context-origin-` | — | Ready — disjoint; owner decided Option B 2026-08-17 | no canary (no behaviour change) | -| S2 | A2 (+A3): composition menu + moderate length | `claude/rag-a2-composition-` | — | Blocked on S1b + S1c | canary pair + `eval:answer-quality` + Gate E | -| S2b | A3: moderate length (if separate review needed) | `claude/rag-a3-length-` | — | Blocked on S2 | — | -| S3 | A4: follow-up suggestion refinement | `claude/rag-a4-follow-ups-` | — | Blocked on S2 + S2b | — | -| S4 | B0: adversarial fixtures + baseline + register | `claude/packet-s4-adversarial-fixtures-5ho5tp` | #2036 | Merged 2026-08-17 (squash `f5b093291`) | Offline only: `check:rag:adversarial-fixtures` 24 cases / 8 categories / 6 canaries; `eval:rag:offline` 24 suites, 597 tests. Baseline `scripts/fixtures/rag-adversarial-baseline.v1.json` marks the three provider-backed gates `pending_owner_run` | -| S5 | B1+B2: telemetry assessment + offline harness | `claude/s5-rag-telemetry-harness-2wvis7` | #2056 | Merged 2026-08-17 (merge `093f934`) | Offline only: `eval:rag:adversarial:offline` 25/25 (24 cases + canary-free report; 3 divergences pinned in `KNOWN_DIVERGENCES`); B1 gap = `verification_latency_ms` behind `RAG_TELEMETRY_EXTENDED` (default false); canary-absence tests green; 44/44 denominator reconciled | -| S6 | B3: Docling lab benchmark | `claude/packet-s6-docling-lab-d6foa6` | #2057 | Open — awaiting review (2026-08-17) | Offline only: `check:docling-lab` 36 fixtures / 10 hostile / 6 canaries + Gate B template valid; `verify:pr-local` heavy plan failed:(none); contract test 20/20; legacy smoke 46 docs, 10/10 hostile contained, canary-clean report. Verdict is a separate owner dispatch of `docling-lab.yml` | -| S7+ | B4 shadow / B5 Ragas / B6 reranker / B7 DSPy | — | — | Gated — owner decision | — | -| #212 T1–T3 | Runtime row contracts (rag.ts, rag-candidate-sources.ts, src/app/api) — sibling stream sharing `src/lib/rag/**` | — | #1946 / #1981 / #2023 | Merged (T3 squash `440a34f71` 2026-08-17) | see the #212 ledger row; RAG surface complete for the cast class | -| #212 T4 | Runtime row contracts: `worker/main.ts` (11 casts) — sibling stream | `claude/ledger-212-tranche-4-worker-q3y6i4` | #2037 | Merged 2026-08-17 (squash `1726537b7`); #212 closed by reconcile PR #2045 | Governance Preflight complete; audit: 1 inbound cast (claim rows, per-row fail-soft) + 2 read-back param casts contracted, 9 outbound/interop left; closes #212 (inbox `done` queued in the PR) | +| Packet | Scope | Branch | PR | State | Canary / evidence refs | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | --------------------- | --------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Guide | Programme guide | `claude/rag-plan-review-guide-vhrls9` | #1895 | Merged 2026-08-13 | docs-only | +| Handover | Multi-session handover + coordination | `claude/rag-plan-review-guide-vhrls9` | #1908 / #2024 | Merged 2026-08-13; coordination layer PR #2024 | docs-only | +| S0 | A1 phase 1: structured fallback diagnostics | `claude/lithium-generation-quality-debug-ji1vce` | #1899 | Merged 2026-08-13 | offline 93/93 focused | +| S1 | A1 phase 2: rung-1 verification-faithfulness fixes | `claude/s1-rag-mitigation-231-86c182` | #2022 | Merged 2026-08-17 (squash `2bd146eed`, landed by content) | 8 pre-fix + 5 post-fix live probes 2026-08-17; offline 583/583; canary pair run 31964560921 (baseline `8f8d111ab`) -> run 32025082010 (`2bd146eed`): recall 1.0/1.0, zero per-case rr regressions, answer gate 44/44 (denominator reconciled by S5; see baseline-record §3); rung-2 measurement in `docs/audit/live-drift-forensics-2026-08.md` §5 | +| S1b | A1 rung 3 (R1): pre-deadline strong routing for dosing class | `claude/s1b-rag-dosing-routing-6u1mik` | #2035 | Merged 2026-08-17 (PR #2035, merge `92f7618`) | canary pair pending: baseline run 32025082010 (`2bd146eed`) -> post-merge dispatch (owner-approved); offline 586/586 + verify:pr-local heavy scope green | +| S1c | A1 residuals R2 + R3: claim-support strictness | `claude/s1c-residuals-r2-r3-4pb1at` | #2052 | PR open 2026-08-17 | offline 613/613 + verify:pr-local heavy scope green; R3 measured before loosening: 87->78 sole-overlap rejections (46->42 unique), zero protective flips; canary pair planned post-merge: baseline run 32039841070 (`92f7618`) -> post-merge dispatch (owner-approved) | +| S1d | A1 final-gate gap recovery: hedged cited low-confidence fast answers must recover extractively, not collapse to a citation-free `provider_source_gap` | `claude/s1d-final-gate-gap-recovery-dxgrn2` | #2054 | In review — PR open 2026-08-17 (`recoverFinalGateGapExtractively` in the finalizer; terminal gaps pinned) | needs post-merge canary pair (baseline run 32039841070, `e6ad0d5db`); offline: verify:pr-local heavy green, eval:rag:offline 604/604, 6 new discriminating fixtures | +| G1 | Governance: provenance tag for document-summary rows (Option B) | `claude/rag-g1-document-context-origin-` | — | Ready — disjoint; owner decided Option B 2026-08-17 | no canary (no behaviour change) | +| S2 | A2 (+A3): composition menu + moderate length | `claude/rag-a2-composition-` | — | Blocked on S1b + S1c | canary pair + `eval:answer-quality` + Gate E | +| S2b | A3: moderate length (if separate review needed) | `claude/rag-a3-length-` | — | Blocked on S2 | — | +| S3 | A4: follow-up suggestion refinement | `claude/rag-a4-follow-ups-` | — | Blocked on S2 + S2b | — | +| S4 | B0: adversarial fixtures + baseline + register | `claude/packet-s4-adversarial-fixtures-5ho5tp` | #2036 | Merged 2026-08-17 (squash `f5b093291`) | Offline only: `check:rag:adversarial-fixtures` 24 cases / 8 categories / 6 canaries; `eval:rag:offline` 24 suites, 597 tests. Baseline `scripts/fixtures/rag-adversarial-baseline.v1.json` marks the three provider-backed gates `pending_owner_run` | +| S5 | B1+B2: telemetry assessment + offline harness | `claude/s5-rag-telemetry-harness-2wvis7` | #2056 | Merged 2026-08-17 (merge `093f934`) | Offline only: `eval:rag:adversarial:offline` 25/25 (24 cases + canary-free report; 3 divergences pinned in `KNOWN_DIVERGENCES`); B1 gap = `verification_latency_ms` behind `RAG_TELEMETRY_EXTENDED` (default false); canary-absence tests green; 44/44 denominator reconciled | +| S6 | B3: Docling lab benchmark | `claude/packet-s6-docling-lab-d6foa6` | #2057 | Open — awaiting review (2026-08-17) | Offline only: `check:docling-lab` 36 fixtures / 10 hostile / 6 canaries + Gate B template valid; `verify:pr-local` heavy plan failed:(none); contract test 20/20; legacy smoke 46 docs, 10/10 hostile contained, canary-clean report. Verdict is a separate owner dispatch of `docling-lab.yml` | +| S7+ | B4 shadow / B5 Ragas / B6 reranker / B7 DSPy | — | — | Gated — owner decision | — | +| #212 T1–T3 | Runtime row contracts (rag.ts, rag-candidate-sources.ts, src/app/api) — sibling stream sharing `src/lib/rag/**` | — | #1946 / #1981 / #2023 | Merged (T3 squash `440a34f71` 2026-08-17) | see the #212 ledger row; RAG surface complete for the cast class | +| #212 T4 | Runtime row contracts: `worker/main.ts` (11 casts) — sibling stream | `claude/ledger-212-tranche-4-worker-q3y6i4` | #2037 | Merged 2026-08-17 (squash `1726537b7`); #212 closed by reconcile PR #2045 | Governance Preflight complete; audit: 1 inbound cast (claim rows, per-row fail-soft) + 2 read-back param casts contracted, 9 outbound/interop left; closes #212 (inbox `done` queued in the PR) | Update rule: the session that opens a packet's PR edits its row (branch, PR number, state) in the same PR. A later session updating another packet may also correct stale rows diff --git a/src/lib/rag/rag-extractive-answer.ts b/src/lib/rag/rag-extractive-answer.ts index a9de572dd9..e103484770 100644 --- a/src/lib/rag/rag-extractive-answer.ts +++ b/src/lib/rag/rag-extractive-answer.ts @@ -54,6 +54,7 @@ import { } from "@/lib/rag/rag-answer-text"; import { cloneAnswer } from "@/lib/rag/rag-cache"; import { ragProviderMode } from "@/lib/rag/rag-provider"; +import { buildSmartRagApiPlan } from "@/lib/smart-rag-api"; import { isLowYieldClinicalText, normalizeInlineBulletGlyphs, @@ -3786,6 +3787,78 @@ function applyProviderLabels(answer: RagAnswer): RagAnswer { }; } +/** + * Fast-route final-gate gap recovery: when a grounded, cited, low-confidence fast + * model answer over strong routine retrieval is only gap-like phrasing, rebuild a + * deterministic source-backed answer from the evidence it already cites instead of + * shipping a citation-free evidence gap. Mirrors the rag.ts generation-fallback + * recovery, which handles the thrown variants of the same failure — today, model + * phrasing alone decides which of the two branches a gap-shaped fast answer takes. + * Returns null to keep the gap terminal: genuinely empty retrieval, strong-route + * gaps, and comparison/dose/threshold classes (whose fallbacks need dedicated + * single-chunk handling) never recover here. The rebuilt candidate is routingMode + * "extractive", so the re-entrant validation finalize cannot re-trigger this + * fast-gated recovery. + */ +function recoverFinalGateGapExtractively( + answer: RagAnswer, + query: string, + queryClass: RagQueryClass, + gapReason: string, +): RagAnswer | null { + if (answer.routingMode !== "fast") return null; + if (!answer.routingReason?.includes("strong_routine_retrieval")) return null; + if ((answer.sources?.length ?? 0) === 0) return null; + if (queryClass === "comparison" || queryClass === "medication_dose_risk" || queryClass === "table_threshold") { + return null; + } + // Band-coherence source conflicts already have a throw-based recovery in rag.ts. + if (answer.routingReason.includes("numeric_band_coherence_gate_source_conflict")) return null; + const recoveryRouteReason = [ + answer.routingReason, + `generation_fallback:${gapReason}`, + "source_backed_extractive_fallback", + `final_quality_gate_source_backed_recovery:${gapReason}`, + ].join("; "); + const candidate = buildExtractiveAnswer({ + query, + queryClass, + results: answer.sources, + quoteCards: answer.quoteCards ?? [], + documentBreakdown: answer.documentBreakdown ?? [], + evidenceSummary: answer.evidenceSummary, + sourceCoverage: answer.sourceCoverage, + conflictsOrGaps: answer.conflictsOrGaps ?? [], + visualEvidence: answer.visualEvidence ?? [], + bestSource: answer.bestSource ?? null, + smartPanel: answer.smartPanel, + relatedDocuments: answer.relatedDocuments ?? [], + routeReason: recoveryRouteReason, + timings: answer.latencyTimings, + }); + if (!candidate.grounded || candidate.confidence === "unsupported" || candidate.citations.length === 0) return null; + if (isBareCrossReferenceAnswer(candidate.answer ?? "")) return null; + const smartApiPlan = buildSmartRagApiPlan({ + query, + queryClass, + results: candidate.sources, + routeMode: "extractive", + routeReason: candidate.routingReason, + conflictsOrGaps: candidate.conflictsOrGaps ?? [], + }); + const merged: RagAnswer = { + ...answer, + ...candidate, + modelUsed: null, + supportedClaims: undefined, + evidenceAssessments: undefined, + smartApiPlan, + responseMode: smartApiPlan.displayMode, + }; + if (!isSafeExtractiveFallbackCandidate(merged, query, queryClass)) return null; + return merged; +} + // Public wrapper: runs quality finalization, then stamps provider/quality labels so the UI can // disclose source-only (lower-quality) answers and verify-against-sources guidance. /** Finalize rag answer quality. */ @@ -3826,8 +3899,12 @@ function finalizeRagAnswerQualityCore(answer: RagAnswer, query: string, queryCla const existingGapAnswer = gapLikeAnswer && (!answer.grounded || answer.routingMode === "strong" || answer.confidence === "low"); if (existingGapAnswer) { - const gapAnswer = finalQualityGapAnswer(query, queryClass); const gapReason = answer.modelUsed ? "provider_source_gap" : "source_gap"; + const recovered = recoverFinalGateGapExtractively(answer, query, queryClass, gapReason); + // Terminates: the recovered answer is routingMode "extractive", so the fast-gated + // recovery cannot re-fire on this re-run. + if (recovered) return finalizeRagAnswerQualityCore(recovered, query, queryClass); + const gapAnswer = finalQualityGapAnswer(query, queryClass); return { ...answer, answer: gapAnswer, diff --git a/tests/extractive-answer-formatting.test.ts b/tests/extractive-answer-formatting.test.ts index 7c5864feb2..bc8b959f34 100644 --- a/tests/extractive-answer-formatting.test.ts +++ b/tests/extractive-answer-formatting.test.ts @@ -350,6 +350,129 @@ describe("provider source-gap lifecycle", () => { }), ).toBe(false); }); + + const dischargeChunks = [ + figureChunk({ + id: "discharge-planning-start", + document_id: "discharge-guidance", + title: "Admission to Discharge for Mental Health Inpatients (NMHS)", + file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf", + page_number: 4, + section_heading: "Discharge planning", + content: + "Clinicians will actively plan effective and timely discharge from the beginning of admission and review the plan throughout the inpatient stay.", + similarity: 0.97, + hybrid_score: 0.97, + }), + figureChunk({ + id: "discharge-documentation", + document_id: "discharge-guidance", + title: "Admission to Discharge for Mental Health Inpatients (NMHS)", + file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf", + page_number: 5, + section_heading: "Discharge documentation", + content: + "The discharge plan must document ongoing care arrangements, communicate the plan with the consumer, and identify follow-up responsibilities.", + similarity: 0.97, + hybrid_score: 0.97, + }), + ]; + + // The S1d defect shape: a hedged, cited, grounded, low-confidence fast answer whose + // lead misses providerSourceGapLeadPattern but whose body matches the finalizer's + // broad gap-like regex ("do not provide specific") and survives sanitizeAnswerText. + const hedgedGapLikeFastAnswer = (overrides: Partial = {}): RagAnswer => + ({ + answer: + "Discharge planning begins at admission and the plan is reviewed during the inpatient stay. The discharge documents do not provide specific timing details.", + grounded: true, + confidence: "low", + citations: dischargeChunks.map((chunk) => citationFromResult(chunk)), + sources: dischargeChunks, + answerSections: [], + quoteCards: [], + bestSource: null, + routingReason: "strong_routine_retrieval", + routingMode: "fast", + modelUsed: "gpt-test", + queryClass: "broad_summary", + ...overrides, + }) as RagAnswer; + const dischargeQuery = "Summarize the discharge guidance"; + + it("rebuilds a source-backed extractive answer when a cited low-confidence fast answer is only gap-like phrasing", () => { + const answer = finalizeRagAnswerQuality(hedgedGapLikeFastAnswer(), dischargeQuery, "broad_summary"); + + expect(answer.routingMode).toBe("extractive"); + expect(answer.grounded).toBe(true); + expect(answer.confidence).not.toBe("unsupported"); + expect(answer.citations.length).toBeGreaterThan(0); + expect(answer.responseMode).toBeDefined(); + expect(answer.responseMode).not.toBe("evidence_gap"); + expect(answer.answer).not.toMatch(/do not provide specific/i); + expect(answer.routingReason).toContain("generation_fallback:provider_source_gap"); + expect(answer.routingReason).toContain("source_backed_extractive_fallback"); + expect(answer.routingReason).toContain("final_quality_gate_source_backed_recovery:provider_source_gap"); + expect(answer.routingReason).not.toMatch(/final_quality_gate:/); + expect(answer.modelUsed).toBeNull(); + expect(answer.answerQualityTier).toBe("source_only"); + }); + + it("keeps a zero-source fast gap-like answer terminal", () => { + const answer = finalizeRagAnswerQuality( + hedgedGapLikeFastAnswer({ sources: [], citations: [] }), + dischargeQuery, + "broad_summary", + ); + + expect(answer.grounded).toBe(false); + expect(answer.confidence).toBe("unsupported"); + expect(answer.citations).toEqual([]); + expect(answer.responseMode).toBe("evidence_gap"); + expect(answer.routingReason).toContain("final_quality_gate:provider_source_gap"); + }); + + it("keeps a strong-route gap-like answer terminal", () => { + const answer = finalizeRagAnswerQuality( + hedgedGapLikeFastAnswer({ routingMode: "strong" }), + dischargeQuery, + "broad_summary", + ); + + expect(answer.grounded).toBe(false); + expect(answer.confidence).toBe("unsupported"); + expect(answer.citations).toEqual([]); + expect(answer.responseMode).toBe("evidence_gap"); + expect(answer.routingReason).toContain("final_quality_gate:provider_source_gap"); + }); + + it("keeps a comparison-class gap-like answer terminal", () => { + const answer = finalizeRagAnswerQuality( + hedgedGapLikeFastAnswer({ queryClass: "comparison" }), + dischargeQuery, + "comparison", + ); + + expect(answer.grounded).toBe(false); + expect(answer.confidence).toBe("unsupported"); + expect(answer.citations).toEqual([]); + expect(answer.responseMode).toBe("evidence_gap"); + expect(answer.routingReason).toContain("final_quality_gate:provider_source_gap"); + }); + + it("does not recover outside strong routine retrieval", () => { + const answer = finalizeRagAnswerQuality( + hedgedGapLikeFastAnswer({ routingReason: "retrieval_gap_or_conflict" }), + dischargeQuery, + "broad_summary", + ); + + expect(answer.grounded).toBe(false); + expect(answer.confidence).toBe("unsupported"); + expect(answer.citations).toEqual([]); + expect(answer.responseMode).toBe("evidence_gap"); + expect(answer.routingReason).toContain("final_quality_gate:provider_source_gap"); + }); }); describe("extractive malformed-fragment signatures", () => { diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 6ecf651565..b8790cd19f 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -340,6 +340,56 @@ describe("RAG structured-output fallback", () => { expect(answer.answer).not.toMatch(/^No current source/i); }); + it("recovers a grounded low-confidence cited source-gap phrasing at the final quality gate without a strong retry", async () => { + // The S1d shape: grounded, cited, confidence "low", substantive lead misses + // providerSourceGapLeadPattern, hedged body matches the finalizer's broad + // gap-like regex ("do not provide specific") and survives sanitizeAnswerText. + // It passes every in-loop fast-failure screen and used to collapse to a + // citation-free evidence_gap in finalizeRagAnswerQualityCore. + const dischargeSources = [ + source({ + id: "discharge-planning-start", + document_id: "discharge-guidance", + title: "Admission to Discharge for Mental Health Inpatients (NMHS)", + file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf", + section_heading: "Discharge planning", + content: + "Clinicians will actively plan effective and timely discharge from the beginning of admission and review the plan throughout the inpatient stay.", + }), + source({ + id: "discharge-documentation", + document_id: "discharge-guidance", + title: "Admission to Discharge for Mental Health Inpatients (NMHS)", + file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf", + section_heading: "Discharge documentation", + content: + "The discharge plan must document ongoing care arrangements, communicate the plan with the consumer, and identify follow-up responsibilities.", + }), + ]; + const answer = await answerFromTextSources("Summarize the discharge guidance", dischargeSources, { + answer: + "Discharge planning begins at admission and the plan is reviewed during the inpatient stay. The discharge documents do not provide specific timing details.", + grounded: true, + confidence: "low", + answerSections: [], + citations: [{ chunk_id: "discharge-planning-start" }, { chunk_id: "discharge-documentation" }], + quoteCards: [], + conflictsOrGaps: [], + }); + + expect(answer.routingMode).toBe("extractive"); + expect(answer.routingReason).toContain("generation_fallback:provider_source_gap"); + expect(answer.routingReason).toContain("source_backed_extractive_fallback"); + expect(answer.routingReason).toContain("final_quality_gate_source_backed_recovery:provider_source_gap"); + expect(answer.routingReason).not.toMatch(/final_quality_gate:/); + expect(answer.grounded).toBe(true); + expect(answer.citations.length).toBeGreaterThan(0); + expect(answer.answer).not.toMatch(/do not provide specific/i); + expect(answer.latencyTimings?.answer_retry_reasons ?? []).not.toContain("fast_source_gap_retry_strong"); + expect(answer.latencyTimings?.answer_retry_reasons ?? []).not.toContain("fast_unsupported_retry_strong"); + expect(answer.latencyTimings?.answer_retry_reasons ?? []).not.toContain("fast_quality_retry_strong"); + }); + it("keeps provider-failed complex comparisons on the source-attributed comparison fallback", async () => { const comparisonFact = (documentId: string, chunkId: string, value: string) => ({ id: `${documentId}-threshold`,