diff --git a/src/lib/rag/rag-extractive-answer.ts b/src/lib/rag/rag-extractive-answer.ts index a9de572dd9..df02664b73 100644 --- a/src/lib/rag/rag-extractive-answer.ts +++ b/src/lib/rag/rag-extractive-answer.ts @@ -3276,6 +3276,7 @@ export function generatedAnswerQualityFailureReason(answer: RagAnswer, query: st return "missing_query_overlap"; } if (hasInvalidModelEvidenceIds(answer)) return "invalid_model_evidence_ids"; + if (answer.unverifiedNumericTokens?.length) return "numeric_faithfulness_gap"; const broadDocumentCoverageRequested = queryClass === "document_lookup" && /(?:\b(?:what|which)\b.{0,100}\b(?:include|included|require|required|requirements?)\b|\b(?:process|procedure)\b|\bhow\b.{0,80}\b(?:handled|managed|performed|completed)\b)/i.test( diff --git a/src/lib/rag/rag-generation-failure.ts b/src/lib/rag/rag-generation-failure.ts new file mode 100644 index 0000000000..b67dbde5f9 --- /dev/null +++ b/src/lib/rag/rag-generation-failure.ts @@ -0,0 +1,56 @@ +const PROVIDER_SAFE_GENERATION_QUALITY_FAILURE_REASONS = new Set([ + "empty_after_sanitize", + "provider_source_gap", + "incomplete_opening_sentence", + "bad_final_answer_quality", + "clinical_answer_quality_issue", + "low_yield_answer", + "fragment_like_answer", + "missing_query_intent", + "missing_query_overlap", + "invalid_model_evidence_ids", + "insufficient_broad_citation_coverage", + "unusable_generated_answer", + "template_like_answer", + "overexpanded_simple_answer", + "claim_support_high_risk_gap", + "material_source_governance_gap", + "numeric_band_coherence_gap", + "numeric_faithfulness_gap", +]); + +/** Reduce generation errors to bounded, provider-safe diagnostic metadata. */ +export function summarizeGenerationFailureReason(error: unknown) { + const message = (error instanceof Error ? error.message : typeof error === "string" ? error : "").trim(); + const normalized = message.toLowerCase(); + const sourceBackedRecovery = normalized.match(/\bsource_backed_extractive_recovery:([a-z0-9_]+)/); + + if (sourceBackedRecovery) return `source_backed_extractive_recovery_${sourceBackedRecovery[1]}`; + if (!normalized) return "generation_failed"; + if (/\bprovider_source_gap\b/.test(normalized)) return "provider_source_gap"; + + const qualityFailure = normalized.match(/^openai generation quality gate failed:\s*([a-z0-9_]+)$/); + if (qualityFailure && PROVIDER_SAFE_GENERATION_QUALITY_FAILURE_REASONS.has(qualityFailure[1])) { + return `generation_quality_failed_${qualityFailure[1]}`; + } + if (normalized.startsWith("openai generation quality gate failed:")) return "generation_quality_failed"; + + if (/\bmax_output_tokens\b/.test(normalized)) return "provider_incomplete_max_output_tokens"; + if (/\bincomplete\b/.test(normalized)) return "provider_incomplete"; + if (/\brate limit|rate_limited|429\b/.test(normalized)) return "provider_rate_limited"; + if (/\btimeout|timed out|deadline|aborted|etimedout\b/.test(normalized)) return "provider_timeout"; + if (/\bauthentication|api key|unauthori[sz]ed|401|403\b/.test(normalized)) return "provider_auth_failed"; + + if (/\bvalidation|quality gate|schema|parse|json\b/.test(normalized)) return "generation_quality_failed"; + if (/\bopenai|provider|model\b/.test(normalized)) return "provider_generation_failed"; + return "generation_failed"; +} + +/** Build the bounded repair instruction for an existing strong-model retry. */ +export function generationQualityRetryInstruction(failureReason: string) { + const numericFaithfulnessInstruction = + failureReason === "numeric_faithfulness_gap" + ? " The previous answer included a numeric token that deterministic verification could not match to its cited retrieved evidence. Include a number, dose, frequency, threshold, or timing only when its exact digits and unit appear in the cited source excerpt; otherwise omit it. Do not infer, convert, calculate, round, or combine figures." + : ""; + return `The previous answer failed deterministic validation (${failureReason}).${numericFaithfulnessInstruction} Return schema-valid output only, with a complete natural clinical synthesis in the answer field. The first sentence must directly answer the question as a full sentence. Every clinical claim must be supported by valid retrieved citation_chunk_id values; do not invent citation IDs. Within one named scale and source, if differently labelled intervals overlap or a range is reversed, omit the entire affected band set; do not quote, repair, or infer any label or value. If a separate sentence or clause states a nonnumeric condition and action independent of the score, answer only with that independently supported condition and action, cite the smallest sufficient directly supporting chunk set, and add a conflict entry; otherwise return a source gap. Avoid template/source-inventory wording and do not include JSON fragments inside text fields. If the evidence cannot support the requested clinical answer, return a concise source-gap answer instead. If the question is a simple definition or direct fact question, answer only that question and return answerSections as an empty array unless a source-gap or safety caveat is essential.`; +} diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index debe71a516..0edf9f52b2 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -36,6 +36,7 @@ import { isSourceOnlyMode, sourceOnlyReason, } from "@/lib/rag/rag-provider"; +import { generationQualityRetryInstruction, summarizeGenerationFailureReason } from "@/lib/rag/rag-generation-failure"; import { allowedChunkMap, citationFromResult as resultCitation, compactCitations } from "@/lib/citations"; import { assessAndEnforceClaimSupport, enforceLabelledNumericBandCoherence } from "@/lib/rag/rag-claim-support"; import { @@ -3375,6 +3376,7 @@ ${qualityRetryInstruction}` "fast_unusable_retry_strong", "fast_template_retry_strong", "fast_quality_retry_strong", + "fast_numeric_faithfulness_retry_strong", ]); const eligibleForRoutineExtractiveRecovery = route.mode === "fast" && @@ -3399,25 +3401,6 @@ ${qualityRetryInstruction}` }); } - /** Summarize generation failure reason. */ - function summarizeGenerationFailureReason(error: unknown) { - const message = (error instanceof Error ? error.message : typeof error === "string" ? error : "").trim(); - const normalized = message.toLowerCase(); - const sourceBackedRecovery = normalized.match(/\bsource_backed_extractive_recovery:([a-z0-9_]+)/); - - if (sourceBackedRecovery) return `source_backed_extractive_recovery_${sourceBackedRecovery[1]}`; - if (!normalized) return "generation_failed"; - if (/\bprovider_source_gap\b/.test(normalized)) return "provider_source_gap"; - if (/\bmax_output_tokens\b/.test(normalized)) return "provider_incomplete_max_output_tokens"; - if (/\bincomplete\b/.test(normalized)) return "provider_incomplete"; - if (/\brate limit|rate_limited|429\b/.test(normalized)) return "provider_rate_limited"; - if (/\btimeout|timed out|deadline|aborted|etimedout\b/.test(normalized)) return "provider_timeout"; - if (/\bauthentication|api key|unauthori[sz]ed|401|403\b/.test(normalized)) return "provider_auth_failed"; - if (/\bvalidation|quality gate|schema|parse|json\b/.test(normalized)) return "generation_quality_failed"; - if (/\bopenai|provider|model\b/.test(normalized)) return "provider_generation_failed"; - return "generation_failed"; - } - /** Build generation fallback answer. */ async function buildGenerationFallbackAnswer( error: unknown, @@ -3598,12 +3581,13 @@ ${qualityRetryInstruction}` !fastSourceGap && !fastAnswerWasTemplateLike && shouldRetryWithStrongAfterFast({ route, answer, results: answerInputResults }); - const fastAnswerFailedQualityGate = + const fastQualityFailureReason = route.mode === "fast" && !fastAnswerWasUnusable && !fastAnswerWasTemplateLike && !fastAnswerWasOverExpanded && - Boolean(generatedAnswerQualityFailureReason(answer, args.query, queryClass)); + generatedAnswerQualityFailureReason(answer, args.query, queryClass); + const fastAnswerFailedQualityGate = Boolean(fastQualityFailureReason); if ( fastAnswerHadInvalidEvidenceIds || fastSourceGap || @@ -3613,19 +3597,22 @@ ${qualityRetryInstruction}` fastAnswerWasOverExpanded || fastAnswerFailedQualityGate ) { - const retryReason = fastAnswerHadInvalidEvidenceIds - ? "fast_invalid_evidence_retry_strong" - : fastSourceGap - ? "fast_source_gap_retry_strong" - : fastAnswerWasUnsupported - ? "fast_unsupported_retry_strong" - : fastAnswerWasUnusable - ? "fast_unusable_retry_strong" - : fastAnswerWasTemplateLike - ? "fast_template_retry_strong" - : fastAnswerWasOverExpanded - ? "fast_overexpanded_simple_retry_strong" - : "fast_quality_retry_strong"; + const retryReason = + fastQualityFailureReason === "numeric_faithfulness_gap" + ? "fast_numeric_faithfulness_retry_strong" + : fastAnswerHadInvalidEvidenceIds + ? "fast_invalid_evidence_retry_strong" + : fastSourceGap + ? "fast_source_gap_retry_strong" + : fastAnswerWasUnsupported + ? "fast_unsupported_retry_strong" + : fastAnswerWasUnusable + ? "fast_unusable_retry_strong" + : fastAnswerWasTemplateLike + ? "fast_template_retry_strong" + : fastAnswerWasOverExpanded + ? "fast_overexpanded_simple_retry_strong" + : "fast_quality_retry_strong"; if (shouldRecoverFastFailureExtractively(retryReason)) { answerRetryCount += 1; answerRetryReasons.push(`fast_source_backed_extractive_recovery:${retryReason}`); @@ -3639,19 +3626,21 @@ ${qualityRetryInstruction}` await args.onProgress?.({ stage: "retrying", message: - retryReason === "fast_invalid_evidence_retry_strong" - ? "Fast answer cited invalid evidence IDs, retrying with the strong model." - : retryReason === "fast_source_gap_retry_strong" - ? "Fast answer returned a source gap despite strong retrieval, retrying with the strong model." - : retryReason === "fast_unsupported_retry_strong" - ? "Fast answer was unsupported, retrying with the strong model." - : retryReason === "fast_unusable_retry_strong" - ? "Fast answer was not usable, retrying with the strong model." - : retryReason === "fast_template_retry_strong" - ? "Fast answer was too template-like, retrying with the strong model." - : retryReason === "fast_overexpanded_simple_retry_strong" - ? "Fast answer over-expanded a simple question, retrying with the strong model." - : "Fast answer failed quality checks, retrying with the strong model.", + retryReason === "fast_numeric_faithfulness_retry_strong" + ? "Fast answer included an unverified figure, retrying with exact numeric-grounding instructions." + : retryReason === "fast_invalid_evidence_retry_strong" + ? "Fast answer cited invalid evidence IDs, retrying with the strong model." + : retryReason === "fast_source_gap_retry_strong" + ? "Fast answer returned a source gap despite strong retrieval, retrying with the strong model." + : retryReason === "fast_unsupported_retry_strong" + ? "Fast answer was unsupported, retrying with the strong model." + : retryReason === "fast_unusable_retry_strong" + ? "Fast answer was not usable, retrying with the strong model." + : retryReason === "fast_template_retry_strong" + ? "Fast answer was too template-like, retrying with the strong model." + : retryReason === "fast_overexpanded_simple_retry_strong" + ? "Fast answer over-expanded a simple question, retrying with the strong model." + : "Fast answer failed quality checks, retrying with the strong model.", mode: "strong", model: env.OPENAI_STRONG_ANSWER_MODEL, reason: routingReason, @@ -3663,6 +3652,10 @@ ${qualityRetryInstruction}` generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { strong: true, maxOutputTokensOverride: strongRetryMaxOutputTokens, + qualityRetryInstruction: + fastQualityFailureReason === "numeric_faithfulness_gap" + ? generationQualityRetryInstruction(fastQualityFailureReason) + : undefined, }); retrievalDiagnostics.routeMode = "strong"; if (generated.truncated) { @@ -3697,7 +3690,7 @@ ${qualityRetryInstruction}` // valid (if imperfect) cited strong answer instead of spending a third generation // and risking a truncation -> unsupported tail. Recorded for observability. answerRetryReasons.push(`strong_quality_repair_skipped_time_budget:${strongQualityFailureReason}`); - } else if (answerNeedsStrongQualityRepair) { + } else if (strongQualityFailureReason) { routingReason = `${routingReason}; strong_quality_retry`; answerRetryCount += 1; answerRetryReasons.push("strong_quality_retry"); @@ -3711,7 +3704,7 @@ ${qualityRetryInstruction}` generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { strong: true, maxOutputTokensOverride: strongRetryMaxOutputTokens, - qualityRetryInstruction: `The previous answer failed deterministic validation (${strongQualityFailureReason}). Return schema-valid output only, with a complete natural clinical synthesis in the answer field. The first sentence must directly answer the question as a full sentence. Every clinical claim must be supported by valid retrieved citation_chunk_id values; do not invent citation IDs. Within one named scale and source, if differently labelled intervals overlap or a range is reversed, omit the entire affected band set; do not quote, repair, or infer any label or value. If a separate sentence or clause states a nonnumeric condition and action independent of the score, answer only with that independently supported condition and action, cite the smallest sufficient directly supporting chunk set, and add a conflict entry; otherwise return a source gap. Avoid template/source-inventory wording and do not include JSON fragments inside text fields. If the evidence cannot support the requested clinical answer, return a concise source-gap answer instead. If the question is a simple definition or direct fact question, answer only that question and return answerSections as an empty array unless a source-gap or safety caveat is essential.`, + qualityRetryInstruction: generationQualityRetryInstruction(strongQualityFailureReason), }); retrievalDiagnostics.routeMode = "strong"; if (generated.truncated) { diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 5d05302bbe..77cffa5018 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -91,8 +91,8 @@ type GeneratedAnswerPayload = { async function answerFromTextSources( query: string, sources: SearchResult[], - generatedAnswer?: GeneratedAnswerPayload | Error, - options: { sourceOnly?: boolean } = {}, + generatedAnswer?: GeneratedAnswerPayload | Error | Array, + options: { sourceOnly?: boolean; onGenerate?: (input: string, index: number) => void } = {}, ) { // `src/lib/env.ts` freezes process.env at module load. The offline vitest wrapper // starts every worker as RAG_PROVIDER_MODE=offline with a blank OpenAI key, so we @@ -116,11 +116,16 @@ async function answerFromTextSources( from: vi.fn(() => new EmptyQuery()), }), })); - const generateStructuredTextResult = vi.fn(async () => { - if (generatedAnswer instanceof Error) throw generatedAnswer; + const generatedAnswers = Array.isArray(generatedAnswer) ? generatedAnswer : [generatedAnswer]; + let generatedAnswerIndex = 0; + const generateStructuredTextResult = vi.fn(async (input: string) => { + options.onGenerate?.(input, generatedAnswerIndex); + const currentGeneratedAnswer = generatedAnswers[Math.min(generatedAnswerIndex, generatedAnswers.length - 1)]; + generatedAnswerIndex += 1; + if (currentGeneratedAnswer instanceof Error) throw currentGeneratedAnswer; return { text: JSON.stringify( - generatedAnswer ?? { + currentGeneratedAnswer ?? { answer: "No current source with specific guidance for this query was found.", grounded: false, confidence: "unsupported", @@ -165,6 +170,59 @@ afterEach(() => { }); describe("RAG structured-output fallback", () => { + it("repairs an unsupported fast-answer figure with the existing strong retry", async () => { + const answer = await answerFromTextSources( + "Lithium dosing?", + [ + source({ + id: "lithium-dose-source", + document_id: "lithium-guideline", + title: "Medication guideline", + file_name: "medication-guideline.pdf", + section_heading: "Lithium initiation", + content: "Start lithium carbonate at 250 mg once daily and review tolerability.", + similarity: 0.94, + hybrid_score: 0.94, + text_rank: 0.09, + }), + ], + [ + { + answer: "Start lithium carbonate at 500 mg once daily.", + grounded: true, + confidence: "high", + answerSections: [], + citations: [{ chunk_id: "lithium-dose-source" }], + quoteCards: [], + conflictsOrGaps: [], + }, + { + answer: "Start lithium carbonate at 250 mg once daily.", + grounded: true, + confidence: "high", + answerSections: [], + citations: [{ chunk_id: "lithium-dose-source" }], + quoteCards: [], + conflictsOrGaps: [], + }, + ], + { + onGenerate: (input, index) => { + if (index !== 1) return; + expect(input).toContain("numeric_faithfulness_gap"); + expect(input).toContain("exact digits and unit appear in the cited source excerpt"); + }, + }, + ); + + expect(answer.routingMode).toBe("strong"); + expect(answer.routingReason).toContain("fast_numeric_faithfulness_retry_strong"); + expect(answer.routingReason).not.toContain("generation_fallback"); + expect(answer.answer.replace(/\*\*/g, "")).toContain("250 mg"); + expect(answer.unverifiedNumericTokens ?? []).toEqual([]); + expect(answer.openAIRequestIds).toEqual(["req_answer_from_text_sources", "req_answer_from_text_sources"]); + }); + it("recovers a cited provider source gap instead of treating nearby citations as a grounded answer", async () => { const dischargeSources = [ source({ @@ -1510,13 +1568,13 @@ describe("RAG structured-output fallback", () => { text_rank: 1, }), ], - new Error("OpenAI generation quality gate failed: labelled numeric band conflict"), + new Error("OpenAI generation quality gate failed: numeric_band_coherence_gap"), ); const deliveredText = `${answer.answer} ${(answer.answerSections ?? []).map((section) => section.body).join(" ")}`.replace(/\*\*/g, ""); expect(answer.grounded).toBe(true); - expect(answer.routingReason).toContain("generation_fallback:generation_quality_failed"); + expect(answer.routingReason).toContain("generation_fallback:generation_quality_failed_numeric_band_coherence_gap"); expect(answer.routingReason).toContain("source_backed_extractive_fallback"); expect(deliveredText).toMatch( /any side effect which is causing distress irrespective of score should be escalated to the treating doctor and reviewed/i, diff --git a/tests/rag-generation-failure.test.ts b/tests/rag-generation-failure.test.ts new file mode 100644 index 0000000000..d3b028e374 --- /dev/null +++ b/tests/rag-generation-failure.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { + generationQualityRetryInstruction, + summarizeGenerationFailureReason, +} from "../src/lib/rag/rag-generation-failure"; + +describe("generation failure diagnostics", () => { + it("preserves an allowlisted deterministic quality-gate reason", () => { + expect( + summarizeGenerationFailureReason(new Error("OpenAI generation quality gate failed: missing_query_intent")), + ).toBe("generation_quality_failed_missing_query_intent"); + }); + + it("preserves an allowlisted post-finalization quality-gate reason", () => { + expect( + summarizeGenerationFailureReason(new Error("OpenAI generation quality gate failed: numeric_faithfulness_gap")), + ).toBe("generation_quality_failed_numeric_faithfulness_gap"); + }); + + it("does not confuse an incomplete-opening quality gate with provider truncation", () => { + expect( + summarizeGenerationFailureReason(new Error("OpenAI generation quality gate failed: incomplete_opening_sentence")), + ).toBe("generation_quality_failed_incomplete_opening_sentence"); + }); + + it("does not expose an unknown quality-gate message", () => { + expect( + summarizeGenerationFailureReason( + new Error("OpenAI generation quality gate failed: incomplete private patient details"), + ), + ).toBe("generation_quality_failed"); + }); + + it("keeps provider failure classification unchanged", () => { + expect(summarizeGenerationFailureReason(new Error("OpenAI provider timed out"))).toBe("provider_timeout"); + }); + + it("adds a bounded numeric-faithfulness repair instruction", () => { + const instruction = generationQualityRetryInstruction("numeric_faithfulness_gap"); + + expect(instruction).toContain("exact digits and unit appear in the cited source excerpt"); + expect(instruction).toContain("Do not infer, convert, calculate, round, or combine figures"); + expect(instruction).toContain("numeric_faithfulness_gap"); + }); +});