fix(rag): retry unverified numeric answers safely

src/lib/rag/rag-extractive-answer.ts

@@ -3276,6 +3276,7 @@ export function generatedAnswerQualityFailureReason(answer: RagAnswer, query: st
     return "missing_query_overlap";
   }
   if (hasInvalidModelEvidenceIds(answer)) return "invalid_model_evidence_ids";
+  if (answer.unverifiedNumericTokens?.length) return "numeric_faithfulness_gap";
   const broadDocumentCoverageRequested =
     queryClass === "document_lookup" &&
     /(?:\b(?:what|which)\b.{0,100}\b(?:include|included|require|required|requirements?)\b|\b(?:process|procedure)\b|\bhow\b.{0,80}\b(?:handled|managed|performed|completed)\b)/i.test(

src/lib/rag/rag-generation-failure.ts

@@ -0,0 +1,56 @@
+const PROVIDER_SAFE_GENERATION_QUALITY_FAILURE_REASONS = new Set([
+  "empty_after_sanitize",
+  "provider_source_gap",
+  "incomplete_opening_sentence",
+  "bad_final_answer_quality",
+  "clinical_answer_quality_issue",
+  "low_yield_answer",
+  "fragment_like_answer",
+  "missing_query_intent",
+  "missing_query_overlap",
+  "invalid_model_evidence_ids",
+  "insufficient_broad_citation_coverage",
+  "unusable_generated_answer",
+  "template_like_answer",
+  "overexpanded_simple_answer",
+  "claim_support_high_risk_gap",
+  "material_source_governance_gap",
+  "numeric_band_coherence_gap",
+  "numeric_faithfulness_gap",
+]);
+
+/** Reduce generation errors to bounded, provider-safe diagnostic metadata. */
+export function summarizeGenerationFailureReason(error: unknown) {
+  const message = (error instanceof Error ? error.message : typeof error === "string" ? error : "").trim();
+  const normalized = message.toLowerCase();
+  const sourceBackedRecovery = normalized.match(/\bsource_backed_extractive_recovery:([a-z0-9_]+)/);
+
+  if (sourceBackedRecovery) return `source_backed_extractive_recovery_${sourceBackedRecovery[1]}`;
+  if (!normalized) return "generation_failed";
+  if (/\bprovider_source_gap\b/.test(normalized)) return "provider_source_gap";
+
+  const qualityFailure = normalized.match(/^openai generation quality gate failed:\s*([a-z0-9_]+)$/);
+  if (qualityFailure && PROVIDER_SAFE_GENERATION_QUALITY_FAILURE_REASONS.has(qualityFailure[1])) {
+    return `generation_quality_failed_${qualityFailure[1]}`;
+  }
+  if (normalized.startsWith("openai generation quality gate failed:")) return "generation_quality_failed";
+
+  if (/\bmax_output_tokens\b/.test(normalized)) return "provider_incomplete_max_output_tokens";
+  if (/\bincomplete\b/.test(normalized)) return "provider_incomplete";
+  if (/\brate limit|rate_limited|429\b/.test(normalized)) return "provider_rate_limited";
+  if (/\btimeout|timed out|deadline|aborted|etimedout\b/.test(normalized)) return "provider_timeout";
+  if (/\bauthentication|api key|unauthori[sz]ed|401|403\b/.test(normalized)) return "provider_auth_failed";
+
+  if (/\bvalidation|quality gate|schema|parse|json\b/.test(normalized)) return "generation_quality_failed";
+  if (/\bopenai|provider|model\b/.test(normalized)) return "provider_generation_failed";
+  return "generation_failed";
+}
+
+/** Build the bounded repair instruction for an existing strong-model retry. */
+export function generationQualityRetryInstruction(failureReason: string) {
+  const numericFaithfulnessInstruction =
+    failureReason === "numeric_faithfulness_gap"
+      ? " The previous answer included a numeric token that deterministic verification could not match to its cited retrieved evidence. Include a number, dose, frequency, threshold, or timing only when its exact digits and unit appear in the cited source excerpt; otherwise omit it. Do not infer, convert, calculate, round, or combine figures."
+      : "";
+  return `The previous answer failed deterministic validation (${failureReason}).${numericFaithfulnessInstruction} Return schema-valid output only, with a complete natural clinical synthesis in the answer field. The first sentence must directly answer the question as a full sentence. Every clinical claim must be supported by valid retrieved citation_chunk_id values; do not invent citation IDs. Within one named scale and source, if differently labelled intervals overlap or a range is reversed, omit the entire affected band set; do not quote, repair, or infer any label or value. If a separate sentence or clause states a nonnumeric condition and action independent of the score, answer only with that independently supported condition and action, cite the smallest sufficient directly supporting chunk set, and add a conflict entry; otherwise return a source gap. Avoid template/source-inventory wording and do not include JSON fragments inside text fields. If the evidence cannot support the requested clinical answer, return a concise source-gap answer instead. If the question is a simple definition or direct fact question, answer only that question and return answerSections as an empty array unless a source-gap or safety caveat is essential.`;
+}

src/lib/rag/rag.ts

@@ -36,6 +36,7 @@ import {
   isSourceOnlyMode,
   sourceOnlyReason,
 } from "@/lib/rag/rag-provider";
+import { generationQualityRetryInstruction, summarizeGenerationFailureReason } from "@/lib/rag/rag-generation-failure";
 import { allowedChunkMap, citationFromResult as resultCitation, compactCitations } from "@/lib/citations";
 import { assessAndEnforceClaimSupport, enforceLabelledNumericBandCoherence } from "@/lib/rag/rag-claim-support";
 import {
@@ -3375,6 +3376,7 @@ ${qualityRetryInstruction}`
       "fast_unusable_retry_strong",
       "fast_template_retry_strong",
       "fast_quality_retry_strong",
+      "fast_numeric_faithfulness_retry_strong",
     ]);
     const eligibleForRoutineExtractiveRecovery =
       route.mode === "fast" &&
@@ -3399,25 +3401,6 @@ ${qualityRetryInstruction}`
     });
   }
 
-  /** Summarize generation failure reason. */
-  function summarizeGenerationFailureReason(error: unknown) {
-    const message = (error instanceof Error ? error.message : typeof error === "string" ? error : "").trim();
-    const normalized = message.toLowerCase();
-    const sourceBackedRecovery = normalized.match(/\bsource_backed_extractive_recovery:([a-z0-9_]+)/);
-
-    if (sourceBackedRecovery) return `source_backed_extractive_recovery_${sourceBackedRecovery[1]}`;
-    if (!normalized) return "generation_failed";
-    if (/\bprovider_source_gap\b/.test(normalized)) return "provider_source_gap";
-    if (/\bmax_output_tokens\b/.test(normalized)) return "provider_incomplete_max_output_tokens";
-    if (/\bincomplete\b/.test(normalized)) return "provider_incomplete";
-    if (/\brate limit|rate_limited|429\b/.test(normalized)) return "provider_rate_limited";
-    if (/\btimeout|timed out|deadline|aborted|etimedout\b/.test(normalized)) return "provider_timeout";
-    if (/\bauthentication|api key|unauthori[sz]ed|401|403\b/.test(normalized)) return "provider_auth_failed";
-    if (/\bvalidation|quality gate|schema|parse|json\b/.test(normalized)) return "generation_quality_failed";
-    if (/\bopenai|provider|model\b/.test(normalized)) return "provider_generation_failed";
-    return "generation_failed";
-  }
-
   /** Build generation fallback answer. */
   async function buildGenerationFallbackAnswer(
     error: unknown,
@@ -3598,12 +3581,13 @@ ${qualityRetryInstruction}`
       !fastSourceGap &&
       !fastAnswerWasTemplateLike &&
       shouldRetryWithStrongAfterFast({ route, answer, results: answerInputResults });
-    const fastAnswerFailedQualityGate =
+    const fastQualityFailureReason =
       route.mode === "fast" &&
       !fastAnswerWasUnusable &&
       !fastAnswerWasTemplateLike &&
       !fastAnswerWasOverExpanded &&
-      Boolean(generatedAnswerQualityFailureReason(answer, args.query, queryClass));
+      generatedAnswerQualityFailureReason(answer, args.query, queryClass);
+    const fastAnswerFailedQualityGate = Boolean(fastQualityFailureReason);
     if (
       fastAnswerHadInvalidEvidenceIds ||
       fastSourceGap ||
@@ -3613,19 +3597,22 @@ ${qualityRetryInstruction}`
       fastAnswerWasOverExpanded ||
       fastAnswerFailedQualityGate
     ) {
-      const retryReason = fastAnswerHadInvalidEvidenceIds
-        ? "fast_invalid_evidence_retry_strong"
-        : fastSourceGap
-          ? "fast_source_gap_retry_strong"
-          : fastAnswerWasUnsupported
-            ? "fast_unsupported_retry_strong"
-            : fastAnswerWasUnusable
-              ? "fast_unusable_retry_strong"
-              : fastAnswerWasTemplateLike
-                ? "fast_template_retry_strong"
-                : fastAnswerWasOverExpanded
-                  ? "fast_overexpanded_simple_retry_strong"
-                  : "fast_quality_retry_strong";
+      const retryReason =
+        fastQualityFailureReason === "numeric_faithfulness_gap"
+          ? "fast_numeric_faithfulness_retry_strong"
+          : fastAnswerHadInvalidEvidenceIds
+            ? "fast_invalid_evidence_retry_strong"
+            : fastSourceGap
+              ? "fast_source_gap_retry_strong"
+              : fastAnswerWasUnsupported
+                ? "fast_unsupported_retry_strong"
+                : fastAnswerWasUnusable
+                  ? "fast_unusable_retry_strong"
+                  : fastAnswerWasTemplateLike
+                    ? "fast_template_retry_strong"
+                    : fastAnswerWasOverExpanded
+                      ? "fast_overexpanded_simple_retry_strong"
+                      : "fast_quality_retry_strong";
       if (shouldRecoverFastFailureExtractively(retryReason)) {
         answerRetryCount += 1;
         answerRetryReasons.push(`fast_source_backed_extractive_recovery:${retryReason}`);
@@ -3639,19 +3626,21 @@ ${qualityRetryInstruction}`
       await args.onProgress?.({
         stage: "retrying",
         message:
-          retryReason === "fast_invalid_evidence_retry_strong"
-            ? "Fast answer cited invalid evidence IDs, retrying with the strong model."
-            : retryReason === "fast_source_gap_retry_strong"
-              ? "Fast answer returned a source gap despite strong retrieval, retrying with the strong model."
-              : retryReason === "fast_unsupported_retry_strong"
-                ? "Fast answer was unsupported, retrying with the strong model."
-                : retryReason === "fast_unusable_retry_strong"
-                  ? "Fast answer was not usable, retrying with the strong model."
-                  : retryReason === "fast_template_retry_strong"
-                    ? "Fast answer was too template-like, retrying with the strong model."
-                    : retryReason === "fast_overexpanded_simple_retry_strong"
-                      ? "Fast answer over-expanded a simple question, retrying with the strong model."
-                      : "Fast answer failed quality checks, retrying with the strong model.",
+          retryReason === "fast_numeric_faithfulness_retry_strong"
+            ? "Fast answer included an unverified figure, retrying with exact numeric-grounding instructions."
+            : retryReason === "fast_invalid_evidence_retry_strong"
+              ? "Fast answer cited invalid evidence IDs, retrying with the strong model."
+              : retryReason === "fast_source_gap_retry_strong"
+                ? "Fast answer returned a source gap despite strong retrieval, retrying with the strong model."
+                : retryReason === "fast_unsupported_retry_strong"
+                  ? "Fast answer was unsupported, retrying with the strong model."
+                  : retryReason === "fast_unusable_retry_strong"
+                    ? "Fast answer was not usable, retrying with the strong model."
+                    : retryReason === "fast_template_retry_strong"
+                      ? "Fast answer was too template-like, retrying with the strong model."
+                      : retryReason === "fast_overexpanded_simple_retry_strong"
+                        ? "Fast answer over-expanded a simple question, retrying with the strong model."
+                        : "Fast answer failed quality checks, retrying with the strong model.",
         mode: "strong",
         model: env.OPENAI_STRONG_ANSWER_MODEL,
         reason: routingReason,
@@ -3663,6 +3652,10 @@ ${qualityRetryInstruction}`
       generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, {
         strong: true,
         maxOutputTokensOverride: strongRetryMaxOutputTokens,
+        qualityRetryInstruction:
+          fastQualityFailureReason === "numeric_faithfulness_gap"
+            ? generationQualityRetryInstruction(fastQualityFailureReason)
+            : undefined,
       });
       retrievalDiagnostics.routeMode = "strong";
       if (generated.truncated) {
@@ -3697,7 +3690,7 @@ ${qualityRetryInstruction}`
       // valid (if imperfect) cited strong answer instead of spending a third generation
       // and risking a truncation -> unsupported tail. Recorded for observability.
       answerRetryReasons.push(`strong_quality_repair_skipped_time_budget:${strongQualityFailureReason}`);
-    } else if (answerNeedsStrongQualityRepair) {
+    } else if (strongQualityFailureReason) {
       routingReason = `${routingReason}; strong_quality_retry`;
       answerRetryCount += 1;
       answerRetryReasons.push("strong_quality_retry");
@@ -3711,7 +3704,7 @@ ${qualityRetryInstruction}`
       generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, {
         strong: true,
         maxOutputTokensOverride: strongRetryMaxOutputTokens,
-        qualityRetryInstruction: `The previous answer failed deterministic validation (${strongQualityFailureReason}). Return schema-valid output only, with a complete natural clinical synthesis in the answer field. The first sentence must directly answer the question as a full sentence. Every clinical claim must be supported by valid retrieved citation_chunk_id values; do not invent citation IDs. Within one named scale and source, if differently labelled intervals overlap or a range is reversed, omit the entire affected band set; do not quote, repair, or infer any label or value. If a separate sentence or clause states a nonnumeric condition and action independent of the score, answer only with that independently supported condition and action, cite the smallest sufficient directly supporting chunk set, and add a conflict entry; otherwise return a source gap. Avoid template/source-inventory wording and do not include JSON fragments inside text fields. If the evidence cannot support the requested clinical answer, return a concise source-gap answer instead. If the question is a simple definition or direct fact question, answer only that question and return answerSections as an empty array unless a source-gap or safety caveat is essential.`,
+        qualityRetryInstruction: generationQualityRetryInstruction(strongQualityFailureReason),
       });
       retrievalDiagnostics.routeMode = "strong";
       if (generated.truncated) {

tests/rag-answer-fallback.test.ts

@@ -91,8 +91,8 @@ type GeneratedAnswerPayload = {
 async function answerFromTextSources(
   query: string,
   sources: SearchResult[],
-  generatedAnswer?: GeneratedAnswerPayload | Error,
-  options: { sourceOnly?: boolean } = {},
+  generatedAnswer?: GeneratedAnswerPayload | Error | Array<GeneratedAnswerPayload | Error>,
+  options: { sourceOnly?: boolean; onGenerate?: (input: string, index: number) => void } = {},
 ) {
   // `src/lib/env.ts` freezes process.env at module load. The offline vitest wrapper
   // starts every worker as RAG_PROVIDER_MODE=offline with a blank OpenAI key, so we
@@ -116,11 +116,16 @@ async function answerFromTextSources(
       from: vi.fn(() => new EmptyQuery()),
     }),
   }));
-  const generateStructuredTextResult = vi.fn(async () => {
-    if (generatedAnswer instanceof Error) throw generatedAnswer;
+  const generatedAnswers = Array.isArray(generatedAnswer) ? generatedAnswer : [generatedAnswer];
+  let generatedAnswerIndex = 0;
+  const generateStructuredTextResult = vi.fn(async (input: string) => {
+    options.onGenerate?.(input, generatedAnswerIndex);
+    const currentGeneratedAnswer = generatedAnswers[Math.min(generatedAnswerIndex, generatedAnswers.length - 1)];
+    generatedAnswerIndex += 1;
+    if (currentGeneratedAnswer instanceof Error) throw currentGeneratedAnswer;
     return {
       text: JSON.stringify(
-        generatedAnswer ?? {
+        currentGeneratedAnswer ?? {
           answer: "No current source with specific guidance for this query was found.",
           grounded: false,
           confidence: "unsupported",
@@ -165,6 +170,59 @@ afterEach(() => {
 });
 
 describe("RAG structured-output fallback", () => {
+  it("repairs an unsupported fast-answer figure with the existing strong retry", async () => {
+    const answer = await answerFromTextSources(
+      "Lithium dosing?",
+      [
+        source({
+          id: "lithium-dose-source",
+          document_id: "lithium-guideline",
+          title: "Medication guideline",
+          file_name: "medication-guideline.pdf",
+          section_heading: "Lithium initiation",
+          content: "Start lithium carbonate at 250 mg once daily and review tolerability.",
+          similarity: 0.94,
+          hybrid_score: 0.94,
+          text_rank: 0.09,
+        }),
+      ],
+      [
+        {
+          answer: "Start lithium carbonate at 500 mg once daily.",
+          grounded: true,
+          confidence: "high",
+          answerSections: [],
+          citations: [{ chunk_id: "lithium-dose-source" }],
+          quoteCards: [],
+          conflictsOrGaps: [],
+        },
+        {
+          answer: "Start lithium carbonate at 250 mg once daily.",
+          grounded: true,
+          confidence: "high",
+          answerSections: [],
+          citations: [{ chunk_id: "lithium-dose-source" }],
+          quoteCards: [],
+          conflictsOrGaps: [],
+        },
+      ],
+      {
+        onGenerate: (input, index) => {
+          if (index !== 1) return;
+          expect(input).toContain("numeric_faithfulness_gap");
+          expect(input).toContain("exact digits and unit appear in the cited source excerpt");
+        },
+      },
+    );
+
+    expect(answer.routingMode).toBe("strong");
+    expect(answer.routingReason).toContain("fast_numeric_faithfulness_retry_strong");
+    expect(answer.routingReason).not.toContain("generation_fallback");
+    expect(answer.answer.replace(/\*\*/g, "")).toContain("250 mg");
+    expect(answer.unverifiedNumericTokens ?? []).toEqual([]);
+    expect(answer.openAIRequestIds).toEqual(["req_answer_from_text_sources", "req_answer_from_text_sources"]);
+  });
+
   it("recovers a cited provider source gap instead of treating nearby citations as a grounded answer", async () => {
     const dischargeSources = [
       source({
@@ -1510,13 +1568,13 @@ describe("RAG structured-output fallback", () => {
           text_rank: 1,
         }),
       ],
-      new Error("OpenAI generation quality gate failed: labelled numeric band conflict"),
+      new Error("OpenAI generation quality gate failed: numeric_band_coherence_gap"),
     );
     const deliveredText =
       `${answer.answer} ${(answer.answerSections ?? []).map((section) => section.body).join(" ")}`.replace(/\*\*/g, "");
 
     expect(answer.grounded).toBe(true);
-    expect(answer.routingReason).toContain("generation_fallback:generation_quality_failed");
+    expect(answer.routingReason).toContain("generation_fallback:generation_quality_failed_numeric_band_coherence_gap");
     expect(answer.routingReason).toContain("source_backed_extractive_fallback");
     expect(deliveredText).toMatch(
       /any side effect which is causing distress irrespective of score should be escalated to the treating doctor and reviewed/i,