diff --git a/.env.example b/.env.example index 6ff8dd9512..1f32a51988 100644 --- a/.env.example +++ b/.env.example @@ -141,6 +141,10 @@ SENTRY_ENVIRONMENT=production # Optional JSON override for app-layer ranking weights (see src/lib/ranking-config.ts). # Omit for current defaults. Example (enable diversity demotion + linear freshness): # RAG_RANKING_CONFIG={"documentDiversityPenalty":0.03,"freshness":{"mode":"linear"}} +# #100 Phase 1: emit the governed retrieval-complete evidence preview as a verified unit +# on the answer stream. Server emission flag; client rendering ships separately. Keep false +# until the offline contract proof has landed and enablement is deliberately staged. +RAG_INCREMENTAL_EVIDENCE_PREVIEW=false # Ambiguity-only structured semantic reranking. Keep false until the retrieval canary is approved. RAG_SEMANTIC_RERANK_ENABLED=false # Append OR-relaxed recall behind weak-but-nonzero strict text matches (P8b extension). diff --git a/docs/branch-review-records/6a5bd9d5cba90563645f1d0ccdbae85ce69990ed735f5494573242f14c7abdc9.record.md b/docs/branch-review-records/6a5bd9d5cba90563645f1d0ccdbae85ce69990ed735f5494573242f14c7abdc9.record.md new file mode 100644 index 0000000000..059ed0671e --- /dev/null +++ b/docs/branch-review-records/6a5bd9d5cba90563645f1d0ccdbae85ce69990ed735f5494573242f14c7abdc9.record.md @@ -0,0 +1 @@ +| 2026-08-13 | claude/rag-incremental-delivery-lpw15e | 387a403a4bf802208420f6847771ada24e1e3eb1 | #100 Phase 0 contract proof + flag-gated Phase 1 evidence preview (stream contract, answer-preview, rag.ts emission) | PR #1909 opened; flag default off, no retrieval/generation behaviour change | verify:pr-local failed:(none); new contract tests 13/13; production-readiness expected demo-mode gap only | diff --git a/docs/outstanding-issues-inbox/75df9b82-7ecc-4aa2-bdac-653da976fe53.json b/docs/outstanding-issues-inbox/75df9b82-7ecc-4aa2-bdac-653da976fe53.json new file mode 100644 index 0000000000..a0552aba81 --- /dev/null +++ b/docs/outstanding-issues-inbox/75df9b82-7ecc-4aa2-bdac-653da976fe53.json @@ -0,0 +1,10 @@ +{ + "version": 1, + "id": "75df9b82-7ecc-4aa2-bdac-653da976fe53", + "createdOn": "2026-08-13", + "action": "update", + "payload": { + "id": "#100", + "detail": "UPDATE 2026-08-13 (PR #1909): Phase 0 offline contract proof and flag-gated Phase 1 server emission implemented (RAG_INCREMENTAL_EVIDENCE_PREVIEW, default false). Remaining: client parsing/rendering phase behind its own flag + verify:ui, then the design's provider-backed acceptance gates before production enablement; Phase 2 stays provider-gated. **Design complete; runtime work remains provider-gated.** [`verified-answer-incremental-delivery-design.md`](verified-answer-incremental-delivery-design.md) records the clinical-governance decision and staged contract: keep the `progress`/`final`/`error` allowlist; disclose bounded, owner-scoped evidence only after the canonical danger-level source-governance refusal permits it, then emit complete answer sections only after each reuses the full production verification boundary; reconcile every preview byte-for-byte with the authoritative `final`; discard all previews on error/cancel/retry; deploy behind separate parse/emission/render flags. Phase 0 contract proof and Phase 1 evidence preview can be developed offline, but visible rollout still needs clinical/browser proof. Phase 2 changes generation architecture and requires explicit approval for answer-quality evals plus a baseline/post live canary pair. **Naive token streaming remains REFUTED:** never re-land `token`, `revising`, provisional prose, or a weaker stream-only verifier. Cross-references #021." + } +} diff --git a/docs/outstanding-issues-inbox/e1506952-64c6-472f-9da4-812f8d69b483.json b/docs/outstanding-issues-inbox/e1506952-64c6-472f-9da4-812f8d69b483.json new file mode 100644 index 0000000000..f1bbbc6b97 --- /dev/null +++ b/docs/outstanding-issues-inbox/e1506952-64c6-472f-9da4-812f8d69b483.json @@ -0,0 +1,10 @@ +{ + "version": 1, + "id": "e1506952-64c6-472f-9da4-812f8d69b483", + "createdOn": "2026-08-13", + "action": "update", + "payload": { + "id": "#310", + "detail": "UPDATE 2026-08-13: PR #1851 head now carries the 1-edit cap itself (typoDistanceLimit >=5 -> 1); the owed regression test (fluoxetine!=duloxetine, prednisone!=prednisolone with both records present, setraline recovery preserved) was pushed to that branch as d2d3256, 17/17 passing. Row closeable when #1851 merges. MEASURED 2026-08-12 by running the matcher itself, not by reading it. PR #1851 adds Damerau-Levenshtein typo recovery to `src/lib/catalog-search.ts` (`fuzzySearchTokenCount`, `boundedTypoDistance`, `typoDistanceLimit`) and folds it into the score. The tier `term.length >= 8 -> 2 edits` is the problem: Damerau counts an adjacent transposition as ONE edit, so `fluoxetine` -> `duloxetine` is distance 2 (substitute f->d, transpose lu->ul) and both are 10 characters. Confirmed hits against the PR's own algorithm: **fluoxetine -> duloxetine** (SSRI vs SNRI, different drugs), **prednisone -> prednisolone** (different drugs). Intended cases also confirmed working: sertraline -> setraline, olanzapine -> olanzepine. The existing guards DO hold — SSRI/SNRI, ADHD/ODD, citalopram/escitalopram, clozapine/clonazepam and quetiapine/olanzapine all correctly return no match. ONE MITIGATION, stated so this is not over-read: terms under 5 characters are excluded entirely. The fuzzy trigger is evaluated independently for each candidate record, so the hazard persists when both the exact drug and a two-edit near-match are present: the exact record receives a literal score while the wrong drug can independently receive a fuzzy score and appear as an additional result. Blast radius is wide because `catalog-search.ts` feeds ELEVEN modules — medications.ts (prescribing), dsm.ts, differentials.ts, differential-stream.ts, universal-search.ts, specifiers-search-index.ts, tools-catalog.ts, form-ranker.ts, service-ranker.ts. TESTED FIX: capping the >=8 tier at 1 edit removes both cross-drug hits and preserves every legitimate typo recovery in the sample — a one-line change to `typoDistanceLimit`. Next: if PR #1851 is still open, raise this on it; if it merged, apply the cap directly and add a test over real catalogue drug names with both the exact and near-match records present, asserting the wrong drug is excluded while the exact drug remains. Stop: do not remove fuzzy search outright — the typo recovery is genuinely useful and the guards are otherwise well judged. Note `classifyPullRequestFiles` returns clinicalRisk:true for this path (governance preflight fires) but ragRanking:false, which is correct — this is catalogue ranking, not the pgvector retrieval path." + } +} diff --git a/src/app/api/answer/stream/route.ts b/src/app/api/answer/stream/route.ts index 1bc0140d40..2353e68393 100644 --- a/src/app/api/answer/stream/route.ts +++ b/src/app/api/answer/stream/route.ts @@ -168,6 +168,10 @@ function streamAnswer( async start(controller) { const streamStartedAt = Date.now(); let completionSent = false; + // Verified units are append-only within one SSE response. Keep this state in + // the stream, never globally or across attempts, so duplicate/out-of-order + // progress callbacks fail closed at the public boundary. + let lastVerifiedUnitSequence: number | null = null; const send = (event: Name, data: AnswerStreamEventMap[Name]) => { try { controller.enqueue(encoder.encode(encodeSse(event, data))); @@ -177,10 +181,11 @@ function streamAnswer( } }; const sendProgress = (event: unknown) => { - const publicEvent = toPublicAnswerProgressEvent(event); + const publicEvent = toPublicAnswerProgressEvent(event, lastVerifiedUnitSequence); if (!publicEvent || (publicEvent.stage === "complete" && completionSent)) return; if (publicEvent.stage === "complete") completionSent = true; send("progress", publicEvent); + if (publicEvent.verifiedUnit) lastVerifiedUnitSequence = publicEvent.verifiedUnit.sequence; }; const sendComplete = () => { sendProgress({ stage: "complete", elapsedMs: Date.now() - streamStartedAt }); diff --git a/src/lib/answer-client-payload.ts b/src/lib/answer-client-payload.ts index 6785af5fd1..6ef28073c2 100644 --- a/src/lib/answer-client-payload.ts +++ b/src/lib/answer-client-payload.ts @@ -53,7 +53,9 @@ const sourceFieldPolicy = { images: "server", } as const satisfies Record; -function trimSourceForClient(source: SearchResult): SearchResult { +/** Exported for the verified evidence preview (#100), which must cross the route + * boundary through the exact same trim as the final payload — never a copy of it. */ +export function trimSourceForClient(source: SearchResult): SearchResult { const trimmed = Object.fromEntries( (Object.keys(sourceFieldPolicy) as Array) .filter((key) => sourceFieldPolicy[key] === "client" && key in source) diff --git a/src/lib/answer-preview.ts b/src/lib/answer-preview.ts new file mode 100644 index 0000000000..308499a6a8 --- /dev/null +++ b/src/lib/answer-preview.ts @@ -0,0 +1,66 @@ +// #100 Phase 1 — verified evidence preview (server-only). +// Builds the single retrieval-complete verified unit defined in +// docs/verified-answer-incremental-delivery-design.md. The unit must reuse the +// production gates, never approximate them: the same danger-level +// source-governance decision that governs the final response, and the exact +// route-boundary source trim the final payload goes through. If the gates +// cannot permit disclosure the preview is simply absent — there is no weaker +// "stream-safe" variant. + +import { trimSourceForClient } from "@/lib/answer-client-payload"; +import { env } from "@/lib/env"; +import { hasDangerSourceGovernanceWarning, sourceGovernanceWarnings } from "@/lib/source-governance"; +import type { VerifiedEvidencePreviewUnit, VerifiedUnit } from "@/lib/answer-stream-contract"; +import type { EvidenceRelevance, SearchResult } from "@/lib/types"; + +export type { VerifiedUnit }; + +const evidencePreviewMaxSources = 12; + +/** + * Build the retrieval-complete evidence preview, or null when nothing may be disclosed. + * + * Deliberately stricter than the final response's refusal: the final gate only refuses + * grounded, supported answers over danger-level sources, because unsupported/evidence-gap + * responses withhold content anyway. At preview time the answer's support level is not yet + * known, so ANY danger-level governance warning suppresses the preview — a preview must + * never disclose source content the final governed response might withhold. + */ +export function buildEvidencePreviewUnit(args: { + results: SearchResult[]; + /** The full source set the final answer may expose; used to fail closed early. */ + governanceResults?: SearchResult[]; + relevance?: EvidenceRelevance | null; +}): VerifiedEvidencePreviewUnit | null { + if (!args.results.length) return null; + const warnings = sourceGovernanceWarnings({ + results: args.governanceResults ?? args.results, + relevance: args.relevance ?? null, + }); + if (hasDangerSourceGovernanceWarning(warnings)) return null; + const selected = args.results.slice(0, evidencePreviewMaxSources); + return { + schemaVersion: 1, + kind: "evidence_preview", + sequence: 0, + sources: selected.map(trimSourceForClient), + selectedContextCount: args.results.length, + }; +} + +/** Keep the ranking event small and keep final-path reconciliation out of the RAG monolith. */ +export function buildEvidencePreviewProgress(args: { + normalResults: SearchResult[]; + fallbackResults: SearchResult[]; + governanceResults: SearchResult[]; + relevance?: EvidenceRelevance | null; +}): { verifiedUnit?: VerifiedEvidencePreviewUnit } { + if (!env.RAG_INCREMENTAL_EVIDENCE_PREVIEW) return {}; + const fallbackIds = new Set(args.fallbackResults.map((result) => result.id)); + const verifiedUnit = buildEvidencePreviewUnit({ + results: args.normalResults.filter((result) => fallbackIds.has(result.id)), + governanceResults: args.governanceResults, + relevance: args.relevance, + }); + return verifiedUnit ? { verifiedUnit } : {}; +} diff --git a/src/lib/answer-progress-public.ts b/src/lib/answer-progress-public.ts index 5df11a4cbd..2833ae21eb 100644 --- a/src/lib/answer-progress-public.ts +++ b/src/lib/answer-progress-public.ts @@ -1,3 +1,5 @@ +import { isDeliverableVerifiedUnit, type VerifiedUnit } from "@/lib/answer-stream-contract"; + export type PublicAnswerProgressStage = | "scoping" | "retrieving" @@ -18,6 +20,9 @@ export type PublicAnswerProgressEvent = { australianSourceCount?: number; waSourceCount?: number; elapsedMs?: number; + /** #100: optional verified unit (already governed + client-trimmed server-side). + * Old clients ignore it; it only crosses the boundary when it validates. */ + verifiedUnit?: VerifiedUnit; }; function safeProgressNumber(value: unknown) { @@ -25,7 +30,10 @@ function safeProgressNumber(value: unknown) { } /** Convert internal RAG progress into the minimal, stable DTO allowed at the browser boundary. */ -export function toPublicAnswerProgressEvent(event: unknown): PublicAnswerProgressEvent | null { +export function toPublicAnswerProgressEvent( + event: unknown, + lastVerifiedUnitSequence: number | null = null, +): PublicAnswerProgressEvent | null { if (!event || typeof event !== "object") return null; const value = event as Record; const resultCount = safeProgressNumber(value.resultCount); @@ -86,9 +94,16 @@ export function toPublicAnswerProgressEvent(event: unknown): PublicAnswerProgres return null; } + // The verified unit crosses the boundary only when it passes the stream contract's + // structural validation; a malformed or oversized unit is dropped, never repaired. + const verifiedUnit = isDeliverableVerifiedUnit(value.verifiedUnit, lastVerifiedUnitSequence) + ? value.verifiedUnit + : undefined; + return { stage, message, + ...(verifiedUnit === undefined ? {} : { verifiedUnit }), ...(resultCount === undefined ? {} : { resultCount }), ...(selectedContextCount === undefined ? {} : { selectedContextCount }), ...(australianSourceCount === undefined ? {} : { australianSourceCount }), diff --git a/src/lib/answer-stream-contract.ts b/src/lib/answer-stream-contract.ts index 97a8b6dc5e..a5d2de32b6 100644 --- a/src/lib/answer-stream-contract.ts +++ b/src/lib/answer-stream-contract.ts @@ -1,4 +1,322 @@ import type { PublicAnswerProgressEvent } from "@/lib/answer-progress-public"; +import type { AnswerSection, Citation, SearchResult } from "@/lib/types"; + +// #100 incremental verified delivery (docs/verified-answer-incremental-delivery-design.md). +// A verified unit is an append-only preview of content that is byte-identical to a subset +// of the authoritative `final` payload. No new SSE event name is introduced: units ride the +// existing `progress` event as an optional field that old clients ignore. +export type VerifiedEvidencePreviewUnit = { + schemaVersion: 1; + kind: "evidence_preview"; + sequence: 0; + /** Client-trimmed sources — the exact trimSourceForClient output used by `final`. */ + sources: SearchResult[]; + selectedContextCount: number; +}; + +export type VerifiedAnswerSectionUnit = { + schemaVersion: 1; + kind: "answer_section"; + sequence: number; + section: AnswerSection; + citations: Citation[]; + supportLevel: string; +}; + +export type VerifiedUnit = VerifiedEvidencePreviewUnit | VerifiedAnswerSectionUnit; + +// A unit is a bounded preview, never a transport for full documents. Sized to the +// client-source snippet policy (≤900 chars/source, ≤12 sources) with headroom. +const verifiedUnitMaxJsonChars = 64_000; +const evidencePreviewMaxSources = 12; +const clientSourceSnippetMaxChars = 900; +const verifiedUnitKinds = new Set(["evidence_preview", "answer_section"]); +const answerSectionKinds = new Set([ + "bottom_line", + "required_actions", + "monitoring_timing", + "medication_dose", + "thresholds", + "escalation_risk", + "contraindications_cautions", + "comparison", + "documentation", + "source_gap", + "visual_evidence", + "quotes", + "verification", +]); +const answerSectionSupportLevels = new Set(["direct", "partial", "nearby", "unsupported"]); +const citationProvenanceValues = new Set([ + "model_selected", + "section_selected", + "exact_quote", + "deterministic_support", + "review_only", + "retrieval_only", +]); +const clientSourceKeys = new Set([ + "id", + "document_id", + "title", + "file_name", + "page_number", + "chunk_index", + "section_heading", + "section_path", + "heading_level", + "parent_heading", + "anchor_id", + "content", + "retrieval_synopsis", + "image_ids", + "similarity", + "similarity_origin", + "text_rank", + "hybrid_score", + "lexical_score", + "rrf_score", + "score_explanation", + "source_strength", + "source_metadata", + "document_labels", + "memory_score", + "relevance", + "match_explanation", + "indexing_quality", + "images", +]); + +function isPlainRecord(value: unknown): value is Record { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function hasOnlyKeys(record: Record, allowedKeys: ReadonlySet): boolean { + return Object.keys(record).every((key) => allowedKeys.has(key)); +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function isInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value); +} + +function isNullableString(value: unknown): value is string | null { + return value === null || typeof value === "string"; +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string"); +} + +function isJsonValue(value: unknown, depth = 0): boolean { + if (value === null || typeof value === "string" || typeof value === "boolean") return true; + if (isFiniteNumber(value)) return true; + if (depth >= 8) return false; + if (Array.isArray(value)) return value.every((item) => isJsonValue(item, depth + 1)); + if (!isPlainRecord(value)) return false; + return Object.values(value).every((item) => item === undefined || isJsonValue(item, depth + 1)); +} + +function isOptionalFiniteNumber(record: Record, key: string): boolean { + return !(key in record) || record[key] === undefined || isFiniteNumber(record[key]); +} + +function isOptionalNullableString(record: Record, key: string): boolean { + return !(key in record) || record[key] === undefined || isNullableString(record[key]); +} + +function isClientSource(value: unknown): value is SearchResult { + if (!isPlainRecord(value) || !hasOnlyKeys(value, clientSourceKeys)) return false; + if ( + typeof value.id !== "string" || + typeof value.document_id !== "string" || + typeof value.title !== "string" || + typeof value.file_name !== "string" || + !(value.page_number === null || isInteger(value.page_number)) || + !isInteger(value.chunk_index) || + !isNullableString(value.section_heading) || + typeof value.content !== "string" || + value.content.length > clientSourceSnippetMaxChars || + !isStringArray(value.image_ids) || + !isFiniteNumber(value.similarity) || + !Array.isArray(value.images) || + value.images.length !== 0 + ) { + return false; + } + if ("section_path" in value && value.section_path !== undefined && !isStringArray(value.section_path)) return false; + if ( + "heading_level" in value && + value.heading_level !== undefined && + !(value.heading_level === null || isInteger(value.heading_level)) + ) { + return false; + } + if (!isOptionalNullableString(value, "parent_heading") || !isOptionalNullableString(value, "anchor_id")) return false; + if ("retrieval_synopsis" in value && value.retrieval_synopsis !== undefined) { + if (!isNullableString(value.retrieval_synopsis)) return false; + if (value.retrieval_synopsis !== null && value.retrieval_synopsis !== value.content) return false; + } + if ( + "similarity_origin" in value && + value.similarity_origin !== undefined && + value.similarity_origin !== "cosine" && + value.similarity_origin !== "synthetic_text" + ) { + return false; + } + for (const key of ["text_rank", "hybrid_score", "rrf_score", "memory_score"] as const) { + if (!isOptionalFiniteNumber(value, key)) return false; + } + if ( + "lexical_score" in value && + value.lexical_score !== undefined && + !(value.lexical_score === null || isFiniteNumber(value.lexical_score)) + ) { + return false; + } + if ( + "source_strength" in value && + value.source_strength !== undefined && + !["strong", "moderate", "limited"].includes(String(value.source_strength)) + ) { + return false; + } + for (const key of [ + "score_explanation", + "source_metadata", + "document_labels", + "relevance", + "match_explanation", + "indexing_quality", + ] as const) { + if (key in value && value[key] !== undefined && !isJsonValue(value[key])) return false; + } + return true; +} + +function isAnswerSection(value: unknown): value is AnswerSection { + if ( + !isPlainRecord(value) || + !hasOnlyKeys(value, new Set(["heading", "body", "citation_chunk_ids", "kind", "supportLevel"])) + ) { + return false; + } + if (typeof value.heading !== "string" || typeof value.body !== "string" || !isStringArray(value.citation_chunk_ids)) { + return false; + } + if ( + "kind" in value && + value.kind !== undefined && + (typeof value.kind !== "string" || !answerSectionKinds.has(value.kind)) + ) { + return false; + } + if ( + "supportLevel" in value && + value.supportLevel !== undefined && + (typeof value.supportLevel !== "string" || !answerSectionSupportLevels.has(value.supportLevel)) + ) { + return false; + } + return true; +} + +function isCitation(value: unknown): value is Citation { + if ( + !isPlainRecord(value) || + !hasOnlyKeys( + value, + new Set([ + "chunk_id", + "document_id", + "title", + "file_name", + "page_number", + "chunk_index", + "similarity", + "source_metadata", + "provenance", + ]), + ) + ) { + return false; + } + if ( + typeof value.chunk_id !== "string" || + typeof value.document_id !== "string" || + typeof value.title !== "string" || + typeof value.file_name !== "string" || + !(value.page_number === null || isInteger(value.page_number)) || + !isInteger(value.chunk_index) + ) { + return false; + } + if (!isOptionalFiniteNumber(value, "similarity")) return false; + if ( + "source_metadata" in value && + value.source_metadata !== undefined && + !(value.source_metadata === null || isJsonValue(value.source_metadata)) + ) { + return false; + } + if ( + "provenance" in value && + value.provenance !== undefined && + (typeof value.provenance !== "string" || !citationProvenanceValues.has(value.provenance)) + ) { + return false; + } + return true; +} + +/** Validate a candidate verified unit at the stream boundary. `lastSequence` is the + * previously accepted sequence in this response (null before the first unit); sequences + * must be strictly increasing within one response and never carry across attempts. + * Anything token-/revising-shaped, unknown, unsized, or out of order is rejected. */ +export function isDeliverableVerifiedUnit(value: unknown, lastSequence: number | null = null): value is VerifiedUnit { + if (!isPlainRecord(value)) return false; + if (value.schemaVersion !== 1) return false; + if (typeof value.kind !== "string" || !verifiedUnitKinds.has(value.kind)) return false; + if (!isInteger(value.sequence) || value.sequence < 0) return false; + if (lastSequence !== null && value.sequence <= lastSequence) return false; + if (value.kind === "evidence_preview") { + if (!hasOnlyKeys(value, new Set(["schemaVersion", "kind", "sequence", "sources", "selectedContextCount"]))) { + return false; + } + if (value.sequence !== 0) return false; + if ( + !Array.isArray(value.sources) || + value.sources.length === 0 || + value.sources.length > evidencePreviewMaxSources + ) { + return false; + } + if (!value.sources.every(isClientSource)) return false; + if (!isInteger(value.selectedContextCount) || value.selectedContextCount < value.sources.length) { + return false; + } + } else { + if (!hasOnlyKeys(value, new Set(["schemaVersion", "kind", "sequence", "section", "citations", "supportLevel"]))) { + return false; + } + if (!isAnswerSection(value.section) || !Array.isArray(value.citations) || !value.citations.every(isCitation)) { + return false; + } + if (typeof value.supportLevel !== "string" || !answerSectionSupportLevels.has(value.supportLevel)) return false; + if (value.section.supportLevel !== undefined && value.section.supportLevel !== value.supportLevel) return false; + } + try { + return JSON.stringify(value).length <= verifiedUnitMaxJsonChars; + } catch { + return false; + } +} export type AnswerStreamEventMap = { progress: PublicAnswerProgressEvent; diff --git a/src/lib/env.ts b/src/lib/env.ts index 604b6e08fd..67d24308f1 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -169,6 +169,14 @@ const envSchema = z.object({ .enum(["true", "false"]) .default("false") .transform((value) => value === "true"), + // #100 Phase 1: emit a governed, client-trimmed evidence preview as a verified unit on + // the answer stream once retrieval + ranking complete. Default OFF: server emission is + // enabled deliberately after the offline contract proof; rendering is a separate client + // flag per docs/verified-answer-incremental-delivery-design.md. + RAG_INCREMENTAL_EVIDENCE_PREVIEW: z + .enum(["true", "false"]) + .default("false") + .transform((value) => value === "true"), RAG_REGISTRY_CORPUS_EMBEDDING: z .enum(["true", "false"]) .default("false") diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index cbe667f079..e09f968c1f 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -56,6 +56,7 @@ import { applyNumericVerification, textReferencesAdjacentBandConflict, } from "@/lib/answer-verification"; +import { buildEvidencePreviewProgress, type VerifiedUnit } from "@/lib/answer-preview"; export { applyNumericVerification, unboldUnverifiedNumbers } from "@/lib/answer-verification"; import { selectModelContextResults, summarizeAustralianSourceSelection } from "@/lib/rag/rag-context-selection"; export { @@ -510,8 +511,8 @@ export type AnswerProgressEvent = { model?: string | null; reason?: string; smartApiPlan?: SmartRagApiPlan; + verifiedUnit?: VerifiedUnit; }; - type AnswerQuestionWithScopeArgs = SearchChunksArgs & { logQuery?: boolean; onProgress?: (event: AnswerProgressEvent) => void | Promise; @@ -3499,6 +3500,12 @@ ${qualityRetryInstruction}` australianSourceCount: modelContextSelectionSummary.australianSelectedCount, waSourceCount: modelContextSelectionSummary.waSelectedCount, usedSupplementaryFallback: modelContextSelectionSummary.usedSupplementaryFallback, + ...buildEvidencePreviewProgress({ + normalResults: modelContextResults, + fallbackResults: generationFallbackResults, + governanceResults: answerInputResults, + relevance, + }), }); // The quality-repair call below may itself fail or truncate. Preserve the first // deterministic verdict so fallback telemetry explains why that retry occurred, diff --git a/tests/answer-incremental-delivery.test.ts b/tests/answer-incremental-delivery.test.ts new file mode 100644 index 0000000000..6789921d74 --- /dev/null +++ b/tests/answer-incremental-delivery.test.ts @@ -0,0 +1,244 @@ +// #100 Phase 0 — offline contract proof for incremental verified delivery +// (docs/verified-answer-incremental-delivery-design.md). These tests must hold before +// any client renders a verified unit: schema/sequence validation, rejection of the +// removed token/revising shapes, governance refusal emitting zero units, owner-boundary +// trimming, and byte-identical preview/final reconciliation at the trim layer. + +import { describe, expect, it } from "vitest"; + +import { toClientAnswerPayload, trimSourceForClient } from "../src/lib/answer-client-payload"; +import { buildEvidencePreviewUnit } from "../src/lib/answer-preview"; +import { toPublicAnswerProgressEvent } from "../src/lib/answer-progress-public"; +import { + isAnswerStreamEventName, + isDeliverableVerifiedUnit, + type VerifiedEvidencePreviewUnit, +} from "../src/lib/answer-stream-contract"; +import type { SearchResult } from "../src/lib/types"; + +function makeSource(overrides: Partial = {}): SearchResult { + return { + id: "chunk-1", + document_id: "doc-1", + title: "Clozapine Monitoring", + file_name: "clozapine.pdf", + page_number: 3, + chunk_index: 1, + section_heading: "Monitoring", + content: "ANC thresholds and FBC monitoring schedule for clozapine.", + image_ids: [], + similarity: 0.82, + // Server-only fields that must never cross the route boundary. + adjacent_context: "SERVER-ONLY adjacent context", + memory_cards: [{ card: "SERVER-ONLY memory card" }], + table_facts: [{ fact: "SERVER-ONLY table fact" }], + document_summary: "SERVER-ONLY document summary", + images: [{ id: "img-1", caption: "SERVER-ONLY caption" }], + ...overrides, + } as unknown as SearchResult; +} + +const previewUnit = (): VerifiedEvidencePreviewUnit => ({ + schemaVersion: 1, + kind: "evidence_preview", + sequence: 0, + sources: [trimSourceForClient(makeSource())], + selectedContextCount: 1, +}); + +const sectionUnit = () => ({ + schemaVersion: 1, + kind: "answer_section", + sequence: 2, + section: { heading: "Monitoring", body: "Check levels.", citation_chunk_ids: ["chunk-1"] }, + citations: [ + { + chunk_id: "chunk-1", + document_id: "doc-1", + title: "Clozapine Monitoring", + file_name: "clozapine.pdf", + page_number: 3, + chunk_index: 1, + }, + ], + supportLevel: "direct", +}); + +describe("verified-unit stream contract (#100 Phase 0)", () => { + it("accepts well-formed evidence and section previews", () => { + expect(isDeliverableVerifiedUnit(previewUnit())).toBe(true); + expect(isDeliverableVerifiedUnit(sectionUnit(), 1)).toBe(true); + }); + + it("rejects unknown schema versions and kinds", () => { + expect(isDeliverableVerifiedUnit({ ...previewUnit(), schemaVersion: 2 })).toBe(false); + expect(isDeliverableVerifiedUnit({ ...previewUnit(), kind: "token" })).toBe(false); + expect(isDeliverableVerifiedUnit({ ...previewUnit(), kind: "revising" })).toBe(false); + expect(isDeliverableVerifiedUnit(null)).toBe(false); + expect(isDeliverableVerifiedUnit("token")).toBe(false); + }); + + it("enforces strictly increasing sequences within one response", () => { + const section = sectionUnit(); + expect(isDeliverableVerifiedUnit(section, 1)).toBe(true); + expect(isDeliverableVerifiedUnit(section, 2)).toBe(false); + expect(isDeliverableVerifiedUnit(section, 3)).toBe(false); + // Evidence previews are pinned to sequence 0 and therefore only valid first. + expect(isDeliverableVerifiedUnit(previewUnit(), null)).toBe(true); + expect(isDeliverableVerifiedUnit(previewUnit(), 0)).toBe(false); + expect(isDeliverableVerifiedUnit({ ...previewUnit(), sequence: 1 })).toBe(false); + }); + + it("rejects empty, over-cap, non-finite, and non-integer evidence previews", () => { + expect(isDeliverableVerifiedUnit({ ...previewUnit(), sources: [] })).toBe(false); + expect( + isDeliverableVerifiedUnit({ + ...previewUnit(), + sources: Array.from({ length: 13 }, (_, index) => trimSourceForClient(makeSource({ id: `chunk-${index}` }))), + selectedContextCount: 13, + }), + ).toBe(false); + expect(isDeliverableVerifiedUnit({ ...previewUnit(), selectedContextCount: Number.POSITIVE_INFINITY })).toBe(false); + expect(isDeliverableVerifiedUnit({ ...previewUnit(), selectedContextCount: 1.5 })).toBe(false); + }); + + it("rejects raw server fields at the stream boundary", () => { + expect( + isDeliverableVerifiedUnit({ + ...previewUnit(), + sources: [makeSource()], + }), + ).toBe(false); + expect( + isDeliverableVerifiedUnit({ + ...previewUnit(), + sources: [{ ...previewUnit().sources[0], adjacent_context: "private generation context" }], + }), + ).toBe(false); + }); + + it("rejects malformed answer sections, citations, and support levels", () => { + const valid = sectionUnit(); + expect(isDeliverableVerifiedUnit({ ...valid, section: { heading: "Monitoring", content: "wrong field" } }, 1)).toBe( + false, + ); + expect(isDeliverableVerifiedUnit({ ...valid, citations: [{ chunk_id: "incomplete" }] }, 1)).toBe(false); + expect(isDeliverableVerifiedUnit({ ...valid, supportLevel: "unverified" }, 1)).toBe(false); + expect(isDeliverableVerifiedUnit({ ...valid, section: { ...valid.section, supportLevel: "partial" } }, 1)).toBe( + false, + ); + }); + + it("rejects unbounded payloads", () => { + const oversized = { + ...previewUnit(), + sources: Array.from({ length: 12 }, (_, index) => + trimSourceForClient( + makeSource({ + id: `chunk-${index}`, + content: "x".repeat(900), + match_explanation: { reasons: ["y".repeat(5_000)] }, + }), + ), + ), + selectedContextCount: 12, + }; + expect(isDeliverableVerifiedUnit(oversized)).toBe(false); + }); + + it("keeps the token and revising SSE event names excluded", () => { + expect(isAnswerStreamEventName("token")).toBe(false); + expect(isAnswerStreamEventName("revising")).toBe(false); + expect(isAnswerStreamEventName("progress")).toBe(true); + }); +}); + +describe("evidence preview builder (#100 Phase 1 server gate)", () => { + it("emits zero units when a danger-level governance warning exists", () => { + const outdated = makeSource({ + source_metadata: { document_status: "outdated" } as SearchResult["source_metadata"], + }); + expect(buildEvidencePreviewUnit({ results: [outdated] })).toBeNull(); + }); + + it("suppresses a preview when another potential final source fails governance", () => { + const safe = makeSource(); + const outdated = makeSource({ + id: "chunk-outdated", + source_metadata: { document_status: "outdated" } as SearchResult["source_metadata"], + }); + + expect(buildEvidencePreviewUnit({ results: [safe], governanceResults: [safe, outdated] })).toBeNull(); + }); + + it("emits zero units for empty retrieval", () => { + expect(buildEvidencePreviewUnit({ results: [] })).toBeNull(); + }); + + it("never lets server-only source fields cross the boundary", () => { + const unit = buildEvidencePreviewUnit({ results: [makeSource()] }); + expect(unit).not.toBeNull(); + const serialized = JSON.stringify(unit); + expect(serialized).not.toContain("SERVER-ONLY"); + const source = unit!.sources[0] as unknown as Record; + expect(source.adjacent_context).toBeUndefined(); + expect(source.memory_cards).toBeUndefined(); + expect(source.table_facts).toBeUndefined(); + expect(source.document_summary).toBeUndefined(); + expect(source.images).toEqual([]); + }); + + it("is byte-identical to the final payload's trim of the same sources", () => { + const results = [makeSource(), makeSource({ id: "chunk-2", content: "y".repeat(2000) })]; + const unit = buildEvidencePreviewUnit({ results }); + const finalPayload = toClientAnswerPayload({ sources: results }); + expect(JSON.stringify(unit!.sources)).toBe(JSON.stringify(finalPayload.sources)); + }); + + it("bounds the preview to the source cap while reporting the full selected count", () => { + const results = Array.from({ length: 20 }, (_, index) => makeSource({ id: `chunk-${index}` })); + const unit = buildEvidencePreviewUnit({ results }); + expect(unit!.sources).toHaveLength(12); + expect(unit!.selectedContextCount).toBe(20); + expect(isDeliverableVerifiedUnit(unit)).toBe(true); + }); +}); + +describe("public progress DTO passthrough", () => { + it("passes a valid verified unit through the ranking stage", () => { + const event = toPublicAnswerProgressEvent({ stage: "ranking", resultCount: 3, verifiedUnit: previewUnit() }); + expect(event?.verifiedUnit).toBeDefined(); + expect(event?.verifiedUnit?.kind).toBe("evidence_preview"); + }); + + it("drops duplicate and out-of-order verified units once a stream has accepted sequence 0", () => { + const first = toPublicAnswerProgressEvent({ stage: "retrieved", verifiedUnit: previewUnit() }); + expect(first?.verifiedUnit?.sequence).toBe(0); + expect( + toPublicAnswerProgressEvent({ stage: "retrieved", verifiedUnit: previewUnit() }, 0)?.verifiedUnit, + ).toBeUndefined(); + expect( + toPublicAnswerProgressEvent( + { + stage: "generating", + verifiedUnit: { ...sectionUnit(), sequence: 1 }, + }, + 0, + )?.verifiedUnit?.sequence, + ).toBe(1); + }); + + it("drops malformed verified units instead of repairing them", () => { + const event = toPublicAnswerProgressEvent({ + stage: "retrieved", + verifiedUnit: { schemaVersion: 1, kind: "token", sequence: 0, text: "raw model prose" }, + }); + expect(event).not.toBeNull(); + expect(event?.verifiedUnit).toBeUndefined(); + }); + + it("emits no verified unit by default (flag off end-to-end)", () => { + const event = toPublicAnswerProgressEvent({ stage: "retrieved", resultCount: 3 }); + expect(event?.verifiedUnit).toBeUndefined(); + }); +}); diff --git a/tests/answer-stream-preview-order.test.ts b/tests/answer-stream-preview-order.test.ts new file mode 100644 index 0000000000..0a24322bb1 --- /dev/null +++ b/tests/answer-stream-preview-order.test.ts @@ -0,0 +1,130 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const answerQuestionWithScope = vi.fn(); +const publicAccessContext = vi.fn(); +const consumeSubjectApiRateLimit = vi.fn(); +const resolveSearchScope = vi.fn(); + +vi.mock("@/lib/env", () => ({ isDemoMode: () => false })); +vi.mock("@/lib/rag/rag", () => ({ answerQuestionWithScope, summarizeDocument: vi.fn() })); +vi.mock("@/lib/public-api-access", () => ({ publicAccessContext })); +vi.mock("@/lib/api-rate-limit", () => ({ + allowRateLimitInMemoryFallbackOnUnavailable: () => false, + consumeSummaryRateLimits: vi.fn(), + consumeSubjectApiRateLimit, + rateLimitJsonResponse: () => new Response(null, { status: 429 }), +})); +vi.mock("@/lib/answer-response", () => ({ + answerDegradedModeSignal: () => ({ active: false }), + buildGovernedAnswerClientResponse: (answer: unknown) => ({ payload: answer, telemetryAnswer: answer }), + buildGovernedDemoAnswerClientResponse: vi.fn(), +})); +vi.mock("@/lib/search-scope", async (importOriginal) => ({ + ...(await importOriginal()), + resolveSearchScope, +})); +vi.mock("@/lib/owner-scope", () => ({ + resolveRetrievalAccessScope: (ownerId?: string) => ({ ownerId }), +})); +vi.mock("@/lib/supabase/admin", () => ({ createAdminClient: () => ({}) })); +vi.mock("@/lib/answer-telemetry", () => ({ logAnswerDiagnostics: vi.fn() })); +vi.mock("@/lib/observability/agent-monitoring", () => ({ setAgentConversationId: vi.fn() })); +vi.mock("@/lib/sse-heartbeat", () => ({ startSseHeartbeat: () => () => undefined })); +vi.mock("@/lib/server-timing", () => ({ + buildServerTimingHeader: () => null, + preambleServerTimingEntries: () => [], +})); +vi.mock("@/lib/answer-feedback-token", () => ({ answerFeedbackMetadata: () => ({}) })); + +const ownerId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + +type ParsedSseFrame = { event: string; data: Record }; + +function parseSseFrames(body: string): ParsedSseFrame[] { + return body + .split(/\n\n+/) + .map((frame) => frame.trim()) + .filter(Boolean) + .flatMap((frame) => { + const lines = frame.split("\n"); + const event = lines.find((line) => line.startsWith("event: "))?.slice("event: ".length); + const data = lines.find((line) => line.startsWith("data: "))?.slice("data: ".length); + if (!event || !data) return []; + return [{ event, data: JSON.parse(data) as Record }]; + }); +} + +beforeEach(() => { + publicAccessContext.mockResolvedValue({ + ownerId, + authenticated: true, + rateLimitSubject: { kind: "owner", id: ownerId }, + }); + consumeSubjectApiRateLimit.mockResolvedValue({ + limited: false, + limit: 100, + remaining: 99, + retryAfterSeconds: 0, + resetAt: new Date(Date.now() + 60_000).toISOString(), + }); + resolveSearchScope.mockResolvedValue({ documentIds: undefined, filters: {}, activeFilterCount: 0, warnings: [] }); +}); + +afterEach(() => { + vi.clearAllMocks(); + vi.resetModules(); +}); + +describe("answer stream verified preview ordering", () => { + it("delivers the ranking preview before generation and final frames", async () => { + answerQuestionWithScope.mockImplementation(async ({ onProgress }: { onProgress?: (event: unknown) => void }) => { + onProgress?.({ + stage: "ranking", + verifiedUnit: { + schemaVersion: 1, + kind: "evidence_preview", + sequence: 0, + sources: [ + { + id: "chunk-1", + document_id: "doc-1", + title: "Clozapine Monitoring", + file_name: "clozapine.pdf", + page_number: 3, + chunk_index: 1, + section_heading: "Monitoring", + content: "ANC thresholds and FBC monitoring schedule for clozapine.", + image_ids: [], + similarity: 0.82, + images: [], + }, + ], + selectedContextCount: 1, + }, + }); + onProgress?.({ stage: "generating" }); + return { answer: "Source-backed answer.", grounded: true, confidence: "high", citations: [], sources: [] }; + }); + + const { POST } = await import("../src/app/api/answer/stream/route"); + const response = await POST( + new Request("http://localhost/api/answer/stream", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query: "clozapine monitoring" }), + }), + ); + const frames = parseSseFrames(await response.text()); + + const rankingIndex = frames.findIndex((frame) => frame.event === "progress" && frame.data.stage === "ranking"); + const generationIndex = frames.findIndex( + (frame) => frame.event === "progress" && frame.data.stage === "generating", + ); + const finalIndex = frames.findIndex((frame) => frame.event === "final"); + const rankingFrame = frames[rankingIndex]; + expect(rankingIndex).toBeGreaterThan(-1); + expect((rankingFrame.data.verifiedUnit as { kind?: string } | undefined)?.kind).toBe("evidence_preview"); + expect(rankingIndex).toBeLessThan(generationIndex); + expect(generationIndex).toBeLessThan(finalIndex); + }); +});