diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 4c51412442..03497796eb 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -661,5 +661,7 @@ Records before 2026-07-28 were written by hand and had drifted: 146 lines carrie | 2026-08-05 | claude/design-system-1616-colors-aw538h | 98b65ae9f222e621da8b5bca75d0b0f25d05ca09 | prlanded | merged, squash 98b65ae verified content-identical to branch tip e66ecaa (empty diff) | vitest ckb-v2-token-contract (26 passed), vitest pwa-manifest (11 passed), live Playwright render check, CI green (pr-required) | | 2026-08-05 | cursor/privacy-page-mockups-2ff6 | 7c82f92a447986178ba12d6f8b7a447bb63e91ef | Run PR sweep | resolved privacy/page conflict with #1621 standalone shell; fixed Devin double scroll-pad + scrollIntoView yank | merge resolved; prettier | | 2026-08-05 | cursor/privacy-page-mockups-2ff6 | 32c4406cb728176933b669779d4f16fd245534bb | Run PR sweep | supersede: desktop index sticky top tracks measured StickySignalChrome height; prior row checks lacked decisive prettier output | prettier --check mockup+ledger: All matched files use Prettier code style!; ResizeObserver sticky chrome height for desktop index | +| 2026-08-06 | claude/implement-97vpz7 | 5ffa042d686a542de3333ebccbd903b6422124a7 | src/lib/rag/rag.ts, src/app/api/search/route.ts, tests/rag-unsupported-short-circuit-cache.test.ts (RAG soft-tail unsupported-short-circuit cache fix + corpus_grounding telemetry exposure) | PR #1646 opened (draft); no retrieval/ranking behaviour change; verified: lint, typecheck, full unit suite (513 files/5413 tests), eval:rag:offline, build, check:bundle-budget | lint,typecheck,test,eval:rag:offline,build,check:bundle-budget | | 2026-08-06 | a24f74fdf0134487a03dce37dd9f1e9bd18502f5 | a24f74fdf0134487a03dce37dd9f1e9bd18502f5 | PR #1614 post-merge RAG index restoration audit | Pass - guard-only migration, no DDL, no ranking/RPC change; pr-policy ragRanking=false so no eval-canary required; 1 P3 doc nit (#248 renumber note says 237->246, row is #248) | check:migration-role; npx vitest run tests/supabase-schema.test.ts (74 passed); check:outstanding-issues | | 2026-08-06 | PR #1614 / codex/restore-rag-indexes-20260804 | a24f74fdf0134487a03dce37dd9f1e9bd18502f5 | PR #1614 post-merge RAG index restoration audit | Pass - guard-only migration, no DDL, no ranking/RPC change; pr-policy ragRanking=false so no eval-canary required; 1 P3 doc nit (#248 renumber note says 237->246, row is #248); supersedes 2026-08-06 row (ref column mistakenly held commit SHA instead of PR ref, breaking ledger:lookup per Devin/Sentry review on PR #1636) | check:migration-role; npx vitest run tests/supabase-schema.test.ts (74 passed); check:outstanding-issues | +| 2026-08-06 | claude/implement-97vpz7 | 00ab7bfd34684bc854d15a3f28987674098a7130 | PR #1646 soft-tail answer-cache skip + soft-tail test hardening | fixed — answer-path soft-tail skip via rag-query-guard helpers; soft-tail fixture pins; duplicate memo test removed; in-corpus assert narrowed; budget 4362 | test:rag-query-guard+unsupported-cache+classifier-memo 22/22,check:maintainability-budgets 4362/4362 | diff --git a/scripts/check-maintainability-budgets.mjs b/scripts/check-maintainability-budgets.mjs index baa30fd526..65de282d21 100644 --- a/scripts/check-maintainability-budgets.mjs +++ b/scripts/check-maintainability-budgets.mjs @@ -7,7 +7,8 @@ const budgets = new Map([ ["src/components/ClinicalDashboard.tsx", 4140], // Evidence coverage, per-request hydration, and second-stage ranking live in // focused rag modules; keep the reclaimed budget so it cannot silently drift back. - ["src/lib/rag/rag.ts", 4351], + // 4362: soft-tail answer-cache skip call site (logic in rag-query-guard.ts). + ["src/lib/rag/rag.ts", 4362], ["src/components/DocumentViewer.tsx", 1734], ["supabase/functions/indexing-v3-agent/index.ts", 2191], ]); diff --git a/src/app/api/search/route.ts b/src/app/api/search/route.ts index 9569f80066..cc465f5c04 100644 --- a/src/app/api/search/route.ts +++ b/src/app/api/search/route.ts @@ -897,6 +897,7 @@ async function buildScopedSearchPayload( weak_source_count: relevance.weakSourceCount, retrieval_strategy: search.telemetry.retrieval_strategy, retrieval_plan: search.telemetry.retrieval_plan, + corpus_grounding: search.telemetry.corpus_grounding, smart_api_intent: smartApiPlan.intent, smart_api_response_mode: smartApiPlan.responseMode, smart_api_display_mode: smartApiPlan.displayMode, diff --git a/src/lib/rag/rag-cache.ts b/src/lib/rag/rag-cache.ts index 276e0f19de..ea42480664 100644 --- a/src/lib/rag/rag-cache.ts +++ b/src/lib/rag/rag-cache.ts @@ -542,6 +542,7 @@ export async function getSharedCachedSearch( shared_cache_status: "hit", shared_cache_miss_reason: null, query_class: payload.telemetry?.query_class, + corpus_grounding: payload.telemetry?.corpus_grounding, vector_candidate_count: payload.telemetry?.vector_candidate_count, text_candidate_count: payload.telemetry?.text_candidate_count, embedding_field_count: payload.telemetry?.embedding_field_count, diff --git a/src/lib/rag/rag-query-guard.ts b/src/lib/rag/rag-query-guard.ts index eb96d97a69..dd994ed160 100644 --- a/src/lib/rag/rag-query-guard.ts +++ b/src/lib/rag/rag-query-guard.ts @@ -32,3 +32,40 @@ export function isUnsupportedSoftTailAnalysis(query: string, analysis: ClinicalQ if (clearlyNonClinicalConsumerPattern.test(query)) return false; return analysis.confidence <= 0.42 && analysis.expandedTerms.length <= 5; } + +/** + * Soft-tail zeros should skip search/answer cache writes only when a nondeterministic + * classifier call could have produced them. Without an API key the classifier path is + * unreachable (`analyzeQueryWithClassifierFallback` returns early), and an + * `"out_of_corpus"` grounding verdict is a deterministic corpus-derived true negative — + * both stay cacheable. + */ +export function shouldSkipUnsupportedSoftTailCacheWrite( + query: string, + analysis: ClinicalQueryAnalysis, + options: { + openAiApiKeyPresent: boolean; + corpusGrounding?: ClinicalQueryAnalysis["corpusGrounding"]; + }, +): boolean { + if (!options.openAiApiKeyPresent) return false; + const grounding = options.corpusGrounding ?? analysis.corpusGrounding; + if (grounding === "out_of_corpus") return false; + return isUnsupportedSoftTailAnalysis(query, analysis); +} + +/** Answer-path counterpart: only skip when the empty unsupported refusal came from the soft-tail short circuit. */ +export function shouldSkipUnsupportedSoftTailAnswerCacheWrite(args: { + resultCount: number; + retrievalStrategy: string | undefined; + query: string; + analysis: ClinicalQueryAnalysis; + openAiApiKeyPresent: boolean; + corpusGrounding?: ClinicalQueryAnalysis["corpusGrounding"]; +}): boolean { + if (args.resultCount > 0 || args.retrievalStrategy !== "unsupported_short_circuit") return false; + return shouldSkipUnsupportedSoftTailCacheWrite(args.query, args.analysis, { + openAiApiKeyPresent: args.openAiApiKeyPresent, + corpusGrounding: args.corpusGrounding, + }); +} diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index c8fda5a1be..b32bc1a404 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -204,6 +204,8 @@ export { retrievalPlanForQueryClass, type SearchChunksArgs, type SearchTelemetry import { clearlyOutsideCorpusMedicalPattern, isUnsupportedSoftTailAnalysis, + shouldSkipUnsupportedSoftTailAnswerCacheWrite, + shouldSkipUnsupportedSoftTailCacheWrite, unavailableDocumentNoisePattern, } from "@/lib/rag/rag-query-guard"; export { shouldShortCircuitUnsupportedSearch } from "@/lib/rag/rag-query-guard"; @@ -951,6 +953,8 @@ type ClassifierVerdict = z.infer; // a query's classification for the whole TTL. The full corpus-grounded relevance fix remains // scoped to RAG optimisation Phase 2. const classifierVerdictMemoTtlMs = 15 * 60 * 1000; +// Finding #11 follow-up: bounds retries for a rejected soft-tail verdict (isUnsupportedSoftTailAnalysis). +const rejectedSoftTailMemoTtlMs = 60 * 1000; const classifierVerdictMemoMaxEntries = 500; const classifierVerdictMemo = new Map(); const classifierVerdictInflight = new Map>(); @@ -970,12 +974,12 @@ function classifierVerdictMemoKey(query: string, analysis: ClinicalQueryAnalysis } /** Store classifier verdict memo. */ -function storeClassifierVerdictMemo(key: string, verdict: ClassifierVerdict) { +function storeClassifierVerdictMemo(key: string, verdict: ClassifierVerdict, ttlMs = classifierVerdictMemoTtlMs) { if (classifierVerdictMemo.size >= classifierVerdictMemoMaxEntries) { const oldestKey = classifierVerdictMemo.keys().next().value; if (oldestKey !== undefined) classifierVerdictMemo.delete(oldestKey); } - classifierVerdictMemo.set(key, { expiresAt: Date.now() + classifierVerdictMemoTtlMs, verdict }); + classifierVerdictMemo.set(key, { expiresAt: Date.now() + ttlMs, verdict }); } /** Reset classifier verdict memo for tests. */ @@ -1152,7 +1156,10 @@ export async function analyzeQueryWithClassifierFallback( try { const verdict = await awaitWithCallerSignal(pending, opts?.signal); - storeClassifierVerdictMemo(memoKey, verdict); + // Finding #11 follow-up: bounded TTL for a rejected soft-tail verdict — see the constant above. + const rejected = verdict.confidence < 0.58 || verdict.queryClass === "unsupported_or_general"; + const softTail = rejected && isUnsupportedSoftTailAnalysis(query, analysis); + storeClassifierVerdictMemo(memoKey, verdict, softTail ? rejectedSoftTailMemoTtlMs : undefined); return applyClassifierVerdict(analysis, verdict); } catch (error) { if ( @@ -1740,7 +1747,14 @@ export async function searchChunksWithTelemetry( telemetry.embedding_skip_reason = "unsupported_short_circuit"; telemetry.retrieval_strategy = "unsupported_short_circuit"; recordSearchScoreTelemetry(telemetry, []); - await setCachedSearch(args, [], telemetry, queryVariants, { indexingVersionAtRetrievalStart }); + // Skip only when a reachable classifier could have produced a nondeterministic soft-tail zero. + if ( + !shouldSkipUnsupportedSoftTailCacheWrite(retrievalQuery, queryAnalysis, { + openAiApiKeyPresent: Boolean(env.OPENAI_API_KEY), + }) + ) { + await setCachedSearch(args, [], telemetry, queryVariants, { indexingVersionAtRetrievalStart }); + } return finishSearch(searchTiming, { results: [] as SearchResult[], telemetry }); } @@ -2947,7 +2961,18 @@ async function answerQuestionWithScopeUncoalesced( }, }); - if (answerRouteResultCanBeCached(routeDeadline)) + // Soft-tail unsupported refusals must not stick in the 5-minute answer cache. + if ( + answerRouteResultCanBeCached(routeDeadline) && + !shouldSkipUnsupportedSoftTailAnswerCacheWrite({ + resultCount: results.length, + retrievalStrategy: search.telemetry.retrieval_strategy, + query: answerFocusQuery, + analysis: queryAnalysis, + openAiApiKeyPresent: Boolean(env.OPENAI_API_KEY), + corpusGrounding: search.telemetry.corpus_grounding, + }) + ) await setCachedAnswer(args, finalizedAnswer, { indexingVersionAtRetrievalStart }); routeDeadline.dispose(); return finalizedAnswer; diff --git a/tests/rag-classifier-memo.test.ts b/tests/rag-classifier-memo.test.ts index 60958362bb..814f45116e 100644 --- a/tests/rag-classifier-memo.test.ts +++ b/tests/rag-classifier-memo.test.ts @@ -21,8 +21,9 @@ async function loadWithClassifierMock(mock: ReturnType) { }); const rag = await import("../src/lib/rag/rag"); const { analyzeClinicalQuery } = await import("../src/lib/clinical-search"); + const { isUnsupportedSoftTailAnalysis } = await import("../src/lib/rag/rag-query-guard"); rag.resetClassifierVerdictMemoForTests(); - return { rag, analyzeClinicalQuery }; + return { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis }; } function classifierResponse(overrides: Record = {}) { @@ -39,20 +40,24 @@ function classifierResponse(overrides: Record = {}) { function fallbackQueryAnalysis( analyzeClinicalQuery: (typeof import("../src/lib/clinical-search"))["analyzeClinicalQuery"], + isUnsupportedSoftTailAnalysis: (typeof import("../src/lib/rag/rag-query-guard"))["isUnsupportedSoftTailAnalysis"], ) { // A bare condition query above the short-query deterministic fallback threshold still needs // the LLM fallback (confidence below 0.58 with class unsupported_or_general). const query = "bipolar disorder long term care"; const analysis = analyzeClinicalQuery(query); expect(analysis.needsClassifierFallback).toBe(true); + // Pin the non-soft-tail shape explicitly so a scoring tweak cannot silently move this + // fixture into the soft-tail bucket (and vice versa) without failing loudly. + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(false); return { query, analysis }; } describe("classifier verdict memoization", () => { it("does not re-call the model for a repeated query and returns an identical verdict", async () => { const mock = vi.fn(async () => classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -73,8 +78,8 @@ describe("classifier verdict memoization", () => { it("memoizes rejected verdicts so a rejection is also deterministic", async () => { const mock = vi.fn(async () => classifierResponse({ confidence: 0.3 })); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -85,10 +90,70 @@ describe("classifier verdict memoization", () => { expect(second).toBe(analysis); }); + // "catatonia" is a bare single-word query: low confidence, no medications/thresholds/title + // terms, few expanded terms — exactly the soft-tail bucket the unsupported short-circuit + // treats specially (isUnsupportedSoftTailAnalysis true), unlike fallbackQueryAnalysis's + // multi-word fixture above (confidence 0.45, just above the 0.42 soft-tail ceiling). + function softTailQueryAnalysis( + analyzeClinicalQuery: (typeof import("../src/lib/clinical-search"))["analyzeClinicalQuery"], + isUnsupportedSoftTailAnalysis: (typeof import("../src/lib/rag/rag-query-guard"))["isUnsupportedSoftTailAnalysis"], + ) { + const query = "catatonia"; + // Mirror production: searchChunksWithTelemetry always passes opts.corpusGrounding, and a + // corpus-grounding verdict of "inconclusive" (no matching document title, so the query can't + // be deterministically rescued) is what routes a bare in-corpus topic like "catatonia" to the + // classifier at all — otherwise the Finding #2 deterministic short-query fallback (rag.ts + // ~1122-1134) rescues it before ever calling the classifier, and these tests would prove + // nothing about the memo. Setting corpusGrounding directly reproduces that gate without + // depending on the corpus-grounding module's live database call. + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; + expect(analysis.needsClassifierFallback).toBe(true); + // Pin soft-tail eligibility explicitly so a scoring tweak cannot silently move this + // fixture out of the soft-tail bucket without failing loudly. + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(true); + return { query, analysis }; + } + + it("still memoizes a rejected soft-tail verdict within its short TTL (bounded, not unlimited retries)", async () => { + const mock = vi.fn(async () => classifierResponse({ confidence: 0.3 })); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = softTailQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); + + const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); + const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); + + expect(mock).toHaveBeenCalledTimes(1); + expect(first).toBe(analysis); + expect(second).toBe(analysis); + }); + + it("re-calls the classifier for a rejected soft-tail verdict after its short TTL expires, so it can recover", async () => { + // The two-call proof Codex review asked for on PR #1646: reject on call 1, recover on call 2 + // — reachable only once the short soft-tail TTL (60s, well under the full 15-minute TTL) + // expires, never on an immediate repeat (see the previous test). + vi.useFakeTimers({ now: new Date("2026-07-06T00:00:00Z") }); + const mock = vi + .fn() + .mockResolvedValueOnce(classifierResponse({ confidence: 0.3 })) + .mockResolvedValueOnce(classifierResponse()); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = softTailQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); + + const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); + vi.setSystemTime(new Date("2026-07-06T00:01:01Z")); + const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); + + expect(mock).toHaveBeenCalledTimes(2); + expect(first).toBe(analysis); + expect(first.queryClass).toBe("unsupported_or_general"); + expect(second.queryClass).toBe("broad_summary"); + expect(second.needsClassifierFallback).toBe(false); + }); + it("sends only supported structural constraints to Structured Outputs", async () => { const mock = vi.fn(async () => classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -106,8 +171,8 @@ describe("classifier verdict memoization", () => { expandedTerms: Array.from({ length: 11 }, (_, index) => `term-${index}`), }), ); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -120,8 +185,8 @@ describe("classifier verdict memoization", () => { it("threads a pseudonymous safety identifier for an authenticated classifier request", async () => { vi.stubEnv("OPENAI_SAFETY_IDENTIFIER_SECRET", "test-secret-that-is-at-least-thirty-two-characters"); const mock = vi.fn(async () => classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); await rag.analyzeQueryWithClassifierFallback(query, analysis, { ownerId: "owner-a" }); @@ -133,8 +198,8 @@ describe("classifier verdict memoization", () => { it("does not memoize transport errors — the next request retries the classifier", async () => { const mock = vi.fn().mockRejectedValueOnce(new Error("timeout")).mockResolvedValueOnce(classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const first = await rag.analyzeQueryWithClassifierFallback(query, analysis); const second = await rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -152,8 +217,8 @@ describe("classifier verdict memoization", () => { resolveCall = resolve; }), ); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const firstPromise = rag.analyzeQueryWithClassifierFallback(query, analysis); const secondPromise = rag.analyzeQueryWithClassifierFallback(query, analysis); @@ -167,8 +232,8 @@ describe("classifier verdict memoization", () => { it("skips the classifier when fallback is not required", async () => { const mock = vi.fn(); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); const fallback = { ...analysis, needsClassifierFallback: false, queryClass: "unsupported_or_general" as const }; const result = await rag.analyzeQueryWithClassifierFallback(query, fallback); @@ -180,8 +245,8 @@ describe("classifier verdict memoization", () => { it("re-calls the model after the memo TTL expires", async () => { vi.useFakeTimers({ now: new Date("2026-07-06T00:00:00Z") }); const mock = vi.fn(async () => classifierResponse()); - const { rag, analyzeClinicalQuery } = await loadWithClassifierMock(mock); - const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery); + const { rag, analyzeClinicalQuery, isUnsupportedSoftTailAnalysis } = await loadWithClassifierMock(mock); + const { query, analysis } = fallbackQueryAnalysis(analyzeClinicalQuery, isUnsupportedSoftTailAnalysis); await rag.analyzeQueryWithClassifierFallback(query, analysis); vi.setSystemTime(new Date("2026-07-06T00:16:00Z")); diff --git a/tests/rag-query-guard-soft-tail-cache.test.ts b/tests/rag-query-guard-soft-tail-cache.test.ts new file mode 100644 index 0000000000..dcab1f0fb3 --- /dev/null +++ b/tests/rag-query-guard-soft-tail-cache.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; + +import { analyzeClinicalQuery } from "@/lib/clinical-search"; +import { + isUnsupportedSoftTailAnalysis, + shouldSkipUnsupportedSoftTailAnswerCacheWrite, + shouldSkipUnsupportedSoftTailCacheWrite, +} from "@/lib/rag/rag-query-guard"; + +describe("shouldSkipUnsupportedSoftTailCacheWrite", () => { + it("skips only when a classifier could have decided a soft-tail inconclusive zero", () => { + const query = "catatonia"; + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(true); + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: true, + }), + ).toBe(true); + }); + + it("does not skip when no classifier is reachable", () => { + const query = "catatonia"; + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: false, + }), + ).toBe(false); + }); + + it("does not skip a deterministic out_of_corpus zero even with a key present", () => { + const query = "catatonia"; + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "out_of_corpus" as const }; + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(true); + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: true, + }), + ).toBe(false); + }); + + it("does not skip deterministic exclusion patterns outside the soft-tail bucket", () => { + const query = "Show me the airport travel policy"; + const analysis = analyzeClinicalQuery(query); + expect(isUnsupportedSoftTailAnalysis(query, analysis)).toBe(false); + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: true, + corpusGrounding: "inconclusive", + }), + ).toBe(false); + }); + + it("honours an explicit corpusGrounding override over analysis.corpusGrounding", () => { + const query = "catatonia"; + const analysis = { ...analyzeClinicalQuery(query), corpusGrounding: "inconclusive" as const }; + + expect( + shouldSkipUnsupportedSoftTailCacheWrite(query, analysis, { + openAiApiKeyPresent: true, + corpusGrounding: "out_of_corpus", + }), + ).toBe(false); + }); + + it("skips the answer-cache write only for empty unsupported_short_circuit soft-tail zeros", () => { + const query = "catatonia"; + const analysis = analyzeClinicalQuery(query); + + expect( + shouldSkipUnsupportedSoftTailAnswerCacheWrite({ + resultCount: 0, + retrievalStrategy: "unsupported_short_circuit", + query, + analysis, + openAiApiKeyPresent: true, + corpusGrounding: "inconclusive", + }), + ).toBe(true); + expect( + shouldSkipUnsupportedSoftTailAnswerCacheWrite({ + resultCount: 1, + retrievalStrategy: "unsupported_short_circuit", + query, + analysis, + openAiApiKeyPresent: true, + corpusGrounding: "inconclusive", + }), + ).toBe(false); + expect( + shouldSkipUnsupportedSoftTailAnswerCacheWrite({ + resultCount: 0, + retrievalStrategy: "text_fast_path", + query, + analysis, + openAiApiKeyPresent: true, + corpusGrounding: "inconclusive", + }), + ).toBe(false); + }); +}); diff --git a/tests/rag-shared-cache.test.ts b/tests/rag-shared-cache.test.ts index 9daaae8965..a74f1eacf3 100644 --- a/tests/rag-shared-cache.test.ts +++ b/tests/rag-shared-cache.test.ts @@ -62,4 +62,67 @@ describe("shared RAG search cache", () => { expect(result).toEqual({ kind: "miss", reason: "unknown_filter_miss" }); expect(sharedCacheReads).toBe(1); }); + + it("forwards corpus_grounding from the stored payload on a shared-cache hit", async () => { + vi.resetModules(); + + vi.doMock("@/lib/env", () => ({ + env: { + RAG_SEARCH_CACHE_TTL_MS: 60_000, + RAG_SEARCH_CACHE_SIZE: 200, + RAG_PERSIST_RAW_QUERY_TEXT: false, + RAG_QUERY_HASH_SECRET: "test-query-hash-secret", + }, + isDemoMode: () => false, + isLocalNoAuthMode: () => false, + })); + vi.doMock("@/lib/deep-memory", () => ({ ragDeepMemoryVersion: "test-rag-version" })); + vi.doMock("@/lib/clinical-search", () => ({ + buildClinicalTextSearchQuery: (query: string) => query.trim(), + })); + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + from: (table: string) => { + const builder = { + select: () => builder, + eq: () => builder, + is: () => builder, + in: () => builder, + or: () => builder, + gt: () => builder, + order: () => builder, + limit: () => builder, + maybeSingle: async () => { + if (table !== "rag_response_cache") return { data: null, error: null }; + return { + data: { + payload: { + results: [], + telemetry: { query_class: "unsupported_or_general", corpus_grounding: "in_corpus_topic" }, + }, + }, + error: null, + }; + }, + then: (resolve: (value: { data: unknown[]; error: null }) => unknown) => + Promise.resolve({ + data: + table === "documents" ? [{ id: "doc-1", updated_at: "2026-07-14T00:00:00.000Z", metadata: {} }] : [], + error: null, + }).then(resolve), + }; + return builder; + }, + }), + })); + + const { getSharedCachedSearch } = await import("../src/lib/rag/rag-cache"); + const result = await getSharedCachedSearch({ + query: "catatonia", + ownerId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + }); + + expect(result?.kind).toBe("hit"); + expect(result?.kind === "hit" && result.telemetry.corpus_grounding).toBe("in_corpus_topic"); + }); }); diff --git a/tests/rag-unsupported-short-circuit-cache.test.ts b/tests/rag-unsupported-short-circuit-cache.test.ts new file mode 100644 index 0000000000..01e7d10405 --- /dev/null +++ b/tests/rag-unsupported-short-circuit-cache.test.ts @@ -0,0 +1,195 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +// Finding #11 follow-up (handover 2026-08-06): the soft-tail unsupported short-circuit can be a +// false negative for genuinely in-corpus bare topics (e.g. "catatonia" is well represented in +// chunk content but no document is *titled* "Catatonia", so corpus grounding returns +// "inconclusive" rather than rescuing it, and the query falls through to the nondeterministic +// LLM classifier). Caching that zero made the false negative sticky for every later caller within +// the cache TTL. searchChunksWithTelemetry (src/lib/rag/rag.ts) must skip the cache write only for +// that specific soft-tail bucket, while still caching the three deterministic exclusion patterns +// (unavailable-document noise, clearly-outside-corpus terms, clearly-non-clinical consumer terms) +// that are stable true negatives. + +class EmptyQuery implements PromiseLike<{ data: unknown[]; error: null }> { + select() { + return this; + } + in() { + return this; + } + eq() { + return this; + } + is() { + return this; + } + neq() { + return this; + } + or() { + return this; + } + order() { + return this; + } + limit() { + return Promise.resolve({ data: [], error: null }); + } + then( + onfulfilled?: ((value: { data: unknown[]; error: null }) => TResult1 | PromiseLike) | null, + onrejected?: ((reason: unknown) => TResult2 | PromiseLike) | null, + ): PromiseLike { + return Promise.resolve({ data: [], error: null }).then(onfulfilled, onrejected); + } +} + +async function loadSearch( + corpusGroundingVerdict: "inconclusive" | "out_of_corpus" | "in_corpus_topic", + options: { openAiApiKey?: string } = {}, +) { + const setCachedSearch = vi.fn(async () => undefined); + + vi.doMock("@/lib/rag/rag-cache", async () => { + const actual = await vi.importActual("@/lib/rag/rag-cache"); + return { + ...actual, + cacheIndexingVersion: vi.fn(async () => "test-indexing-version"), + getCachedSearch: vi.fn(async () => null), + getSharedCachedSearch: vi.fn(async () => null), + setCachedSearch, + }; + }); + vi.doMock("@/lib/rag/rag-retrieval-variants", async () => { + const actual = await vi.importActual( + "@/lib/rag/rag-retrieval-variants", + ); + return { + ...actual, + fetchEnabledRagAliases: vi.fn(async () => []), + }; + }); + vi.doMock("@/lib/corpus-grounding", () => ({ + classifyCorpusGrounding: vi.fn(async () => ({ verdict: corpusGroundingVerdict, anchorTerms: [], absentTerms: [] })), + })); + // A rejected classifier verdict — only reachable, and only mock-invoked, when the test opts + // into a non-empty OPENAI_API_KEY below (Devin review: the classifier is unreachable at all + // without a key, so that case must stay deterministic and cacheable). + vi.doMock("@/lib/openai", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + generateParsedTextResult: vi.fn(async () => ({ + parsed: { queryClass: "unsupported_or_general", confidence: 0.3, reasons: ["test"], expandedTerms: [] }, + })), + }; + }); + vi.doMock("@/lib/rag/rag-provider", () => ({ + isSourceOnlyMode: () => true, + allowsAutoDegrade: () => true, + sourceOnlyReason: () => "source_only", + classifyProviderFailure: () => "provider_failure", + SOURCE_ONLY_EMBEDDING_SKIP_REASON: "source_only", + })); + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc: vi.fn(async () => ({ data: [], error: null })), + from: vi.fn(() => new EmptyQuery()), + }), + })); + vi.stubEnv("OPENAI_API_KEY", options.openAiApiKey ?? ""); + + const { searchChunksWithTelemetry } = await import("../src/lib/rag/rag"); + return { searchChunksWithTelemetry, setCachedSearch }; +} + +const ownerId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + +describe("unsupported short-circuit cache write", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.resetModules(); + vi.unstubAllEnvs(); + }); + + it("does not cache a zero result for the soft-tail bucket when a classifier could have decided it (OPENAI_API_KEY set)", async () => { + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive", { + openAiApiKey: "test-key", + }); + + const result = await searchChunksWithTelemetry({ + query: "catatonia", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).not.toHaveBeenCalled(); + }, 60_000); + + it("caches a soft-tail zero when no classifier was ever reachable (no OPENAI_API_KEY — deterministic)", async () => { + // Devin review on PR #1646: without a key, analyzeQueryWithClassifierFallback returns before + // any classifier call (rag.ts:1139), so the "inconclusive" corpus-grounding verdict is the + // whole story — same query, same DB state, same empty result every time. Skipping the cache + // write here bought nothing but repeat classifyCorpusGrounding + trigram-RPC cost. + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive"); + + const result = await searchChunksWithTelemetry({ + query: "catatonia", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).toHaveBeenCalledTimes(1); + }, 60_000); + + it("still caches the deterministic unavailable-document-noise short circuit", async () => { + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("inconclusive"); + + const result = await searchChunksWithTelemetry({ + query: "Show me the airport travel policy", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).toHaveBeenCalledTimes(1); + }, 60_000); + + it("still caches a deterministic out-of-corpus zero even though it looks soft-tail-shaped", async () => { + // Devin review on PR #1646: isUnsupportedSoftTailAnalysis only looks at the query text and + // deterministic analysis, not queryAnalysis.corpusGrounding, so a real "out_of_corpus" verdict + // (a deterministic, corpus-derived true negative reached without any LLM call) must be + // excluded from the soft-tail cache-write skip, or every repeat re-runs classifyCorpusGrounding + // and the trigram-correction RPC for a query that will always come back empty. + const { searchChunksWithTelemetry, setCachedSearch } = await loadSearch("out_of_corpus"); + + const result = await searchChunksWithTelemetry({ + query: "catatonia", + ownerId, + lexicalOnly: true, + }); + + expect(result.results).toEqual([]); + expect(result.telemetry.retrieval_strategy).toBe("unsupported_short_circuit"); + expect(setCachedSearch).toHaveBeenCalledTimes(1); + }, 60_000); + + it("rescues an in-corpus bare topic before ever reaching the short circuit", async () => { + const { searchChunksWithTelemetry } = await loadSearch("in_corpus_topic"); + + const result = await searchChunksWithTelemetry({ + query: "catatonia", + ownerId, + lexicalOnly: true, + }); + + // Pin the rescue itself. Do not assert on setCachedSearch here: with an empty mocked + // Supabase client the retrieval path may write nothing for unrelated reasons, so a + // "not called" expectation would pass for the wrong reason if candidates later appear. + expect(result.telemetry.retrieval_strategy).not.toBe("unsupported_short_circuit"); + }, 60_000); +});