diff --git a/docs/branch-review-records/a8521fd05bde007f6203e4cdd7fee80e20e0dbfc7f8d3d99fc588dfef30b50fe.record.md b/docs/branch-review-records/a8521fd05bde007f6203e4cdd7fee80e20e0dbfc7f8d3d99fc588dfef30b50fe.record.md new file mode 100644 index 0000000000..9fd2a0b3fe --- /dev/null +++ b/docs/branch-review-records/a8521fd05bde007f6203e4cdd7fee80e20e0dbfc7f8d3d99fc588dfef30b50fe.record.md @@ -0,0 +1 @@ +| 2026-08-17 | claude/s1c-residuals-r2-r3-4pb1at | beb7298a8cdac29b568bc425e9736c9415729e9b | packet S1c: rag-claim-support R2 normative-norm disjunct + R3 adjacent atom-free topic lending, tests | PR #2052 open; offline 613/613; verify:pr-local heavy scope green; R3 measured 87->78 sole-overlap rejections, zero protective flips | vitest rag-claim-support 157/157; eval:rag:offline 613/613; check:rag:fixtures 36 golden/25 suites; verify:pr-local (lint, typecheck, test, build) green; check:production-readiness offline provider gaps only | diff --git a/docs/design-system/adoption-manifest.json b/docs/design-system/adoption-manifest.json index d4adc0225f..6f34baf01c 100644 --- a/docs/design-system/adoption-manifest.json +++ b/docs/design-system/adoption-manifest.json @@ -1210,7 +1210,11 @@ "preview": ".design-sync/previews/Progress.tsx", "previewValid": true }, - "testFiles": ["tests/design-sync-visual-exports.test.ts", "tests/ui-v2-components.dom.test.tsx"], + "testFiles": [ + "tests/design-sync-visual-exports.test.ts", + "tests/rag-claim-support.test.ts", + "tests/ui-v2-components.dom.test.tsx" + ], "baseline": { "targetLayer": "v2", "liveLayer": "v2", diff --git a/docs/outstanding-issues-inbox/632c50f9-6e6a-4247-8b75-11572137e579.json b/docs/outstanding-issues-inbox/632c50f9-6e6a-4247-8b75-11572137e579.json new file mode 100644 index 0000000000..cdb050022c --- /dev/null +++ b/docs/outstanding-issues-inbox/632c50f9-6e6a-4247-8b75-11572137e579.json @@ -0,0 +1,10 @@ +{ + "version": 2, + "id": "632c50f9-6e6a-4247-8b75-11572137e579", + "createdOn": "2026-08-17", + "action": "done", + "payload": { + "id": "#6BG9X2", + "outcome": "Fixed in PR #2052 (packet S1c): R2 descriptive-norm disjunct in normativeDirectiveActions (digit-anchored, descriptiveContext-guarded, adversarial negatives for care-record prose, unrelated imperatives, and incidental norm-adjacent action words) and R3 adjacent atom-free-segment topic lending confined to the overlap clause with all other gates single-segment. Measured offline before loosening: 87 -> 78 sole-overlap rejections (46 -> 42 unique), zero protective fixture flips across the 613-test offline corpus; discriminating negatives pin cross-bullet dose mis-binding, non-adjacent synthesis, and alien-topic claims. Post-merge canary pair owner-approved. Note: request file emitted via the repo inbox schema because issues:done rejects ULID display ids (issueRowFingerprint is numeric-only)." + } +} diff --git a/docs/rag-improvement/HANDOVER.md b/docs/rag-improvement/HANDOVER.md index 75ac30fd79..c8ddb44594 100644 --- a/docs/rag-improvement/HANDOVER.md +++ b/docs/rag-improvement/HANDOVER.md @@ -74,7 +74,7 @@ generation-quality verdict on fallback`), merged 2026-08-13 — structured | S0 | A1 phase 1: structured fallback diagnostics | `claude/lithium-generation-quality-debug-ji1vce` | #1899 | Merged 2026-08-13 | offline 93/93 focused | | S1 | A1 phase 2: rung-1 verification-faithfulness fixes | `claude/s1-rag-mitigation-231-86c182` | #2022 | Merged 2026-08-17 (squash `2bd146eed`, landed by content) | 8 pre-fix + 5 post-fix live probes 2026-08-17; offline 583/583; canary pair run 31964560921 (baseline `8f8d111ab`) -> run 32025082010 (`2bd146eed`): recall 1.0/1.0, zero per-case rr regressions, answer gate 44/44 (denominator reconciled by S5; see baseline-record §3); rung-2 measurement in `docs/audit/live-drift-forensics-2026-08.md` §5 | | S1b | A1 rung 3 (R1): pre-deadline strong routing for dosing class | `claude/s1b-rag-dosing-routing-6u1mik` | #2035 | Merged 2026-08-17 (PR #2035, merge `92f7618`) | canary pair pending: baseline run 32025082010 (`2bd146eed`) -> post-merge dispatch (owner-approved); offline 586/586 + verify:pr-local heavy scope green | -| S1c | A1 residuals R2 + R3: claim-support strictness | `claude/rag-a1-r2-r3-claim-support-` | — | Ready — dispatch now (S1b merged, canary pair green 2026-08-17) | needs canary pair | +| S1c | A1 residuals R2 + R3: claim-support strictness | `claude/s1c-residuals-r2-r3-4pb1at` | #2052 | PR open 2026-08-17 | offline 613/613 + verify:pr-local heavy scope green; R3 measured before loosening: 87->78 sole-overlap rejections (46->42 unique), zero protective flips; canary pair planned post-merge: baseline run 32039841070 (`92f7618`) -> post-merge dispatch (owner-approved) | | S1d | A1 final-gate gap recovery: hedged cited low-confidence fast answers must recover extractively, not collapse to a citation-free `provider_source_gap` | `claude/rag-a1-final-gate-gap-recovery-` | — | Ready — dispatch now; parallel-safe with S1c (different file); lands before S2 | needs canary pair; evidence: canary runs 32038751592 (red) vs 32039841070 (green), 3/3 live probes on the extractive branch | | G1 | Governance: provenance tag for document-summary rows (Option B) | `claude/rag-g1-document-context-origin-` | — | Ready — disjoint; owner decided Option B 2026-08-17 | no canary (no behaviour change) | | S2 | A2 (+A3): composition menu + moderate length | `claude/rag-a2-composition-` | — | Blocked on S1b + S1c | canary pair + `eval:answer-quality` + Gate E | diff --git a/src/lib/rag/rag-claim-support.ts b/src/lib/rag/rag-claim-support.ts index 714848fb84..b4df666079 100644 --- a/src/lib/rag/rag-claim-support.ts +++ b/src/lib/rag/rag-claim-support.ts @@ -416,7 +416,15 @@ function normativeDirectiveActions(value: string) { "i", ).test(value) || (!descriptiveContext && - new RegExp(String.raw`\b(?:is|are)\s+${boundedDirectiveAdverbs}(?:${definition.forms})\b`, "i").test(value)); + new RegExp(String.raw`\b(?:is|are)\s+${boundedDirectiveAdverbs}(?:${definition.forms})\b`, "i").test(value)) || + // Descriptive guideline norm: "the usual/recommended dose … is " + // (S1c R2). The digit anchor and the premodifier bound keep incidental + // norm-adjacent action words ("a typical error is a repeated dose …") inert. + (!descriptiveContext && + new RegExp( + String.raw`\b(?:usual|recommended|typical|standard|initial)\b[^.!?;:\n]{0,30}?\b(?:${definition.forms})\b\s+(?:[a-z-]+\s+){0,2}dos(?:e|age|ing|es)\b[^.!?;:\n]{0,60}?\b(?:is|are)\b[^.!?;:\n]{0,16}?\d`, + "i", + ).test(value)); if (hasNormativeSignal) actions.add(definition.action); } return actions; @@ -467,7 +475,7 @@ export function sourceEvidenceText(source: SearchResult) { .join(" "); } -function evidenceTextSupportsClaim(claim: string, evidence: string) { +function evidenceTextSupportsClaim(claim: string, evidence: string, adjacentTopicText?: string) { const claimEntities = entities(claim); const evidenceEntities = entities(evidence); if (claimEntities.size > 0 && [...claimEntities].some((entity) => !evidenceEntities.has(entity))) return false; @@ -482,6 +490,11 @@ function evidenceTextSupportsClaim(claim: string, evidence: string) { const claimTopics = topicTokens(claim); const evidenceTopics = topicTokens(evidence); + // S1c R3: a claim synthesising two adjacent source bullets may count topic tokens + // from an immediately adjacent atom-free segment. Only this overlap ratio widens — + // entities, polarity, directives, safety dimensions, the trigger check, and the + // value-atom containment above all still hold against the single segment. + for (const token of topicTokens(adjacentTopicText ?? "")) evidenceTopics.add(token); const matchedTopics = [...claimTopics].filter((token) => evidenceTopics.has(token)).length; return claimTopics.size === 0 || matchedTopics / claimTopics.size >= 0.5; } @@ -502,7 +515,7 @@ function usesSourceBoundComparisonReflow(source: SearchResult, claim: string) { return knownPolicySource && knownRequirementClaim; } -function sourceEvidenceClaimSegments(source: SearchResult, claim: string) { +function sourceEvidenceClaimSegmentGroups(source: SearchResult, claim: string) { const split = (value: string | null | undefined, context?: string | null, reflowVisualLines = false) => { const rawValue = reflowWrappedAgitationDoseLines(reflowWrappedEscalationRecipientLines(value ?? "")); // Always rejoin PDF visual line wraps before sentence-splitting. Splitting raw extracted @@ -538,11 +551,14 @@ function sourceEvidenceClaimSegments(source: SearchResult, claim: string) { sourceLabel: [source.title, source.file_name].filter(Boolean).join(" "), content: source.content ?? "", }); + // Segments are grouped by source representation so adjacency means "next to each + // other in the same representation", never a flat-list coincidence across e.g. the + // content/synopsis boundary. return [ - ...(atomicClozapineRedRange ? [`${sourceContext}. ${atomicClozapineRedRange}`] : []), - ...split(source.content, sourceContext, reflowComparisonContent), - ...split(source.retrieval_synopsis, sourceContext), - ...(source.table_facts ?? []).flatMap((fact) => + ...(atomicClozapineRedRange ? [[`${sourceContext}. ${atomicClozapineRedRange}`]] : []), + split(source.content, sourceContext, reflowComparisonContent), + split(source.retrieval_synopsis, sourceContext), + ...(source.table_facts ?? []).map((fact) => split( [[fact.row_label, fact.clinical_parameter, fact.threshold_value].filter(Boolean).join(" "), fact.action] .filter(Boolean) @@ -550,13 +566,26 @@ function sourceEvidenceClaimSegments(source: SearchResult, claim: string) { fact.table_title, ), ), - ...split(source.index_unit?.content, source.index_unit?.title), - ]; + split(source.index_unit?.content, source.index_unit?.title), + ].filter((group) => group.length > 0); } function sourceSupportsClaim(claim: string, source: SearchResult) { if (!isHighRiskClaim(claim)) return evidenceTextSupportsClaim(claim, sourceEvidenceText(source)); - return sourceEvidenceClaimSegments(source, claim).some((evidence) => evidenceTextSupportsClaim(claim, evidence)); + return sourceEvidenceClaimSegmentGroups(source, claim).some((group) => + group.some((evidence, index) => { + // S1c R3: lend only the immediately adjacent segments' topic tokens, and only + // from segments carrying no clinical value atoms of their own — an atom-bearing + // neighbour is a competing value context (e.g. the other population's dose), not + // supplementary prose, and lending its topics would let a claim bind this + // segment's value to the neighbour's condition. + const adjacentTopicText = [group[index - 1], group[index + 1]] + .filter((neighbour): neighbour is string => Boolean(neighbour)) + .filter((neighbour) => extractClinicalValueAtoms(neighbour).length === 0) + .join(" "); + return evidenceTextSupportsClaim(claim, evidence, adjacentTopicText || undefined); + }), + ); } /** diff --git a/tests/rag-claim-support.test.ts b/tests/rag-claim-support.test.ts index 2ebcb21d00..426cb2e76a 100644 --- a/tests/rag-claim-support.test.ts +++ b/tests/rag-claim-support.test.ts @@ -132,6 +132,159 @@ describe("deterministic claim support", () => { expect(claims[0]?.supportingChunkIds).toEqual(["wrapped-dose-bullet"]); }); + it("supports an imperative dosing claim against descriptive guideline norm phrasing (S1c R2)", () => { + // R2: "The usual … starting dose … is 500 mg" is a descriptive norm. The imperative + // claim expects a normative "start" signal, which no prior pattern produced. + const emhs = source( + "emhs-lithium-dosing", + [ + "Dosage and administration", + "", + "• The usual oral starting dose for adults is 500 mg nocte and for patients over 65 years it", + "is 250 mg nocte.", + "• Dose should be titrated against target serum levels.", + ].join("\n"), + { title: "Lithium Clinical Guideline(EMHS)", file_name: "Lithium Clinical Guideline(EMHS).pdf" }, + ); + + expect(sourceDirectlySupportsAnswerText("Start lithium at 500 mg nocte.", emhs)).toBe(true); + }); + + it("keeps rejecting imperative claims whose only echo is descriptive care-record prose (S1c R2 negative)", () => { + const chartRecord = source( + "chart-record", + "Progress notes: the patient was started on lithium 500 mg nocte at this visit.", + ); + + expect(sourceDirectlySupportsAnswerText("Start lithium at 500 mg nocte.", chartRecord)).toBe(false); + }); + + it("does not let a norm phrase for one action lend normativity to an unrelated imperative (S1c R2 negatives)", () => { + const sameSentence = source( + "stop-same-sentence", + "The usual practice after abrupt stopping of lithium is that the dose is reduced.", + ); + const crossSentence = source( + "stop-cross-sentence", + "The usual oral starting dose for adults is 500 mg nocte. Abrupt stopping is associated with relapse.", + { title: "Lithium Clinical Guideline(EMHS)" }, + ); + + expect(sourceDirectlySupportsAnswerText("Stop lithium.", sameSentence)).toBe(false); + expect(sourceDirectlySupportsAnswerText("Stop lithium.", crossSentence)).toBe(false); + }); + + it("does not read incidental norm-adjacent action words as normative directives (S1c R2 negatives)", () => { + const repeatedDoseWarning = source( + "repeated-dose-warning", + "A typical error is a repeated dose when the schedule is unclear.", + ); + const monitoredPatientDescription = source( + "monitored-patient", + "The typical patient monitored on this dose is reviewed weekly.", + ); + const startingWeightRecord = source("starting-weight", "The usual practice is to record the starting weight."); + + expect(sourceDirectlySupportsAnswerText("Repeat the dose.", repeatedDoseWarning)).toBe(false); + expect(sourceDirectlySupportsAnswerText("Monitor the dose weekly.", monitoredPatientDescription)).toBe(false); + expect(sourceDirectlySupportsAnswerText("Start lithium at 500 mg nocte.", startingWeightRecord)).toBe(false); + }); + + it("still requires the claimed medication itself in the dosing evidence (S1c R2 negative)", () => { + const emhs = source( + "emhs-lithium-entity", + "• The usual oral starting dose for adults is 500 mg nocte and for patients over 65 years it\nis 250 mg nocte.", + { title: "Lithium Clinical Guideline(EMHS)", file_name: "Lithium Clinical Guideline(EMHS).pdf" }, + ); + + expect(sourceDirectlySupportsAnswerText("Start sertraline at 500 mg nocte.", emhs)).toBe(false); + }); + + it("requires normative evidence for a descriptive-norm claim rather than incidental care history (S1c R2 pin)", () => { + // R2 also grows the CLAIM side: a descriptive-norm claim now expects normative + // evidence, so an incidental care-history sentence stops counting as support. + const careHistory = source("care-history", "Patients were started on an average dose of 500 mg nocte."); + + expect(sourceDirectlySupportsAnswerText("The usual starting dose is 500 mg nocte.", careHistory)).toBe(false); + }); + + it("supports a claim synthesising two adjacent source bullets when one segment carries every atom (S1c R3)", () => { + const emhs = source( + "emhs-lithium-synthesis", + [ + "Dosage and administration", + "", + "• The usual oral starting dose for adults is 500 mg nocte and for patients over 65 years it", + "is 250 mg nocte.", + "• Dose should be titrated against target serum levels.", + ].join("\n"), + { title: "Lithium Clinical Guideline(EMHS)", file_name: "Lithium Clinical Guideline(EMHS).pdf" }, + ); + + expect( + sourceDirectlySupportsAnswerText( + "Start lithium at 500 mg nocte and titrate the dose against target serum levels.", + emhs, + ), + ).toBe(true); + }); + + it("does not borrow topics from an adjacent bullet that carries its own competing dose atoms (S1c R3 negative)", () => { + // Cross-bullet value mis-binding: the over-65 population must not be bound to the + // adult dose merely because the two bullets are adjacent. + const perPopulationBullets = source( + "per-population-bullets", + [ + "• Adults: the usual starting dose is 500 mg nocte.", + "• Patients over 65 years: the usual starting dose is 250 mg nocte.", + ].join("\n"), + { title: "Lithium Clinical Guideline(EMHS)", file_name: "Lithium Clinical Guideline(EMHS).pdf" }, + ); + + expect( + sourceDirectlySupportsAnswerText( + "For patients over 65 years, start lithium at 500 mg nocte.", + perPopulationBullets, + ), + ).toBe(false); + }); + + it("does not synthesise support across non-adjacent segments (S1c R3 negative)", () => { + const spacedBullets = source( + "spaced-bullets", + [ + "• The usual oral starting dose for adults is 500 mg nocte.", + "• Dose should be titrated against target serum levels.", + "• Take samples 12 hours after the last evening dose.", + "• Annual thyroid function testing is recommended.", + ].join("\n"), + { title: "Lithium Clinical Guideline(EMHS)", file_name: "Lithium Clinical Guideline(EMHS).pdf" }, + ); + + expect( + sourceDirectlySupportsAnswerText( + "Start lithium at 500 mg nocte and arrange annual thyroid function testing.", + spacedBullets, + ), + ).toBe(false); + }); + + it("keeps rejecting an alien-topic claim whose atoms coincidentally match the dosing bullet (S1c R3 negative)", () => { + const emhs = source( + "emhs-lithium-alien-topic", + [ + "• The usual oral starting dose for adults is 500 mg nocte and for patients over 65 years it", + "is 250 mg nocte.", + "• Dose should be titrated against target serum levels.", + ].join("\n"), + { title: "Lithium Clinical Guideline(EMHS)", file_name: "Lithium Clinical Guideline(EMHS).pdf" }, + ); + + expect( + sourceDirectlySupportsAnswerText("Start lithium at 500 mg nocte to prevent migraine recurrence.", emhs), + ).toBe(false); + }); + it("keeps a wrapped escalation recipient in the directly supporting source segment", () => { const wrappedRule = source( "wrapped-escalation-rule",