diff --git a/data/outstanding-issues-snapshot.json b/data/outstanding-issues-snapshot.json index e61c4caf84..ff5cd51854 100644 --- a/data/outstanding-issues-snapshot.json +++ b/data/outstanding-issues-snapshot.json @@ -10,7 +10,7 @@ "p2": 49, "p3": 31, "queued": 7, - "pending": 84, + "pending": 85, "resolved": 431 }, "queue": [ @@ -884,6 +884,12 @@ "summary": "#GQ5X8T: NOT REPRODUCED ON PINNED LINUX; WINDOWS-SPECIFIC CAUSE UNRESOLVED. Eleven retained PR #2313 ubuntu-24.04 Lighthouse artifacts used Lighthouse 12.8.2 and pinned HeadlessChrome/151.0.0.0. In the nine-run synchronized-baseline subset, mobile-root LCP was 2260.791-2300.923ms with median 2287.574ms versus 2274.017ms baseline (+13.557ms, +0.60%); TBT median was 430.602ms versus 436.600ms baseline (-1.37%); CLS was 0 throughout. Across all eleven heads LCP was 2221.675-2300.923ms, never reproducing the Windows 7422-8032ms state. The exact local 883f1007 control / 7cf37b53 treatment snapshots had identical lock hashes, but the local host stayed at 42.3-62.9% CPU and the control build hit the 10-minute Linux cap after compiling, before measurement; that attempt is environment-blocked and was not rerun. This excludes a repeatable cross-platform regression but cannot distinguish Windows host noise from a Windows-specific client regression. No product, budget, tolerance, request-count, or proven CLS change was made. Reopen if a pinned Linux exact-head run breaches repeatably or a synchronized quiet-host Windows control reproduces the slowdown. Evidence: docs/evidence/mobile-root-timing-control-2026-08-26.md.", "created_at": "2026-08-26" }, + { + "request_id": "224cda1c-ae72-4921-b8aa-e6b3e315796a", + "action": "update", + "summary": "#ZBAC9D: detail → documents.owner_id is ON DELETE SET NULL (supabase/schema.sql) while a null owner independently means 'public corpus' in retrieval. CORRECTION (Codex on PR #2494): deleting an auth user does NOT silently republish private documents — documents_require_publication_approval (BEFORE INSERT OR UPDATE) runs guard_document_publication_transition, which rejects every non-null-to-null owner transition without matching publication evidence, so the user-delete transaction fails instead of orphaning/republishing. Distinguish blocked user deletion from any pre-existing orphan rows. RE-VERIFIED against main and the P2 -> P1 escalation still holds on the retrieval hazard, not on a live delete-to-publish path. (1) The retrieval path is NOT closed at any layer. public.retrieval_owner_matches still resolves the public sentinel to 'row_owner_id is null' with no public_corpus condition, and grep for 'public_corpus' returns ZERO occurrences across the entire retrieval chain: rag/rag-candidate-sources.ts, corpus-grounding.ts, deep-memory.ts, document-enrichment.ts, owner-scope.ts, clinical-search.ts and query-privacy.ts. Any ownerless row (pre-existing orphan or armed hole) is therefore retrievable as public corpus and citable in an answer to an anonymous query. (2) The helper written to close exactly this hazard is DEAD CODE. src/lib/documents/is-public-document.ts requires owner_id === null AND metadata.public_corpus === true, and tests/documents-is-public.test.ts:22 pins the precise case ('Orphaned after user delete' -> false), but grep across src/ and worker/ finds NO production caller outside its own module and the barrel re-export. Same shape as #A3AHCX. (3) The only genuinely closed application read path is the REST document read: withOwnerReadScope in src/lib/public-api-access.ts applies .is('owner_id', null).eq('metadata->>public_corpus','true'). NEXT, in order: (a) one read-only live count of documents where owner_id is null and metadata->>'public_corpus' is distinct from 'true' — this decides whether the hole is currently leaking or merely armed, and needs owner approval as a provider-backed read; (b) close the retrieval path, which is a genuine retrieval behaviour change on a protected RAG surface and so needs the RAG flag plus a live eval-canary pair; (c) add a database proof that deleting an owner of an unapproved private document fails and creates no ownerless indexed row, then in an approved production window change documents.owner_id to ON DELETE RESTRICT (or an explicit visibility column) and align the RPC predicate. Note the offline proof that no table whose null owner means public declares ON DELETE SET NULL cannot be added green today — it would fail on documents.owner_id, so it is the guard that accompanies (c), not a step that precedes it.; priority → P1; source → repo-wide audit at 3ed1932 (six domain reviewers); re-verified and expanded against main a633eb6e3 by the database coordination chat 2026-09-01; deletion-path correction per Codex review on PR #2494", + "created_at": "2026-09-01" + }, { "request_id": "256a5104-df13-44d3-9734-645e08b2b0df", "action": "cancel", diff --git a/data/repo-awareness-snapshot.json b/data/repo-awareness-snapshot.json index dd4998adfd..97f1ab3851 100644 --- a/data/repo-awareness-snapshot.json +++ b/data/repo-awareness-snapshot.json @@ -1,8 +1,8 @@ { "version": "repo-awareness-snapshot-v1", "captured_revision": { - "sha": "612e9735c0096c046fad01a1b3a3b23482b3fc07", - "committed_at": "2026-09-01T17:10:51+08:00" + "sha": "bf4489206167d41e78e629e10194a3856616afea", + "committed_at": "2026-09-01T18:36:13+08:00" }, "routes": { "modes": [ @@ -573,6 +573,11 @@ "file": "src/app/mockups/development/page.tsx", "area": "mockup" }, + { + "path": "/mockups/development/clinical-answer-failures", + "file": "src/app/mockups/development/clinical-answer-failures/page.tsx", + "area": "mockup" + }, { "path": "/mockups/development/clinical-trust", "file": "src/app/mockups/development/clinical-trust/page.tsx", @@ -1420,9 +1425,9 @@ ], "counts": { "modes": 15, - "pages": 198, + "pages": 199, "product_pages": 63, - "mockup_pages": 135, + "mockup_pages": 136, "redirects": 17, "api": 60 } diff --git a/docs/site-map.md b/docs/site-map.md index 16f8a66675..dd7923767d 100644 --- a/docs/site-map.md +++ b/docs/site-map.md @@ -1199,6 +1199,7 @@ This file is generated by `npm run docs:update` (or `npm run sitemap:update` dir - `/mockups/caring-contacts/templates` - Route discovered from app directory Source: `src/app/mockups/caring-contacts/templates/page.tsx`. - `/mockups/caring-contacts/templates/[pathwayId]` - Route discovered from app directory Source: `src/app/mockups/caring-contacts/templates/[pathwayId]/page.tsx`. - `/mockups/development` - Route discovered from app directory Source: `src/app/mockups/development/page.tsx`. +- `/mockups/development/clinical-answer-failures` - Route discovered from app directory Source: `src/app/mockups/development/clinical-answer-failures/page.tsx`. - `/mockups/development/clinical-trust` - Route discovered from app directory Source: `src/app/mockups/development/clinical-trust/page.tsx`. - `/mockups/development/documentation` - Route discovered from app directory Source: `src/app/mockups/development/documentation/page.tsx`. - `/mockups/development/ingestion` - Route discovered from app directory Source: `src/app/mockups/development/ingestion/page.tsx`. diff --git a/src/app/mockups/development/clinical-answer-failures/page.tsx b/src/app/mockups/development/clinical-answer-failures/page.tsx new file mode 100644 index 0000000000..4f745d437a --- /dev/null +++ b/src/app/mockups/development/clinical-answer-failures/page.tsx @@ -0,0 +1,124 @@ +import type { Metadata } from "next"; + +import { PanelPageShell } from "@/components/developer-area/hub/panel-page-shell"; +import { CountTile, META_CLASS, PanelSection } from "@/components/developer-area/hub/panel-primitives"; +import { referencedQuestionCount, resolveClinicalAnswerFailures } from "@/lib/developer-area/clinical-answer-failures"; +import { loadLedgerSnapshot, resolveFreshness } from "@/lib/developer-area/ledger-snapshot"; + +export const metadata: Metadata = { + title: "Answer failures · Developer · PsychSift", + description: "Open ledger items that name one of the repository's clinical eval questions.", +}; + +export default function DeveloperClinicalAnswerFailuresPage() { + const snapshot = loadLedgerSnapshot(); + const now = new Date(); + const freshness = resolveFreshness(snapshot, now); + const failures = resolveClinicalAnswerFailures(snapshot); + const questions = referencedQuestionCount(failures); + + return ( + +
+ + +
+ + {/* + * The scope caveat is page content, not a footnote, and it is deliberately + * above the list rather than below it. This page reports the intersection + * of two records — the task ledger and the eval case list — and that + * intersection is narrower than "clinical answer quality". Anyone reading + * an empty list here must not conclude the answer engine is well. + */} +

+ Every open item in the task ledger that names one of the repository’s clinical eval questions by its case + id. That is a narrower thing than a picture of answer quality: a recorded problem that does not name a case is + not listed here, and a question that is failing right now but that nobody has written up is not either. An empty + list means nothing is recorded against a named question, not that every question answers well. +

+ {/* + * The correction from review of #2498, and the reason the questions below + * are labelled as referenced rather than affected. An item names a case for + * more than one reason: `#J8SJQ9` names the discharge-documentation case as + * the *contrast* that legitimately answers with a source pointer. Nothing in + * the text separates that from a case being reported broken, so the page + * asserts at the level it can stand behind -- the item -- and leaves the + * reader to open it. + */} +

+ The questions under each item are the ones its text names, not a verdict on each question. An item may + name a question as the contrast that is behaving correctly, so read the item before concluding that a question + listed here answers badly. +

+ + + {failures.length > 0 ? ( + + ) : ( + /* + * In words, never a blank container: an empty result and a failed load + * look identical otherwise, and here the difference matters more than + * usual because the empty state is the reassuring-looking one. + */ +
+

+ No open ledger item names a clinical eval question. +

+

+ That is the ledger being quiet, not the answer engine being proven well. Answer quality itself is measured + by the eval runs, which this page does not read. +

+
+ )} + + + ); +} diff --git a/src/app/mockups/development/page.tsx b/src/app/mockups/development/page.tsx index 68b2fb6818..ccc1b77f05 100644 --- a/src/app/mockups/development/page.tsx +++ b/src/app/mockups/development/page.tsx @@ -1,10 +1,12 @@ import type { Metadata } from "next"; +import Link from "next/link"; import { ShieldAlert } from "lucide-react"; import { DeveloperHubNavHeader } from "@/components/developer-area/developer-hub-nav-header"; import { EnvironmentStrip } from "@/components/developer-area/hub/environment-strip"; import { PanelCard } from "@/components/developer-area/hub/panel-card"; import { inPageAnchor } from "@/components/in-page-nav/in-page-nav-classes"; +import { resolveClinicalAnswerFailures } from "@/lib/developer-area/clinical-answer-failures"; import { resolveHubEnvironmentFacts } from "@/lib/developer-area/environment-facts"; import { panelsInGroup, type HubPanelGroup } from "@/lib/developer-area/hub-panels"; import { loadLedgerSnapshot } from "@/lib/developer-area/ledger-snapshot"; @@ -52,6 +54,7 @@ const GROUPS: { id: HubPanelGroup; anchor: string; label: string }[] = [ export default async function DeveloperHubPage() { const snapshot = loadLedgerSnapshot(); const environment = await resolveHubEnvironmentFacts(); + const clinicalAnswerFailures = resolveClinicalAnswerFailures(snapshot); return ( <> @@ -113,6 +116,25 @@ export default async function DeveloperHubPage() {

) : null} + {/* + * Warning, not danger: these are recorded problems against named + * clinical questions, and today they are P2/P3 rather than blocking. + * Painting them the same red as the blocking band would teach the reader + * to discount both. The band exists at all because the same facts sit + * unread inside an 81-item ledger, which is the whole reason this panel + * was built. + */} + {clinicalAnswerFailures.length > 0 ? ( + + {clinicalAnswerFailures.length} recorded {clinicalAnswerFailures.length === 1 ? "problem" : "problems"}{" "} + against a named clinical question. + + ) : null} + {GROUPS.map((group) => { const panels = panelsInGroup(group.id); // An empty group must render no anchor at all. `useResolvedPageSections` diff --git a/src/lib/developer-area/clinical-answer-failures.ts b/src/lib/developer-area/clinical-answer-failures.ts new file mode 100644 index 0000000000..cab8955b2b --- /dev/null +++ b/src/lib/developer-area/clinical-answer-failures.ts @@ -0,0 +1,113 @@ +import { answerQualityEvalCases, ragEvalCases } from "@/lib/rag/rag-eval-cases"; + +import type { LedgerOpenItem, LedgerSnapshot } from "./ledger-snapshot"; + +export type NamedEvalCase = { id: string; question: string }; + +export type ClinicalAnswerFailure = { + item: LedgerOpenItem; + /** + * The eval cases this item's text NAMES. A mention is a reference, not a + * verdict on that case -- see `resolveClinicalAnswerFailures` for why the + * difference is load-bearing, and never relabel this field as the cases the + * item reports broken. + */ + cases: NamedEvalCase[]; +}; + +const PRIORITY_ORDER: Record = { P1: 0, P2: 1, P3: 2 }; + +/** + * Every eval case the repository names, keyed by id, with duplicates collapsed. + * + * Read-only. This module imports a protected RAG surface and must never write to + * one: the case list is the repository's own record of which clinical questions + * the answer engine is held to, and duplicating it here would create a second + * source of truth that drifts silently. + */ +function allEvalCases(): NamedEvalCase[] { + const byId = new Map(); + for (const testCase of [...ragEvalCases, ...answerQualityEvalCases]) { + if (!byId.has(testCase.id)) byId.set(testCase.id, { id: testCase.id, question: testCase.question }); + } + return [...byId.values()]; +} + +/** + * Whether `haystack` names `id` as a whole token rather than as part of a longer + * one. + * + * This is the difference between a correct panel and a quietly wrong one. Case + * ids nest: `discharge-documentation` is a real case and is also a substring of + * the real case `quality-discharge-documentation`. A plain `includes` therefore + * reports the short case as failing whenever the long one is mentioned, and the + * panel would show a clinical question that nothing said was broken. Requiring + * the surrounding characters to fall outside the id alphabet removes that whole + * class without needing to know which pairs happen to nest today. + */ +function namesCase(haystack: string, id: string): boolean { + const isIdCharacter = (character: string | undefined) => character !== undefined && /[a-z0-9-]/.test(character); + let from = 0; + for (;;) { + const at = haystack.indexOf(id, from); + if (at === -1) return false; + if (!isIdCharacter(haystack[at - 1]) && !isIdCharacter(haystack[at + id.length])) return true; + from = at + 1; + } +} + +/** + * The open ledger items that name at least one of the repository's clinical eval + * cases, newest concern first by priority. + * + * **What this is, and what it deliberately is not.** It is the set of recorded + * problems whose text names a clinical question. It is *not* a complete picture + * of clinical answer quality: an item that describes a bad answer without naming + * a case is invisible here, and so is a case that is failing right now but that + * nobody has written an item about. + * + * **A named case is a REFERENCE, not an assertion that the case is failing, and + * the two cannot be told apart from the text.** Raised in review of PR #2498 + * against a real item: `#J8SJQ9` is about `quality-antipsychotic-metabolic-monitoring`, + * and names `quality-discharge-documentation` only as the *contrast* -- the case + * that "deliberately drops mustContainAny" because a source pointer is a + * legitimate answer there. Reporting that as a failure states the opposite of + * what the ledger says, about a clinical question. + * + * Two tempting fixes were rejected against the real data. Restricting the match + * to the `source` field fixes `#J8SJQ9` and **hides both genuinely broken + * questions in `#S4R2W3`**, which names them only in its detail prose; on this + * panel a false negative is worse than a loose one. Guessing intent from the + * surrounding words is the fragile heuristic this panel exists to avoid. So the + * assertion is made at the level it is sound at -- the item -- and named cases + * are presented as references. The panel's wording must match that, and a + * curated per-case association would need the ledger to record one explicitly. + * + * The matching is derived rather than curated on purpose. A hand-kept list of + * ledger ids would be exact on the day it was written and wrong a month later, + * with nothing to catch the drift. + */ +export function resolveClinicalAnswerFailures(snapshot: LedgerSnapshot): ClinicalAnswerFailure[] { + const cases = allEvalCases(); + + return snapshot.open + .map((item) => { + const haystack = `${item.summary} ${item.detail} ${item.source}`; + return { item, cases: cases.filter((testCase) => namesCase(haystack, testCase.id)) }; + }) + .filter((failure) => failure.cases.length > 0) + .sort((a, b) => { + const byPriority = (PRIORITY_ORDER[a.item.priority] ?? 99) - (PRIORITY_ORDER[b.item.priority] ?? 99); + return byPriority !== 0 ? byPriority : a.item.id.localeCompare(b.item.id); + }); +} + +/** + * The distinct clinical questions REFERENCED across the open items. Named for + * what it counts: these are questions the items mention, not questions proven to + * be failing. It was called `affectedQuestionCount` until review of PR #2498 + * showed the panel was counting a contrast case as affected. + */ +export function referencedQuestionCount(failures: ClinicalAnswerFailure[]): number { + return new Set(failures.flatMap((failure) => failure.cases.map((testCase) => testCase.id))).size; +} diff --git a/src/lib/developer-area/hub-panels.ts b/src/lib/developer-area/hub-panels.ts index f54fab427e..3ed5e69161 100644 --- a/src/lib/developer-area/hub-panels.ts +++ b/src/lib/developer-area/hub-panels.ts @@ -57,6 +57,19 @@ export const HUB_PANELS: readonly HubPanel[] = [ phase: 1, href: "/mockups/development/clinical-trust", }, + // Named for its evidence, not for its subject. It reports open ledger items + // that name one of the repository's own clinical eval cases -- not "every + // clinical answer problem", which it has no way to know. A panel on a clinical + // system that implies coverage it does not have is worse than no panel, so the + // narrower name is deliberate and should not be "improved" into a broader one. + { + id: "clinical-answer-failures", + name: "Answer failures", + summary: "Open problems recorded against a named clinical question", + group: "clinical", + phase: 1, + href: "/mockups/development/clinical-answer-failures", + }, // Kept, unlike the five removed above, and settled: the owner ruled on // 2026-08-26 that the hazard register belongs in the developer hub rather // than as a separate clinical-safety surface. It was never a removal diff --git a/tests/developer-clinical-answer-failures-page.dom.test.tsx b/tests/developer-clinical-answer-failures-page.dom.test.tsx new file mode 100644 index 0000000000..32a72de3e2 --- /dev/null +++ b/tests/developer-clinical-answer-failures-page.dom.test.tsx @@ -0,0 +1,146 @@ +import { render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import DeveloperClinicalAnswerFailuresPage from "@/app/mockups/development/clinical-answer-failures/page"; + +// PanelPageShell's back control is a ContextualBackLink, which calls +// next/navigation's useRouter for its history-aware click handler. Outside an +// app-router tree that throws, so every render here needs the router mocked. +vi.mock("next/navigation", () => ({ + usePathname: () => "/mockups/development/clinical-answer-failures", + useRouter: () => ({ push: vi.fn(), replace: vi.fn(), back: vi.fn(), prefetch: vi.fn() }), +})); + +/** + * The derivation is mocked so this file stays a *page* contract — what is + * rendered for a given set of failures. Its correctness (which items count, and + * the whole-token matching that keeps a nested case id off the screen) is owned + * by `tests/developer-clinical-answer-failures.test.ts`. + */ +const failures = vi.hoisted(() => ({ + value: [] as { + item: { + id: string; + priority: string; + type: string; + summary: string; + detail: string; + source: string; + added: string; + }; + cases: { id: string; question: string }[]; + }[], +})); + +vi.mock("@/lib/developer-area/clinical-answer-failures", () => ({ + resolveClinicalAnswerFailures: () => failures.value, + referencedQuestionCount: (list: { cases: { id: string }[] }[]) => + new Set(list.flatMap((failure) => failure.cases.map((testCase) => testCase.id))).size, +})); + +function failure(id: string, summary: string, cases: { id: string; question: string }[]) { + return { + item: { id, priority: "P2", type: "issue", summary, detail: "", source: "", added: "2026-08-22" }, + cases, + }; +} + +afterEach(() => { + failures.value = []; +}); + +describe("developer clinical answer failures page", () => { + it("renders inside the shared shell and stamps the ledger as its source", () => { + render(); + expect(screen.getByTestId("developer-clinical-answer-failures")).toBeInTheDocument(); + expect(screen.getByTestId("developer-clinical-answer-failures-back")).toHaveAttribute( + "href", + "/mockups/development", + ); + expect(screen.getByTestId("developer-hub-freshness")).toHaveTextContent(/Ledger/); + }); + + it("shows the clinical question in words, not only its case id", () => { + // The id is what makes the match checkable; the question is the only part a + // reader can act on. Losing the question would leave a page of slugs. + failures.value = [ + failure("#J8SJQ9", "Metabolic monitoring returns a stub", [ + { + id: "quality-antipsychotic-metabolic-monitoring", + question: "What metabolic monitoring is required for antipsychotics?", + }, + ]), + ]; + render(); + + const entry = screen.getByTestId("developer-clinical-answer-failure-#J8SJQ9"); + expect(entry).toHaveTextContent("What metabolic monitoring is required for antipsychotics?"); + expect(entry).toHaveTextContent("quality-antipsychotic-metabolic-monitoring"); + expect(entry).toHaveTextContent("Metabolic monitoring returns a stub"); + }); + + it("counts items and referenced questions separately, because one item can name several", () => { + failures.value = [ + failure("#S4R2W3", "Two questions answer with a bare title list", [ + { id: "quality-agitation-im-route", question: "Q1" }, + { id: "quality-duress-pathway", question: "Q2" }, + ]), + ]; + render(); + + expect(screen.getByTestId("developer-clinical-answer-failures-count-items")).toHaveTextContent("1"); + expect(screen.getByTestId("developer-clinical-answer-failures-count-questions")).toHaveTextContent("2"); + }); + + /** + * Raised in review of PR #2498. An item names a case for more than one reason + * — `#J8SJQ9` names the discharge-documentation case as the contrast that + * legitimately answers with a source pointer — so the page must not present a + * named question as a question proven broken. This pins the wording that keeps + * the claim at the level the data supports. + */ + it("presents named questions as references rather than as verdicts", async () => { + failures.value = [ + failure("#J8SJQ9", "Metabolic monitoring returns a stub", [ + { id: "quality-antipsychotic-metabolic-monitoring", question: "What metabolic monitoring is required?" }, + { id: "quality-discharge-documentation", question: "What discharge documentation is required?" }, + ]), + ]; + const { container } = render(); + const text = container.textContent ?? ""; + + expect(screen.getByTestId("developer-clinical-answer-failure-#J8SJQ9")).toHaveTextContent( + "Questions this item names", + ); + expect(text).toMatch(/not a verdict on each question/); + expect(text).toMatch(/name a question as the contrast that is behaving correctly/); + expect(screen.getByTestId("developer-clinical-answer-failures-count-questions")).toHaveTextContent( + "questions referenced", + ); + }); + + /** + * The empty state is the dangerous one on this page: it is the reading that + * looks like good news. It must say what it does not know, in words, rather + * than render a blank container that an empty list and a failed load would + * both produce. + */ + it("says what an empty list does not prove", () => { + render(); + + const empty = screen.getByTestId("developer-clinical-answer-failures-empty"); + expect(empty).toHaveTextContent(/No open ledger item names a clinical eval question/); + expect(empty).toHaveTextContent(/not the answer engine being proven well/); + }); + + it("states the limits of its own coverage above the list, not as a footnote", () => { + failures.value = [failure("#J8SJQ9", "A failure", [{ id: "quality-duress-pathway", question: "Q" }])]; + const { container } = render(); + + const text = container.textContent ?? ""; + expect(text).toMatch(/narrower thing than a picture of answer quality/); + // The caveat has to precede the evidence it qualifies; a reader who stops + // after the first entry must already have read it. + expect(text.indexOf("narrower thing")).toBeLessThan(text.indexOf("A failure")); + }); +}); diff --git a/tests/developer-clinical-answer-failures.test.ts b/tests/developer-clinical-answer-failures.test.ts new file mode 100644 index 0000000000..40b973093e --- /dev/null +++ b/tests/developer-clinical-answer-failures.test.ts @@ -0,0 +1,221 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { LedgerOpenItem, LedgerSnapshot } from "@/lib/developer-area/ledger-snapshot"; + +// The panel this feeds is clinical-facing, so both directions of its matching +// are load-bearing. Listing a question nothing reported puts a false clinical +// failure on screen; missing one that was reported hides a real answer defect. +// The nesting case below is not hypothetical: `discharge-documentation` and +// `patient-safety-plan` are both real eval case ids AND substrings of other real +// ones, and a plain `includes` reported both as failing on the committed ledger. + +afterEach(() => { + vi.resetModules(); +}); + +function item(overrides: Partial = {}): LedgerOpenItem { + return { + id: "#AAA111", + priority: "P2", + type: "issue", + summary: "", + detail: "", + source: "", + added: "2026-08-01", + ...overrides, + }; +} + +function snapshotOf(open: LedgerOpenItem[]): LedgerSnapshot { + return { + version: "outstanding-issues-snapshot-v1", + ledger_revision: null, + counts: { open: open.length, p1: 0, p2: 0, p3: 0, queued: 0, pending: 0, resolved: 0 }, + queue: [], + open, + pending: [], + }; +} + +async function loadWithCases(cases: { id: string; question: string }[]) { + vi.doMock("@/lib/rag/rag-eval-cases", () => ({ + ragEvalCases: cases, + answerQualityEvalCases: [], + })); + return import("../src/lib/developer-area/clinical-answer-failures"); +} + +describe("resolveClinicalAnswerFailures", () => { + it("lists an open item that names a clinical eval case, with the question a reader can understand", async () => { + const { resolveClinicalAnswerFailures } = await loadWithCases([ + { id: "quality-antipsychotic-metabolic-monitoring", question: "What metabolic monitoring is required?" }, + ]); + + const failures = resolveClinicalAnswerFailures( + snapshotOf([ + item({ + summary: "Metabolic monitoring returns a stub", + source: "canary run 1; quality-antipsychotic-metabolic-monitoring", + }), + ]), + ); + + expect(failures).toHaveLength(1); + expect(failures[0].cases).toEqual([ + { id: "quality-antipsychotic-metabolic-monitoring", question: "What metabolic monitoring is required?" }, + ]); + }); + + it("does not report a case whose id is only a fragment of the id actually named", async () => { + const { resolveClinicalAnswerFailures } = await loadWithCases([ + { id: "discharge-documentation", question: "Short case that must not be implicated" }, + { id: "quality-discharge-documentation", question: "The case that was actually named" }, + ]); + + const failures = resolveClinicalAnswerFailures( + snapshotOf([item({ source: "fails targeting: quality-discharge-documentation" })]), + ); + + expect(failures[0].cases.map((testCase) => testCase.id)).toEqual(["quality-discharge-documentation"]); + }); + + it("still reports the short case when the text names it in its own right", async () => { + // The other half of the same contract. A boundary rule that never matched + // the shorter id would trade a false positive for a false negative, which on + // this panel is the worse of the two. + const { resolveClinicalAnswerFailures } = await loadWithCases([ + { id: "discharge-documentation", question: "Short case" }, + { id: "quality-discharge-documentation", question: "Long case" }, + ]); + + const failures = resolveClinicalAnswerFailures( + snapshotOf([item({ source: "both named: discharge-documentation and quality-discharge-documentation" })]), + ); + + expect(failures[0].cases.map((testCase) => testCase.id).sort()).toEqual([ + "discharge-documentation", + "quality-discharge-documentation", + ]); + }); + + it("searches the summary, the detail and the source, because items record the case id in any of them", async () => { + const { resolveClinicalAnswerFailures } = await loadWithCases([{ id: "quality-duress-pathway", question: "Q" }]); + + for (const field of ["summary", "detail", "source"] as const) { + const failures = resolveClinicalAnswerFailures(snapshotOf([item({ [field]: "affects quality-duress-pathway" })])); + expect(failures, `case id in ${field} was not found`).toHaveLength(1); + } + }); + + it("ignores an open item that names no case at all", async () => { + const { resolveClinicalAnswerFailures } = await loadWithCases([{ id: "quality-duress-pathway", question: "Q" }]); + + expect(resolveClinicalAnswerFailures(snapshotOf([item({ summary: "Bundle budget baseline is stale" })]))).toEqual( + [], + ); + }); + + it("orders blocking items above the rest rather than leaving ledger order to chance", async () => { + const { resolveClinicalAnswerFailures } = await loadWithCases([{ id: "quality-duress-pathway", question: "Q" }]); + + const failures = resolveClinicalAnswerFailures( + snapshotOf([ + item({ id: "#P3ONLY", priority: "P3", source: "quality-duress-pathway" }), + item({ id: "#P1FIRST", priority: "P1", source: "quality-duress-pathway" }), + item({ id: "#P2MID", priority: "P2", source: "quality-duress-pathway" }), + ]), + ); + + expect(failures.map((failure) => failure.item.id)).toEqual(["#P1FIRST", "#P2MID", "#P3ONLY"]); + }); + + it("counts each referenced question once even when several items name it", async () => { + const { resolveClinicalAnswerFailures, referencedQuestionCount } = await loadWithCases([ + { id: "quality-duress-pathway", question: "Q1" }, + { id: "quality-agitation-im-route", question: "Q2" }, + ]); + + const failures = resolveClinicalAnswerFailures( + snapshotOf([ + item({ id: "#ONE", source: "quality-duress-pathway quality-agitation-im-route" }), + item({ id: "#TWO", source: "quality-duress-pathway" }), + ]), + ); + + expect(failures).toHaveLength(2); + expect(referencedQuestionCount(failures)).toBe(2); + }); + + /** + * The shape review of PR #2498 caught, reproduced from the real `#J8SJQ9` + * detail: an item about one case names a second only as the *contrast* that + * legitimately answers with a source pointer. + * + * The assertion is that BOTH come back, and that is not a bug being enshrined. + * Nothing in the text distinguishes a contrast from a report, so the module + * returns references and the page presents them as references. Restricting the + * match to the `source` field was tried against the real ledger and rejected: + * it fixes this item and hides both genuinely broken questions in `#S4R2W3`, + * which names them only in its detail prose. + * + * If a later change makes this return one case, it must be because the ledger + * gained an explicit per-case association — not because a heuristic guessed. + */ + it("returns a contrast case as a reference, and does not pretend to know it is failing", async () => { + const { resolveClinicalAnswerFailures } = await loadWithCases([ + { id: "quality-antipsychotic-metabolic-monitoring", question: "What metabolic monitoring is required?" }, + { id: "quality-discharge-documentation", question: "What discharge documentation is required?" }, + ]); + + const failures = resolveClinicalAnswerFailures( + snapshotOf([ + item({ + id: "#J8SJQ9", + summary: "Antipsychotic metabolic monitoring returns a source-backed stub instead of a written answer", + detail: + "All four cases carrying that flag document the same rationale: the corpus has no single authoritative " + + "source, so a source pointer is a legitimate answer, and quality-discharge-documentation deliberately " + + "drops mustContainAny for exactly that reason. quality-antipsychotic-metabolic-monitoring is the " + + "opposite case - it names expectedFiles and an authoritative source exists.", + source: "canary run 32589154243; quality-antipsychotic-metabolic-monitoring", + }), + ]), + ); + + expect(failures[0].cases.map((testCase) => testCase.id).sort()).toEqual([ + "quality-antipsychotic-metabolic-monitoring", + "quality-discharge-documentation", + ]); + }); + + /** + * Against the real eval case list and the real committed ledger, asserted as a + * property rather than as a fixed expectation: a hard-coded list of today's + * three items would fail the next time the ledger legitimately changes, and + * would be "fixed" by pasting in whatever the code then produced, which proves + * nothing. What must always hold is that every reported case was named as a + * whole token by the item reporting it. + */ + it("reports only whole-token matches when run against the repository's real data", async () => { + // `vi.resetModules()` clears the module cache but NOT the mock registry, so + // without this `doUnmock` the "real data" below silently ran against + // whichever `loadWithCases` fixture the previous test registered — a check + // that could not fail. Caught by mutating the matcher to a plain `includes` + // and watching this test stay green while the synthetic one went red. + vi.doUnmock("@/lib/rag/rag-eval-cases"); + vi.resetModules(); + const { resolveClinicalAnswerFailures } = await import("../src/lib/developer-area/clinical-answer-failures"); + const { loadLedgerSnapshot } = await import("../src/lib/developer-area/ledger-snapshot"); + + for (const { item: openItem, cases } of resolveClinicalAnswerFailures(loadLedgerSnapshot())) { + const haystack = `${openItem.summary} ${openItem.detail} ${openItem.source}`; + for (const testCase of cases) { + expect( + new RegExp(`(^|[^a-z0-9-])${testCase.id}([^a-z0-9-]|$)`).test(haystack), + `${openItem.id} reports ${testCase.id}, which its text does not name as a whole token`, + ).toBe(true); + expect(testCase.question.length, `${testCase.id} has no question text to show a reader`).toBeGreaterThan(0); + } + } + }); +}); diff --git a/tests/developer-hub-page.dom.test.tsx b/tests/developer-hub-page.dom.test.tsx index 638f4eb914..d869ef0ad3 100644 --- a/tests/developer-hub-page.dom.test.tsx +++ b/tests/developer-hub-page.dom.test.tsx @@ -47,6 +47,18 @@ vi.mock("@/lib/developer-area/environment-facts", () => ({ resolveHubEnvironmentFacts: async () => environment.value, })); +/** + * The band is the whole reason the panel was built: the same facts already sit + * in the ledger this page renders, buried among eighty-odd open items. Mocked + * here so both the present and absent branches are reachable regardless of what + * the committed ledger happens to hold today. + */ +const clinicalAnswerFailures = vi.hoisted(() => ({ value: [] as { item: { id: string } }[] })); + +vi.mock("@/lib/developer-area/clinical-answer-failures", () => ({ + resolveClinicalAnswerFailures: () => clinicalAnswerFailures.value, +})); + /** * Only `counts.p1` is overridden, and only on top of the *real* committed * snapshot — so the band is exercised against the shape the route actually @@ -356,3 +368,35 @@ describe("developer hub page — environment strip", () => { expect(screen.getByTestId("developer-hub-environment-strip")).toHaveTextContent("account unknown"); }); }); + +describe("developer hub page — clinical answer failure band", () => { + afterEach(() => { + clinicalAnswerFailures.value = []; + }); + + it("links recorded clinical answer problems straight to their panel", async () => { + clinicalAnswerFailures.value = [{ item: { id: "#J8SJQ9" } }, { item: { id: "#S4R2W3" } }]; + render(await DeveloperHubPage()); + + const band = screen.getByTestId("developer-hub-clinical-answer-failures-band"); + expect(band).toHaveTextContent("2 recorded problems against a named clinical question."); + expect(band).toHaveAttribute("href", "/mockups/development/clinical-answer-failures"); + }); + + it("says problem, not problems, for a single one", async () => { + clinicalAnswerFailures.value = [{ item: { id: "#J8SJQ9" } }]; + render(await DeveloperHubPage()); + + expect(screen.getByTestId("developer-hub-clinical-answer-failures-band")).toHaveTextContent( + "1 recorded problem against a named clinical question.", + ); + }); + + it("renders nothing rather than a reassuring all-clear when none are recorded", async () => { + // Same rule the blocking band follows. An explicit "none recorded" here + // would read as "answers are fine", which this data cannot support. + render(await DeveloperHubPage()); + + expect(screen.queryByTestId("developer-hub-clinical-answer-failures-band")).toBeNull(); + }); +});