From 748b8530ac9a2092c7e4f2d6e41f05d65083f36a Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Sun, 26 Jul 2026 10:40:34 +0800 Subject: [PATCH 01/11] Implement audit recommendations for provenance and safety --- package.json | 1 + scripts/verify-locality-metadata.ts | 61 ++++++++ scripts/verify-pr-local.mjs | 2 +- src/app/api/search/interaction/route.ts | 7 + .../clinical-dashboard/answer-content.tsx | 13 +- .../answer-result-surface.tsx | 1 + .../clinical-dashboard/answer-thread-turn.tsx | 1 + .../clinical-dashboard/evidence-panels.tsx | 8 +- .../prior-answer-turn-surface.tsx | 1 + .../clinical-dashboard/source-actions.tsx | 44 +++++- src/lib/citations.ts | 12 +- src/lib/evidence.ts | 30 +++- src/lib/source-authority-metadata.ts | 17 ++- src/lib/source-governance.ts | 139 ++++++++++-------- src/lib/source-metadata.ts | 8 +- src/lib/source-text-sanitizer.ts | 2 +- src/lib/types.ts | 30 ++-- 17 files changed, 281 insertions(+), 96 deletions(-) create mode 100644 scripts/verify-locality-metadata.ts diff --git a/package.json b/package.json index e12a23e117..7740a54d65 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "check:migration-role": "node scripts/check-hosted-migration-role.mjs", "check:function-grants": "node scripts/check-function-grants.mjs", "check:owner-scope": "node scripts/check-owner-scope-api.mjs", + "check:locality-metadata": "node scripts/run-tsx.mjs scripts/verify-locality-metadata.ts", "recover:ingestion": "node scripts/run-tsx.mjs scripts/recover-ingestion-queue.ts", "registry:seed": "node scripts/run-tsx.mjs scripts/seed-registry-records.ts", "registry:embed": "node scripts/run-tsx.mjs scripts/embed-registry-records.ts", diff --git a/scripts/verify-locality-metadata.ts b/scripts/verify-locality-metadata.ts new file mode 100644 index 0000000000..e425408b88 --- /dev/null +++ b/scripts/verify-locality-metadata.ts @@ -0,0 +1,61 @@ +import * as nextEnv from "@next/env"; +import { auditSourceAuthorityDocuments } from "@/lib/source-authority-metadata"; + +const loadEnvConfig = + nextEnv.loadEnvConfig ?? + (nextEnv as unknown as { default?: { loadEnvConfig?: typeof nextEnv.loadEnvConfig } }).default?.loadEnvConfig; + +async function loadAdminClient() { + const { createAdminClient } = await import("@/lib/supabase/admin"); + return createAdminClient(); +} + +export async function main() { + if (loadEnvConfig) { + loadEnvConfig(process.cwd()); + } + + const supabase = await loadAdminClient(); + const documents: any[] = []; + const pageSize = 1000; + + for (let from = 0; ; from += pageSize) { + const { data, error } = await supabase + .from("documents") + .select("id,title,file_name,source_path,status,metadata") + .eq("status", "indexed") + .order("id", { ascending: true }) + .range(from, from + pageSize - 1); + + if (error) throw new Error(error.message); + documents.push(...(data ?? [])); + if (!data || data.length < pageSize) break; + } + + const report = auditSourceAuthorityDocuments(documents); + + if (!report.passed) { + console.error("FAIL: Source authority metadata verification failed."); + if (report.missing_australian_locality_count > 0) { + console.error(`- Missing Australian locality metadata: ${report.missing_australian_locality_count} documents`); + } + if (report.authority_conflict_count > 0) { + console.error(`- Authority conflicts: ${report.authority_conflict_count} documents`); + } + process.exitCode = 1; + } else { + console.log("PASS: Source authority metadata verification passed."); + } +} + +if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`.replace(/\\/g, "/")) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} else if (process.argv[1] && process.argv[1].endsWith("verify-locality-metadata.ts")) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/verify-pr-local.mjs b/scripts/verify-pr-local.mjs index 5455e97403..cfa274e0de 100644 --- a/scripts/verify-pr-local.mjs +++ b/scripts/verify-pr-local.mjs @@ -7,7 +7,7 @@ import { acquireHeavyRunLock } from "./test-run-lock.mjs"; const isWindows = process.platform === "win32"; const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const baseScripts = ["check:runtime", "format:changed", "lint", "typecheck", "test"]; +const baseScripts = ["check:runtime", "format:changed", "lint", "typecheck", "test", "check:locality-metadata"]; function parseArgs(args) { const options = { dryRun: false, extended: false, files: undefined }; diff --git a/src/app/api/search/interaction/route.ts b/src/app/api/search/interaction/route.ts index 9e6d7bc28d..e501ff5636 100644 --- a/src/app/api/search/interaction/route.ts +++ b/src/app/api/search/interaction/route.ts @@ -30,6 +30,12 @@ const interactionSchema = z title: z.string().trim().max(240).optional(), queryClass: z.string().trim().max(80).optional(), crossMode: crossModeTargetSchema.optional(), + citationTelemetry: z.object({ + provenance: z.string().optional(), + source_strength: z.string().optional(), + similarity: z.number().optional(), + document_status: z.string().optional(), + }).optional(), }) .refine((body) => Boolean(body.documentId || body.crossMode), { message: "Either documentId or a crossMode target is required.", @@ -154,6 +160,7 @@ export async function POST(request: Request) { metadata: { interaction: "source_open", ...queryPrivacyMetadata(body.query), + ...(body.citationTelemetry && { citation_telemetry: body.citationTelemetry }), }, }); if (insertError) throw new Error(insertError.message); diff --git a/src/components/clinical-dashboard/answer-content.tsx b/src/components/clinical-dashboard/answer-content.tsx index 93247322a1..b18dceab58 100644 --- a/src/components/clinical-dashboard/answer-content.tsx +++ b/src/components/clinical-dashboard/answer-content.tsx @@ -20,7 +20,7 @@ import { subtleStatusPill, textMuted, } from "@/components/ui-primitives"; -import { sourceResultHref } from "@/components/clinical-dashboard/source-actions"; +import { sourceResultHref, logSourceOpen } from "@/components/clinical-dashboard/source-actions"; import { cleanDisplayTitle, comparableAnswerText, @@ -380,12 +380,14 @@ function capsulePreviewSources( } function SourcePreviewContent({ + query, previewSources, quoteText, copiedQuote, onCopyQuote, showHeader = true, }: { + query?: string; previewSources: CapsulePreviewSource[]; quoteText?: string | null; copiedQuote: boolean; @@ -447,6 +449,7 @@ function SourcePreviewContent({ query && logSourceOpen(query, source)} data-testid="source-capsule-preview-row" className="flex min-h-12 items-center rounded-md text-sm font-semibold leading-5 text-[color:var(--text-heading)] transition hover:text-[color:var(--clinical-accent)] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[color:var(--focus)]" aria-label={`Open source ${cleanDisplayTitle(source.title)}, page ${source.pageNumber ?? "not available"}`} @@ -471,6 +474,7 @@ function SourcePreviewContent({ query && logSourceOpen(query, source)} className={cn( index === 0 ? "inline-flex min-h-12 items-center gap-1.5 rounded-md border border-[color:var(--border)] bg-[color:var(--surface-raised)] px-2.5 text-xs font-semibold text-[color:var(--text)] shadow-[var(--shadow-inset)] transition hover:border-[color:var(--clinical-accent-border)]" @@ -495,6 +499,7 @@ function SourcePreviewContent({ {primaryPreviewSource ? ( query && logSourceOpen(query, primaryPreviewSource)} className={chatMicroAction} aria-label={`Open source page for ${primaryPreviewSource.title}`} > @@ -528,6 +533,7 @@ function SourcePreviewContent({ {primaryPreviewSource ? ( query && logSourceOpen(query, primaryPreviewSource)} className="inline-flex min-h-8 items-center gap-1.5 rounded-md px-2 text-[color:var(--clinical-accent)] transition hover:bg-[color:var(--clinical-accent-soft)] focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[color:var(--focus)]" > Evidence details @@ -543,6 +549,7 @@ function SourcePreviewContent({ * Displays a sanitized clinical answer with source status, source previews, and copy actions. * * @param text - The raw answer text to display. + * @param query - The user's query context for logging. * @param preformatted - Whether to preserve the supplied formatting during display processing. * @param sourceCount - The number of direct sources associated with the answer. * @param sourceOnly - Whether to show a notice that the answer was assembled solely from source passages. @@ -555,6 +562,7 @@ function SourcePreviewContent({ */ export function NaturalLanguageAnswer({ text, + query, preformatted = false, sourceCount, sourceOnly, @@ -567,6 +575,7 @@ export function NaturalLanguageAnswer({ // Raw answer text (server bold intact); this component owns display // sanitization so can render the high-yield emphasis. text: string; + query?: string; preformatted?: boolean; sourceCount: number; sourceOnly: boolean; @@ -702,6 +711,7 @@ export function NaturalLanguageAnswer({ anchorRef={sourceCapsuleRef} >
query && logCitationOpen(query, finding.citation)} className="inline-flex min-h-tap min-w-0 items-center gap-1 text-xs font-semibold text-[color:var(--primary)] transition hover:underline focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-[color:var(--focus)] lg:min-h-8" aria-label={`Open source ${formatSafetyFindingLabel(finding)}`} > @@ -1304,12 +1305,14 @@ export function QuoteCards({ onCopyQuotes, onFollowUp, onScopeDocument, + query, }: { quotes: QuoteCard[]; copiedQuotes: boolean; onCopyQuotes: () => void; onFollowUp?: (quote: QuoteCard) => void; onScopeDocument: (documentId: string) => void; + query?: string; }) { return (
@@ -1364,6 +1367,7 @@ export function QuoteCards({ documentId={quote.document_id} onScopeDocument={onScopeDocument} onFollowUp={onFollowUp ? () => onFollowUp(quote) : undefined} + onOpenSource={() => query && logCitationOpen(query, quote, quote.source_strength)} divider={false} />
diff --git a/src/components/clinical-dashboard/prior-answer-turn-surface.tsx b/src/components/clinical-dashboard/prior-answer-turn-surface.tsx index dbd330be06..f2f2c45639 100644 --- a/src/components/clinical-dashboard/prior-answer-turn-surface.tsx +++ b/src/components/clinical-dashboard/prior-answer-turn-surface.tsx @@ -85,6 +85,7 @@ export function PriorAnswerTurnSurface({ <> void; onFollowUp?: () => void; + onOpenSource?: () => void; imageCount?: number; divider?: boolean; }) { return (
- +