diff --git a/.github/workflows/live-web-vitals.yml b/.github/workflows/live-web-vitals.yml new file mode 100644 index 0000000000..27993037ba --- /dev/null +++ b/.github/workflows/live-web-vitals.yml @@ -0,0 +1,140 @@ +name: Live Web Vitals baseline + +# Ledger #017 gates every client payload decision (#012/#013/#016) behind +# reproducible mobile/desktop Web-Vitals evidence, and the 2026-07-28 latency +# audit added seven more findings behind the same gate. Nothing in the repo +# could produce that evidence: in-sandbox runtime vitals are blocked because the +# production server hard-requires Supabase secrets, and local Playwright timings +# rank routes against each other without discharging #017, which asks for the +# real origin. +# +# This runs Lighthouse against the deployed domain and keeps the JSON, so the +# baseline is reproducible after payload work rather than a one-off screenshot. +# +# Deliberately dispatch-only — no schedule. It is heavier than the plain GETs in +# live-domain-monitor.yml, and the decision it feeds is made a handful of times a +# year, not every six hours. Like that monitor it uses no secrets and no +# providers beyond what any anonymous visitor triggers. +# +# The decision rule is written down BEFORE the numbers are read, so the gate +# cannot be rationalised after the fact: +# mobile LCP < 2.5s AND CLS < 0.1 AND INP < 200ms on every route +# -> close #017 "metrics acceptable"; only the explicitly measured payload +# findings gated by #017 become WONTFIX (e.g. #013 route-chunk weight). +# #016's motion/CSS/waterfall/caching/dynamic-import items stay open +# unless they have separate evidence — LCP/CLS do not validate them. +# any breach +# -> only the breaching route's findings become actionable, ranked by +# measured contribution +# Record the verdict in docs/outstanding-issues.md against #017 either way. + +on: + workflow_dispatch: + inputs: + routes: + description: "Comma-separated routes to measure" + required: false + # Every entry must be a real page route. `/documents` is not one — the + # documents segment holds only `search`, `source` and `[id]` with no + # `page.tsx` (see docs/site-map.md), so it would have measured the 404 + # document. `/documents/search` is the canonical documents-mode route. + default: "/,/therapy-compass,/documents/search,/dsm,/forms" + +permissions: + contents: read + +concurrency: + group: live-web-vitals + cancel-in-progress: false + +jobs: + measure: + name: Lighthouse against the live domain + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + # Repository variable overrides the default (e.g. a staging cutover), + # matching live-domain-monitor.yml. + LIVE_DOMAIN_URL: ${{ vars.LIVE_DOMAIN_URL || 'https://psychiatry.tools' }} + ROUTES: ${{ inputs.routes }} + # Pinned exactly, not `lighthouse@12`. That range is >=12.0.0 <13.0.0-0, + # so a patch published between a baseline run and its follow-up would + # change metric collection independently of the application and silently + # decalibrate the comparison this workflow exists to make. The resolved + # version is recorded in summary.json so a baseline states its own tooling. + LIGHTHOUSE_VERSION: "12.8.2" + steps: + - name: Check out the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The job only reads files; nothing pushes. Keeping the token in the + # local git config for the rest of the job buys nothing (zizmor + # artipacked). + persist-credentials: false + + - name: Use the repository Node runtime + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + + - name: Normalize the configured origin + run: | + set -euo pipefail + # LIVE_DOMAIN_URL may be configured with a trailing slash (common for + # "site root" variables). Appending "/forms" would then yield + # `https://…//forms`, which the summariser rejects as the wrong page + # after the server canonicalises the double slash. + normalized="${LIVE_DOMAIN_URL%/}" + echo "LIVE_DOMAIN_URL=$normalized" >> "$GITHUB_ENV" + echo "origin -> $normalized" + + - name: Confirm the target is reachable before spending a Lighthouse run + run: | + set -euo pipefail + code="$(curl -sSL -o /dev/null -w '%{http_code}' --max-time 30 --retry 2 --retry-delay 10 --retry-all-errors "$LIVE_DOMAIN_URL/")" + echo "root -> $code" + if [ "$code" != "200" ]; then + echo "::error::$LIVE_DOMAIN_URL/ returned $code — not measuring" + exit 1 + fi + + - name: Measure each route on mobile and desktop + run: | + set -euo pipefail + mkdir -p web-vitals + IFS=',' read -ra route_list <<< "$ROUTES" + for strategy in mobile desktop; do + for route in "${route_list[@]}"; do + route="$(echo "$route" | xargs)" + [ -n "$route" ] || continue + # Filename-safe slug: "/" -> root, "/a/b" -> a-b + slug="$(echo "$route" | sed 's|^/||; s|/|-|g')" + [ -n "$slug" ] || slug="root" + out="web-vitals/${strategy}-${slug}" + echo "::group::$strategy $route" + # One flaky route must not discard the whole run, so a failure is + # a warning here; the summary step fails if NOTHING was produced. + npx --yes "lighthouse@$LIGHTHOUSE_VERSION" "$LIVE_DOMAIN_URL$route" \ + --output=json --output-path="${out}.json" \ + --preset="$([ "$strategy" = desktop ] && echo desktop || echo perf)" \ + --only-categories=performance \ + --chrome-flags="--headless=new --no-sandbox --disable-dev-shm-usage" \ + --max-wait-for-load=60000 \ + --quiet || echo "::warning::lighthouse failed for $strategy $route" + echo "::endgroup::" + done + done + + # ROUTES is passed so a route whose Lighthouse run failed above is counted + # as a breach. Grading only the reports that happen to exist would let a + # partially failed run read as "every mobile route passed". + - name: Summarise LCP / CLS / TBT against the decision rule + run: node scripts/summarise-web-vitals.mjs web-vitals "$ROUTES" + + - name: Upload the reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: live-web-vitals + path: web-vitals/ + retention-days: 30 diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 130a2de3dc..2740e31d48 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -1270,6 +1270,7 @@ Records before 2026-07-28 were written by hand and had drifted: 146 lines carrie | 2026-07-29 | PR #1377 / claude/latency-findings-impl-s8g01v | 80df35f6cebe5f8a29dbbe98c960c114c64de8c7 | PR #1377 CI/review babysit | Re-synced main after #1378 (DIRTY=staleness). Codex P1+P2 threads resolved. Hosted CI green on d7aa4c6c prior tip; re-running after sync. | prior tip CI: PR required success; Production UI/Unit/Build/Static/Migration success; CircleCI success; merge-tree clean | | 2026-07-29 | PR #1377 / claude/latency-findings-impl-s8g01v | 8b8d4b8952fc96401116af9e34604c2a3e6e53b4 | PR #1377 CI/review babysit | Merged #1376 from main with real conflicts: kept admission-before-scope + L1-2 REFUTED docs; took #1376 invalidation epochs / empty-scope Server-Timing / stream signal. Codex threads resolved earlier. MERGEABLE; CI re-running. | vitest preamble+rag-cache-invalidation 7/7; check:rag:fixtures 36/21; prior tip PR-required green before #1376 land | | 2026-07-29 | claude/clinical-design-system-update-e34ca9 | 0cdae091ad92f40e0ad7335b3e2d396c44188a4f | PR #1375 conflict fix + Bugbot | FIXED second CONFLICTING after #1378: took main removal of SelectedDocumentEvidencePanel; retained tracking-eyebrow on surviving document-search-results. Prior DocumentViewerRail + form-detail settlement retained. MERGEABLE; CI re-running. | local: document-search-record-fault + design-token tests; merge-tree CLEAN; prior Production UI PASS on 6903f51f; form-detail e2e 2/2. | +| 2026-07-29 | claude/latency-fixes-2026-07-29 | 96a4c76da12b4478539b44fdc01a59ccfe791890 | prlanded | PR #1376 merged via squash. Content diff against the squash commit is empty and six probes confirmed on main (preambleServerTimingEntries, LoadingPanel fallbacks, Supabase preconnect, Refutation 6, audit corrections, tableFactListProjection). No orphaned commits despite pushing after auto-merge was armed; disarm-push-rearm was used. | verify:pr-local exit 0 on fresh npm ci: 422 files / 4272 tests passed, 3 skipped. prettier clean; docs:check-links 1333 refs; docs:check-index OK. verify:ui NOT run (heavy lock contended). No provider-backed gates. | | 2026-07-29 | cursor/recent-pr-bugfixes-f30d | 2f48884e1a65e8747164cc4fc8107b4b521f0941 | pr-1374-ci-fix | FIXED: account-switch signed-URL cache clear before setSession; merged main; resolved Codex P1 + CodeRabbit privacy/ui-smoke threads. CI was green pre-push. | vitest auth-signed-url+privacy 22/8; merge-tree clean; eslint touched files | | 2026-07-29 | cursor/recent-pr-bugfixes-f30d | 96eaf8768ffc369c4fb4ec406f9ea2b00443b734 | pr-1374-merge-main-staleness | merged origin/main; merge-tree clean; GitHub DIRTY was staleness | merge-tree-clean; push | | 2026-07-29 | PR #1378 / codex/remove-source-overlays (squash) | 6f2f1aa259ad7b554b3bac4e6c24adf2f8d28436 | PR #1378 babysit | MERGED via squash auto-merge. Supersedes prior closeout row that recorded pre-squash tip c3feb4cea34dd1a0d0d675df3e063c95174f2504 (unreachable after squash). Hosted required checks green; unresolved threads 0; Bugbot no open findings. | Hosted PR required/Production UI/Static/Unit/Build/Safety/PR policy SUCCESS; verify:cheap 4273 pass; squash SHA 6f2f1aa2 resolvable | @@ -1301,5 +1302,6 @@ Records before 2026-07-28 were written by hand and had drifted: 146 lines carrie | 2026-07-29 | cursor/page-anchored-search-composer-30ee | 7ff134ca7f614db527b8d142676640305533669d | branch-cleanup-deletion-pending | DELETION PENDING — content proven fully on main. Merge-base with main is 79d1c879 and tree(merge-base) equals tree(tip): git diff --name-only 79d1c879 7ff134ca reports 0 files, so the tip introduces nothing beyond a state already in main. Backs no open PR. Deletion blocked by HTTP 403; HEAD recorded here so the later branch-cleanup row can be keyed to it after the ref is gone. Supersedes the aggregate main-keyed row at 855aa291, which recorded no candidate HEADs. | local git only for the content proof: git merge-base, git diff --name-only (0 files), git diff --name-only origin/main... (0 files), full history after git fetch --unshallow. PROVIDER-BACKED evidence obtained (GitHub reads, authorised repo-scope): open-PR head cross-check via GitHub API against PRs #1374/#1377/#1384/#1385/#1386/#1387, and GitHub MCP capability inspection. PROVIDER-BACKED MUTATION ATTEMPTED AND REJECTED: git push origin --delete returned HTTP 403; no branch was deleted. No OpenAI/Supabase/hosted-CI calls. | | 2026-07-29 | cursor/pr-1379-babysit-ledger-9365 | be2de03f855cb7fdfccea4bb74d05eb4c9bf6c61 | branch-cleanup-deletion-pending | DELETION PENDING — content proven fully on main. Merge-base with main is b2740480 and tree(merge-base) equals tree(tip): git diff --name-only b2740480 be2de03f reports 0 files, so the tip introduces nothing beyond a state already in main. Backs no open PR. Deletion blocked by HTTP 403; HEAD recorded here so the later branch-cleanup row can be keyed to it after the ref is gone. Supersedes the aggregate main-keyed row at 855aa291, which recorded no candidate HEADs. | local git only for the content proof: git merge-base, git diff --name-only (0 files), git diff --name-only origin/main... (0 files), full history after git fetch --unshallow. PROVIDER-BACKED evidence obtained (GitHub reads, authorised repo-scope): open-PR head cross-check via GitHub API against PRs #1374/#1377/#1384/#1385/#1386/#1387, and GitHub MCP capability inspection. PROVIDER-BACKED MUTATION ATTEMPTED AND REJECTED: git push origin --delete returned HTTP 403; no branch was deleted. No OpenAI/Supabase/hosted-CI calls. | | 2026-07-29 | claude/latency-findings-impl-s8g01v | 9e2ee65ca0bcce45a3cb6a0539e265ec8d961582 | PR #1377 latency findings — #098 stale offline-harness references | Codex P2 confirmed and fixed: the #098 row in docs/outstanding-issues.md still named test-cache-path.mjs and check-rag-fixtures.mjs as the offline fixtures for the round-trip counting harness. Neither exercises a RAG request (cache paths; fixture-manifest validation), so a harness built on them would count nothing. The audit doc carried the retraction at :358 but this row did not - the same local-retraction pattern flagged in two prior rounds. Now names eval-rag-offline.mjs, test-rag-offline.mjs, rag-offline-contract.mjs and the contract fixture, all verified present, with the correction recorded inline. Docs only. | prettier --check clean; docs:check-links 1363; docs:check-scripts 390; grep confirms no stale refs remain | +| 2026-07-29 | 1391 | ff4ddfabb8d8594da0c09e04dc37e9286745140b | PR #1391 merge | merged as 39f2bcea. Three review P2s fixed: both ui-smoke retries made idempotent (toPass re-runs after a late-landing click; the mode-menu one clicked a toggle and oscillated) plus a second guard so the <768px branch can recover at all, and #111 archived instead of left open-and-done. One Production UI red on ui-phone-scroll.spec.ts:574 was proven a flake before re-running: the delta from the passing head baecef05 was two ui-smoke guards and one docs row, neither reachable from that spec, and the PR touches no document-viewer phone-scroll surface. Re-run green | verify:cheap exit 0 (429 files / 4404 tests); design-token orphan guard proven red on a reintroduced class; PR required success with UI_RESULT success on re-run | | 2026-07-29 | 1391 | baecef05cac86c4d52af895d483a33ba3c40cd61 | PR #1391 review | reviewed clean — text-4xs retirement confirmed against globals.css (--text-3xs 0.625rem present, --text-4xs absent); orphan guard proven to fail on a reintroduced class; six Playwright retries all retry action-plus-effect so a genuine regression still fails. Resolved the outstanding-issues #108/#109 double-allocation (renumbered to #110/#111, marker to 112) and recorded #111 done | verify:cheap exit 0 (429 files / 4404 tests); design-token-contract 28 passed; check:branch-review-ledger passed | | 2026-07-29 | 1374 | c14edb9c6f0bdbbfb147752503e016f2543fd803 | PR #1374 review + merge | merged as 3704007c — DocumentViewer identity-bound state clear (P1) implemented and verified red without it; all 12 review threads resolved | verify:cheap exit 0 (429 files / 4403 tests); PR required success; Production UI success | diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index 07c77de0ab..42722a1b4b 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -108,7 +108,7 @@ removed after current-main verification; it is not missing recommended work. | #011 | P3 | task | Auth DB-connection allocation is operator-only | Supabase Auth (GoTrue) is capped at ~10 absolute DB connections (Supabase perf advisor). Switch to **percentage-based** allocation in the Supabase **dashboard** before the first compute scale-up — **not settable via SQL/MCP** (operator-owned). Verify via a staging soak + an approval-gated read-only advisor re-check. | `docs/auth-connection-cap-runbook.md`; `docs/process-hardening.md` (Known follow-up debts) | 2026-07-21 | | #013 | P3 | rec | Route-chunk + mockup catalogue JSON weight | `build:analyze`: `/specifiers` ships `specifiers-search-index.json` (~180 KB parsed), `/forms` ships `forms-catalog.json` (~132 KB), `/formulation` ships `formulation-content.json` (~52 KB, client-side local search — needs index/full split or a search endpoint, architectural). All route-scoped (not initial bundle). Also `*-mockups.tsx` (~100 KB across chunks) build though `/mockups` 404s in prod — exclude from the prod artifact. | session 2026-07-21 (build:analyze) | 2026-07-21 | | #016 | P3 | rec | "Big but not easy" structural + motion perf | Deferred larger levers: (a) nonce-CSP forces every product route to `╞Æ Dynamic` (zero static generation) — evaluate Partial Prerendering / static shells for the static clinical catalogues (DSM/differentials/therapy/specifiers/formulation); (b) sidebar expand/collapse animates `grid-template-columns` (biggest smoothness cost, motion-gated — needs a transform-overlay rethink); (c) Therapy Compass fetches 692 KB / 2.5 MB JSON client-side (defer until interaction + confirm brotli); (d) settings/setup/admin dialogs static-imported into the home chunk (`next/dynamic` them); (e) `src/app/(search-app)/layout.tsx:4` imports 71.6 KB of Therapy-Compass-only CSS in the ROUTE-GROUP layout, making it render-blocking on `/`, `/documents`, `/forms`, `/dsm` and every mode home; (f) `shared-search-app-shell.tsx:8` statically imports the `therapy-compass` barrel, pulling `workspace.tsx` + `bindings.tsx` + `nav.tsx` into every `(search-app)` route; (g) three client waterfalls (`use-app-preferences.ts:156-182`, `ClinicalDashboard.tsx:977-1069`, `signed-image.tsx:60-84` + `use-signed-image-url.ts:39`) and the paint offenders in `globals.css` beyond the sidebar grid — three stacked `backdrop-filter` passes on an always-mounted translating element (`:709-748`), `box-shadow` inside a `transition` list (`:677-684`), and `@keyframes shimmer` animating `background-position` on the shared `Skeleton` (`:2289-2296`). **CORRECTED 2026-07-29 on (c):** the Therapy Compass filenames are unversioned and Next serves `/public` with an ETag, so only the FIRST visit pays 690.6 KB / 2,470 KB — repeat visits pay ~4 revalidation round trips. The fix is content-hashed filenames + `immutable` (touching `scripts/build-therapies-index.mjs` and `check:therapy-data-index`), NOT a bare `Cache-Control` line. See `docs/audit/latency-audit-2026-07-28.md` L3-1/L3-2/L3-3/L3-6/L3-7. | session 2026-07-21 (build route table + design audit) | 2026-07-21 | -| #017 | P3 | task | Field Web-Vitals baseline via live Lighthouse | In-sandbox runtime vitals were blocked (prod server hard-requires Supabase secrets; dev-mode CLS measured excellent at 0.00–0.04, content-first pages 0.000). Run Lighthouse against `psychiatry.tools` for real LCP/INP/CLS to prioritize #012–#016 by measured impact rather than reasoning. | session 2026-07-21 (measurement pass) | 2026-07-21 | +| #017 | P3 | task | Field Web-Vitals baseline via live Lighthouse | In-sandbox runtime vitals were blocked (prod server hard-requires Supabase secrets; dev-mode CLS measured excellent at 0.00–0.04, content-first pages 0.000). Run Lighthouse against `psychiatry.tools` for real LCP/INP/CLS to prioritize #012–#016 by measured impact rather than reasoning. **Instrument landed (PR #1385):** `live-web-vitals.yml` is dispatch-only, takes no secrets and runs nothing until someone dispatches it — which is itself a live production action needing explicit approval. `scripts/summarise-web-vitals.mjs` holds the decision rule, committed before any numbers were read: mobile LCP < 2500 ms AND CLS < 0.1 on every route (plus INP < 200 ms from CrUX) closes this row and makes only the explicitly measured **payload** findings gated by #017 WONTFIX — e.g. the route-chunk/catalogue weight in #013. It does **not** close #016 wholesale: #016's motion, CSS, client-waterfall, caching, and dynamic-import items stay open unless they have separate evidence, because Lighthouse LCP/CLS do not validate those. Any breach makes only that route's findings actionable. It fails closed — a missing run, a null metric, a route-slug collision, and a measurement that landed on a different URL than requested (redirect, dropped or reordered query) are all breaches. **Read the first dispatch as a measurement to be sanity-checked, not as an oracle:** the grading logic took eight rounds of review corrections to reach this shape, each fix locally right and globally incomplete, so cross-check the emitted table against the raw Lighthouse JSON artifact before recording a verdict — most of all a PASS, which would close #017 and the explicitly measured payload findings. Lighthouse cannot measure INP in lab conditions, so that clause of the rule is confirmed from CrUX field data and any pass is provisional on it. | session 2026-07-21 (measurement pass) | 2026-07-21 | | #018 | P2 | task | Split the lithium, ADHD and metabolic residuals by mechanism | Current evidence keeps the mechanisms separate. **Lithium — closed within this item:** the row/atom-aware subject guard, foreign-parameter rejection and query-specific range promotion returned `0.5–1.0 mmol/L` with correct targeting/citation; the full retrieval canary remained 36/36 with recall 1.0 and zero per-case RR regressions, and the full answer canary passed every blocking gate. **ADHD — open corpus debt:** `CG.MHSP.ADHD.pdf` is absent from the hosted corpus and the retrieved chart exposes `accessible_table_count=0`; repair corpus/fixture or ingestion evidence rather than weakening extractive budgets. **Metabolic — open structured-evidence debt:** the standalone plural classifier worsened the live answer and was reverted; obtain auditable schedule text/table evidence before another candidate. | targeted live lithium/ADHD/metabolic evidence 2026-07-27; `docs/evidence/rag-reliability-evidence-2026-07-27.md`; refuted approaches | 2026-07-21 | | #021 | P3 | rec | E-3d H2 residual: strong/comparison generation discards | approx. 6 generation attempts per full 44-case run still fail the final quality gate and fall to extractive on strong-route comparison/complex shapes (the designed-conservative outcome). PARKED: weakest cost/benefit on the queue — a wave (approx. $2-4 pair + reviewer cycle) to shave seconds off a few hard cases. Revisit only if latency/waste complaints or a cheaper lever appears. | E-3c design record; runs #59-#61 diagnostics | 2026-07-21 | | #022 | P2 | task | Source-governance metadata refresh (operator) | The selected policy is now encoded locally as auditable `third_party_reference_attested` evidence with policy version, reviewer qualification, evidence references and append-only review history. It deliberately preserves `clinical_validation_status=unverified`; malformed, stale or non-BMJ evidence remains review debt. Migration `20260727010000_bmj_third_party_source_attestation.sql` is prepared but was **not applied**. The ten most visible local-document candidates are captured in `docs/evidence/rag-top-local-review-manifest-2026-07-26.json` with `attestation_applied=false`; qualified human review, deliberate hosted apply/attestation, and warning-rate remeasurement remain operator work. | governance worklist; local policy/migration tests; top-ten evidence manifest | 2026-07-21 | @@ -158,7 +158,7 @@ Move resolved rows here with the resolution date and a one-line outcome. Keep th | ---- | ----- | -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | | #087 | issue | `npm run check:knip` reported pre-existing dependency findings | NOT A DEFECT — false positive. The findings (unused `rimraf`/`tsx`/`@testing-library/dom`, unlisted `playwright-core`) only appeared because the worktree had no `node_modules` of its own and tooling resolved from the parent checkout. After `npm ci` in the same worktree, `npm run check:knip` exits 0 with no ignore-list change. Durable lesson: never act on a knip finding from a worktree that has not been installed. | 2026-07-28 | | #089 | issue | Branch-review-ledger hygiene deferred from PR #1275 | Closed by the 2026-07-28 hygiene pass. The MD056 cell-count and duplicate-row findings dispositioned as "belongs in a dedicated main ledger hygiene pass" on PR #1275 are fixed: 140 mojibake lines restored byte-exact from git history, 6 residual separators repaired, 46 exact duplicates removed, 21 wrong-width rows normalised, and 4 heading+bullet records converted to table rows — 1067 records, all six cells. Root cause closed too: `npm run ledger:lookup` / `ledger:append` (`scripts/branch-review-ledger.mjs`) replace hand-written rows, and `check:branch-review-ledger` now fails on mojibake, cell width, heading records, impossible dates, table gaps, and (from 2026-07-29) unresolvable HEADs and near-duplicates. | 2026-07-28 | -| #111 | issue | `ui-overlap` phone-inset case still flakes under load | RESOLVED 2026-07-29 (PR #1391). The inset measurement now retries inside `expect(async () => …).toPass({ timeout: 15_000 })` (`ui-overlap.spec.ts:159`) — the same retry-the-measurement shape PR #1375 applied to the eight overlap cases. The 2px symmetry tolerance and the assertions themselves are byte-for-byte unchanged, so a genuinely asymmetric header still fails once the retry budget is spent; only a transient mid-remount sample settles. Full spec green across 4 consecutive runs (14 passed each). Do not relax the tolerance if this recurs — the assertion is the point; make the measurement robust instead. Source: PR #1375 flake triage; session 2026-07-29 | +| #111 | issue | `ui-overlap` phone-inset case still flakes under load | RESOLVED 2026-07-29 (PR #1391). The inset measurement now retries inside `expect(async () => …).toPass({ timeout: 15_000 })` (`ui-overlap.spec.ts:159`) — the same retry-the-measurement shape PR #1375 applied to the eight overlap cases. The 2px symmetry tolerance and the assertions themselves are byte-for-byte unchanged, so a genuinely asymmetric header still fails once the retry budget is spent; only a transient mid-remount sample settles. Full spec green across 4 consecutive runs (14 passed each). Do not relax the tolerance if this recurs — the assertion is the point; make the measurement robust instead. Source: PR #1375 flake triage; session 2026-07-29 | 2026-07-29 | | #012 | rec | Slim the lazy cross-mode differentials chunk | Precomputed a trimmed index (`src/data/cross-mode-differentials-index.json` via `scripts/build-cross-mode-differentials-index.mjs`) so the lazily-loaded cross-mode chunk imports a ~53 KB catalog instead of statically pulling the ~1.2 MB differentials snapshot (only that dynamic path reached it). A drift test plus `check:cross-mode-index` (in verify:cheap) lock the index to the live projection. | 2026-07-27 | | #029 | issue | Residual answer-quality fallback stubs | Closed after fixing each causal cluster independently. Active-community ED, community-home-visit, clozapine blood-threshold/typo, discharge source-gap recovery, and Best Practice Prescription now use narrowly validated, source-bound answers or auditable recovery; cited provider refusal prose can no longer masquerade as grounded, and terminal gaps retain no claim citations. The final 44-case gate reported 30/30 substantive grounded supported answers, 14/14 unsupported correct, zero review fallbacks, zero citation/numeric failures, and zero route-ceiling failures. Measurement still reports review fallback separately and denies targeting credit for echoed boilerplate. | 2026-07-27 | diff --git a/scripts/summarise-web-vitals.mjs b/scripts/summarise-web-vitals.mjs new file mode 100644 index 0000000000..cd89b056c4 --- /dev/null +++ b/scripts/summarise-web-vitals.mjs @@ -0,0 +1,378 @@ +#!/usr/bin/env node +// Summarises the Lighthouse JSON produced by .github/workflows/live-web-vitals.yml +// into a markdown table plus a machine-readable summary.json, and states the +// ledger #017 verdict. +// +// Lives here rather than inline in the workflow so the escaping is not at the +// mercy of YAML-inside-shell-inside-node quoting, and so the thresholds can be +// unit-tested. +// +// Lab metrics only. Lighthouse cannot measure INP in lab conditions (it is an +// interaction metric); TBT is its lab proxy. The #017 rule has three clauses — +// LCP, CLS and INP — so this script can satisfy at most two of them and NEVER +// emits an #017 closure on its own. The best it reports is "LCP and CLS are +// within threshold; obtain INP from CrUX before closing #017". + +import { readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +/** #017 decision rule, fixed before any numbers are read. */ +export const WEB_VITALS_THRESHOLDS = { lcpMs: 2500, cls: 0.1 }; + +/** + * #017 asks for evidence from the production origin specifically + * (`docs/outstanding-issues.md`). `LIVE_DOMAIN_URL` can point the workflow at a + * staging cutover, which is useful for a dry run but cannot discharge #017. + */ +export const CANONICAL_ORIGIN = "https://psychiatry.tools"; + +/** Origins actually loaded, taken from the reports rather than the input URL. */ +export function measuredOrigins(rows) { + const origins = new Set(); + for (const row of rows) { + if (!row?.url) continue; + try { + origins.add(new URL(row.url).origin); + } catch { + origins.add(String(row.url)); + } + } + return [...origins].sort(); +} + +/** + * Whether this run may state an #017 verdict at all. Read from each report's own + * final URL, not from `LIVE_DOMAIN_URL`, so a redirect to another host is caught + * as well as a deliberate staging override. + */ +export function isProductionVerdict(rows) { + const origins = measuredOrigins(rows); + return origins.length > 0 && origins.every((origin) => origin === CANONICAL_ORIGIN); +} + +/** Extract the fields we report from one Lighthouse JSON report. */ +export function summariseReport(run, report) { + const audits = report?.audits ?? {}; + const numeric = (id) => { + const value = audits[id]?.numericValue; + return typeof value === "number" && Number.isFinite(value) ? value : null; + }; + return { + run, + url: report?.finalDisplayedUrl ?? report?.requestedUrl ?? null, + // Kept alongside the final URL so a redirect can be detected. Measuring + // /login instead of the requested route yields perfectly good numbers for + // the wrong page, and the filename alone cannot reveal that. + requestedUrl: report?.requestedUrl ?? null, + runtimeError: report?.runtimeError?.code ?? null, + // The browser that produced these numbers. Chrome comes from the runner + // image, not from `LIGHTHOUSE_VERSION`, so it moves independently of the + // pinned tooling and a metric shift can originate there rather than in the + // application. `main()` records the distinct values in summary.json. + chromeVersion: report?.environment?.hostUserAgent ?? null, + performanceScore: report?.categories?.performance?.score ?? null, + lcpMs: numeric("largest-contentful-paint"), + cls: numeric("cumulative-layout-shift"), + tbtMs: numeric("total-blocking-time"), + fcpMs: numeric("first-contentful-paint"), + }; +} + +/** + * Filename slug for a route, matching the sed expression in the workflow: + * "/" -> root, "/a" -> a, "/a/b" -> a-b. + */ +export function routeSlug(route) { + const trimmed = String(route ?? "").trim(); + if (!trimmed) return null; + return trimmed.replace(/^\//, "").replaceAll("/", "-") || "root"; +} + +/** The strategies the workflow measures. #017 asks for mobile AND desktop evidence. */ +export const WEB_VITALS_STRATEGIES = ["mobile", "desktop"]; + +/** The requested routes' slugs, in dispatch order, blanks dropped. */ +export function routeSlugs(routes) { + return (Array.isArray(routes) ? routes : String(routes ?? "").split(",")).map(routeSlug).filter(Boolean); +} + +/** + * Requested routes whose slugs collide. The workflow names each report + * `-.json`, so `/a/b` and `/a-b` both write `a-b` and the second + * run overwrites the first. The surviving file then satisfies the expected-run + * check for both routes, and one requested route is silently never measured. + * The filename scheme simply cannot represent both, so the evidence can never be + * complete and this is fatal rather than a warning. + */ +export function collidingRouteSlugs(routes) { + const counts = new Map(); + for (const slug of routeSlugs(routes)) counts.set(slug, (counts.get(slug) ?? 0) + 1); + return [...counts.entries()].filter(([, count]) => count > 1).map(([slug]) => slug); +} + +/** Every `-` run name the workflow was asked to produce. */ +export function expectedRuns(routes, strategies = WEB_VITALS_STRATEGIES) { + const slugs = routeSlugs(routes); + return strategies.flatMap((strategy) => slugs.map((slug) => `${strategy}-${slug}`)); +} + +/** The `mobile-` runs — the threshold rule in #017 is graded on mobile only. */ +export function expectedMobileRuns(routes) { + return expectedRuns(routes, ["mobile"]); +} + +/** + * Requested runs of EITHER strategy that produced no report. The threshold + * verdict is mobile-only, but #017 asks for reproducible mobile *and* desktop + * evidence (`docs/outstanding-issues.md`), so a run with every mobile report and + * no desktop report is not a baseline and must not be reported as one. + */ +export function missingRuns(rows, routes) { + const seen = new Set(rows.map((row) => row.run)); + return expectedRuns(routes).filter((run) => !seen.has(run)); +} + +/** A report is usable evidence only if it carries both graded numbers. */ +export function hasUsableMetrics(row) { + return row != null && row.lcpMs !== null && row.cls !== null; +} + +/** + * Path AND query of a URL, normalised, for comparing requested vs final. The + * query matters: `/documents/search?q=depression` and `/documents/search` are + * different pages to measure — one renders results, the other an empty state — + * so a redirect that drops or rewrites the query must not pass as the requested + * route. Params are sorted so a reordering redirect is not a false rejection. + */ +function normalisedTarget(value) { + if (!value) return null; + try { + const { pathname, searchParams } = new URL(value); + const path = pathname.length > 1 ? pathname.replace(/\/+$/, "") : "/"; + const params = [...searchParams.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + // Encode each component. Joining the decoded entries as `key=val&…` lets + // `/forms?q=alpha%26run%3D1` (one value containing `&`/`=`) alias + // `/forms?q=alpha&run=1` (two params) — different pages, same string after + // decode. URLSearchParams.toString() keeps the boundary. + const query = new URLSearchParams(params).toString(); + return query ? `${path}?${query}` : path; + } catch { + return null; + } +} + +/** + * Whether the report measured the page that was asked for. An origin check alone + * is not enough: a route that redirects to `/login`, to a same-origin error page + * or to any other path still produces clean metrics on `psychiatry.tools`, and + * the filename cannot reveal that the numbers describe a different page. A + * Lighthouse `runtimeError` is treated the same way — the run did not measure + * what it claims to. + */ +export function measuredRequestedPage(row) { + if (row == null || row.runtimeError) return false; + const requested = normalisedTarget(row.requestedUrl); + // Older reports carry no requestedUrl; absent evidence of a redirect is not + // evidence of none, but neither is it a reason to reject a report that has + // no requested URL recorded at all — those are caught by hasUsableMetrics. + if (requested === null) return true; + return normalisedTarget(row.url) === requested; +} + +/** + * Everything that makes the evidence incomplete rather than merely bad: a + * requested run with no report, a report with no LCP/CLS number, or a route + * whose slug collides so its report was overwritten. All must fail the step — a + * problem that is only rendered in the table still exits 0, and #017 exists + * precisely because this repo has acted on unmeasured latency claims before. + * + * This walks the expected route-by-strategy matrix DIRECTLY rather than asking + * `mobileBreaches`. Delegating to that mobile-only helper is what produced three + * successive holes here — desktop reports, then desktop metrics — each fixed one + * instance at a time. Completeness is a property of the whole matrix, so it is + * computed over the whole matrix. + * + * An over-threshold number is deliberately NOT here: that is a real measurement + * and a real verdict, not missing evidence. + */ +export function incompleteEvidence(rows, routes) { + const byRun = new Map(rows.map((row) => [row.run, row])); + const expected = expectedRuns(routes); + const problems = new Set(collidingRouteSlugs(routes).map((slug) => `route slug collision: ${slug}`)); + for (const run of expected) { + const row = byRun.get(run); + if (!hasUsableMetrics(row) || !measuredRequestedPage(row)) problems.add(run); + } + // With no route list to check against, an absence of mobile reports entirely + // is still not a pass — the guard must not be bypassable by omitting the arg. + if (expected.length === 0 && !rows.some((row) => row.run.startsWith("mobile-") && hasUsableMetrics(row))) { + problems.add("mobile-*"); + } + return [...problems].sort(); +} + +/** + * Mobile runs that breach the rule. Three ways to breach, all fail-closed: + * a metric is missing, a metric is over threshold, or the run produced no + * report at all. The last one matters because the workflow deliberately + * downgrades a per-route Lighthouse failure to a warning, so a run that never + * happened would otherwise be silently absent from the pass verdict — and #017 + * exists precisely because this repo has acted on unmeasured latency claims + * before. An absent number is not evidence of passing; nor is an absent run. + */ +export function mobileBreaches(rows, routes) { + const present = rows.filter((row) => row.run.startsWith("mobile-")); + const seen = new Set(present.map((row) => row.run)); + const expected = expectedMobileRuns(routes); + const missing = expected + .filter((run) => !seen.has(run)) + .map((run) => ({ run, reason: "no Lighthouse report produced", missingReport: true, missingMetric: false })); + const failed = present + .filter( + (row) => + row.lcpMs === null || + row.cls === null || + row.lcpMs >= WEB_VITALS_THRESHOLDS.lcpMs || + row.cls >= WEB_VITALS_THRESHOLDS.cls, + ) + .map((row) => { + const missingMetric = row.lcpMs === null || row.cls === null; + return { + ...row, + reason: missingMetric ? "report has no LCP or CLS number" : "outside the threshold", + missingReport: false, + missingMetric, + }; + }); + // With no expected list to check against, zero mobile reports is still not a + // pass — never let an all-desktop directory read as "every mobile route ok". + if (expected.length === 0 && present.length === 0) { + return [ + { run: "mobile-*", reason: "no mobile Lighthouse report produced", missingReport: true, missingMetric: false }, + ]; + } + return [...missing, ...failed]; +} + +export function renderTable(rows, routes) { + const format = (value, digits = 0) => (value === null ? "n/a" : value.toFixed(digits)); + const lines = [ + "| run | perf | LCP ms | CLS | TBT ms | FCP ms |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ...rows.map( + (row) => + `| ${row.run} | ${format((row.performanceScore ?? 0) * 100)} | ${format(row.lcpMs)} | ` + + `${format(row.cls, 3)} | ${format(row.tbtMs)} | ${format(row.fcpMs)} |`, + ), + ]; + const breaches = mobileBreaches(rows, routes); + const incomplete = incompleteEvidence(rows, routes); + const productionVerdict = isProductionVerdict(rows); + + // Disqualifiers are resolved BEFORE any threshold or actionability prose, and + // that ordering is the point. Both were previously computed independently of + // the verdict branch and appended afterwards, so a run could assert "every + // mobile route is within threshold" in bold and then admit two paragraphs + // later that the evidence was incomplete, or call a staging breach + // "actionable" for #017. Whichever line a reader takes away, one of them was + // false. A run that cannot produce a verdict must not emit verdict prose at + // all — the measurements are still shown, explicitly subordinated. + const disqualifiers = []; + if (incomplete.length > 0) { + disqualifiers.push(`${incomplete.length} requested run(s) produced no usable report: ${incomplete.join(", ")}`); + } + if (!productionVerdict) { + disqualifiers.push( + `#017 asks for evidence from ${CANONICAL_ORIGIN}; this run measured ` + + `${measuredOrigins(rows).join(", ") || "an unknown origin"}`, + ); + } + + lines.push(""); + if (disqualifiers.length > 0) { + lines.push( + `**This run is NOT an #017 verdict.** ${disqualifiers.join(". ")}. ` + + "Record nothing against #017 from this run; fix the cause and rerun.", + ); + lines.push(""); + lines.push( + breaches.length === 0 + ? "_For reference only — the reports that were produced are within the thresholds. " + + "This is a measurement, not a verdict._" + : `_For reference only — ${breaches.length} of the reports that were produced are outside the thresholds: ` + + `${breaches.map((breach) => `${breach.run} (${breach.reason})`).join(", ")}. ` + + "This is a measurement, not a verdict, and does not make any finding actionable._", + ); + return lines.join("\n"); + } + + lines.push( + breaches.length === 0 + ? // Deliberately NOT a closure. The #017 rule has three clauses — LCP, + // CLS and INP < 200ms — and Lighthouse cannot measure INP in lab + // conditions at all (it is an interaction metric; TBT is the lab + // proxy). This run therefore satisfies two of three and cannot close + // #017 by itself. Announcing closure here and asking the operator to + // "confirm INP afterward" put the bold verdict before the evidence, + // which is the exact failure mode #017 exists to prevent. + `**Every mobile route is within LCP < ${WEB_VITALS_THRESHOLDS.lcpMs}ms and CLS < ${WEB_VITALS_THRESHOLDS.cls}. ` + + "This is NOT yet an #017 closure.** The #017 rule also requires INP < 200ms, which is field data this run " + + "does not collect — Lighthouse cannot measure INP in lab conditions. Obtain INP from CrUX for these routes; " + + "only if it is available AND under 200ms does #017 close as metrics-acceptable and the gated payload " + + "findings become WONTFIX. An unavailable or >=200ms INP leaves #017 open." + : `**${breaches.length} mobile route(s) breach the rule: ` + + `${breaches.map((breach) => `${breach.run} (${breach.reason})`).join(", ")}.** ` + + "Only findings on those routes become actionable, ranked by measured contribution. " + + "A route with no report is a breach, not a pass — rerun it before recording any #017 verdict.", + ); + return lines.join("\n"); +} + +function main() { + const directory = process.argv[2] ?? "web-vitals"; + // The routes the workflow was asked to measure, so a route whose Lighthouse + // run failed is reported as a breach rather than silently omitted. + const routes = process.argv[3] ?? process.env.ROUTES ?? ""; + const files = readdirSync(directory) + .filter((file) => file.endsWith(".json") && file !== "summary.json") + .sort(); + if (files.length === 0) { + console.log("::error::no Lighthouse output produced"); + process.exit(1); + } + const rows = files.map((file) => + summariseReport(file.replace(/\.json$/, ""), JSON.parse(readFileSync(join(directory, file), "utf8"))), + ); + // Record the measurement environment alongside the numbers: a baseline is + // only comparable to a follow-up produced by the same implementation AND the + // same browser. `LIGHTHOUSE_VERSION` is pinned in the workflow, but Chrome + // comes from the `ubuntu-24.04` runner image and moves underneath it, so a + // rendering or metric change can originate in the browser rather than the + // application. Lighthouse reports the build it drove in `environment. + // hostUserAgent`; taking it from the reports rather than from the runner + // records what actually measured, and disagreement across reports is itself + // worth seeing. Compare baselines only when both fields match. + const lighthouseVersion = process.env.LIGHTHOUSE_VERSION ?? "unpinned"; + const chromeVersions = [...new Set(rows.map((row) => row.chromeVersion).filter(Boolean))].sort(); + writeFileSync( + join(directory, "summary.json"), + `${JSON.stringify({ lighthouseVersion, chromeVersions, routes, rows }, null, 2)}\n`, + ); + const table = renderTable(rows, routes); + console.log(table); + if (process.env.GITHUB_STEP_SUMMARY) { + writeFileSync(process.env.GITHUB_STEP_SUMMARY, `## Live Web Vitals\n\n${table}\n`, { flag: "a" }); + } + const incomplete = incompleteEvidence(rows, routes); + if (incomplete.length > 0) { + console.log( + `::error::${incomplete.length} requested run(s) produced no usable report — evidence is incomplete: ` + + `${incomplete.join(", ")}`, + ); + process.exit(1); + } +} + +if (import.meta.url === `file://${process.argv[1]}` || process.argv[1]?.endsWith("summarise-web-vitals.mjs")) { + main(); +} diff --git a/tests/summarise-web-vitals.test.ts b/tests/summarise-web-vitals.test.ts new file mode 100644 index 0000000000..830767dd63 --- /dev/null +++ b/tests/summarise-web-vitals.test.ts @@ -0,0 +1,363 @@ +import { describe, expect, it } from "vitest"; + +import { + WEB_VITALS_THRESHOLDS, + expectedMobileRuns, + expectedRuns, + collidingRouteSlugs, + incompleteEvidence, + isProductionVerdict, + measuredRequestedPage, + missingRuns, + mobileBreaches, + renderTable, + routeSlug, + summariseReport, +} from "../scripts/summarise-web-vitals.mjs"; + +// Kept in step with the workflow's `routes` dispatch default. Every entry must +// be a real page route in docs/site-map.md: a bare `/documents` has no +// `page.tsx`, so measuring it would have profiled the 404 document. +const DEFAULT_ROUTES = "/,/therapy-compass,/documents/search,/dsm,/forms"; + +function row(run: string, lcpMs: number | null, cls: number | null) { + const url = `https://psychiatry.tools/${run}`; + return { + run, + url, + requestedUrl: url, + runtimeError: null, + performanceScore: 0.99, + lcpMs, + cls, + tbtMs: 10, + fcpMs: 500, + }; +} + +describe("routeSlug", () => { + it("matches the filename slugs the workflow's sed expression produces", () => { + expect(routeSlug("/")).toBe("root"); + expect(routeSlug("/dsm")).toBe("dsm"); + expect(routeSlug("/therapy-compass")).toBe("therapy-compass"); + expect(routeSlug("/a/b")).toBe("a-b"); + expect(routeSlug(" /forms ")).toBe("forms"); + expect(routeSlug("")).toBeNull(); + }); +}); + +describe("expectedMobileRuns", () => { + it("derives one mobile run per requested route", () => { + expect(expectedMobileRuns(DEFAULT_ROUTES)).toEqual([ + "mobile-root", + "mobile-therapy-compass", + "mobile-documents-search", + "mobile-dsm", + "mobile-forms", + ]); + }); +}); + +describe("mobileBreaches", () => { + it("passes when every requested mobile route reports inside the thresholds", () => { + const rows = expectedMobileRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); + }); + + it("treats a missing metric as a breach", () => { + const rows = expectedMobileRuns(DEFAULT_ROUTES).map((run) => + run === "mobile-dsm" ? row(run, null, 0.01) : row(run, 1200, 0.01), + ); + const breaches = mobileBreaches(rows, DEFAULT_ROUTES); + expect(breaches.map((breach) => breach.run)).toEqual(["mobile-dsm"]); + }); + + it("treats an over-threshold metric as a breach", () => { + const rows = expectedMobileRuns(DEFAULT_ROUTES).map((run) => + run === "mobile-forms" ? row(run, WEB_VITALS_THRESHOLDS.lcpMs + 1, 0.01) : row(run, 1200, 0.01), + ); + expect(mobileBreaches(rows, DEFAULT_ROUTES).map((breach) => breach.run)).toEqual(["mobile-forms"]); + }); + + // The regression this file exists for: the workflow downgrades a per-route + // Lighthouse failure to a warning, so a mobile route can produce no report at + // all. Grading only the reports that exist reported a clean pass and would + // have closed #017 on evidence that was never collected. + it("treats a requested mobile route with no report as a breach", () => { + const rows = [row("desktop-root", 900, 0.005)]; + const breaches = mobileBreaches(rows, DEFAULT_ROUTES); + expect(breaches).toHaveLength(5); + expect(breaches.every((breach) => breach.missingReport)).toBe(true); + expect(renderTable(rows, DEFAULT_ROUTES)).not.toContain("Every mobile route is within"); + }); + + it("does not read an all-desktop directory as a pass even with no route list", () => { + const breaches = mobileBreaches([row("desktop-root", 900, 0.005)], ""); + expect(breaches).toHaveLength(1); + expect(breaches[0].missingReport).toBe(true); + expect(renderTable([row("desktop-root", 900, 0.005)], "")).not.toContain("Every mobile route is within"); + }); +}); + +// #017 asks for reproducible mobile AND desktop evidence, and a run that exits 0 +// is a run whose verdict gets recorded. Both gaps below rendered a clean pass and +// exited 0 before this: a complete mobile sweep with no desktop reports at all, +// and a mobile report that exists but carries no LCP/CLS number. +describe("incompleteEvidence", () => { + const allRuns = () => expectedRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + + it("is empty when every requested mobile and desktop run reported", () => { + expect(incompleteEvidence(allRuns(), DEFAULT_ROUTES)).toEqual([]); + }); + + it("covers both strategies, so a missing desktop report is incomplete evidence", () => { + const mobileOnly = expectedMobileRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + expect(mobileBreaches(mobileOnly, DEFAULT_ROUTES)).toEqual([]); // thresholds all pass + expect(missingRuns(mobileOnly, DEFAULT_ROUTES)).toEqual([ + "desktop-root", + "desktop-therapy-compass", + "desktop-documents-search", + "desktop-dsm", + "desktop-forms", + ]); + expect(incompleteEvidence(mobileOnly, DEFAULT_ROUTES)).toHaveLength(5); + expect(renderTable(mobileOnly, DEFAULT_ROUTES)).toContain("NOT an #017 verdict"); + }); + + it("treats a present report with no LCP/CLS number as incomplete, not merely a breach", () => { + const rows = allRuns().map((r) => (r.run === "mobile-dsm" ? { ...r, lcpMs: null } : r)); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-dsm"]); + }); + + it("does not treat an over-threshold measurement as incomplete evidence", () => { + const rows = allRuns().map((r) => + r.run === "mobile-forms" ? { ...r, lcpMs: WEB_VITALS_THRESHOLDS.lcpMs + 1 } : r, + ); + expect(mobileBreaches(rows, DEFAULT_ROUTES).map((b) => b.run)).toEqual(["mobile-forms"]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); + }); + + // Completeness is a property of the whole matrix. Checking metric validity + // only on mobile left a desktop report with null metrics reading as evidence. + it("rejects a desktop report that exists but carries no LCP/CLS number", () => { + const rows = allRuns().map((r) => (r.run === "desktop-forms" ? { ...r, cls: null } : r)); + expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); // mobile verdict is clean + expect(missingRuns(rows, DEFAULT_ROUTES)).toEqual([]); // the file is present + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["desktop-forms"]); + }); + + // `/a/b` and `/a-b` both slug to `a-b`, so the second Lighthouse run + // overwrites the first and one requested route is never retained — while the + // surviving file satisfies the expected-run check for both. + it("rejects colliding route slugs, which silently drop a requested route", () => { + const colliding = "/a/b,/a-b"; + expect(collidingRouteSlugs(colliding)).toEqual(["a-b"]); + const rows = [row("mobile-a-b", 1200, 0.01), row("desktop-a-b", 1200, 0.01)]; + expect(missingRuns(rows, colliding)).toEqual([]); // every expected key is "present" + expect(incompleteEvidence(rows, colliding)).toContain("route slug collision: a-b"); + }); + + it("accepts distinct routes that do not collide", () => { + expect(collidingRouteSlugs(DEFAULT_ROUTES)).toEqual([]); + }); + + // An origin check alone passes a route that redirected to another page on the + // same host — an auth bounce to /login, a same-origin error page, a mistyped + // route. The numbers are clean but they describe a different page, and the + // filename cannot reveal it. + it("rejects a run that redirected to a different same-origin path", () => { + const rows = allRuns().map((r) => (r.run === "mobile-dsm" ? { ...r, url: "https://psychiatry.tools/login" } : r)); + expect(isProductionVerdict(rows)).toBe(true); // origin is still canonical + expect(measuredRequestedPage(rows.find((r) => r.run === "mobile-dsm"))).toBe(false); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-dsm"]); + }); + + it("rejects a run that reported a Lighthouse runtime error", () => { + const rows = allRuns().map((r) => (r.run === "desktop-forms" ? { ...r, runtimeError: "NO_FCP" } : r)); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["desktop-forms"]); + }); + + // /documents/search?q=depression and /documents/search are different pages to + // measure: one renders results, the other an empty state. + it("rejects a redirect that drops the query string", () => { + const requested = "https://psychiatry.tools/documents/search?q=depression"; + const rows = allRuns().map((r) => + r.run === "mobile-documents-search" + ? { ...r, requestedUrl: requested, url: "https://psychiatry.tools/documents/search" } + : r, + ); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-documents-search"]); + }); + + it("accepts the same query with params in a different order", () => { + const rows = allRuns().map((r) => + r.run === "mobile-dsm" + ? { + ...r, + requestedUrl: "https://psychiatry.tools/dsm?a=1&b=2", + url: "https://psychiatry.tools/dsm?b=2&a=1", + } + : r, + ); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); + }); + + // Decoded `key=val&…` concatenation aliases a value that itself contains + // separators with a multi-param query. Encode the components so a rewrite + // from one page to another cannot pass as the requested route. + it("rejects a rewrite that splits an encoded separator into extra params", () => { + const requested = "https://psychiatry.tools/forms?q=alpha%26run%3D1"; + const rewritten = "https://psychiatry.tools/forms?q=alpha&run=1"; + expect( + measuredRequestedPage({ + ...row("mobile-forms", 1200, 0.01), + requestedUrl: requested, + url: rewritten, + }), + ).toBe(false); + const rows = allRuns().map((r) => + r.run === "mobile-forms" ? { ...r, requestedUrl: requested, url: rewritten } : r, + ); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-forms"]); + }); + + it("accepts the same encoded query value on both requested and final URLs", () => { + const url = "https://psychiatry.tools/forms?q=alpha%26run%3D1"; + const rows = allRuns().map((r) => (r.run === "mobile-forms" ? { ...r, requestedUrl: url, url } : r)); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); + }); + + it("tolerates a trailing-slash difference between requested and final URL", () => { + const rows = allRuns().map((r) => (r.run === "mobile-forms" ? { ...r, url: `${r.requestedUrl}/` } : r)); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); + }); +}); + +// LIVE_DOMAIN_URL can point the workflow at a staging cutover. That is a useful +// dry run, but #017 asks for evidence from the production origin, so such a run +// must not emit the closure sentence. Read from each report's own final URL so a +// redirect to another host is caught too, not just a deliberate override. +describe("production-origin gate", () => { + const staging = (run: string) => ({ + run, + url: `https://staging.psychiatry.tools/${run}`, + performanceScore: 0.99, + lcpMs: 1200, + cls: 0.01, + tbtMs: 10, + fcpMs: 500, + }); + + it("states the #017 closure only for the canonical production origin", () => { + const rows = expectedRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + expect(isProductionVerdict(rows)).toBe(true); + expect(renderTable(rows, DEFAULT_ROUTES)).toContain("NOT yet an #017 closure"); + }); + + it("refuses an #017 verdict for a staging override even when every threshold passes", () => { + const rows = expectedRuns(DEFAULT_ROUTES).map(staging); + expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); // thresholds all pass + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); // evidence is complete + expect(isProductionVerdict(rows)).toBe(false); + const table = renderTable(rows, DEFAULT_ROUTES); + expect(table).not.toContain("NOT yet an #017 closure"); + expect(table).toContain("NOT an #017 verdict"); + expect(table).toContain("https://staging.psychiatry.tools"); + }); + + it("refuses a verdict when only some reports left the canonical origin", () => { + const rows = expectedRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + expect(isProductionVerdict([...rows.slice(1), staging("mobile-root")])).toBe(false); + }); +}); + +describe("summariseReport", () => { + it("reads the lab metrics and treats a non-finite value as absent", () => { + const summary = summariseReport("mobile-root", { + finalDisplayedUrl: "https://psychiatry.tools/", + categories: { performance: { score: 0.97 } }, + audits: { + "largest-contentful-paint": { numericValue: 1234 }, + "cumulative-layout-shift": { numericValue: 0.02 }, + "total-blocking-time": { numericValue: 55 }, + "first-contentful-paint": { numericValue: Number.NaN }, + }, + }); + expect(summary).toMatchObject({ run: "mobile-root", lcpMs: 1234, cls: 0.02, tbtMs: 55, fcpMs: null }); + }); +}); + +// A run that cannot produce a verdict must not emit verdict prose at all. Both +// the threshold sentence and the "findings become actionable" sentence used to +// be reachable while the run was separately disqualified — for incomplete +// evidence or for a non-production origin — so the summary asserted a verdict in +// bold and then contradicted it. Whichever line a reader took away, one was +// false. These pin that the disqualification is the ONLY verdict-shaped claim. +describe("verdict prose is gated on the run being able to produce a verdict", () => { + const staging = (run: string, lcpMs = 1200) => { + const url = `https://staging.psychiatry.tools/${run}`; + return { ...row(run, lcpMs, 0.01), url, requestedUrl: url }; + }; + + it("suppresses the threshold claim when a run measured the wrong page", () => { + const rows = expectedRuns(DEFAULT_ROUTES).map((run) => + run === "mobile-dsm" ? { ...row(run, 1200, 0.01), url: "https://psychiatry.tools/login" } : row(run, 1200, 0.01), + ); + // The redirected run still carries passing numbers, so nothing breaches. + expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-dsm"]); + + const table = renderTable(rows, DEFAULT_ROUTES); + expect(table).toContain("NOT an #017 verdict"); + expect(table).toContain("mobile-dsm"); + // The two claims that must not survive a disqualified run. + expect(table).not.toContain("Every mobile route is within"); + expect(table).not.toContain("NOT yet an #017 closure"); + // The measurement is still reported, explicitly subordinated. + expect(table).toContain("For reference only"); + }); + + it("does not call a staging breach actionable", () => { + const rows = expectedRuns(DEFAULT_ROUTES).map((run) => staging(run, run === "mobile-forms" ? 3000 : 1200)); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); // evidence is complete… + expect(isProductionVerdict(rows)).toBe(false); // …but not from production + expect(mobileBreaches(rows, DEFAULT_ROUTES)).toHaveLength(1); + + const table = renderTable(rows, DEFAULT_ROUTES); + expect(table).toContain("NOT an #017 verdict"); + expect(table).toContain("https://staging.psychiatry.tools"); + expect(table).not.toContain("become actionable, ranked by measured contribution"); + // The breach is still visible, as a measurement rather than a verdict. + expect(table).toContain("mobile-forms"); + expect(table).toContain("This is a measurement, not a verdict"); + }); + + it("reports both disqualifiers together rather than only the first", () => { + const rows = expectedRuns(DEFAULT_ROUTES) + .filter((run) => !run.startsWith("desktop-")) + .map((run) => staging(run)); + const table = renderTable(rows, DEFAULT_ROUTES); + expect(table).toContain("produced no usable report"); + expect(table).toContain("asks for evidence from https://psychiatry.tools"); + }); +}); + +// Chrome ships with the runner image, not with the pinned LIGHTHOUSE_VERSION, so +// it moves underneath a baseline and a metric shift can originate in the browser +// rather than the application. summariseReport keeps what actually drove the run. +describe("measurement environment", () => { + it("records the Chrome build that produced the report", () => { + const summary = summariseReport("mobile-root", { + requestedUrl: "https://psychiatry.tools/", + finalDisplayedUrl: "https://psychiatry.tools/", + environment: { hostUserAgent: "Mozilla/5.0 … Chrome/141.0.0.0 Safari/537.36" }, + categories: { performance: { score: 0.99 } }, + audits: {}, + }); + expect(summary.chromeVersion).toBe("Mozilla/5.0 … Chrome/141.0.0.0 Safari/537.36"); + }); + + it("leaves the field null when the report does not state it", () => { + expect(summariseReport("mobile-root", { audits: {} }).chromeVersion).toBeNull(); + }); +});