diff --git a/.github/workflows/live-web-vitals.yml b/.github/workflows/live-web-vitals.yml index 27993037ba..8e897cec5f 100644 --- a/.github/workflows/live-web-vitals.yml +++ b/.github/workflows/live-web-vitals.yml @@ -18,7 +18,8 @@ name: Live Web Vitals baseline # # The decision rule is written down BEFORE the numbers are read, so the gate # cannot be rationalised after the fact: -# mobile LCP < 2.5s AND CLS < 0.1 AND INP < 200ms on every route +# mobile MEDIAN LCP < 2.5s AND CLS < 0.1 AND INP < 200ms on every route, +# with the samples behind each median all on one side of the line # -> close #017 "metrics acceptable"; only the explicitly measured payload # findings gated by #017 become WONTFIX (e.g. #013 route-chunk weight). # #016's motion/CSS/waterfall/caching/dynamic-import items stay open @@ -39,6 +40,10 @@ on: # `page.tsx` (see docs/site-map.md), so it would have measured the 404 # document. `/documents/search` is the canonical documents-mode route. default: "/,/therapy-compass,/documents/search,/dsm,/forms" + samples: + description: "Lighthouse runs per route/strategy (median is graded)" + required: false + default: "3" permissions: contents: read @@ -57,6 +62,12 @@ jobs: # matching live-domain-monitor.yml. LIVE_DOMAIN_URL: ${{ vars.LIVE_DOMAIN_URL || 'https://psychiatry.tools' }} ROUTES: ${{ inputs.routes }} + # Ledger #114: one run per route cannot measure dispersion, so it cannot + # detect the "evidence is too noisy" condition #017 says to stop on, and a + # route near the threshold resolves on variance alone. The summariser + # grades the MEDIAN of these and refuses to grade a cell whose samples + # straddle the line. + SAMPLES: ${{ inputs.samples || '3' }} # Pinned exactly, not `lighthouse@12`. That range is >=12.0.0 <13.0.0-0, # so a patch published between a baseline run and its follow-up would # change metric collection independently of the application and silently @@ -88,6 +99,27 @@ jobs: echo "LIVE_DOMAIN_URL=$normalized" >> "$GITHUB_ENV" echo "origin -> $normalized" + - name: Reject a sample count too small to grade + run: | + set -euo pipefail + # `samples` is a free-text dispatch input, so `samples=1` is accepted + # by the form and would produce one report per cell — a "median" of + # one, a zero-width range, and a straddle check that can never fire. + # The summariser refuses the same count, but refusing HERE means the + # operator finds out in seconds instead of after a full measurement + # pass against the live domain. + case "$SAMPLES" in + ''|*[!0-9]*) + echo "::error::samples must be a positive integer, got '$SAMPLES'" + exit 1 + ;; + esac + if [ "$SAMPLES" -lt 3 ]; then + echo "::error::samples=$SAMPLES cannot be graded — #017 needs at least 3 runs per cell so a median has dispersion behind it" + exit 1 + fi + echo "samples -> $SAMPLES per route/strategy" + - name: Confirm the target is reachable before spending a Lighthouse run run: | set -euo pipefail @@ -110,8 +142,9 @@ jobs: # Filename-safe slug: "/" -> root, "/a/b" -> a-b slug="$(echo "$route" | sed 's|^/||; s|/|-|g')" [ -n "$slug" ] || slug="root" - out="web-vitals/${strategy}-${slug}" - echo "::group::$strategy $route" + for sample in $(seq 1 "$SAMPLES"); do + out="web-vitals/${strategy}-${slug}-${sample}" + echo "::group::$strategy $route (sample $sample/$SAMPLES)" # One flaky route must not discard the whole run, so a failure is # a warning here; the summary step fails if NOTHING was produced. npx --yes "lighthouse@$LIGHTHOUSE_VERSION" "$LIVE_DOMAIN_URL$route" \ @@ -120,8 +153,9 @@ jobs: --only-categories=performance \ --chrome-flags="--headless=new --no-sandbox --disable-dev-shm-usage" \ --max-wait-for-load=60000 \ - --quiet || echo "::warning::lighthouse failed for $strategy $route" + --quiet || echo "::warning::lighthouse failed for $strategy $route sample $sample" echo "::endgroup::" + done done done @@ -129,7 +163,7 @@ jobs: # as a breach. Grading only the reports that happen to exist would let a # partially failed run read as "every mobile route passed". - name: Summarise LCP / CLS / TBT against the decision rule - run: node scripts/summarise-web-vitals.mjs web-vitals "$ROUTES" + run: node scripts/summarise-web-vitals.mjs web-vitals "$ROUTES" "$SAMPLES" - name: Upload the reports if: always() diff --git a/docs/outstanding-issues.md b/docs/outstanding-issues.md index 9fece3f88f..df563eaea4 100644 --- a/docs/outstanding-issues.md +++ b/docs/outstanding-issues.md @@ -108,7 +108,7 @@ removed after current-main verification; it is not missing recommended work. | #011 | P3 | task | Auth DB-connection allocation is operator-only | Supabase Auth (GoTrue) is capped at ~10 absolute DB connections (Supabase perf advisor). Switch to **percentage-based** allocation in the Supabase **dashboard** before the first compute scale-up — **not settable via SQL/MCP** (operator-owned). Verify via a staging soak + an approval-gated read-only advisor re-check. | `docs/auth-connection-cap-runbook.md`; `docs/process-hardening.md` (Known follow-up debts) | 2026-07-21 | | #013 | P3 | rec | Route-chunk + mockup catalogue JSON weight | `build:analyze`: `/specifiers` ships `specifiers-search-index.json` (~180 KB parsed), `/forms` ships `forms-catalog.json` (~132 KB), `/formulation` ships `formulation-content.json` (~52 KB, client-side local search — needs index/full split or a search endpoint, architectural). All route-scoped (not initial bundle). Also `*-mockups.tsx` (~100 KB across chunks) build though `/mockups` 404s in prod — exclude from the prod artifact. | session 2026-07-21 (build:analyze) | 2026-07-21 | | #016 | P3 | rec | "Big but not easy" structural + motion perf | Deferred larger levers: (a) nonce-CSP forces every product route to `╞Æ Dynamic` (zero static generation) — evaluate Partial Prerendering / static shells for the static clinical catalogues (DSM/differentials/therapy/specifiers/formulation); (b) sidebar expand/collapse animates `grid-template-columns` (biggest smoothness cost, motion-gated — needs a transform-overlay rethink); (c) Therapy Compass fetches 692 KB / 2.5 MB JSON client-side (defer until interaction + confirm brotli); (d) settings/setup/admin dialogs static-imported into the home chunk (`next/dynamic` them); (e) `src/app/(search-app)/layout.tsx:4` imports 71.6 KB of Therapy-Compass-only CSS in the ROUTE-GROUP layout, making it render-blocking on `/`, `/documents`, `/forms`, `/dsm` and every mode home; (f) `shared-search-app-shell.tsx:8` statically imports the `therapy-compass` barrel, pulling `workspace.tsx` + `bindings.tsx` + `nav.tsx` into every `(search-app)` route; (g) three client waterfalls (`use-app-preferences.ts:156-182`, `ClinicalDashboard.tsx:977-1069`, `signed-image.tsx:60-84` + `use-signed-image-url.ts:39`) and the paint offenders in `globals.css` beyond the sidebar grid — three stacked `backdrop-filter` passes on an always-mounted translating element (`:709-748`), `box-shadow` inside a `transition` list (`:677-684`), and `@keyframes shimmer` animating `background-position` on the shared `Skeleton` (`:2289-2296`). **CORRECTED 2026-07-29 on (c):** the Therapy Compass filenames are unversioned and Next serves `/public` with an ETag, so only the FIRST visit pays 690.6 KB / 2,470 KB — repeat visits pay ~4 revalidation round trips. The fix is content-hashed filenames + `immutable` (touching `scripts/build-therapies-index.mjs` and `check:therapy-data-index`), NOT a bare `Cache-Control` line. See `docs/audit/latency-audit-2026-07-28.md` L3-1/L3-2/L3-3/L3-6/L3-7. | session 2026-07-21 (build route table + design audit) | 2026-07-21 | -| #017 | P3 | task | Field Web-Vitals baseline via live Lighthouse | In-sandbox runtime vitals were blocked (prod server hard-requires Supabase secrets; dev-mode CLS measured excellent at 0.00–0.04, content-first pages 0.000). Run Lighthouse against `psychiatry.tools` for real LCP/INP/CLS to prioritize #012–#016 by measured impact rather than reasoning. **Instrument landed (PR #1385):** `live-web-vitals.yml` is dispatch-only, takes no secrets and runs nothing until someone dispatches it — which is itself a live production action needing explicit approval. `scripts/summarise-web-vitals.mjs` holds the decision rule, committed before any numbers were read: mobile LCP < 2500 ms AND CLS < 0.1 on every route (plus INP < 200 ms from CrUX) closes this row and makes only the explicitly measured **payload** findings gated by #017 WONTFIX — e.g. the route-chunk/catalogue weight in #013. It does **not** close #016 wholesale: #016's motion, CSS, client-waterfall, caching, and dynamic-import items stay open unless they have separate evidence, because Lighthouse LCP/CLS do not validate those. Any breach makes only that route's findings actionable. It fails closed — a missing run, a null metric, a route-slug collision, and a measurement that landed on a different URL than requested (redirect, dropped or reordered query) are all breaches. **Read the first dispatch as a measurement to be sanity-checked, not as an oracle:** the grading logic took eight rounds of review corrections to reach this shape, each fix locally right and globally incomplete, so cross-check the emitted table against the raw Lighthouse JSON artifact before recording a verdict — most of all a PASS, which would close #017 and the explicitly measured payload findings. Lighthouse cannot measure INP in lab conditions, so that clause of the rule is confirmed from CrUX field data and any pass is provisional on it. | session 2026-07-21 (measurement pass) | 2026-07-21 | +| #017 | P3 | task | Field Web-Vitals baseline via live Lighthouse | In-sandbox runtime vitals were blocked (prod server hard-requires Supabase secrets; dev-mode CLS measured excellent at 0.00–0.04, content-first pages 0.000). Run Lighthouse against `psychiatry.tools` for real LCP/INP/CLS to prioritize #012–#016 by measured impact rather than reasoning. **Instrument landed (PR #1385):** `live-web-vitals.yml` is dispatch-only, takes no secrets and runs nothing until someone dispatches it — which is itself a live production action needing explicit approval. `scripts/summarise-web-vitals.mjs` holds the decision rule, committed before any numbers were read: mobile LCP < 2500 ms AND CLS < 0.1 on every route (plus INP < 200 ms from CrUX) closes this row and makes only the explicitly measured **payload** findings gated by #017 WONTFIX — e.g. the route-chunk/catalogue weight in #013. It does **not** close #016 wholesale: #016's motion, CSS, client-waterfall, caching, and dynamic-import items stay open unless they have separate evidence, because Lighthouse LCP/CLS do not validate those. Any breach makes only that route's findings actionable. It fails closed — a missing run, a null metric, a route-slug collision, and a measurement that landed on a different URL than requested (redirect, dropped or reordered query) are all breaches. **Sampled since PR for #114:** each route/strategy cell is measured `SAMPLES` times (default 3), the MEDIAN is graded, and a cell whose samples land on both sides of a threshold is reported as incomplete evidence rather than resolved either way — so the "evidence is too noisy" stop rule can finally fire. A mid-run Chrome change also disqualifies the verdict. **Read the first dispatch as a measurement to be sanity-checked, not as an oracle:** the grading logic took eight rounds of review corrections to reach this shape, each fix locally right and globally incomplete, so cross-check the emitted table against the raw Lighthouse JSON artifact before recording a verdict — most of all a PASS, which would close #017 and the explicitly measured payload findings. Lighthouse cannot measure INP in lab conditions, so that clause of the rule is confirmed from CrUX field data and any pass is provisional on it. | session 2026-07-21 (measurement pass) | 2026-07-21 | | #018 | P2 | task | Split the lithium, ADHD and metabolic residuals by mechanism | Current evidence keeps the mechanisms separate. **Lithium — closed within this item:** the row/atom-aware subject guard, foreign-parameter rejection and query-specific range promotion returned `0.5–1.0 mmol/L` with correct targeting/citation; the full retrieval canary remained 36/36 with recall 1.0 and zero per-case RR regressions, and the full answer canary passed every blocking gate. **ADHD — open corpus debt:** `CG.MHSP.ADHD.pdf` is absent from the hosted corpus and the retrieved chart exposes `accessible_table_count=0`; repair corpus/fixture or ingestion evidence rather than weakening extractive budgets. **Metabolic — open structured-evidence debt:** the standalone plural classifier worsened the live answer and was reverted; obtain auditable schedule text/table evidence before another candidate. | targeted live lithium/ADHD/metabolic evidence 2026-07-27; `docs/evidence/rag-reliability-evidence-2026-07-27.md`; refuted approaches | 2026-07-21 | | #021 | P3 | rec | E-3d H2 residual: strong/comparison generation discards | approx. 6 generation attempts per full 44-case run still fail the final quality gate and fall to extractive on strong-route comparison/complex shapes (the designed-conservative outcome). PARKED: weakest cost/benefit on the queue — a wave (approx. $2-4 pair + reviewer cycle) to shave seconds off a few hard cases. Revisit only if latency/waste complaints or a cheaper lever appears. | E-3c design record; runs #59-#61 diagnostics | 2026-07-21 | | #022 | P2 | task | Source-governance metadata refresh (operator) | The selected policy is now encoded locally as auditable `third_party_reference_attested` evidence with policy version, reviewer qualification, evidence references and append-only review history. It deliberately preserves `clinical_validation_status=unverified`; malformed, stale or non-BMJ evidence remains review debt. Migration `20260727010000_bmj_third_party_source_attestation.sql` is prepared but was **not applied**. The ten most visible local-document candidates are captured in `docs/evidence/rag-top-local-review-manifest-2026-07-26.json` with `attestation_applied=false`; qualified human review, deliberate hosted apply/attestation, and warning-rate remeasurement remain operator work. | governance worklist; local policy/migration tests; top-ten evidence manifest | 2026-07-21 | diff --git a/scripts/summarise-web-vitals.mjs b/scripts/summarise-web-vitals.mjs index cd89b056c4..faf2e1c9cc 100644 --- a/scripts/summarise-web-vitals.mjs +++ b/scripts/summarise-web-vitals.mjs @@ -110,26 +110,151 @@ export function collidingRouteSlugs(routes) { return [...counts.entries()].filter(([, count]) => count > 1).map(([slug]) => slug); } -/** Every `-` run name the workflow was asked to produce. */ -export function expectedRuns(routes, strategies = WEB_VITALS_STRATEGIES) { +/** + * Samples taken per route/strategy. Ledger #114: a SINGLE Lighthouse run cannot + * measure dispersion, so it cannot detect the "evidence is too noisy" condition + * #017 itself tells the operator to stop on — and against a hard threshold, a + * route near the line resolves to a pass or a breach on run-to-run variance + * alone, invisibly. Three is the `lighthouse-ci` default and the smallest count + * that yields a median rather than a mean of two. + */ +export const WEB_VITALS_SAMPLES = 3; + +/** + * The fewest samples that may be GRADED, as opposed to merely defaulted to. + * + * The default above is only a default: `samples` is a free-text + * `workflow_dispatch` input, so an operator can dispatch `samples=1` and get + * back exactly the single-run matrix this change exists to abolish — one report + * per cell, a "median" of one, a range of zero width, and therefore a + * straddle-check that can never fire. Worse, the output would look identical to + * a sound run and could be recorded against #017 as a verdict. + * + * Two samples are no better for the purpose: the median is their mean, and two + * points cannot distinguish a stable cell from a noisy one whose runs happened + * to land close together. Three is the `lighthouse-ci` default and the smallest + * count that yields a real median. + */ +export const WEB_VITALS_MIN_SAMPLES = 3; + +/** Why this sample count cannot be graded, or `null` if it can. */ +export function samplesRefusal(samples) { + if (!Number.isInteger(samples) || samples < WEB_VITALS_MIN_SAMPLES) { + return ( + `samples=${samples} cannot be graded — #017 needs at least ${WEB_VITALS_MIN_SAMPLES} runs per cell. ` + + "Below that there is no dispersion to measure, so a cell near a threshold resolves on variance alone " + + "and the straddle check can never fire" + ); + } + return null; +} + +/** + * Every `-` CELL the workflow was asked to measure. A cell is + * the thing graded; its samples are the evidence behind the grade. + */ +export function expectedCells(routes, strategies = WEB_VITALS_STRATEGIES) { const slugs = routeSlugs(routes); return strategies.flatMap((strategy) => slugs.map((slug) => `${strategy}-${slug}`)); } -/** The `mobile-` runs — the threshold rule in #017 is graded on mobile only. */ -export function expectedMobileRuns(routes) { - return expectedRuns(routes, ["mobile"]); +/** Every `--` report the workflow was asked to produce. */ +export function expectedRuns(routes, strategies = WEB_VITALS_STRATEGIES, samples = WEB_VITALS_SAMPLES) { + return expectedCells(routes, strategies).flatMap((cell) => + Array.from({ length: samples }, (_, index) => `${cell}-${index + 1}`), + ); +} + +/** The `mobile-` cells — the threshold rule in #017 is graded on mobile only. */ +export function expectedMobileCells(routes) { + return expectedCells(routes, ["mobile"]); +} + +/** The cell a report belongs to: `mobile-dsm-2` -> `mobile-dsm`. */ +export function cellOf(run) { + return String(run ?? "").replace(/-\d+$/, ""); +} + +/** Middle value; the mean of the middle two for an even count. */ +export function median(values) { + const sorted = [...values] + .filter((value) => typeof value === "number" && Number.isFinite(value)) + .sort((a, b) => a - b); + if (sorted.length === 0) return null; + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +} + +/** + * Collapse each cell's samples into the graded figure plus the spread behind it. + * + * The median is graded rather than the mean: one pathological cold-cache sample + * should not drag a cell across a threshold, which is the whole reason for + * taking more than one. The min/max are retained because they, not the median, + * are what says whether the median can be trusted. + */ +export function aggregateCells(rows, samples = WEB_VITALS_SAMPLES) { + const grouped = new Map(); + for (const row of rows) { + const cell = cellOf(row.run); + if (!grouped.has(cell)) grouped.set(cell, []); + grouped.get(cell).push(row); + } + return new Map( + [...grouped.entries()].map(([cell, cellRows]) => { + // Metric validity only. A report that measured the WRONG page still has + // real numbers, so excluding it here would turn a redirect into a + // "no usable metric" breach and blur two different failures. + // `incompleteEvidence` owns the did-it-measure-the-right-page check. + const usable = cellRows.filter((row) => hasUsableMetrics(row)); + const lcps = usable.map((row) => row.lcpMs); + const clss = usable.map((row) => row.cls); + return [ + cell, + { + cell, + rows: cellRows, + usable, + expectedSamples: samples, + lcpMs: median(lcps), + cls: median(clss), + lcpMin: lcps.length ? Math.min(...lcps) : null, + lcpMax: lcps.length ? Math.max(...lcps) : null, + clsMin: clss.length ? Math.min(...clss) : null, + clsMax: clss.length ? Math.max(...clss) : null, + }, + ]; + }), + ); +} + +/** + * Whether a cell's samples sit on both sides of a threshold. + * + * This is the check #114 exists for. When the spread straddles the line, the + * median is an artefact of which samples happened to land where — the same + * route would grade differently on a rerun, so the run has measured noise + * rather than the site. That is missing evidence, not a verdict, and the + * operator is told to rerun rather than handed a number. + */ +export function straddlesThreshold(summary) { + if (summary == null) return false; + const spans = (min, max, limit) => min !== null && max !== null && min < limit && max >= limit; + return ( + spans(summary.lcpMin, summary.lcpMax, WEB_VITALS_THRESHOLDS.lcpMs) || + spans(summary.clsMin, summary.clsMax, WEB_VITALS_THRESHOLDS.cls) + ); } /** - * Requested runs of EITHER strategy that produced no report. The threshold + * Requested reports of EITHER strategy that were never produced. The threshold * verdict is mobile-only, but #017 asks for reproducible mobile *and* desktop * evidence (`docs/outstanding-issues.md`), so a run with every mobile report and * no desktop report is not a baseline and must not be reported as one. */ -export function missingRuns(rows, routes) { +export function missingRuns(rows, routes, samples = WEB_VITALS_SAMPLES) { const seen = new Set(rows.map((row) => row.run)); - return expectedRuns(routes).filter((run) => !seen.has(run)); + return expectedRuns(routes, WEB_VITALS_STRATEGIES, samples).filter((run) => !seen.has(run)); } /** A report is usable evidence only if it carries both graded numbers. */ @@ -195,14 +320,27 @@ export function measuredRequestedPage(row) { * An over-threshold number is deliberately NOT here: that is a real measurement * and a real verdict, not missing evidence. */ -export function incompleteEvidence(rows, routes) { +export function incompleteEvidence(rows, routes, samples = WEB_VITALS_SAMPLES) { const byRun = new Map(rows.map((row) => [row.run, row])); - const expected = expectedRuns(routes); + const expected = expectedRuns(routes, WEB_VITALS_STRATEGIES, samples); const problems = new Set(collidingRouteSlugs(routes).map((slug) => `route slug collision: ${slug}`)); for (const run of expected) { const row = byRun.get(run); if (!hasUsableMetrics(row) || !measuredRequestedPage(row)) problems.add(run); } + // Ledger #114. A cell whose samples land on both sides of a threshold has + // measured noise, not the site: the median is then an artefact of which + // samples fell where, and a rerun would grade it differently. Reporting a + // verdict from that is exactly the "evidence too noisy" case #017 says to + // stop on, so it is incomplete evidence rather than a pass or a breach. + for (const [cell, summary] of aggregateCells(rows, samples)) { + if (straddlesThreshold(summary)) { + problems.add( + `${cell} (samples straddle a threshold: LCP ${formatRange(summary.lcpMin, summary.lcpMax, 0)}ms, ` + + `CLS ${formatRange(summary.clsMin, summary.clsMax, 3)} — too noisy to grade)`, + ); + } + } // With no route list to check against, an absence of mobile reports entirely // is still not a pass — the guard must not be bypassable by omitting the arg. if (expected.length === 0 && !rows.some((row) => row.run.startsWith("mobile-") && hasUsableMetrics(row))) { @@ -211,6 +349,21 @@ export function incompleteEvidence(rows, routes) { return [...problems].sort(); } +/** + * Distinct Chrome builds across the reports. More than one means the runner + * image rolled mid-dispatch, so half the matrix was measured by a different + * browser and the cells are not comparable even to each other. + */ +export function chromeBuilds(rows) { + return [...new Set(rows.map((row) => row.chromeVersion).filter(Boolean))].sort(); +} + +/** `1200` for a single value, `1180\u20131320` for a spread. */ +export function formatRange(min, max, digits) { + if (min === null || max === null) return "n/a"; + return min === max ? min.toFixed(digits) : `${min.toFixed(digits)}\u2013${max.toFixed(digits)}`; +} + /** * Mobile runs that breach the rule. Three ways to breach, all fail-closed: * a metric is missing, a metric is over threshold, or the run produced no @@ -220,30 +373,42 @@ export function incompleteEvidence(rows, routes) { * exists precisely because this repo has acted on unmeasured latency claims * before. An absent number is not evidence of passing; nor is an absent run. */ -export function mobileBreaches(rows, routes) { - const present = rows.filter((row) => row.run.startsWith("mobile-")); - const seen = new Set(present.map((row) => row.run)); - const expected = expectedMobileRuns(routes); +export function mobileBreaches(rows, routes, samples = WEB_VITALS_SAMPLES) { + const cells = aggregateCells(rows, samples); + const present = [...cells.keys()].filter((cell) => cell.startsWith("mobile-")); + const expected = expectedMobileCells(routes); const missing = expected - .filter((run) => !seen.has(run)) - .map((run) => ({ run, reason: "no Lighthouse report produced", missingReport: true, missingMetric: false })); - const failed = present + .filter((cell) => !cells.has(cell)) + .map((cell) => ({ run: cell, reason: "no Lighthouse report produced", missingReport: true, missingMetric: false })); + + // Graded on the MEDIAN of the cell's samples, not on any one report. A cell + // whose spread straddles the threshold is not graded here at all — it is + // incomplete evidence (see `incompleteEvidence`), because a verdict drawn + // from it would flip on a rerun. + const failed = expected + .filter((cell) => cells.has(cell)) + .map((cell) => cells.get(cell)) .filter( - (row) => - row.lcpMs === null || - row.cls === null || - row.lcpMs >= WEB_VITALS_THRESHOLDS.lcpMs || - row.cls >= WEB_VITALS_THRESHOLDS.cls, + (summary) => + summary.lcpMs === null || + summary.cls === null || + summary.lcpMs >= WEB_VITALS_THRESHOLDS.lcpMs || + summary.cls >= WEB_VITALS_THRESHOLDS.cls, ) - .map((row) => { - const missingMetric = row.lcpMs === null || row.cls === null; + .map((summary) => { + const missingMetric = summary.lcpMs === null || summary.cls === null; return { - ...row, - reason: missingMetric ? "report has no LCP or CLS number" : "outside the threshold", + ...summary, + run: summary.cell, + reason: missingMetric + ? "no usable LCP or CLS across its samples" + : `median outside the threshold (LCP ${formatRange(summary.lcpMin, summary.lcpMax, 0)}ms, ` + + `CLS ${formatRange(summary.clsMin, summary.clsMax, 3)})`, missingReport: false, missingMetric, }; }); + // With no expected list to check against, zero mobile reports is still not a // pass — never let an all-desktop directory read as "every mobile route ok". if (expected.length === 0 && present.length === 0) { @@ -254,19 +419,24 @@ export function mobileBreaches(rows, routes) { return [...missing, ...failed]; } -export function renderTable(rows, routes) { +export function renderTable(rows, routes, samples = WEB_VITALS_SAMPLES) { const format = (value, digits = 0) => (value === null ? "n/a" : value.toFixed(digits)); + // One line per CELL, not per report: the graded median with the spread it came + // from, so a reader can see at a glance whether the number is stable. A table + // of raw samples would bury that; the samples are all in the artifact. + const cells = aggregateCells(rows, samples); const lines = [ - "| run | perf | LCP ms | CLS | TBT ms | FCP ms |", + "| run | samples | LCP ms (median) | LCP range | CLS (median) | CLS range |", "| --- | ---: | ---: | ---: | ---: | ---: |", - ...rows.map( - (row) => - `| ${row.run} | ${format((row.performanceScore ?? 0) * 100)} | ${format(row.lcpMs)} | ` + - `${format(row.cls, 3)} | ${format(row.tbtMs)} | ${format(row.fcpMs)} |`, + ...[...cells.values()].map( + (summary) => + `| ${summary.cell} | ${summary.usable.length}/${summary.expectedSamples} | ` + + `${format(summary.lcpMs)} | ${formatRange(summary.lcpMin, summary.lcpMax, 0)} | ` + + `${format(summary.cls, 3)} | ${formatRange(summary.clsMin, summary.clsMax, 3)} |`, ), ]; - const breaches = mobileBreaches(rows, routes); - const incomplete = incompleteEvidence(rows, routes); + const breaches = mobileBreaches(rows, routes, samples); + const incomplete = incompleteEvidence(rows, routes, samples); const productionVerdict = isProductionVerdict(rows); // Disqualifiers are resolved BEFORE any threshold or actionability prose, and @@ -287,6 +457,21 @@ export function renderTable(rows, routes) { `${measuredOrigins(rows).join(", ") || "an unknown origin"}`, ); } + // Same reasoning as the two above, and it belongs in the same list rather + // than only in main()'s exit: a mid-run browser change was already fatal to + // the JOB, but the step summary still carried the threshold sentence, and the + // summary is what outlives the run. A disqualified run must not print a + // verdict-shaped claim anywhere. + const builds = chromeBuilds(rows); + if (builds.length > 1) { + disqualifiers.push(`reports came from ${builds.length} different Chrome builds: ${builds.join(" | ")}`); + } + // Same list, same reason: a run measured with too few samples per cell is not + // a verdict, and the step summary is what outlives the run. main() also exits + // non-zero, but a caller that only renders the table must not be handed + // verdict prose built on one sample per cell. + const refusal = samplesRefusal(samples); + if (refusal) disqualifiers.push(refusal); lines.push(""); if (disqualifiers.length > 0) { @@ -333,6 +518,18 @@ function main() { // The routes the workflow was asked to measure, so a route whose Lighthouse // run failed is reported as a breach rather than silently omitted. const routes = process.argv[3] ?? process.env.ROUTES ?? ""; + // How many samples each cell was asked for, so a dispatch that raised the + // count is graded against what it actually requested rather than the default. + const samples = Number.parseInt(process.argv[4] ?? process.env.SAMPLES ?? "", 10) || WEB_VITALS_SAMPLES; + // Refuse BEFORE reading a single report. `samples` is a free-text dispatch + // input, so `samples=1` reaches here as a valid integer and would otherwise + // grade a one-run-per-cell matrix — the exact thing this change removes — + // while emitting output indistinguishable from a sound run. + const refusal = samplesRefusal(samples); + if (refusal) { + console.log(`::error::${refusal}`); + process.exit(1); + } const files = readdirSync(directory) .filter((file) => file.endsWith(".json") && file !== "summary.json") .sort(); @@ -353,17 +550,28 @@ function main() { // records what actually measured, and disagreement across reports is itself // worth seeing. Compare baselines only when both fields match. const lighthouseVersion = process.env.LIGHTHOUSE_VERSION ?? "unpinned"; - const chromeVersions = [...new Set(rows.map((row) => row.chromeVersion).filter(Boolean))].sort(); + const chromeVersions = chromeBuilds(rows); writeFileSync( join(directory, "summary.json"), - `${JSON.stringify({ lighthouseVersion, chromeVersions, routes, rows }, null, 2)}\n`, + `${JSON.stringify({ lighthouseVersion, samples, chromeVersions, routes, rows }, null, 2)}\n`, ); - const table = renderTable(rows, routes); + const table = renderTable(rows, routes, samples); console.log(table); if (process.env.GITHUB_STEP_SUMMARY) { writeFileSync(process.env.GITHUB_STEP_SUMMARY, `## Live Web Vitals\n\n${table}\n`, { flag: "a" }); } - const incomplete = incompleteEvidence(rows, routes); + // A runner image that rolled mid-dispatch means half the matrix was measured + // by a different browser, so the cells are not comparable even to each other. + // Recording the versions makes a cross-RUN comparison checkable at read time; + // this makes a within-run split fatal, which is the part that can be enforced. + if (chromeVersions.length > 1) { + console.log( + `::error::reports came from ${chromeVersions.length} different Chrome builds — ` + + `the cells are not comparable: ${chromeVersions.join(" | ")}`, + ); + process.exit(1); + } + const incomplete = incompleteEvidence(rows, routes, samples); if (incomplete.length > 0) { console.log( `::error::${incomplete.length} requested run(s) produced no usable report — evidence is incomplete: ` + diff --git a/tests/summarise-web-vitals.test.ts b/tests/summarise-web-vitals.test.ts index 830767dd63..a661255fe5 100644 --- a/tests/summarise-web-vitals.test.ts +++ b/tests/summarise-web-vitals.test.ts @@ -1,15 +1,24 @@ import { describe, expect, it } from "vitest"; import { + WEB_VITALS_MIN_SAMPLES, + WEB_VITALS_SAMPLES, WEB_VITALS_THRESHOLDS, - expectedMobileRuns, + samplesRefusal, + aggregateCells, + cellOf, + chromeBuilds, + expectedCells, + expectedMobileCells, expectedRuns, collidingRouteSlugs, incompleteEvidence, isProductionVerdict, measuredRequestedPage, + median, missingRuns, mobileBreaches, + straddlesThreshold, renderTable, routeSlug, summariseReport, @@ -20,6 +29,21 @@ import { // `page.tsx`, so measuring it would have profiled the 404 document. const DEFAULT_ROUTES = "/,/therapy-compass,/documents/search,/dsm,/forms"; +/** All `WEB_VITALS_SAMPLES` reports for one cell, every sample identical. */ +function cell(name: string, lcpMs: number | null, cls: number | null) { + return Array.from({ length: WEB_VITALS_SAMPLES }, (_, index) => row(`${name}-${index + 1}`, lcpMs, cls)); +} + +/** The report names a cell expands to: "mobile-dsm" -> mobile-dsm-1..3. */ +function samplesOf(...names: string[]) { + return names.flatMap((name) => Array.from({ length: WEB_VITALS_SAMPLES }, (_, i) => `${name}-${i + 1}`)); +} + +/** One cell's reports with an explicit value per sample, for spread cases. */ +function samples(name: string, lcps: (number | null)[], clsValues?: (number | null)[]) { + return lcps.map((lcpMs, index) => row(`${name}-${index + 1}`, lcpMs, clsValues?.[index] ?? 0.01)); +} + function row(run: string, lcpMs: number | null, cls: number | null) { const url = `https://psychiatry.tools/${run}`; return { @@ -46,9 +70,9 @@ describe("routeSlug", () => { }); }); -describe("expectedMobileRuns", () => { +describe("expectedMobileCells", () => { it("derives one mobile run per requested route", () => { - expect(expectedMobileRuns(DEFAULT_ROUTES)).toEqual([ + expect(expectedMobileCells(DEFAULT_ROUTES)).toEqual([ "mobile-root", "mobile-therapy-compass", "mobile-documents-search", @@ -60,21 +84,21 @@ describe("expectedMobileRuns", () => { describe("mobileBreaches", () => { it("passes when every requested mobile route reports inside the thresholds", () => { - const rows = expectedMobileRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + const rows = expectedMobileCells(DEFAULT_ROUTES).flatMap((name) => cell(name, 1200, 0.01)); expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); }); it("treats a missing metric as a breach", () => { - const rows = expectedMobileRuns(DEFAULT_ROUTES).map((run) => - run === "mobile-dsm" ? row(run, null, 0.01) : row(run, 1200, 0.01), + const rows = expectedMobileCells(DEFAULT_ROUTES).map((run) => + cellOf(run) === "mobile-dsm" ? row(run, null, 0.01) : row(run, 1200, 0.01), ); const breaches = mobileBreaches(rows, DEFAULT_ROUTES); expect(breaches.map((breach) => breach.run)).toEqual(["mobile-dsm"]); }); it("treats an over-threshold metric as a breach", () => { - const rows = expectedMobileRuns(DEFAULT_ROUTES).map((run) => - run === "mobile-forms" ? row(run, WEB_VITALS_THRESHOLDS.lcpMs + 1, 0.01) : row(run, 1200, 0.01), + const rows = expectedMobileCells(DEFAULT_ROUTES).map((run) => + cellOf(run) === "mobile-forms" ? row(run, WEB_VITALS_THRESHOLDS.lcpMs + 1, 0.01) : row(run, 1200, 0.01), ); expect(mobileBreaches(rows, DEFAULT_ROUTES).map((breach) => breach.run)).toEqual(["mobile-forms"]); }); @@ -104,34 +128,30 @@ describe("mobileBreaches", () => { // exited 0 before this: a complete mobile sweep with no desktop reports at all, // and a mobile report that exists but carries no LCP/CLS number. describe("incompleteEvidence", () => { - const allRuns = () => expectedRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + const allRuns = () => expectedCells(DEFAULT_ROUTES).flatMap((name) => cell(name, 1200, 0.01)); it("is empty when every requested mobile and desktop run reported", () => { expect(incompleteEvidence(allRuns(), DEFAULT_ROUTES)).toEqual([]); }); it("covers both strategies, so a missing desktop report is incomplete evidence", () => { - const mobileOnly = expectedMobileRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + const mobileOnly = expectedMobileCells(DEFAULT_ROUTES).flatMap((name) => cell(name, 1200, 0.01)); expect(mobileBreaches(mobileOnly, DEFAULT_ROUTES)).toEqual([]); // thresholds all pass - expect(missingRuns(mobileOnly, DEFAULT_ROUTES)).toEqual([ - "desktop-root", - "desktop-therapy-compass", - "desktop-documents-search", - "desktop-dsm", - "desktop-forms", - ]); - expect(incompleteEvidence(mobileOnly, DEFAULT_ROUTES)).toHaveLength(5); + expect(missingRuns(mobileOnly, DEFAULT_ROUTES)).toEqual( + samplesOf("desktop-root", "desktop-therapy-compass", "desktop-documents-search", "desktop-dsm", "desktop-forms"), + ); + expect(incompleteEvidence(mobileOnly, DEFAULT_ROUTES)).toHaveLength(5 * WEB_VITALS_SAMPLES); expect(renderTable(mobileOnly, DEFAULT_ROUTES)).toContain("NOT an #017 verdict"); }); it("treats a present report with no LCP/CLS number as incomplete, not merely a breach", () => { - const rows = allRuns().map((r) => (r.run === "mobile-dsm" ? { ...r, lcpMs: null } : r)); - expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-dsm"]); + const rows = allRuns().map((r) => (cellOf(r.run) === "mobile-dsm" ? { ...r, lcpMs: null } : r)); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(samplesOf("mobile-dsm")); }); it("does not treat an over-threshold measurement as incomplete evidence", () => { const rows = allRuns().map((r) => - r.run === "mobile-forms" ? { ...r, lcpMs: WEB_VITALS_THRESHOLDS.lcpMs + 1 } : r, + cellOf(r.run) === "mobile-forms" ? { ...r, lcpMs: WEB_VITALS_THRESHOLDS.lcpMs + 1 } : r, ); expect(mobileBreaches(rows, DEFAULT_ROUTES).map((b) => b.run)).toEqual(["mobile-forms"]); expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); @@ -140,10 +160,10 @@ describe("incompleteEvidence", () => { // Completeness is a property of the whole matrix. Checking metric validity // only on mobile left a desktop report with null metrics reading as evidence. it("rejects a desktop report that exists but carries no LCP/CLS number", () => { - const rows = allRuns().map((r) => (r.run === "desktop-forms" ? { ...r, cls: null } : r)); + const rows = allRuns().map((r) => (cellOf(r.run) === "desktop-forms" ? { ...r, cls: null } : r)); expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); // mobile verdict is clean expect(missingRuns(rows, DEFAULT_ROUTES)).toEqual([]); // the file is present - expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["desktop-forms"]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(samplesOf("desktop-forms")); }); // `/a/b` and `/a-b` both slug to `a-b`, so the second Lighthouse run @@ -152,7 +172,7 @@ describe("incompleteEvidence", () => { it("rejects colliding route slugs, which silently drop a requested route", () => { const colliding = "/a/b,/a-b"; expect(collidingRouteSlugs(colliding)).toEqual(["a-b"]); - const rows = [row("mobile-a-b", 1200, 0.01), row("desktop-a-b", 1200, 0.01)]; + const rows = [...cell("mobile-a-b", 1200, 0.01), ...cell("desktop-a-b", 1200, 0.01)]; expect(missingRuns(rows, colliding)).toEqual([]); // every expected key is "present" expect(incompleteEvidence(rows, colliding)).toContain("route slug collision: a-b"); }); @@ -166,15 +186,17 @@ describe("incompleteEvidence", () => { // route. The numbers are clean but they describe a different page, and the // filename cannot reveal it. it("rejects a run that redirected to a different same-origin path", () => { - const rows = allRuns().map((r) => (r.run === "mobile-dsm" ? { ...r, url: "https://psychiatry.tools/login" } : r)); + const rows = allRuns().map((r) => + cellOf(r.run) === "mobile-dsm" ? { ...r, url: "https://psychiatry.tools/login" } : r, + ); expect(isProductionVerdict(rows)).toBe(true); // origin is still canonical - expect(measuredRequestedPage(rows.find((r) => r.run === "mobile-dsm"))).toBe(false); - expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-dsm"]); + expect(measuredRequestedPage(rows.find((r) => cellOf(r.run) === "mobile-dsm"))).toBe(false); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(samplesOf("mobile-dsm")); }); it("rejects a run that reported a Lighthouse runtime error", () => { - const rows = allRuns().map((r) => (r.run === "desktop-forms" ? { ...r, runtimeError: "NO_FCP" } : r)); - expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["desktop-forms"]); + const rows = allRuns().map((r) => (cellOf(r.run) === "desktop-forms" ? { ...r, runtimeError: "NO_FCP" } : r)); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(samplesOf("desktop-forms")); }); // /documents/search?q=depression and /documents/search are different pages to @@ -182,16 +204,16 @@ describe("incompleteEvidence", () => { it("rejects a redirect that drops the query string", () => { const requested = "https://psychiatry.tools/documents/search?q=depression"; const rows = allRuns().map((r) => - r.run === "mobile-documents-search" + cellOf(r.run) === "mobile-documents-search" ? { ...r, requestedUrl: requested, url: "https://psychiatry.tools/documents/search" } : r, ); - expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-documents-search"]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(samplesOf("mobile-documents-search")); }); it("accepts the same query with params in a different order", () => { const rows = allRuns().map((r) => - r.run === "mobile-dsm" + cellOf(r.run) === "mobile-dsm" ? { ...r, requestedUrl: "https://psychiatry.tools/dsm?a=1&b=2", @@ -216,19 +238,19 @@ describe("incompleteEvidence", () => { }), ).toBe(false); const rows = allRuns().map((r) => - r.run === "mobile-forms" ? { ...r, requestedUrl: requested, url: rewritten } : r, + cellOf(r.run) === "mobile-forms" ? { ...r, requestedUrl: requested, url: rewritten } : r, ); - expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-forms"]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(samplesOf("mobile-forms")); }); it("accepts the same encoded query value on both requested and final URLs", () => { const url = "https://psychiatry.tools/forms?q=alpha%26run%3D1"; - const rows = allRuns().map((r) => (r.run === "mobile-forms" ? { ...r, requestedUrl: url, url } : r)); + const rows = allRuns().map((r) => (cellOf(r.run) === "mobile-forms" ? { ...r, requestedUrl: url, url } : r)); expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); }); it("tolerates a trailing-slash difference between requested and final URL", () => { - const rows = allRuns().map((r) => (r.run === "mobile-forms" ? { ...r, url: `${r.requestedUrl}/` } : r)); + const rows = allRuns().map((r) => (cellOf(r.run) === "mobile-forms" ? { ...r, url: `${r.requestedUrl}/` } : r)); expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); }); }); @@ -249,7 +271,7 @@ describe("production-origin gate", () => { }); it("states the #017 closure only for the canonical production origin", () => { - const rows = expectedRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + const rows = expectedCells(DEFAULT_ROUTES).flatMap((name) => cell(name, 1200, 0.01)); expect(isProductionVerdict(rows)).toBe(true); expect(renderTable(rows, DEFAULT_ROUTES)).toContain("NOT yet an #017 closure"); }); @@ -266,7 +288,7 @@ describe("production-origin gate", () => { }); it("refuses a verdict when only some reports left the canonical origin", () => { - const rows = expectedRuns(DEFAULT_ROUTES).map((run) => row(run, 1200, 0.01)); + const rows = expectedCells(DEFAULT_ROUTES).flatMap((name) => cell(name, 1200, 0.01)); expect(isProductionVerdict([...rows.slice(1), staging("mobile-root")])).toBe(false); }); }); @@ -301,11 +323,13 @@ describe("verdict prose is gated on the run being able to produce a verdict", () it("suppresses the threshold claim when a run measured the wrong page", () => { const rows = expectedRuns(DEFAULT_ROUTES).map((run) => - run === "mobile-dsm" ? { ...row(run, 1200, 0.01), url: "https://psychiatry.tools/login" } : row(run, 1200, 0.01), + cellOf(run) === "mobile-dsm" + ? { ...row(run, 1200, 0.01), url: "https://psychiatry.tools/login" } + : row(run, 1200, 0.01), ); // The redirected run still carries passing numbers, so nothing breaches. expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); - expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-dsm"]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(samplesOf("mobile-dsm")); const table = renderTable(rows, DEFAULT_ROUTES); expect(table).toContain("NOT an #017 verdict"); @@ -318,7 +342,7 @@ describe("verdict prose is gated on the run being able to produce a verdict", () }); it("does not call a staging breach actionable", () => { - const rows = expectedRuns(DEFAULT_ROUTES).map((run) => staging(run, run === "mobile-forms" ? 3000 : 1200)); + const rows = expectedRuns(DEFAULT_ROUTES).map((run) => staging(run, cellOf(run) === "mobile-forms" ? 3000 : 1200)); expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); // evidence is complete… expect(isProductionVerdict(rows)).toBe(false); // …but not from production expect(mobileBreaches(rows, DEFAULT_ROUTES)).toHaveLength(1); @@ -361,3 +385,198 @@ describe("measurement environment", () => { expect(summariseReport("mobile-root", { audits: {} }).chromeVersion).toBeNull(); }); }); + +// Ledger #114. A single Lighthouse run cannot measure dispersion, so it cannot +// detect the "evidence is too noisy" condition #017 tells the operator to stop +// on — and against a hard threshold, a route near the line resolves to a pass +// or a breach on run-to-run variance alone, invisibly. Each cell is now +// sampled, the MEDIAN is graded, and a spread that crosses the line is refused. +describe("sampling", () => { + it("expects every sample of every route/strategy cell", () => { + expect(WEB_VITALS_SAMPLES).toBeGreaterThanOrEqual(3); + expect(expectedCells(DEFAULT_ROUTES)).toHaveLength(10); + expect(expectedRuns(DEFAULT_ROUTES)).toHaveLength(10 * WEB_VITALS_SAMPLES); + expect(expectedRuns(DEFAULT_ROUTES).slice(0, 3)).toEqual(["mobile-root-1", "mobile-root-2", "mobile-root-3"]); + expect(cellOf("mobile-documents-search-2")).toBe("mobile-documents-search"); + // A slug ending in a digit must not lose its last character to the sample + // suffix: only the trailing "-" the workflow appends is stripped. + expect(cellOf("mobile-covid-19-3")).toBe("mobile-covid-19"); + }); + + it("takes the median, so one bad sample cannot decide a cell", () => { + expect(median([1200, 9000, 1300])).toBe(1300); + expect(median([1000, 2000])).toBe(1500); + expect(median([])).toBeNull(); + + // 9000ms in one of three samples is a cold cache, not a regression. + const rows = expectedCells(DEFAULT_ROUTES).flatMap((name) => + name === "mobile-dsm" ? samples(name, [1200, 9000, 1300]) : cell(name, 1200, 0.01), + ); + expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); + expect(aggregateCells(rows).get("mobile-dsm")?.lcpMs).toBe(1300); + }); + + it("refuses to grade a cell whose samples straddle the threshold", () => { + // 2400 / 2600 around a 2500ms line: the median says pass, but a rerun would + // say breach. That is measured noise, not a verdict. + const rows = expectedCells(DEFAULT_ROUTES).flatMap((name) => + name === "mobile-forms" ? samples(name, [2400, 2600, 2450]) : cell(name, 1200, 0.01), + ); + const summary = aggregateCells(rows).get("mobile-forms"); + expect(summary?.lcpMs).toBe(2450); // median is under the line… + expect(straddlesThreshold(summary)).toBe(true); // …but the spread crosses it + + // So it is incomplete evidence, NOT a pass and NOT a breach. + expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); + const incomplete = incompleteEvidence(rows, DEFAULT_ROUTES); + expect(incomplete).toHaveLength(1); + expect(incomplete[0]).toContain("mobile-forms"); + expect(incomplete[0]).toContain("too noisy to grade"); + + const table = renderTable(rows, DEFAULT_ROUTES); + expect(table).toContain("NOT an #017 verdict"); + expect(table).not.toContain("Every mobile route is within"); + }); + + it("catches a straddle on CLS as well as LCP", () => { + const rows = expectedCells(DEFAULT_ROUTES).flatMap((name) => + name === "mobile-root" ? samples(name, [1200, 1200, 1200], [0.09, 0.11, 0.095]) : cell(name, 1200, 0.01), + ); + expect(straddlesThreshold(aggregateCells(rows).get("mobile-root"))).toBe(true); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)[0]).toContain("too noisy to grade"); + }); + + it("does not call a consistent result noisy", () => { + // Spread well clear of the line on one side is a stable measurement, and a + // guard that fired here would make every run unverdictable. + const rows = expectedCells(DEFAULT_ROUTES).flatMap((name) => + name === "mobile-dsm" ? samples(name, [1100, 1400, 1250]) : cell(name, 1200, 0.01), + ); + expect(straddlesThreshold(aggregateCells(rows).get("mobile-dsm"))).toBe(false); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); + expect(renderTable(rows, DEFAULT_ROUTES)).toContain("NOT yet an #017 closure"); + }); + + it("still breaches when the whole spread is over the line", () => { + // Consistently slow is a real, reportable verdict — not noise. + const rows = expectedCells(DEFAULT_ROUTES).flatMap((name) => + name === "mobile-forms" ? samples(name, [3000, 3200, 3100]) : cell(name, 1200, 0.01), + ); + expect(straddlesThreshold(aggregateCells(rows).get("mobile-forms"))).toBe(false); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); + const breaches = mobileBreaches(rows, DEFAULT_ROUTES); + expect(breaches.map((breach) => breach.run)).toEqual(["mobile-forms"]); + expect(breaches[0]?.reason).toContain("3000"); + }); + + it("treats a missing sample as incomplete even when the others are fine", () => { + const rows = expectedRuns(DEFAULT_ROUTES) + .filter((run) => run !== "mobile-dsm-2") + .map((run) => row(run, 1200, 0.01)); + expect(missingRuns(rows, DEFAULT_ROUTES)).toEqual(["mobile-dsm-2"]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual(["mobile-dsm-2"]); + }); + + it("shows the median and the spread it came from", () => { + const rows = expectedCells(DEFAULT_ROUTES).flatMap((name) => + name === "mobile-dsm" ? samples(name, [1100, 1400, 1250]) : cell(name, 1200, 0.01), + ); + const table = renderTable(rows, DEFAULT_ROUTES); + // One row per cell with its sample count, not one row per report. + expect(table).toContain("| mobile-dsm | 3/3 |"); + expect(table).toContain("1250"); + expect(table).toContain("1100–1400"); + }); +}); + +// Chrome ships with the runner image, not with the pinned LIGHTHOUSE_VERSION. +// Two builds in one dispatch means half the matrix was measured by a different +// browser, so the cells are not comparable even to each other. +describe("Chrome build drift", () => { + const withBuild = (rows: ReturnType[], build: string) => + rows.map((entry) => ({ ...entry, chromeVersion: build })); + + it("disqualifies the verdict when the runner changed browser mid-run", () => { + const clean = expectedCells(DEFAULT_ROUTES).flatMap((name) => cell(name, 1200, 0.01)); + const rows = withBuild(clean, "Chrome/141").map((entry) => + entry.run === "mobile-dsm-3" ? { ...entry, chromeVersion: "Chrome/142" } : entry, + ); + expect(chromeBuilds(rows)).toEqual(["Chrome/141", "Chrome/142"]); + + // Every threshold passes and the evidence is complete — only the browser + // split makes this unverdictable, so this isolates that gate. + expect(mobileBreaches(rows, DEFAULT_ROUTES)).toEqual([]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES)).toEqual([]); + + const table = renderTable(rows, DEFAULT_ROUTES); + expect(table).toContain("NOT an #017 verdict"); + expect(table).toContain("2 different Chrome builds"); + // The regression this guards: the job already failed on a split build, but + // the step summary still carried the threshold sentence, and the summary is + // what outlives the run. + expect(table).not.toContain("Every mobile route is within"); + }); + + it("says nothing about the browser when every report used the same one", () => { + const rows = withBuild( + expectedCells(DEFAULT_ROUTES).flatMap((name) => cell(name, 1200, 0.01)), + "Chrome/141", + ); + expect(chromeBuilds(rows)).toEqual(["Chrome/141"]); + expect(renderTable(rows, DEFAULT_ROUTES)).toContain("NOT yet an #017 closure"); + }); +}); + +describe("a sample count too small to grade", () => { + // The default is only a default. `samples` is a free-text `workflow_dispatch` + // input, so an operator can dispatch `samples=1` and get back exactly the + // single-run matrix ledger #114 exists to abolish — and, without this guard, + // output indistinguishable from a sound run. + it("refuses one sample per cell", () => { + expect(samplesRefusal(1)).toContain("cannot be graded"); + expect(samplesRefusal(1)).toContain(`at least ${WEB_VITALS_MIN_SAMPLES} runs per cell`); + }); + + it("refuses two, where the median is just the mean of the pair", () => { + expect(samplesRefusal(2)).toContain("cannot be graded"); + }); + + it("accepts the documented default and anything above it", () => { + expect(samplesRefusal(WEB_VITALS_SAMPLES)).toBeNull(); + expect(samplesRefusal(WEB_VITALS_MIN_SAMPLES)).toBeNull(); + expect(samplesRefusal(9)).toBeNull(); + }); + + it("refuses a count that is not a positive whole number", () => { + for (const value of [0, -3, 2.5, Number.NaN]) { + expect(samplesRefusal(value)).toContain("cannot be graded"); + } + }); + + // The decisive case. With one sample per cell every threshold passes, the + // evidence is "complete" for the count requested, and the spread is zero — so + // nothing else in the pipeline objects. Only this gate stands between that + // matrix and a verdict sentence a reader could record against #017. + it("emits no verdict prose for a one-sample-per-cell run that otherwise looks clean", () => { + const rows = expectedCells(DEFAULT_ROUTES).map((name) => row(`${name}-1`, 1200, 0.01)); + + expect(mobileBreaches(rows, DEFAULT_ROUTES, 1)).toEqual([]); + expect(incompleteEvidence(rows, DEFAULT_ROUTES, 1)).toEqual([]); + // The heart of it: with one sample the cell's range has zero width, so the + // straddle check — the whole mechanism #114 added — cannot ever fire. + expect(straddlesThreshold(aggregateCells(rows, 1).get("mobile-root"))).toBe(false); + + const table = renderTable(rows, DEFAULT_ROUTES, 1); + expect(table).toContain("NOT an #017 verdict"); + expect(table).toContain("at least 3 runs per cell"); + expect(table).not.toContain("Every mobile route is within"); + expect(table).not.toContain("NOT yet an #017 closure"); + }); + + it("still renders the measurements, explicitly subordinated", () => { + const rows = expectedCells(DEFAULT_ROUTES).map((name) => row(`${name}-1`, 1200, 0.01)); + const table = renderTable(rows, DEFAULT_ROUTES, 1); + expect(table).toContain("mobile-dsm"); + expect(table).toContain("This is a measurement, not a verdict"); + }); +});