test(ci): enforce single-browser lighthouse baseline refresh and unif…

docs/testing.md

@@ -214,8 +214,9 @@ indistinguishable from an application regression. Both Lighthouse jobs therefore
 managed Chromium through the shared `./.github/actions/setup-lighthouse-chromium` composite action —
 never the ambient runner-image Chrome, which is not pinned per commit (the fleet was observed serving
 HeadlessChrome/150 and /151 to jobs minutes apart on 2026-08-07). Drift is reported as one collapsed
-instruction rather than one sentence per route, but the verdict is unchanged: incomplete evidence
-still fails, independently of `enforce`.
+instruction when uniform across all expected runs, but mixed browser versions, partial legacy rows,
+or incomplete evidence retain per-run diagnostics so distinct facts are not obscured. The verdict is
+unchanged: incomplete evidence still fails, independently of `enforce`.
 
 Because the numbers must come from that pinned browser, refresh the baseline **from a CI runner**,
 never a developer machine:

scripts/check-lighthouse-budget.mjs

@@ -92,7 +92,9 @@ export function incompleteBudgetEvidence(rows, budget, { ignoreBaseline = false
   // below. The verdict is identical either way; only the message changes.
   const drift = new Map();
 
-  for (const run of expectedBudgetRuns(budget)) {
+  const expectedRuns = expectedBudgetRuns(budget);
+
+  for (const run of expectedRuns) {
     const row = byRun.get(run);
     if (!row) {
       problems.add(`${run}: no Lighthouse report produced`);
@@ -139,7 +141,7 @@ export function incompleteBudgetEvidence(rows, budget, { ignoreBaseline = false
   // `compareToLighthouseBudget` still returns `fail` on a non-empty result,
   // independently of `enforce`.
   const driftPairs = new Set([...drift.values()].map(({ before, after }) => JSON.stringify([before, after])));
-  if (drift.size > 0 && problems.size === 0 && driftPairs.size === 1) {
+  if (drift.size > 0 && drift.size === expectedRuns.length && problems.size === 0 && driftPairs.size === 1) {
     const [{ before, after }] = drift.values();
     return [
       `browser drift on ${drift.size} run(s): the baseline was measured by ${before}, this run used ${after} — ` +
@@ -334,8 +336,104 @@ export function readReports(directory) {
     );
 }
 
+/**
+ * Validate that a baseline object contains exactly one distinct browser version
+ * across all its recorded rows.
+ */
+export function validateBaselineBrowserVersions(baseline) {
+  const rows = Object.values(baseline ?? {});
+  if (rows.length === 0) return { ok: false, versions: [], error: "no baseline rows recorded" };
+  const versions = [
+    ...new Set(rows.map((row) => row.chromeVersion).filter((v) => typeof v === "string" && v.length > 0)),
+  ];
+  if (rows.some((row) => typeof row.chromeVersion !== "string" || !row.chromeVersion)) {
+    return {
+      ok: false,
+      versions,
+      error: `some rows are missing a recorded browser version; found ${versions.length} version(s)`,
+    };
+  }
+  if (versions.length !== 1) {
+    return {
+      ok: false,
+      versions,
+      error: `expected exactly one baseline Chrome version across all rows; found ${versions.length}`,
+    };
+  }
+  return { ok: true, versions, error: null };
+}
+
+export function selfTest() {
+  const sampleBudget = {
+    routes: ["/", "/therapy-compass"],
+    strategies: ["mobile", "desktop"],
+    baseline: null,
+  };
+  const runs = expectedBudgetRuns(sampleBudget);
+  if (runs.length !== 4) throw new Error(`selfTest failed: expected 4 runs, got ${runs.length}`);
+
+  const sampleRows = runs.map((run) => ({
+    run,
+    url: `http://localhost:4461/${run}`,
+    requestedUrl: `http://localhost:4461/${run}`,
+    runtimeError: null,
+    performanceScore: 0.99,
+    lcpMs: 1000,
+    cls: 0,
+    tbtMs: 100,
+    fcpMs: 500,
+    chromeVersion: "HeadlessChrome/140",
+  }));
+
+  const cleanProblems = incompleteBudgetEvidence(sampleRows, sampleBudget);
+  if (cleanProblems.length !== 0) {
+    throw new Error(`selfTest failed: clean matrix produced problems: ${cleanProblems.join(", ")}`);
+  }
+
+  const staleBaseline = baselineFromRows(sampleRows.map((r) => ({ ...r, chromeVersion: "HeadlessChrome/131" })));
+  const uniformDrift = incompleteBudgetEvidence(sampleRows, { ...sampleBudget, baseline: staleBaseline });
+  if (uniformDrift.length !== 1 || !uniformDrift[0].includes("browser drift on 4 run(s)")) {
+    throw new Error(`selfTest failed: uniform drift not collapsed to 1 message: ${uniformDrift.join(", ")}`);
+  }
+
+  const mixedBaseline = {
+    ...staleBaseline,
+    [runs[0]]: { ...staleBaseline[runs[0]], chromeVersion: null },
+  };
+  const nonUniformDrift = incompleteBudgetEvidence(sampleRows, { ...sampleBudget, baseline: mixedBaseline });
+  if (nonUniformDrift.length !== 3) {
+    throw new Error(
+      `selfTest failed: non-uniform drift did not retain per-run diagnostics: ${nonUniformDrift.join(", ")}`,
+    );
+  }
+
+  const validationSuccess = validateBaselineBrowserVersions(staleBaseline);
+  if (!validationSuccess.ok || validationSuccess.versions.length !== 1) {
+    throw new Error("selfTest failed: valid baseline rejected by validateBaselineBrowserVersions");
+  }
+  const validationFailure = validateBaselineBrowserVersions(mixedBaseline);
+  if (validationFailure.ok) {
+    throw new Error("selfTest failed: mixed baseline accepted by validateBaselineBrowserVersions");
+  }
+
+  const gradeResult = gradeRun(sampleRows[0], { lcpMs: 500, cls: 0, tbtMs: 100 });
+  if (gradeResult.length === 0) throw new Error("selfTest failed: gradeRun did not flag regression");
+
+  const comparison = compareToLighthouseBudget(sampleRows, { ...sampleBudget, baseline: staleBaseline });
+  if (comparison.status !== "fail") throw new Error("selfTest failed: drifted baseline did not fail comparison");
+
+  const majorityDecision = majorityBreachDecision([true, true, false]);
+  if (!majorityDecision?.breached) throw new Error("selfTest failed: majority breach decision incorrect");
+
+  console.log("check:lighthouse-budget self-test passed.");
+}
+
 function main() {
   const argv = process.argv.slice(2);
+  if (argv.includes("--self-test")) {
+    selfTest();
+    return;
+  }
   const update = argv.includes("--update");
   const requireReports = argv.includes("--require-reports");
   const asJson = argv.includes("--json");
@@ -377,9 +475,15 @@ function main() {
       console.error(`::error::refusing to update the baseline from incomplete evidence: ${measurementGaps.join("; ")}`);
       process.exit(1);
     }
+    const nextBaseline = baselineFromRows(rows);
+    const validation = validateBaselineBrowserVersions(nextBaseline);
+    if (!validation.ok) {
+      console.error(`::error::refusing to update the baseline: ${validation.error}.`);
+      process.exit(1);
+    }
     const next = {
       ...budget,
-      baseline: baselineFromRows(rows),
+      baseline: nextBaseline,
       updatedAt: new Date().toISOString(),
     };
     writeFileSync(BUDGET_PATH, `${JSON.stringify(next, null, 2)}\n`);

tests/check-lighthouse-budget.test.ts

@@ -15,6 +15,7 @@ import {
   numericBreachConfirmationRuns,
   readReports,
   renderBudgetTable,
+  validateBaselineBrowserVersions,
 } from "../scripts/check-lighthouse-budget.mjs";
 import { measurementFailureReason } from "../scripts/lighthouse-measurement-outcome.mjs";
 import { deadlineAfter, processTimeoutMs, remainingMs } from "../scripts/lighthouse-time-budget.mjs";
@@ -232,6 +233,23 @@ describe("incompleteBudgetEvidence — completeness derived from what is graded"
     expect(incompleteBudgetEvidence(rows, budget({ baseline: legacy }))).toEqual([]);
   });
 
+  it("keeps drift per run when some baseline rows lack a recorded browser version", () => {
+    // When older baselines contain a mix of versioned and unversioned rows,
+    // drift is not uniform across all expected runs and must list per run.
+    const rows = completeRows();
+    const mixed = Object.fromEntries(
+      Object.entries(baselineFromRows(rows)).map(([run, entry], index) => [
+        run,
+        { ...(entry as object), chromeVersion: index === 0 ? null : "HeadlessChrome/131" },
+      ]),
+    );
+    const problems = incompleteBudgetEvidence(rows, budget({ baseline: mixed }));
+
+    expect(problems).toHaveLength(9);
+    expect(problems.every((problem: string) => problem.includes("measured by a different browser"))).toBe(true);
+    expect(problems[0]).not.toContain("browser drift on");
+  });
+
   it("rejects colliding route slugs before anything is measured", () => {
     // `/a/b` and `/a-b` both write `a-b.json`, so the second overwrites the first and
     // the survivor would satisfy the expected-run check for both pages.
@@ -593,3 +611,49 @@ describe("renderBudgetTable", () => {
     expect(renderBudgetTable(rows, result)).toContain("within tolerance of the committed baseline");
   });
 });
+
+describe("validateBaselineBrowserVersions", () => {
+  it("accepts a baseline with exactly one browser version across all rows", () => {
+    const baseline = baselineFromRows(completeRows());
+    const result = validateBaselineBrowserVersions(baseline);
+
+    expect(result.ok).toBe(true);
+    expect(result.versions).toEqual(["HeadlessChrome/140"]);
+    expect(result.error).toBeNull();
+  });
+
+  it("rejects an empty baseline", () => {
+    const result = validateBaselineBrowserVersions({});
+
+    expect(result.ok).toBe(false);
+    expect(result.error).toContain("no baseline rows recorded");
+  });
+
+  it("rejects a baseline with mixed browser versions", () => {
+    const rows = completeRows();
+    const mixed = Object.fromEntries(
+      Object.entries(baselineFromRows(rows)).map(([run, entry], index) => [
+        run,
+        { ...(entry as object), chromeVersion: index % 2 === 0 ? "HeadlessChrome/140" : "HeadlessChrome/141" },
+      ]),
+    );
+    const result = validateBaselineBrowserVersions(mixed);
+
+    expect(result.ok).toBe(false);
+    expect(result.error).toContain("expected exactly one baseline Chrome version");
+  });
+
+  it("rejects a baseline where some rows are missing a browser version", () => {
+    const rows = completeRows();
+    const partial = Object.fromEntries(
+      Object.entries(baselineFromRows(rows)).map(([run, entry], index) => [
+        run,
+        { ...(entry as object), chromeVersion: index === 0 ? null : "HeadlessChrome/140" },
+      ]),
+    );
+    const result = validateBaselineBrowserVersions(partial);
+
+    expect(result.ok).toBe(false);
+    expect(result.error).toContain("some rows are missing a recorded browser version");
+  });
+});