BigSimmo · BigSimmo · Aug 8, 2026 · Aug 8, 2026
@@ -716,12 +716,26 @@ jobs:
         with:
           persist-credentials: false
 
-      # Lighthouse drives the Chrome that ships in the ubuntu-24.04 runner image, so
-      # no browser install is needed here (matching live-web-vitals.yml).
-      - name: Setup Node and dependencies
-        uses: ./.github/actions/setup-node-cached
+      # Pin Chromium through Playwright (same install path as Production UI) so the
+      # budget is not tied to whichever HeadlessChrome the ubuntu-24.04 image ships.
+      # Runner Chrome major bumps were failing every UI PR as "evidence incomplete"
+      # even while enforce is false. live-web-vitals.yml still uses the image Chrome
+      # because it grades the deployed origin, not a committed relative baseline.
+      - name: Setup UI e2e environment
+        uses: ./.github/actions/setup-ui-e2e
+
+      - name: Resolve Playwright Chromium path
+        id: chromium
+        shell: bash
+        run: |
+          CHROME_PATH="$(node -e "const {chromium}=require('playwright'); process.stdout.write(chromium.executablePath())")"
+          test -x "$CHROME_PATH"
+          echo "path=$CHROME_PATH" >> "$GITHUB_OUTPUT"
+          echo "Using Playwright Chromium at $CHROME_PATH"
 
       - name: Measure routes and grade against the baseline
+        env:
+          CHROME_PATH: ${{ steps.chromium.outputs.path }}
         run: npm run verify:lighthouse -- --keep --dir lighthouse
 
       - name: Upload Lighthouse reports

@@ -121,7 +121,11 @@ runs; promote it by adding it to `pr-required` and dropping that flag together.
 ## Performance budget
 
 `npm run verify:lighthouse` builds and serves an isolated production app in demo mode, measures the
-routes in `lighthouse-budget.json` on mobile and desktop, and grades the result. It is a **relative**
+routes in `lighthouse-budget.json` on mobile and desktop, and grades the result. CI pins Chromium
+through Playwright (`CHROME_PATH`) so the budget is not tied to the ubuntu runner image Chrome —
+that pin is the durable fix for browser-identity drift (no per-route retry; retries only lengthen
+CI). While `enforce` is false, a residual Chrome identity drift warns and still grades; once
+`enforce` flips true, drift fails closed until the baseline is refreshed. It is a **relative**
 gate: absolute web-vitals thresholds are meaningless against a localhost server with no network
 latency, so each route is compared to a committed known-good baseline with a per-metric tolerance,
 following the same shape as `check:bundle-budget`.

@@ -1,5 +1,5 @@
 {
-  "$comment": "Pre-merge Lighthouse budget, graded against a LOCAL production build by scripts/check-lighthouse-budget.mjs. This is a relative gate: absolute web-vitals thresholds are meaningless without network latency, so it compares each route to a committed known-good baseline. Refresh intentionally with `npm run check:lighthouse-budget -- --update` after an intentional, known-good run. Distinct from .github/workflows/live-web-vitals.yml, which measures the deployed origin for ledger #017 and cannot block a merge. Starts with enforce:false and no baseline so the first CI runs report without blocking; set enforce:true once the baseline has held across a few runs.",
+  "$comment": "Pre-merge Lighthouse budget, graded against a LOCAL production build by scripts/check-lighthouse-budget.mjs. This is a relative gate: absolute web-vitals thresholds are meaningless without network latency, so it compares each route to a committed known-good baseline. Refresh intentionally with `npm run check:lighthouse-budget -- --update` after an intentional, known-good run. Distinct from .github/workflows/live-web-vitals.yml, which measures the deployed origin for ledger #017 and cannot block a merge. CI measures with Playwright Chromium (CHROME_PATH), not the ubuntu runner image Chrome. While enforce is false, a Chrome identity drift warns and still grades; once enforce is true, drift fails closed until the baseline is refreshed. set enforce:true once the baseline has held across a few runs.",
   "enforce": false,
   "$lighthouseVersion": "Pinned exactly, not `lighthouse@12`: a patch published between a baseline run and its follow-up would silently change the measurement. Kept in step with LIGHTHOUSE_VERSION in .github/workflows/live-web-vitals.yml by tests/check-lighthouse-budget.test.ts so the two Lighthouse entry points cannot drift apart.",
   "lighthouseVersion": "12.8.2",

@@ -109,19 +109,41 @@ export function incompleteBudgetEvidence(rows, budget) {
       problems.add(`${run}: no baseline row recorded — refresh with --update`);
       continue;
     }
-    // Numbers are only comparable when the same browser produced them. Chrome comes
-    // from the runner image and moves independently of the pinned Lighthouse
-    // version, so a browser bump is otherwise indistinguishable from an application
+    // Numbers are only comparable when the same browser produced them. Chrome on
+    // ubuntu-24.04 runners moves independently of the pinned Lighthouse version,
+    // so a browser bump is otherwise indistinguishable from an application
     // regression. summarise-web-vitals.mjs makes the same point about its baselines.
+    //
+    // While `enforce` is false the advisory CI job must not go red on every runner
+    // Chrome bump — that noise blocked signal on every UI PR. Soft-skip the drift
+    // check here; compareToLighthouseBudget still surfaces it as a warning so the
+    // baseline can be refreshed deliberately. Once enforce flips true, drift stays
+    // fail-closed and requires `check:lighthouse-budget -- --update`.
     if (before.chromeVersion && row.chromeVersion && before.chromeVersion !== row.chromeVersion) {
-      problems.add(
-        `${run}: baseline measured by a different browser (${before.chromeVersion} vs ${row.chromeVersion}) — refresh with --update`,
-      );
+      const drift = `${run}: baseline measured by a different browser (${before.chromeVersion} vs ${row.chromeVersion}) — refresh with --update`;
+      if (budget?.enforce) problems.add(drift);
     }
   }
   return [...problems].sort();
 }
 
+/** Browser identity mismatches that are advisory while enforce is false. */
+export function browserDriftWarnings(rows, budget) {
+  if (budget?.enforce) return [];
+  const baseline = budget?.baseline ?? null;
+  if (!baseline || Object.keys(baseline).length === 0) return [];
+  const warnings = [];
+  for (const row of rows ?? []) {
+    const before = baseline[row.run];
+    if (!before?.chromeVersion || !row.chromeVersion) continue;
+    if (before.chromeVersion === row.chromeVersion) continue;
+    warnings.push(
+      `${row.run}: baseline measured by a different browser (${before.chromeVersion} vs ${row.chromeVersion}) — refresh with --update`,
+    );
+  }
+  return warnings.sort();
+}
+
 /** Grade one run against its baseline. Returns the breaches, empty when within tolerance. */
 export function gradeRun(row, baselineRow, tolerance = DEFAULT_TOLERANCE) {
   if (!baselineRow) return [];
@@ -175,11 +197,23 @@ export function compareToLighthouseBudget(rows, budget) {
   const baseline = budget?.baseline ?? null;
   const enforce = Boolean(budget?.enforce);
   const incomplete = incompleteBudgetEvidence(rows, budget);
+  const browserDrift = browserDriftWarnings(rows, budget);
 
   // Incompleteness is fatal regardless of `enforce`: there is nothing to grade, so
-  // "warn" would report a pass for a route that was never measured.
+  // "warn" would report a pass for a route that was never measured. Missing reports
+  // and wrong-page measurements stay here; Chrome drift alone does not while the
+  // gate is advisory (see incompleteBudgetEvidence).
   if (incomplete.length > 0) {
-    return { status: "fail", reason: "evidence incomplete", breaches: [], incomplete, baseline, enforce, tolerance };
+    return {
+      status: "fail",
+      reason: "evidence incomplete",
+      breaches: [],
+      incomplete,
+      browserDrift,
+      baseline,
+      enforce,
+      tolerance,
+    };
   }
 
   if (!baseline || Object.keys(baseline).length === 0) {
@@ -188,21 +222,44 @@ export function compareToLighthouseBudget(rows, budget) {
       reason: "no baseline recorded — run with --update after a known-good build",
       breaches: [],
       incomplete,
+      browserDrift,
       baseline,
       enforce,
       tolerance,
     };
   }
 
   const breaches = rows.flatMap((row) => gradeRun(row, baseline[row.run], tolerance));
+  if (breaches.length === 0 && browserDrift.length === 0) {
+    return {
+      status: "ok",
+      reason: "within tolerance",
+      breaches,
+      incomplete,
+      browserDrift,
+      baseline,
+      enforce,
+      tolerance,
+    };
+  }
   if (breaches.length === 0) {
-    return { status: "ok", reason: "within tolerance", breaches, incomplete, baseline, enforce, tolerance };
+    return {
+      status: "warn",
+      reason: `browser drift on ${browserDrift.length} run(s) — refresh baseline with --update`,
+      breaches,
+      incomplete,
+      browserDrift,
+      baseline,
+      enforce,
+      tolerance,
+    };
   }
   return {
     status: enforce ? "fail" : "warn",
     reason: `${breaches.length} metric(s) outside tolerance`,
     breaches,
     incomplete,
+    browserDrift,
     baseline,
     enforce,
     tolerance,
@@ -250,6 +307,11 @@ export function renderBudgetTable(rows, result) {
     lines.push(`**Evidence incomplete.** ${result.incomplete.join("; ")}. Nothing is graded from this run.`);
     return lines.join("\n");
   }
+  if ((result.browserDrift ?? []).length > 0) {
+    lines.push(
+      `**Browser drift (advisory while enforce is false).** ${result.browserDrift.join("; ")}. Numbers are still graded; refresh the baseline after pinning a stable Chromium.`,
+    );
+  }
   if (result.status === "warn" && result.breaches.length === 0) {
     lines.push(`_${result.reason}._`);
     return lines.join("\n");
@@ -343,6 +405,9 @@ function main() {
     writeFileSync(process.env.GITHUB_STEP_SUMMARY, `## Lighthouse budget\n\n${table}\n`, { flag: "a" });
   }
 
+  for (const drift of result.browserDrift ?? []) {
+    console.log(`::warning::check:lighthouse-budget — ${drift}`);
+  }
   if (result.status === "fail") {
     console.error(`::error::check:lighthouse-budget failed — ${result.reason}`);
     process.exit(1);

@@ -310,6 +310,10 @@ try {
   });
   await waitForServer(baseUrl, server);
 
+  // Prefer an explicit CHROME_PATH. CI pins Playwright Chromium so the budget is
+  // not tied to whichever HeadlessChrome the ubuntu runner image happens to ship —
+  // that version churn was failing every UI PR as "evidence incomplete". No per-route
+  // retry: a pinned browser is the durable fix; retries only lengthen CI.
   const chromePath = process.env.CHROME_PATH ?? process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH ?? "";
   const failures = [];
 

@@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest";
 import {
   DEFAULT_TOLERANCE,
   baselineFromRows,
+  browserDriftWarnings,
   compareToLighthouseBudget,
   expectedBudgetRuns,
   gradeRun,
@@ -133,15 +134,28 @@ describe("incompleteBudgetEvidence — completeness derived from what is graded"
     expect(result.reason).toBe("evidence incomplete");
   });
 
-  it("rejects a baseline measured by a different browser", () => {
+  it("rejects a baseline measured by a different browser when enforcing", () => {
     const rows = completeRows();
     const stale = baselineFromRows(rows.map((entry: Row) => ({ ...entry, chromeVersion: "HeadlessChrome/131" })));
-    const problems = incompleteBudgetEvidence(rows, budget({ baseline: stale }));
+    const problems = incompleteBudgetEvidence(rows, budget({ baseline: stale, enforce: true }));
 
     expect(problems).toHaveLength(10);
     expect(problems[0]).toContain("measured by a different browser");
   });
 
+  it("does not treat browser drift as incomplete evidence while advisory", () => {
+    // Runner Chrome major bumps were failing every UI PR as "evidence incomplete"
+    // even with enforce:false. Drift must warn and still grade until enforce flips.
+    const rows = completeRows();
+    const stale = baselineFromRows(rows.map((entry: Row) => ({ ...entry, chromeVersion: "HeadlessChrome/131" })));
+
+    expect(incompleteBudgetEvidence(rows, budget({ baseline: stale, enforce: false }))).toEqual([]);
+    expect(browserDriftWarnings(rows, budget({ baseline: stale, enforce: false }))).toHaveLength(10);
+    expect(browserDriftWarnings(rows, budget({ baseline: stale, enforce: false }))[0]).toContain(
+      "measured by a different browser",
+    );
+  });
+
   it("accepts a baseline that recorded no browser identity at all", () => {
     // Older baselines predate the field; absent is not the same as mismatched.
     const rows = completeRows();
@@ -242,6 +256,18 @@ describe("compareToLighthouseBudget", () => {
     expect(result.breaches).toHaveLength(1);
   });
 
+  it("warns on browser drift while still grading when enforce is false", () => {
+    const rows = completeRows();
+    const stale = baselineFromRows(rows.map((entry: Row) => ({ ...entry, chromeVersion: "HeadlessChrome/131" })));
+    const result = compareToLighthouseBudget(rows, budget({ baseline: stale, enforce: false }));
+
+    expect(result.status).toBe("warn");
+    expect(result.reason).toMatch(/browser drift/);
+    expect(result.incomplete).toEqual([]);
+    expect(result.breaches).toEqual([]);
+    expect(result.browserDrift).toHaveLength(10);
+  });
+
   it("fails on incomplete evidence even when not enforcing", () => {
     // An ungraded route counted as a pass is the failure mode this repo has
     // already acted on; `enforce: false` must not downgrade it.
@@ -326,6 +352,24 @@ describe("committed lighthouse-budget.json", () => {
     expect(runner).not.toMatch(/spawnSync\(\s*"npx",/);
     expect(runner).not.toMatch(/spawnSync\(\s*"npx\.cmd",/);
   });
+
+  it("does not retry failed Lighthouse routes — the Playwright Chromium pin is the durable fix", () => {
+    const runner = readFileSync(path.join(process.cwd(), "scripts", "run-lighthouse-budget.mjs"), "utf8");
+
+    expect(runner).not.toContain("maxAttempts");
+    expect(runner).not.toContain("retrying once");
+    expect(runner).toContain("CHROME_PATH");
+  });
+
+  it("pins the CI job to Playwright Chromium rather than the ubuntu runner image Chrome", () => {
+    const workflow = readFileSync(path.join(process.cwd(), ".github", "workflows", "ci.yml"), "utf8");
+    const job = workflow.split("lighthouse-budget:")[1]?.split(/\n  [a-z0-9-]+:/)[0] ?? "";
+
+    expect(job).toContain("setup-ui-e2e");
+    expect(job).toContain("CHROME_PATH");
+    expect(job).toContain("chromium.executablePath()");
+    expect(job).not.toMatch(/no browser install is needed/);
+  });
 });
 
 describe("renderBudgetTable", () => {
@@ -346,6 +390,17 @@ describe("renderBudgetTable", () => {
     expect(renderBudgetTable(rows, result)).toContain("enforce` is false");
   });
 
+  it("surfaces browser drift while still showing graded numbers", () => {
+    const rows = completeRows();
+    const stale = baselineFromRows(rows.map((entry: Row) => ({ ...entry, chromeVersion: "HeadlessChrome/131" })));
+    const result = compareToLighthouseBudget(rows, budget({ baseline: stale, enforce: false }));
+
+    const table = renderBudgetTable(rows, result);
+    expect(table).toContain("Browser drift");
+    expect(table).toContain("refresh the baseline");
+    expect(table).not.toContain("Evidence incomplete");
+  });
+
   it("states the pass explicitly when every route is within tolerance", () => {
     const rows = completeRows();
     const result = compareToLighthouseBudget(rows, budget({ baseline: baselineFromRows(rows) }));