diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10e004b5f1..7d7724e8fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -716,12 +716,26 @@ jobs: with: persist-credentials: false - # Lighthouse drives the Chrome that ships in the ubuntu-24.04 runner image, so - # no browser install is needed here (matching live-web-vitals.yml). - - name: Setup Node and dependencies - uses: ./.github/actions/setup-node-cached + # Pin Chromium through Playwright (same install path as Production UI) so the + # budget is not tied to whichever HeadlessChrome the ubuntu-24.04 image ships. + # Runner Chrome major bumps were failing every UI PR as "evidence incomplete" + # even while enforce is false. live-web-vitals.yml still uses the image Chrome + # because it grades the deployed origin, not a committed relative baseline. + - name: Setup UI e2e environment + uses: ./.github/actions/setup-ui-e2e + + - name: Resolve Playwright Chromium path + id: chromium + shell: bash + run: | + CHROME_PATH="$(node -e "const {chromium}=require('playwright'); process.stdout.write(chromium.executablePath())")" + test -x "$CHROME_PATH" + echo "path=$CHROME_PATH" >> "$GITHUB_OUTPUT" + echo "Using Playwright Chromium at $CHROME_PATH" - name: Measure routes and grade against the baseline + env: + CHROME_PATH: ${{ steps.chromium.outputs.path }} run: npm run verify:lighthouse -- --keep --dir lighthouse - name: Upload Lighthouse reports diff --git a/docs/testing.md b/docs/testing.md index 6801fb9272..81cae0595e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -121,7 +121,11 @@ runs; promote it by adding it to `pr-required` and dropping that flag together. ## Performance budget `npm run verify:lighthouse` builds and serves an isolated production app in demo mode, measures the -routes in `lighthouse-budget.json` on mobile and desktop, and grades the result. It is a **relative** +routes in `lighthouse-budget.json` on mobile and desktop, and grades the result. CI pins Chromium +through Playwright (`CHROME_PATH`) so the budget is not tied to the ubuntu runner image Chrome — +that pin is the durable fix for browser-identity drift (no per-route retry; retries only lengthen +CI). While `enforce` is false, a residual Chrome identity drift warns and still grades; once +`enforce` flips true, drift fails closed until the baseline is refreshed. It is a **relative** gate: absolute web-vitals thresholds are meaningless against a localhost server with no network latency, so each route is compared to a committed known-good baseline with a per-metric tolerance, following the same shape as `check:bundle-budget`. diff --git a/lighthouse-budget.json b/lighthouse-budget.json index 112dedb74e..ac7a91ac05 100644 --- a/lighthouse-budget.json +++ b/lighthouse-budget.json @@ -1,5 +1,5 @@ { - "$comment": "Pre-merge Lighthouse budget, graded against a LOCAL production build by scripts/check-lighthouse-budget.mjs. This is a relative gate: absolute web-vitals thresholds are meaningless without network latency, so it compares each route to a committed known-good baseline. Refresh intentionally with `npm run check:lighthouse-budget -- --update` after an intentional, known-good run. Distinct from .github/workflows/live-web-vitals.yml, which measures the deployed origin for ledger #017 and cannot block a merge. Starts with enforce:false and no baseline so the first CI runs report without blocking; set enforce:true once the baseline has held across a few runs.", + "$comment": "Pre-merge Lighthouse budget, graded against a LOCAL production build by scripts/check-lighthouse-budget.mjs. This is a relative gate: absolute web-vitals thresholds are meaningless without network latency, so it compares each route to a committed known-good baseline. Refresh intentionally with `npm run check:lighthouse-budget -- --update` after an intentional, known-good run. Distinct from .github/workflows/live-web-vitals.yml, which measures the deployed origin for ledger #017 and cannot block a merge. CI measures with Playwright Chromium (CHROME_PATH), not the ubuntu runner image Chrome. While enforce is false, a Chrome identity drift warns and still grades; once enforce is true, drift fails closed until the baseline is refreshed. set enforce:true once the baseline has held across a few runs.", "enforce": false, "$lighthouseVersion": "Pinned exactly, not `lighthouse@12`: a patch published between a baseline run and its follow-up would silently change the measurement. Kept in step with LIGHTHOUSE_VERSION in .github/workflows/live-web-vitals.yml by tests/check-lighthouse-budget.test.ts so the two Lighthouse entry points cannot drift apart.", "lighthouseVersion": "12.8.2", diff --git a/scripts/check-lighthouse-budget.mjs b/scripts/check-lighthouse-budget.mjs index 4b1b4cbecb..82abfd94b4 100644 --- a/scripts/check-lighthouse-budget.mjs +++ b/scripts/check-lighthouse-budget.mjs @@ -109,19 +109,41 @@ export function incompleteBudgetEvidence(rows, budget) { problems.add(`${run}: no baseline row recorded — refresh with --update`); continue; } - // Numbers are only comparable when the same browser produced them. Chrome comes - // from the runner image and moves independently of the pinned Lighthouse - // version, so a browser bump is otherwise indistinguishable from an application + // Numbers are only comparable when the same browser produced them. Chrome on + // ubuntu-24.04 runners moves independently of the pinned Lighthouse version, + // so a browser bump is otherwise indistinguishable from an application // regression. summarise-web-vitals.mjs makes the same point about its baselines. + // + // While `enforce` is false the advisory CI job must not go red on every runner + // Chrome bump — that noise blocked signal on every UI PR. Soft-skip the drift + // check here; compareToLighthouseBudget still surfaces it as a warning so the + // baseline can be refreshed deliberately. Once enforce flips true, drift stays + // fail-closed and requires `check:lighthouse-budget -- --update`. if (before.chromeVersion && row.chromeVersion && before.chromeVersion !== row.chromeVersion) { - problems.add( - `${run}: baseline measured by a different browser (${before.chromeVersion} vs ${row.chromeVersion}) — refresh with --update`, - ); + const drift = `${run}: baseline measured by a different browser (${before.chromeVersion} vs ${row.chromeVersion}) — refresh with --update`; + if (budget?.enforce) problems.add(drift); } } return [...problems].sort(); } +/** Browser identity mismatches that are advisory while enforce is false. */ +export function browserDriftWarnings(rows, budget) { + if (budget?.enforce) return []; + const baseline = budget?.baseline ?? null; + if (!baseline || Object.keys(baseline).length === 0) return []; + const warnings = []; + for (const row of rows ?? []) { + const before = baseline[row.run]; + if (!before?.chromeVersion || !row.chromeVersion) continue; + if (before.chromeVersion === row.chromeVersion) continue; + warnings.push( + `${row.run}: baseline measured by a different browser (${before.chromeVersion} vs ${row.chromeVersion}) — refresh with --update`, + ); + } + return warnings.sort(); +} + /** Grade one run against its baseline. Returns the breaches, empty when within tolerance. */ export function gradeRun(row, baselineRow, tolerance = DEFAULT_TOLERANCE) { if (!baselineRow) return []; @@ -175,11 +197,23 @@ export function compareToLighthouseBudget(rows, budget) { const baseline = budget?.baseline ?? null; const enforce = Boolean(budget?.enforce); const incomplete = incompleteBudgetEvidence(rows, budget); + const browserDrift = browserDriftWarnings(rows, budget); // Incompleteness is fatal regardless of `enforce`: there is nothing to grade, so - // "warn" would report a pass for a route that was never measured. + // "warn" would report a pass for a route that was never measured. Missing reports + // and wrong-page measurements stay here; Chrome drift alone does not while the + // gate is advisory (see incompleteBudgetEvidence). if (incomplete.length > 0) { - return { status: "fail", reason: "evidence incomplete", breaches: [], incomplete, baseline, enforce, tolerance }; + return { + status: "fail", + reason: "evidence incomplete", + breaches: [], + incomplete, + browserDrift, + baseline, + enforce, + tolerance, + }; } if (!baseline || Object.keys(baseline).length === 0) { @@ -188,6 +222,7 @@ export function compareToLighthouseBudget(rows, budget) { reason: "no baseline recorded — run with --update after a known-good build", breaches: [], incomplete, + browserDrift, baseline, enforce, tolerance, @@ -195,14 +230,36 @@ export function compareToLighthouseBudget(rows, budget) { } const breaches = rows.flatMap((row) => gradeRun(row, baseline[row.run], tolerance)); + if (breaches.length === 0 && browserDrift.length === 0) { + return { + status: "ok", + reason: "within tolerance", + breaches, + incomplete, + browserDrift, + baseline, + enforce, + tolerance, + }; + } if (breaches.length === 0) { - return { status: "ok", reason: "within tolerance", breaches, incomplete, baseline, enforce, tolerance }; + return { + status: "warn", + reason: `browser drift on ${browserDrift.length} run(s) — refresh baseline with --update`, + breaches, + incomplete, + browserDrift, + baseline, + enforce, + tolerance, + }; } return { status: enforce ? "fail" : "warn", reason: `${breaches.length} metric(s) outside tolerance`, breaches, incomplete, + browserDrift, baseline, enforce, tolerance, @@ -250,6 +307,11 @@ export function renderBudgetTable(rows, result) { lines.push(`**Evidence incomplete.** ${result.incomplete.join("; ")}. Nothing is graded from this run.`); return lines.join("\n"); } + if ((result.browserDrift ?? []).length > 0) { + lines.push( + `**Browser drift (advisory while enforce is false).** ${result.browserDrift.join("; ")}. Numbers are still graded; refresh the baseline after pinning a stable Chromium.`, + ); + } if (result.status === "warn" && result.breaches.length === 0) { lines.push(`_${result.reason}._`); return lines.join("\n"); @@ -343,6 +405,9 @@ function main() { writeFileSync(process.env.GITHUB_STEP_SUMMARY, `## Lighthouse budget\n\n${table}\n`, { flag: "a" }); } + for (const drift of result.browserDrift ?? []) { + console.log(`::warning::check:lighthouse-budget — ${drift}`); + } if (result.status === "fail") { console.error(`::error::check:lighthouse-budget failed — ${result.reason}`); process.exit(1); diff --git a/scripts/run-lighthouse-budget.mjs b/scripts/run-lighthouse-budget.mjs index 9e8ff1ce83..334fe3a171 100644 --- a/scripts/run-lighthouse-budget.mjs +++ b/scripts/run-lighthouse-budget.mjs @@ -310,6 +310,10 @@ try { }); await waitForServer(baseUrl, server); + // Prefer an explicit CHROME_PATH. CI pins Playwright Chromium so the budget is + // not tied to whichever HeadlessChrome the ubuntu runner image happens to ship — + // that version churn was failing every UI PR as "evidence incomplete". No per-route + // retry: a pinned browser is the durable fix; retries only lengthen CI. const chromePath = process.env.CHROME_PATH ?? process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH ?? ""; const failures = []; diff --git a/tests/check-lighthouse-budget.test.ts b/tests/check-lighthouse-budget.test.ts index 31b974118d..8d5b994346 100644 --- a/tests/check-lighthouse-budget.test.ts +++ b/tests/check-lighthouse-budget.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { DEFAULT_TOLERANCE, baselineFromRows, + browserDriftWarnings, compareToLighthouseBudget, expectedBudgetRuns, gradeRun, @@ -133,15 +134,28 @@ describe("incompleteBudgetEvidence — completeness derived from what is graded" expect(result.reason).toBe("evidence incomplete"); }); - it("rejects a baseline measured by a different browser", () => { + it("rejects a baseline measured by a different browser when enforcing", () => { const rows = completeRows(); const stale = baselineFromRows(rows.map((entry: Row) => ({ ...entry, chromeVersion: "HeadlessChrome/131" }))); - const problems = incompleteBudgetEvidence(rows, budget({ baseline: stale })); + const problems = incompleteBudgetEvidence(rows, budget({ baseline: stale, enforce: true })); expect(problems).toHaveLength(10); expect(problems[0]).toContain("measured by a different browser"); }); + it("does not treat browser drift as incomplete evidence while advisory", () => { + // Runner Chrome major bumps were failing every UI PR as "evidence incomplete" + // even with enforce:false. Drift must warn and still grade until enforce flips. + const rows = completeRows(); + const stale = baselineFromRows(rows.map((entry: Row) => ({ ...entry, chromeVersion: "HeadlessChrome/131" }))); + + expect(incompleteBudgetEvidence(rows, budget({ baseline: stale, enforce: false }))).toEqual([]); + expect(browserDriftWarnings(rows, budget({ baseline: stale, enforce: false }))).toHaveLength(10); + expect(browserDriftWarnings(rows, budget({ baseline: stale, enforce: false }))[0]).toContain( + "measured by a different browser", + ); + }); + it("accepts a baseline that recorded no browser identity at all", () => { // Older baselines predate the field; absent is not the same as mismatched. const rows = completeRows(); @@ -242,6 +256,18 @@ describe("compareToLighthouseBudget", () => { expect(result.breaches).toHaveLength(1); }); + it("warns on browser drift while still grading when enforce is false", () => { + const rows = completeRows(); + const stale = baselineFromRows(rows.map((entry: Row) => ({ ...entry, chromeVersion: "HeadlessChrome/131" }))); + const result = compareToLighthouseBudget(rows, budget({ baseline: stale, enforce: false })); + + expect(result.status).toBe("warn"); + expect(result.reason).toMatch(/browser drift/); + expect(result.incomplete).toEqual([]); + expect(result.breaches).toEqual([]); + expect(result.browserDrift).toHaveLength(10); + }); + it("fails on incomplete evidence even when not enforcing", () => { // An ungraded route counted as a pass is the failure mode this repo has // already acted on; `enforce: false` must not downgrade it. @@ -326,6 +352,24 @@ describe("committed lighthouse-budget.json", () => { expect(runner).not.toMatch(/spawnSync\(\s*"npx",/); expect(runner).not.toMatch(/spawnSync\(\s*"npx\.cmd",/); }); + + it("does not retry failed Lighthouse routes — the Playwright Chromium pin is the durable fix", () => { + const runner = readFileSync(path.join(process.cwd(), "scripts", "run-lighthouse-budget.mjs"), "utf8"); + + expect(runner).not.toContain("maxAttempts"); + expect(runner).not.toContain("retrying once"); + expect(runner).toContain("CHROME_PATH"); + }); + + it("pins the CI job to Playwright Chromium rather than the ubuntu runner image Chrome", () => { + const workflow = readFileSync(path.join(process.cwd(), ".github", "workflows", "ci.yml"), "utf8"); + const job = workflow.split("lighthouse-budget:")[1]?.split(/\n [a-z0-9-]+:/)[0] ?? ""; + + expect(job).toContain("setup-ui-e2e"); + expect(job).toContain("CHROME_PATH"); + expect(job).toContain("chromium.executablePath()"); + expect(job).not.toMatch(/no browser install is needed/); + }); }); describe("renderBudgetTable", () => { @@ -346,6 +390,17 @@ describe("renderBudgetTable", () => { expect(renderBudgetTable(rows, result)).toContain("enforce` is false"); }); + it("surfaces browser drift while still showing graded numbers", () => { + const rows = completeRows(); + const stale = baselineFromRows(rows.map((entry: Row) => ({ ...entry, chromeVersion: "HeadlessChrome/131" }))); + const result = compareToLighthouseBudget(rows, budget({ baseline: stale, enforce: false })); + + const table = renderBudgetTable(rows, result); + expect(table).toContain("Browser drift"); + expect(table).toContain("refresh the baseline"); + expect(table).not.toContain("Evidence incomplete"); + }); + it("states the pass explicitly when every route is within tolerance", () => { const rows = completeRows(); const result = compareToLighthouseBudget(rows, budget({ baseline: baselineFromRows(rows) }));