diff --git a/.claude/skills/gates/SKILL.md b/.claude/skills/gates/SKILL.md index 0a597a59b7..64325def52 100644 --- a/.claude/skills/gates/SKILL.md +++ b/.claude/skills/gates/SKILL.md @@ -24,8 +24,8 @@ Check these before believing any result. for exactly this reason — if installed packages do not match `package-lock.json`, treat any test, lint, or typecheck result as void until `npm ci` has run. Its own failure message says as much. - **`verify:cheap` stops at the first failing check.** Everything after that point never ran. Do not - describe the change as broadly verified when the gate died at check 2 of 32. -- **`format:check` is required in CI but is not part of `verify:cheap`.** A locally green + describe the change as broadly verified when the gate died at check 2 of 33. +- **Changed-file formatting is required in CI but is not part of `verify:cheap`.** A locally green `verify:cheap` can still fail CI on formatting. Run `npx prettier --write ` before pushing — scoped to your files, never `prettier --write .`, which sweeps the whole tree. - **Piping a gate into `tail` or `head` masks its exit code.** In Bash, capture `${PIPESTATUS[0]}`, @@ -34,16 +34,18 @@ Check these before believing any result. ## Pick the smallest gate that can fail Match the gate to what actually changed. Running a broader gate is not more rigorous if it cannot -observe the change; running a narrower one is not sloppy if it can. - -| Change | Gate that can actually fail | -| ----------------------------- | --------------------------------------------------------------- | -| Markdown / docs only | `prettier --check`, `docs:check-links`, `docs:check-index` | -| Source, config, tests | `verify:cheap` | -| Before PR handoff | `verify:pr-local` | -| UI, styling, routing, a11y | `npm run ensure` then `verify:ui` | -| Phone chrome | `verify:phone-chrome` (narrower than `verify:ui`; run it first) | -| Release or handoff confidence | `verify:release` | +observe the change; running a narrower one is not sloppy if it can. Add a second gate only when it +covers a distinct plausible regression and the incremental confidence justifies its cost. + +| Change | Gate that can actually fail | +| --------------------------- | --------------------------------------------------------------- | +| Markdown / docs only | `prettier --check`, `docs:check-links`, `docs:check-index` | +| Localised source behavior | `test:focused -- --files ` | +| Cross-module/unknown scope | `verify:cheap` | +| Before PR handoff | `verify:pr-local` (risk-routed; inspect with `--dry-run`) | +| UI, styling, routing, a11y | `npm run ensure`, affected journey, broad UI only when shared | +| Phone chrome | `verify:phone-chrome` (narrower than `verify:ui`; run it first) | +| Explicit release confidence | `verify:release` (provider approval still required) | `lint`, `typecheck`, and `test` cannot observe a markdown-only change. Say so rather than running them for appearance. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01d89936b9..10e004b5f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,22 +32,22 @@ jobs: name: Change scope runs-on: ubuntu-24.04 timeout-minutes: 5 - # Only outputs a job actually reads are exported. `source_changed`, - # `workflow_changed` and `changed_files` were exported here and consumed by - # nothing, which reads as live wiring and invites a job to be gated on a - # value no one maintains. The script still computes all three — `docs_only` - # derives from the first two — they are simply not job outputs (#139). + # Export only signals consumed by job/step routing. Unknown non-document + # paths fail closed to static_heavy_changed in ci-change-scope.mjs. outputs: - docs_only: ${{ steps.scope.outputs.docs_only }} + docs_changed: ${{ steps.scope.outputs.docs_changed }} + static_heavy_changed: ${{ steps.scope.outputs.static_heavy_changed }} coverage_changed: ${{ steps.scope.outputs.coverage_changed }} ui_changed: ${{ steps.scope.outputs.ui_changed }} advisory_ui_changed: ${{ steps.scope.outputs.advisory_ui_changed }} db_changed: ${{ steps.scope.outputs.db_changed }} container_changed: ${{ steps.scope.outputs.container_changed }} rag_eval_changed: ${{ steps.scope.outputs.rag_eval_changed }} + workflow_changed: ${{ steps.scope.outputs.workflow_changed }} codex_autofix_changed: ${{ steps.scope.outputs.codex_autofix_changed }} build_changed: ${{ steps.scope.outputs.build_changed }} lockfile_changed: ${{ steps.scope.outputs.lockfile_changed }} + pr_policy_body_present: ${{ steps.scope.outputs.pr_policy_body_present }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -69,7 +69,8 @@ jobs: sync-pr-policy-body: name: Sync PR policy body - if: github.event_name == 'pull_request' + needs: changes + if: github.event_name == 'pull_request' && needs.changes.outputs.pr_policy_body_present == 'true' runs-on: ubuntu-24.04 timeout-minutes: 5 permissions: @@ -135,8 +136,6 @@ jobs: timeout-minutes: 20 permissions: contents: read - # Read-only Actions access for the eval-canary liveness probe below. - actions: read steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -144,48 +143,6 @@ jobs: fetch-depth: 0 persist-credentials: false - # GitHub cron fires are best-effort, and the canary's own failure-issue step only reacts - # to runs that HAPPEN and fail — since #923 moved the canary cadence from daily to - # weekly (Sunday 18:00 UTC), a silently dropped fire would go unnoticed for a week or - # more. PR traffic runs many times a day, so a warn-only staleness probe here surfaces a - # dropped weekly canary within hours instead of weeks. Never fails the job — API hiccups - # and empty histories degrade to a warning at most. - - name: Eval-canary liveness (warn if stale) - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - try { - const { data } = await github.rest.actions.listWorkflowRuns({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: "eval-canary.yml", - status: "completed", - per_page: 1, - }); - const latest = data.workflow_runs?.[0]; - // updated_at ~= completion time for a completed run (the REST payload carries no - // completed_at); run_started_at is the fallback for older/partial payloads. - const finishedAt = Date.parse(latest?.updated_at ?? latest?.run_started_at ?? ""); - if (!latest) { - core.warning("Eval-canary liveness: no completed canary runs found."); - } else if (!Number.isFinite(finishedAt)) { - core.info("Eval-canary liveness: latest completed run has no parseable timestamp; skipping staleness check."); - } else { - const ageDays = (Date.now() - finishedAt) / 86_400_000; - if (ageDays > 8) { - core.warning( - `Eval-canary liveness: last completed canary run was ${ageDays.toFixed(1)} days ago (${latest.html_url}). ` + - "The weekly Sunday 18:00 UTC schedule may have been dropped by GitHub - dispatch one manually " + - "(provider-backed; needs explicit approval).", - ); - } else { - core.info(`Eval-canary liveness: last completed run ${ageDays.toFixed(1)} days ago.`); - } - } - } catch (error) { - core.warning(`Eval-canary liveness check skipped: ${error.message}`); - } - - name: Setup Node and dependencies uses: ./.github/actions/setup-node-cached @@ -196,70 +153,110 @@ jobs: run: npm run check:installed-lock-parity - name: Upload-limit parity + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:upload-limit-parity - name: GitHub Actions pin check + if: needs.changes.outputs.workflow_changed == 'true' run: npm run check:github-actions - name: CI scope self-test run: npm run check:ci-scope + - name: Verification-plan self-test + run: npm run check:verification-plan + - name: Pinned Gitleaks range self-test + if: needs.changes.outputs.workflow_changed == 'true' run: npm run check:gitleaks-pinned - name: CI triage self-test + if: needs.changes.outputs.workflow_changed == 'true' run: npm run check:ci-triage - name: PR policy self-test + if: needs.changes.outputs.workflow_changed == 'true' run: npm run check:pr-policy - name: Gate-manifest self-test + if: needs.changes.outputs.workflow_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:gate-manifest - name: Branch review ledger integrity + # Also run for static_heavy (including .gitattributes-only edits and the + # scheduled full-run sentinel): both checkers validate merge attributes. + if: needs.changes.outputs.docs_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:branch-review-ledger - name: Outstanding-issues ledger integrity + if: needs.changes.outputs.docs_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true' env: OUTSTANDING_ISSUES_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} run: npm run check:outstanding-issues - name: PR mergeability workflow contract + if: needs.changes.outputs.workflow_changed == 'true' run: npm run check:pr-mergeability + - name: Focused CI workflow contracts + if: needs.changes.outputs.workflow_changed == 'true' + run: npm run test:ci-workflows + + - name: Codex auto-resolve workflow guard + if: needs.changes.outputs.codex_autofix_changed == 'true' + run: npm run check:codex-autofix-workflow + - name: Codebase index coverage + if: needs.changes.outputs.docs_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true' run: npm run docs:check-index - name: Documentation inventory drift + if: needs.changes.outputs.docs_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true' run: npm run docs:check-inventory - name: Check docs scripts + if: needs.changes.outputs.docs_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true' run: npm run docs:check-scripts - name: Check docs links + if: needs.changes.outputs.docs_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true' run: npm run docs:check-links - name: Dependency and duplicate-export hygiene + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:knip - name: Maintainability hotspot budgets + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:maintainability-budgets - - name: Format check + - name: Changed-file format check + if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }} + run: npm run format:changed + + - name: Scheduled full-tree format drift + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: npm run format:check # Design-system guards that were previously only in the local verify:cheap # chain (offline + fast). Without them a stray type/icon/brand drift merged # green because no workflow ran them (there is no test backstop either). - name: Type scale guard + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:type-scale - name: Icon scale guard + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:icon-scale - name: Brand asset check + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run brand:check - name: Asset linting + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:assets # More generated-output drift detectors that were local-only in @@ -268,42 +265,51 @@ jobs: # green because no workflow ran them. The gate-manifest self-test now fails # CI if this list drifts from verify:cheap again. - name: Site map drift + if: needs.changes.outputs.docs_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true' run: npm run sitemap:check - name: Therapy data index drift + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:therapy-data-index - name: Cross-mode differentials index drift + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:cross-mode-index - name: Design-system contract + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:design-system-contract # Prevent hosted SQL/tooling from assuming a platform-reserved role while # byte-pinning the single immutable historical migration exception. - name: Hosted migration-role guard + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:migration-role # Fails if a SECURITY DEFINER public function is left executable by # PUBLIC/anon (privilege-escalation / cross-tenant read surface). - name: Function-grant guard + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:function-grants # Fails if a src/app/api handler queries an owner-scoped table without a # recognised owner filter (defense-in-depth tenancy guard; audit D2). - name: Owner-scope guard + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run check:owner-scope - name: Lint + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run lint - name: Typecheck + if: needs.changes.outputs.static_heavy_changed == 'true' run: npm run typecheck safety: name: Safety and config checks needs: changes - if: needs.changes.outputs.docs_only != 'true' + if: needs.changes.outputs.static_heavy_changed == 'true' runs-on: ubuntu-24.04 timeout-minutes: 15 steps: @@ -322,8 +328,8 @@ jobs: # A PR can only introduce a new (possibly-vulnerable) dependency when a # manifest/lockfile changes, so the audit blocks the merge gate only then - # (retried to absorb transient npm-registry flake). For every other change - # it runs advisory, and the weekly scheduled full run still enforces drift. + # (retried to absorb transient npm-registry flake). The weekly scheduled + # full run still enforces registry drift without querying it on every PR. - name: Dependency audit (blocking on dependency changes) if: needs.changes.outputs.lockfile_changed == 'true' run: | @@ -334,23 +340,15 @@ jobs: done npm audit --omit=dev --audit-level=high - - name: Dependency audit (advisory) - if: needs.changes.outputs.lockfile_changed != 'true' - continue-on-error: true - run: npm audit --omit=dev --audit-level=high - - name: Edge function typecheck run: npm run check:edge:functions - name: Production readiness (CI-safe) run: npm run check:production-readiness:ci - - name: Codex auto-resolve workflow guard - if: needs.changes.outputs.codex_autofix_changed == 'true' - run: npm run check:codex-autofix-workflow - - # Fixtures for ordinary non-docs PRs; full offline contracts (includes - # fixtures) when retrieval/answer surfaces change. + # Fixtures for executable changes; full offline contracts (includes + # fixtures) when retrieval/answer surfaces change. Recognised docs and + # workflow-only changes skip this whole job. - name: Offline RAG fixture and manifest validation if: needs.changes.outputs.rag_eval_changed != 'true' run: npm run check:rag:fixtures @@ -814,7 +812,7 @@ jobs: steps: - name: Verify required in-scope jobs env: - DOCS_ONLY: ${{ needs.changes.outputs.docs_only }} + STATIC_HEAVY_CHANGED: ${{ needs.changes.outputs.static_heavy_changed }} COVERAGE_CHANGED: ${{ needs.changes.outputs.coverage_changed }} UI_CHANGED: ${{ needs.changes.outputs.ui_changed }} DB_CHANGED: ${{ needs.changes.outputs.db_changed }} @@ -891,10 +889,10 @@ jobs: require_success "changes" "$CHANGES_RESULT" require_success "static-pr" "$STATIC_RESULT" - if [ "$DOCS_ONLY" = "true" ]; then - require_skipped_or_success "safety" "$SAFETY_RESULT" - else + if [ "$STATIC_HEAVY_CHANGED" = "true" ]; then require_success "safety" "$SAFETY_RESULT" + else + require_skipped_or_success "safety" "$SAFETY_RESULT" fi if [ "$COVERAGE_CHANGED" = "true" ]; then diff --git a/.github/workflows/ops-digest.yml b/.github/workflows/ops-digest.yml index 4ab355f42d..31b4f40b13 100644 --- a/.github/workflows/ops-digest.yml +++ b/.github/workflows/ops-digest.yml @@ -23,12 +23,76 @@ permissions: issues: write jobs: + # GitHub cron fires are best-effort, and the canary's own failure issue only + # reacts to runs that happen and fail. Check liveness once with the daily ops + # cadence instead of making the Actions API call on every pull request. + eval-canary-liveness: + name: Eval-canary liveness + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read + contents: read + outputs: + stale: ${{ steps.liveness.outputs.stale }} + message: ${{ steps.liveness.outputs.message }} + steps: + - name: Warn if the weekly canary is stale + id: liveness + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const set = (stale, message) => { + core.setOutput("stale", stale ? "true" : "false"); + core.setOutput("message", message); + if (stale) core.warning(message); + else core.info(message); + }; + try { + const { data } = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "eval-canary.yml", + status: "completed", + per_page: 1, + }); + const latest = data.workflow_runs?.[0]; + // updated_at approximates completion time; run_started_at is the + // fallback for older or partial REST payloads. + const finishedAt = Date.parse(latest?.updated_at ?? latest?.run_started_at ?? ""); + if (!latest) { + set(true, "Eval-canary liveness: no completed canary runs found."); + } else if (!Number.isFinite(finishedAt)) { + set( + false, + "Eval-canary liveness: latest completed run has no parseable timestamp; skipping staleness check.", + ); + } else { + const ageDays = (Date.now() - finishedAt) / 86_400_000; + if (ageDays > 8) { + set( + true, + `Eval-canary liveness: last completed canary run was ${ageDays.toFixed(1)} days ago (${latest.html_url}). ` + + "The weekly Sunday 18:00 UTC schedule may have been dropped by GitHub - dispatch one manually " + + "(provider-backed; needs explicit approval).", + ); + } else { + set(false, `Eval-canary liveness: last completed run ${ageDays.toFixed(1)} days ago.`); + } + } + } catch (error) { + set(true, `Eval-canary liveness check failed: ${error.message}`); + } + ops-digest: runs-on: ubuntu-24.04 timeout-minutes: 10 + needs: eval-canary-liveness env: PROD_HEALTH_URL: ${{ vars.PROD_HEALTH_URL }} HEALTH_DEEP_PROBE_SECRET: ${{ secrets.HEALTH_DEEP_PROBE_SECRET }} + EVAL_CANARY_STALE: ${{ needs.eval-canary-liveness.outputs.stale }} + EVAL_CANARY_MESSAGE: ${{ needs.eval-canary-liveness.outputs.message }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -60,9 +124,14 @@ jobs: script: | const fs = require("fs"); const label = "ops-digest"; - const body = fs.readFileSync("digest.md", "utf8"); + const canaryStale = process.env.EVAL_CANARY_STALE === "true"; + const canaryMessage = process.env.EVAL_CANARY_MESSAGE || ""; + let body = fs.readFileSync("digest.md", "utf8"); + if (canaryStale && canaryMessage) { + body = `## Eval-canary liveness\n\n⚠ ${canaryMessage}\n\n${body}`; + } const status = "${{ steps.digest.outputs.status }}" || "unknown"; - const alerting = "${{ steps.digest.outputs.alerting }}" === "true"; + const alerting = "${{ steps.digest.outputs.alerting }}" === "true" || canaryStale; const { data: existing } = await github.rest.issues.listForRepo({ owner: context.repo.owner, repo: context.repo.repo, @@ -95,7 +164,8 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, - body: `⚠ Attention (status=${status}, alerting=${alerting}) — ${new Date().toISOString()}\n\n${body}`, + body: `⚠ Attention (status=${status}, alerting=${alerting}${canaryStale ? ", eval-canary-stale=true" : ""}) — ${new Date().toISOString()}\n\n${body}`, }); } if (status !== "ok") core.warning(`Ops digest status: ${status}`); + if (canaryStale) core.warning(canaryMessage || "Eval-canary liveness reported stale."); diff --git a/AGENTS.md b/AGENTS.md index 770c41f8e8..b200dfafe9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -181,16 +181,29 @@ Babysit / Run PR ledger policy: do not push a tip whose sole delta is a babysit # Process hardening phases -- For non-trivial source/config/test changes, prefer `npm run verify:cheap` as the first broad gate and `npm run verify:pr-local` before PR handoff when the change is ready. The PR-local gate runs the full unit suite once, then conditionally adds the production build/client-bundle scan and RAG fixture/manifest validation. Browser, dependency-audit, Docker/Supabase replay, and provider-backed checks remain separate gates. Use `npm run verify:pr-local -- --dry-run --files ` to inspect selection without running commands. The broader `--extended` plan is dry-run only unless explicit approval is reflected by `ALLOW_EXTENDED_PR_LOCAL=true`. +- **Verification principle:** run the smallest check capable of detecting a plausible regression introduced by the current diff. Before starting a check, identify the failure class it covers, whether a successful check already covered that class, whether a cheaper focused check offers comparable detection, and whether the incremental confidence justifies the runtime, resource use, and repository-lock contention. If there is no plausible changed failure path, do not run the check. + +| Tier | Use when | Default evidence | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| 0 — No test command | Explanation, planning, prompt writing, read-only inspection, or no repository change | No test, build, server, or baseline command | +| 1 — Static/focused | Documentation, comments, metadata, or narrow non-behavioural configuration | Relevant format, docs, syntax, generated-file, or diff check only | +| 2 — Focused behavioural | A localized helper, component, contract, or test change | Directly affected unit/DOM/contract test; add typecheck only when the edit can affect compilation or a type contract | +| 3 — Domain gate | Shared UI/routing, dependencies, security, privacy, RAG, clinical output, production configuration, or another cross-cutting domain | The smallest applicable repository/domain selector, focused journey, or contract gate | +| 4 — Broad handoff | The diff crosses multiple subsystems, cannot be bounded reliably, or the task explicitly requires PR/release confidence | One appropriate broad gate, selected rather than stacked by default | + +- Do not run a broad baseline routinely before localized work, and do not select `verify:cheap` merely because a change is described as “non-trivial.” Use `npm run verify:cheap` once when cross-module risk warrants a broad offline gate. Use `npm run verify:pr-local` when a change is ready for PR handoff: it now classifies the changed paths, runs focused documentation/workflow contracts for recognised low-risk scopes, and fails closed to lint, typecheck, the full unit suite, RAG fixture validation, and relevant build/domain gates for executable or unknown scope. If the diff has not changed, do not run `verify:cheap` first merely to repeat the same coverage. +- Do not stack focused tests, full tests, typecheck, lint, build, and browser checks unless each catches a distinct plausible regression. Do not rerun an unchanged successful gate. A deliberately skipped low-yield broad gate is not automatically verification debt; report the skipped check and its risk-based reason concisely. +- Use dry-run selectors before expensive gates when scope is uncertain. `npm run verify:pr-local -- --dry-run --files ` inspects PR-local selection without running commands. The broader `--extended` plan is dry-run only unless explicit approval is reflected by `ALLOW_EXTENDED_PR_LOCAL=true`. +- CI uses the same fail-closed scope model: recognised docs and workflow/policy-only changes run focused contracts; executable product/test/config, dependency, database, container, RAG, security-sensitive, mixed, or unknown paths retain the applicable heavy jobs. Do not broaden a path trigger or restore an always-on heavy job without evidence that the focused route misses a realistic failure class. Scheduled drift/release checks and the always-reporting `PR required` aggregate remain safety backstops. - Let the repository run coordinator control cross-worktree verification. It permits at most two focused Vitest/read-only typecheck leases from different worktrees; full Vitest, coverage, lint, build, Playwright, and live-provider tests remain exclusive. Do not install while a repository test, build, lint, typecheck, or server command is active. Avoid aggressive short-interval polling, and do not repeat an unchanged full gate after it passes. -- For UI, frontend, browser, routing, styling, reduced-motion, or forced-colors changes, run `npm run ensure` before browser work and use `npm run verify:ui` as the Chromium UI gate. For phone-chrome changes, run `npm run verify:phone-chrome` first: it checks installed-lock parity, selects the affected browser/PWA owners and exact journeys, and adds `verify:ui` last only when shared chrome foundations make the broad gate necessary. Inspect uncertain scope with `-- --dry-run`. -- **Run `npm run format` and commit the result before every push.** `format:check` is in neither `npm run test`, `npm run typecheck`, nor `npm run lint`, so the ordinary loop reports green while `Static PR checks` fails on `prettier --check .`. Three CI failures on 2026-07-30 came from exactly this (two of them on `ci/circleci: verify`, since removed from the repo by PR #1412). Two traps beyond simply running it: +- For UI, frontend, browser, routing, styling, reduced-motion, or forced-colors behaviour changes, run `npm run ensure` before browser work and prove the changed owner or journey first. Use `npm run verify:ui` when shared UI foundations changed or PR/handoff policy requires the complete Chromium gate, not as an automatic addition after focused proof. For phone-chrome changes, run `npm run verify:phone-chrome` first: it checks installed-lock parity, selects the affected browser/PWA owners and exact journeys, and adds `verify:ui` last only when shared chrome foundations make the broad gate necessary. Inspect uncertain scope with `-- --dry-run`. Chromium evidence does not close physical Safari or installed-PWA acceptance gaps. +- **Run `npm run format` and commit the result before every push.** Formatting is in neither `npm run test`, `npm run typecheck`, nor `npm run lint`, so the ordinary loop can report green while the changed-file CI check or exact-commit pre-push guard fails. Three CI failures on 2026-07-30 came from exactly this (two of them on `ci/circleci: verify`, since removed from the repo by PR #1412). Two traps beyond simply running it: - **Formatting without committing does nothing for the push.** A push sends commits, not your working tree, so formatting after committing leaves the unformatted blob on the branch. Amend or add a follow-up commit. - **A per-file check is not the repository-wide check.** `prettier --check ` on the source file you edited passes while a doc or ledger edit in the same push fails; that was the missed file twice out of three. `.githooks/pre-push` carries the guard, and since 2026-07-30 it checks the pushed commit where CI checks it: `guard-push.mjs` puts the pushed SHA in a temporary `git worktree` with an exact-lock `node_modules` linked in and runs Prettier there, so neither the working tree's contents nor its prettier config can vouch for the commit, and a dynamic `prettier.config.*` still loads. An isolated worktree without local dependencies may reuse Prettier only from a registered worktree with a byte-identical lockfile and matching installed Prettier version; if none exists, the guard blocks with the explicit `npm ci --include=dev` remediation instead of skipping formatting. A push that changes prettier policy (`.prettierrc*`, `.prettierignore`, `.editorconfig`, or a `package.json` carrying a `prettier` field) escalates to a whole-tree `prettier --check .`, because a policy change alters the verdict for files the push never touched. But `core.hooksPath` is set by this checkout's `npm install`, so an agent pushing from its own environment bypasses the hook entirely and only CI catches the break — which is why the rule above is still a rule. -- For release or handoff confidence, use `npm run verify:release`; this includes the full Playwright project set. +- For explicit release confidence, use `npm run verify:release` once; this includes the full Playwright project set and retains all provider-approval requirements. Ordinary local completion or PR handoff does not by itself authorize or require this release gate. - For clinical ingestion, answer generation, source governance, privacy, production-readiness, or environment changes, run the smallest relevant domain check plus `npm run check:production-readiness`. - For pull requests that touch ingestion, answer generation, search/ranking, source rendering, document access, privacy, production env, or clinical output, complete the clinical governance preflight in `.github/pull_request_template.md`. - Track known verification debts and staged process improvements in `docs/process-hardening.md` instead of relying on chat-only memory. diff --git a/CLAUDE.md b/CLAUDE.md index 943e90a4fa..58dc0275d7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -139,17 +139,17 @@ npm run dev # direct dev server on the project-stable port Verification pyramid — run the **smallest gate that covers the change**, then widen: -| Gate | What it is | -| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `npm run test:focused -- --files ` | Source-only iteration. Fails closed for deleted files and test infrastructure — then run `npm run test`. | -| `npm run verify:cheap` | The broad local gate: 29 static/consistency gates + `lint` + `typecheck` + full offline unit suite | -| `npm run verify:pr-local` | Closest local mirror of the PR gate; adds format and conditional build / client-bundle scan / RAG fixture validation. `-- --dry-run --files ` shows selection without running. | -| `npm run verify:ui` | Chromium production journeys. Run `npm run ensure` first. | -| `npm run verify:phone-chrome` | Phone-chrome changes; selects affected owners/journeys before escalating to `verify:ui` | -| `npm run verify:release` | Full build + all browsers + readiness. **Provider-backed — needs approval.** | - -`verify:cheap` deliberately does **not** run `format:check`, which is why the installed -pre-push hook (`.githooks/pre-push` → `scripts/guard-push.mjs`) blocks on unformatted files. +| Gate | What it is | +| ----------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `npm run test:focused -- --files ` | Source-only iteration. Fails closed for deleted files and test infrastructure — then run `npm run test`. | +| `npm run verify:cheap` | The broad local gate: 30 static/consistency gates + `lint` + `typecheck` + full offline unit suite; use for cross-module risk, not automatically | +| `npm run verify:pr-local` | Risk-routed PR mirror: focused docs/workflow contracts for recognised light scope, fail-closed heavy checks for executable or unknown scope. `-- --dry-run --files ` shows selection. | +| `npm run verify:ui` | Chromium production journeys. Run `npm run ensure` first. | +| `npm run verify:phone-chrome` | Phone-chrome changes; selects affected owners/journeys before escalating to `verify:ui` | +| `npm run verify:release` | Full build + all browsers + readiness. **Provider-backed — needs approval.** | + +`verify:cheap` deliberately does **not** run formatting, which is why changed-file CI and the +installed pre-push hook (`.githooks/pre-push` → `scripts/guard-push.mjs`) block on unformatted files. It also guards the auto-merge race on `claude/*` branches and drift-manifest staleness. Each guard has a documented override env var. @@ -158,9 +158,10 @@ output, source governance) additionally want the smallest relevant domain check `npm run check:production-readiness`. CI (`.github/workflows/ci.yml`) is risk-scoped: a `changes` job classifies paths, `static-pr` -always runs, and `pr-required` is the single always-reporting required aggregate. Heavier -jobs (coverage, build, Chromium, Supabase migration replay, Docker builds) run only when -their file scope applies. +always runs a small baseline and conditionally selects docs/workflow or heavy static checks, +and `pr-required` is the single always-reporting required aggregate. Coverage, safety/RAG, +build, Chromium, Supabase migration replay, and Docker builds run only when their file scope +applies; unknown non-document paths fail closed to heavy scope. ## Conventions the gates enforce diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 6acfc9aa84..893f53949b 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -2,6 +2,14 @@ This document turns the current process review into phased, durable repo practice. It separates changes that already take effect from work that should stay explicit until it is implemented. +## Risk-routed local and CI verification (2026-08-02) + +- `ci-change-scope.mjs` is the shared fail-closed classifier for local PR handoff and hosted CI. Only recognised documentation and workflow/policy paths take a light route; product code, tests, executable config, dependencies, database/container/RAG surfaces, mixed scope, and unknown non-document paths retain heavy verification. +- `verify:pr-local` no longer treats every handoff as lint + typecheck + full unit + RAG fixtures. It always checks runtime, installed-lock parity, and changed-file formatting, then selects documentation checks, focused workflow contracts, or the heavy executable plan. Its self-test makes those routing decisions a repository contract. +- `static-pr` remains required but step-routes the same signals. Workflow-only edits run focused workflow tests instead of full coverage and safety/RAG; documentation changes run documentation integrity checks; heavy/unknown changes retain lint, typecheck, coverage, safety/RAG, and all applicable build/UI/database/container jobs. `PR required` remains `if: always()` and requires every in-scope job, including `safety` whenever `static_heavy_changed` is true. +- Repeated low-yield provider work was removed from ordinary PRs: dependency audit runs when a lockfile/npm configuration can change the dependency tree (and on the scheduled full-run sentinel), eval-canary liveness moved to the daily Ops Digest cadence, and PR-body synchronization runs only when `PR_POLICY_BODY.md` exists. +- The operating rule is incremental value, not a fixed command count: each added check must cover a distinct plausible failure path. Never rerun an unchanged pass, and do not stack broad gates when one suitable gate already covers the risk. + ## Multi-worktree reconciliation hardening (2026-07-23) The cloud-chat reconciliation postmortem and complete issue/fix matrix are in @@ -206,11 +214,11 @@ API rather than estimated: ## Phase 1 - Active now -- `npm run verify:cheap` is the default broad local gate for source/config/test changes: `check:runtime`, `sitemap:check`, lint, typecheck, and unit tests. -- `npm run verify:pr-local` is the closest local mirror of the normal PR gate: runtime, format, lint, typecheck, one full unit run, conditional build, and RAG fixture/manifest validation when changed-file scope requires it. Local scope resolves against the repository default base rather than a feature-branch upstream; set `PR_BASE_REF` explicitly for release-targeted PRs. +- `npm run verify:cheap` is the broad offline local gate for cross-module risk: `check:runtime`, `sitemap:check`, lint, typecheck, and unit tests. It is selected, not automatic for every source/config/test edit. +- `npm run verify:pr-local` is the risk-routed local mirror of the normal PR gate: runtime, installed-lock parity, changed-file format, then focused docs/workflow contracts or the fail-closed executable plan with lint, typecheck, one full unit run, conditional build, and RAG fixture/manifest validation. Local scope resolves against the repository default base rather than a feature-branch upstream; set `PR_BASE_REF` explicitly for release-targeted PRs. - `npm run verify:ui` is the complete required production Chromium gate: `check:runtime` plus all non-quarantined production journeys (`test:e2e:pr`). - `npm run verify:release` is the release-confidence gate: `check:runtime`, lint, typecheck, unit tests, build, full Playwright browser matrix, `check:production-readiness`, `governance:release`, and `eval:quality:release` (the last step needs live Supabase and OpenAI keys). -- CI uses a risk-scoped PR gate: `changes` classifies paths, `static-pr` always runs runtime/action/scope/format/lint/typecheck checks, and `pr-required` is the single always-reporting required aggregate. One full unit run with coverage, build, both Docker image builds, one required production Chromium invocation, migration replay, safety/config, Codex workflow validation, and RAG fixture validation run only when their file scopes apply. UI PRs also run one non-blocking advisory Chromium invocation for quarantined and mockup journeys. The external `Supabase Preview` check may still replay migrations on branch databases when enabled. A gated `release-browser-matrix` job runs the full Playwright browser set on `main`, `release/*`, manual dispatch, and the weekly schedule. +- CI uses a risk-scoped PR gate: `changes` classifies paths, `static-pr` always runs runtime/install parity, scope/plan self-tests, and changed-file formatting, and `pr-required` is the single always-reporting required aggregate. Focused documentation or workflow contracts run for recognised light scopes. Lint/typecheck, one full unit run with coverage, safety/RAG, build, both Docker image builds, required production Chromium, and migration replay run only when their file scopes apply; unknown non-document paths fail closed to this heavy route. UI PRs also run one non-blocking advisory Chromium invocation for quarantined and mockup journeys. The external `Supabase Preview` check may still replay migrations on branch databases when enabled. A gated `release-browser-matrix` job runs the full Playwright browser set on `main`, `release/*`, manual dispatch, and the weekly schedule. - `tests/ui-accessibility.spec.ts` covers reduced-motion and forced-colors dashboard usability so those modes are no longer only reviewed by inspection. - `tests/ui-tools.spec.ts` covers the Applications dashboard mode at mobile and desktop sizes, including the `/applications` compatibility redirect. - `AGENTS.md` now points future agents to these gates and to this document. @@ -392,7 +400,7 @@ passes `p_worker_id`. Ordered apply steps, R17 manual `CONCURRENTLY` index, and ## PR merge gate: risk-scoped CI + required aggregate (2026-07-10) - CI now has one always-reporting required aggregate: `CI / PR required`. The aggregate depends on `changes`, `static-pr`, `safety`, `coverage`, `build`, `ui-critical-fast`, `ui-critical`, and `db-reset-verify`, then enforces only the jobs whose scopes apply. The aggregate keeps `if: always()` and labels concurrency `cancelled` distinctly from real failures (#095 / PR #1409); it stays red because a skipped required check would count as passing. -- `static-pr` is the deterministic baseline for every PR: runtime, action pin check, CI scope self-test, format, lint, and typecheck. Coverage is the one required full unit run. Build, safety/config (fixtures always; `eval:rag:offline` when `rag_eval_changed`), production UI, and migration replay are independent jobs so reruns stay focused. Coverage includes source, tests, package/test-runner configuration, while process-only documentation does not trigger builds. +- `static-pr` is the deterministic always-reporting baseline for every PR: runtime/install parity, scope and verification-plan self-tests, and changed-file formatting always run. Documentation and workflow-only scopes add focused contracts; executable or unknown scope adds lint, typecheck, coverage, and safety/config/RAG. Build, production UI, and migration replay remain independent jobs so reruns stay focused. - `db-reset-verify` is path-scoped to Supabase migrations/schema/`src/lib/supabase` and drift tooling — not every API route. Do not also require an external Supabase Preview replay unless the repo owner intentionally wants duplicate migration replay. - `ui-critical` retains its job ID for branch-protection compatibility and still runs the full required production Chromium suite (`test:e2e:pr`). On pull requests / merge_group, `ui-critical-fast` runs `@critical` first for fail-fast signal. `src/app/api/**` does not set `ui_changed`. `ui-advisory` runs quarantined and mockup journeys together when UI scope applies. A JUnit failure is considered a known flake only when its exact spec/title matches the validated ledger. The full browser matrix remains main/release/manual/scheduled and depends on static/build/UI success — not on `pr-required` — so a blocking scheduled dependency audit cannot skip Firefox/WebKit (#023 structural half). - Secret Scan pins Gitleaks to the workflow event base/head SHAs and the checked-out commit (`scripts/run-gitleaks-pinned.mjs`) so a concurrent push cannot invalidate the range (#097). diff --git a/docs/scripts-index.md b/docs/scripts-index.md index cc985ef45f..20f0856504 100644 --- a/docs/scripts-index.md +++ b/docs/scripts-index.md @@ -1,6 +1,6 @@ # Scripts index -Curated map of `scripts/` (200 files) and the `package.json` script surface (213 entries), +Curated map of `scripts/` (200 files) and the `package.json` script surface (215 entries), grouped by purpose. This is orientation, not an exhaustive per-file listing — the authoritative command list is `package.json`, and `npm run docs:check-scripts` verifies every `npm run ` referenced in docs resolves to a real script. `npm run docs:update` refreshes the exact counts above. diff --git a/docs/testing.md b/docs/testing.md index dbf9bf8794..4b390088d1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -14,6 +14,12 @@ Ordinary Vitest and Playwright runs remove OpenAI, Supabase, database, and E2E c **Provider-backed boundary:** `test:live`, `eval:quality`, `eval:retrieval:quality`, `verify:release`, `check:supabase-project`, and other OpenAI/Supabase/hosted workflows need **explicit user approval** before agents run them (see root `AGENTS.md`). Prefer offline gates (`verify:cheap`, `verify:pr-local`, `eval:rag:offline`) unless that approval is in the task. +## Risk-based selection + +Start with the cheapest check that can fail for the changed behavior. Add another check only when it covers a distinct plausible regression that the existing evidence does not. Documentation and policy changes normally need formatting, documentation, syntax, or focused contract checks; localized behavior needs its directly affected test; cross-cutting or uncertain executable changes escalate to the relevant domain or broad gate. Do not routinely stack focused tests, the full unit suite, lint, typecheck, build, and browser checks, and do not rerun an unchanged passing gate. + +`npm run verify:pr-local -- --dry-run --files ` shows the local plan. Recognised documentation and workflow/policy-only scopes stay focused. Product code, tests, executable configuration, dependencies, database/container surfaces, mixed scope, and unknown non-document paths fail closed to the heavy plan. Provider, physical-device, and release-only acceptance remain separate and require their normal approval or task context. + Codex Cloud agents remain provider-free. Run authenticated Supabase tests through the manual `.github/workflows/authenticated-live-tests.yml` workflow, which requires the explicit `run-authenticated-live-tests` dispatch confirmation, records the run against the @@ -29,11 +35,12 @@ test requests, and production rate-limit row updates. | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | | `npm run test:focused -- --files ` | Local iteration using Vitest related-file selection. It fails closed for deleted files, test infrastructure, configuration, or an empty/unsafe mapping. | | `npm run test` | Complete offline unit suite. | +| `npm run test:ci-workflows` | Focused offline contracts for CI, authenticated-workflow, Codex-autofix, and eval-canary workflow changes. | | `npm run test:live` | Explicit provider suite; requires `ALLOW_PROVIDER_TESTS=true`. | | `npm run test:e2e:pr` | Required production Chromium journeys and visual-artifact smoke, excluding mockups and quarantined tests. | | `npm run test:e2e:advisory` | Quarantined and mockup journeys in one advisory invocation. | | `npm run verify:cheap` | Broad offline local gate: runtime/config checks, lint, typecheck, and the full unit suite. | -| `npm run verify:pr-local` | PR-like local gate. Formatting is checked on the changed set, the full unit suite runs once, and RAG scope adds fixture/manifest validation. | +| `npm run verify:pr-local` | Risk-routed PR-like local gate. Recognised docs/workflow scopes stay focused; executable or unknown scope adds lint, typecheck, full unit, and domains. | | `npm run verify:phone-chrome` | Smart phone-chrome gate: lock parity, affected contracts, browser/PWA owners and exact journeys, then full UI only for shared foundations. | | `npm run verify:ui` | Complete required production Chromium gate. | | `npm run test:e2e:style-contract` | Focused rendered-effect assertions for the unlayered classes in `globals.css` (also runs inside `test:e2e:pr`). | @@ -137,7 +144,11 @@ Neither uses secrets or providers. ## CI topology -PR CI keeps static checks separate from one required full unit run with coverage. UI scope runs a fail-fast `@critical` Chromium job on pull requests, then one required full production Chromium invocation (`test:e2e:pr`) for non-quarantined journeys, plus one advisory invocation for quarantined and mockup journeys. `src/app/api/**` does not set `ui_changed` or `db_changed` — API handlers stay on unit/coverage (and offline RAG when retrieval-scoped). The `PR required` aggregate keeps `if: always()` and distinguishes `cancelled` from `failure` in its messages (stays red; a skipped required check would count as passing). Secret Scan pins Gitleaks to the workflow event base/head SHAs and the checked-out commit, and verifies the linux_x64 release tarball against a pinned SHA-256 before install. The weekly `release-browser-matrix` depends on static/build/UI success, not on the full aggregate, so a blocking scheduled dependency audit cannot skip Firefox/WebKit. Container scope calls the reusable Docker workflow and requires both app and worker image builds through the `pr-required` aggregate. Build, migration, safety/RAG, and release behavior remain independently scoped. +PR CI uses the same fail-closed classifier as `verify:pr-local`. `static-pr` always proves runtime/install parity, the classifier and verification-plan invariants, and changed-file formatting. Recognised documentation changes add documentation integrity checks; recognised workflow/policy-only changes add action/policy self-tests and `test:ci-workflows`. Executable, test, build/config, dependency, database/container, mixed, or unknown non-document paths set `static_heavy_changed`, retaining lint, typecheck, safety/config/RAG, and the full unit coverage job. Build, migration, Docker, and browser jobs remain separately scoped. A dependency audit blocks on lockfile/npm-config changes and the scheduled full-run sentinel, instead of making a low-value registry request on every PR. + +UI scope runs a fail-fast `@critical` Chromium job on pull requests, then one required full production Chromium invocation (`test:e2e:pr`) for non-quarantined journeys, plus one advisory invocation for quarantined and mockup journeys. `src/app/api/**` does not set `ui_changed` or `db_changed` — API handlers stay on unit/coverage (and offline RAG when retrieval-scoped). The `PR required` aggregate keeps `if: always()` and distinguishes `cancelled` from `failure` in its messages (stays red; a skipped required check would count as passing). Secret Scan pins Gitleaks to the workflow event base/head SHAs and the checked-out commit, and verifies the linux_x64 release tarball against a pinned SHA-256 before install. The weekly `release-browser-matrix` depends on static/build/UI success, not on the full aggregate. Container scope calls the reusable Docker workflow and requires both app and worker image builds through the aggregate. + +PR body synchronization is skipped unless the checked-out head actually contains `PR_POLICY_BODY.md`. The eval-canary liveness API probe runs once with the daily Ops Digest cadence rather than on every PR. These remove repeated provider-side work without weakening a required result. Two further jobs are advisory (`continue-on-error`, deliberately outside `pr-required`): `visual-baseline` on UI scope and `lighthouse-budget` on UI-or-build scope. Both upload their evidence on every run, pass or fail, because the artifact is the whole point on a first run — the baselines to adopt and the reports to grade. Promote either to required by adding it to `pr-required` and removing `continue-on-error` in the same edit. @@ -151,5 +162,5 @@ Before opening a UI PR, confirm: - **Accessibility** ([design-system §7](./design-system.md)): keyboard operable, visible focus, accessible names on icon controls, live regions for async status, and reduced motion honoured — scripted `scrollTo`/`scrollIntoView` go through `resolveScrollBehavior` (`src/lib/scroll-behavior.ts`), never a hard-coded `behavior: "smooth"`. - **Tests.** Add a `.dom.test.tsx` for changed component behaviour (see "Component tests" above) and update the E2E journeys for changed flows. - **Unlayered CSS.** If the change adds a class rule outside `@layer` that sets a border, background, colour, shadow or outline, `tests/style-contract-registry.test.ts` will fail until it is registered. Add a rendered-effect contract rather than an exemption where the rule matters visually — see "Visual regression and style contracts". -- **Verify** ([design-system §9](./design-system.md)): run `npm run verify:cheap`, then `npm run verify:pr-local` before handoff; run `npm run ensure` before browser work and `npm run verify:ui` for UI/routing/styling changes, plus a manual dark-mode + forced-colors spot check on touched surfaces. +- **Verify** ([design-system §9](./design-system.md)): follow the risk tiers in root `AGENTS.md`. Prove changed component behaviour with the focused DOM test first; run `npm run ensure` before browser work and use the narrowest affected journey. Select one appropriate broad handoff gate when the diff crosses owners, cannot be bounded, or applicable PR/handoff policy requires it; do not routinely stack `verify:cheap`, `verify:pr-local`, and `verify:ui`. Add a manual dark-mode + forced-colors spot check when those rendered states can plausibly change. - Architecture and state-ownership conventions: [`docs/frontend-architecture.md`](./frontend-architecture.md). diff --git a/package.json b/package.json index 55c6670f97..4083af3737 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "test:coverage": "node scripts/run-vitest.mjs run --coverage", "test:coverage:node": "node scripts/run-vitest.mjs run --project=node --coverage", "test:coverage:ui": "node scripts/run-vitest.mjs run --project=jsdom --coverage", + "test:ci-workflows": "node scripts/run-vitest.mjs run tests/ci-cache-safety.test.ts tests/authenticated-live-workflow.test.ts tests/codex-autofix-workflow.test.ts tests/eval-canary-workflow.test.ts tests/container-ci-contract.test.ts tests/test-runner-safety.test.ts tests/installed-lock-parity.test.ts tests/railway-config.test.ts tests/ingestion-autopilot.test.ts tests/check-lighthouse-budget.test.ts tests/offline-release-profile.test.ts", "test:e2e": "node scripts/run-playwright.mjs", "test:e2e:all": "node scripts/run-playwright.mjs", "test:e2e:accessibility": "node scripts/run-playwright.mjs tests/ui-accessibility.spec.ts --project=chromium", @@ -53,7 +54,7 @@ "clean:worktree": "node scripts/clean-worktree.mjs", "verify:preflight": "npm run check:installed-lock-parity && npm run typecheck && npm run verify:cheap && npm run clean:worktree", "verify:cheap": "npm run verify:cheap:internal", - "verify:cheap:internal": "npm run check:runtime && npm run check:installed-lock-parity && npm run check:upload-limit-parity && npm run check:github-actions && npm run check:ci-scope && npm run check:gitleaks-pinned && npm run check:ci-triage && npm run check:pr-policy && npm run check:gate-manifest && npm run check:branch-review-ledger && npm run check:outstanding-issues && npm run check:pr-mergeability && npm run sitemap:check && npm run docs:check-index && npm run docs:check-inventory && npm run docs:check-scripts && npm run docs:check-links && npm run check:knip && npm run check:maintainability-budgets && npm run brand:check && npm run check:assets && npm run check:therapy-data-index && npm run check:cross-mode-index && npm run check:type-scale && npm run check:icon-scale && npm run check:design-system-contract && npm run check:migration-role && npm run check:function-grants && npm run check:owner-scope && npm run lint && npm run typecheck && npm run test", + "verify:cheap:internal": "npm run check:runtime && npm run check:installed-lock-parity && npm run check:upload-limit-parity && npm run check:github-actions && npm run check:ci-scope && npm run check:verification-plan && npm run check:gitleaks-pinned && npm run check:ci-triage && npm run check:pr-policy && npm run check:gate-manifest && npm run check:branch-review-ledger && npm run check:outstanding-issues && npm run check:pr-mergeability && npm run sitemap:check && npm run docs:check-index && npm run docs:check-inventory && npm run docs:check-scripts && npm run docs:check-links && npm run check:knip && npm run check:maintainability-budgets && npm run brand:check && npm run check:assets && npm run check:therapy-data-index && npm run check:cross-mode-index && npm run check:type-scale && npm run check:icon-scale && npm run check:design-system-contract && npm run check:migration-role && npm run check:function-grants && npm run check:owner-scope && npm run lint && npm run typecheck && npm run test", "verify:pr-local": "node scripts/verify-pr-local.mjs", "verify:phone-chrome": "node scripts/verify-phone-chrome.mjs", "audit:final-merge": "node scripts/final-merge-audit.mjs", @@ -67,6 +68,7 @@ "check:knip:exports": "knip --no-progress --exports --no-exit-code", "check:maintainability-budgets": "node scripts/check-maintainability-budgets.mjs", "check:ci-scope": "node scripts/ci-change-scope.mjs --self-test", + "check:verification-plan": "node scripts/verify-pr-local.mjs --self-test", "check:gitleaks-pinned": "node scripts/run-gitleaks-pinned.mjs --self-test", "check:ci-triage": "node scripts/ci-triage.mjs --self-test", "check:gate-manifest": "node scripts/check-gate-manifest.mjs", diff --git a/scripts/check-format-changed.mjs b/scripts/check-format-changed.mjs index c8e99a857a..3be4c4d144 100644 --- a/scripts/check-format-changed.mjs +++ b/scripts/check-format-changed.mjs @@ -1,11 +1,35 @@ #!/usr/bin/env node -import { existsSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { childProcessExitCode } from "./child-process-result.mjs"; const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +/** + * Does this path decide Prettier's verdict for files other than itself? + * Mirrors `scripts/guard-push.mjs` so CI and the pre-push hook escalate the same + * policy-file set (a changed-paths-only check cannot see tree-wide drift). + */ +function carriesPrettierField(filePath) { + try { + return JSON.parse(readFileSync(filePath, "utf8")).prettier !== undefined; + } catch { + // Unparseable: assume it is policy rather than assume it is not. + return true; + } +} + +function isPrettierPolicyFile(filePath) { + const base = path.basename(filePath); + if (/^(?:\.prettierrc(?:\..+)?|prettier\.config\.(?:js|cjs|mjs|ts)|\.prettierignore|\.editorconfig)$/.test(base)) { + return true; + } + if (base !== "package.json") return false; + return carriesPrettierField(path.join(projectRoot, filePath)); +} + const scopeResult = spawnSync(process.execPath, ["scripts/ci-change-scope.mjs", "--json"], { cwd: projectRoot, encoding: "utf8", @@ -20,7 +44,18 @@ if (changedFiles.length === 0) { } const prettierBin = path.join(projectRoot, "node_modules", "prettier", "bin", "prettier.cjs"); -const result = spawnSync(process.execPath, [prettierBin, "--check", "--ignore-unknown", "--", ...changedFiles], { +const escalateToFullTree = changedFiles.some((file) => isPrettierPolicyFile(file)); +const prettierArgs = escalateToFullTree + ? ["--check", "--ignore-unknown", "."] + : ["--check", "--ignore-unknown", "--", ...changedFiles]; + +if (escalateToFullTree) { + console.log( + "Prettier policy file changed; escalating to whole-tree format check " + "(matches scripts/guard-push.mjs).", + ); +} + +const result = spawnSync(process.execPath, [prettierBin, ...prettierArgs], { cwd: projectRoot, stdio: "inherit", }); diff --git a/scripts/ci-change-scope.mjs b/scripts/ci-change-scope.mjs index 63e35d8493..f4e445ca52 100644 --- a/scripts/ci-change-scope.mjs +++ b/scripts/ci-change-scope.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node import { execFileSync } from "node:child_process"; -import { appendFileSync, readFileSync } from "node:fs"; +import { appendFileSync, existsSync, readFileSync } from "node:fs"; const zeroSha = /^0{40}$/; @@ -19,7 +19,9 @@ const fullRunSentinelFiles = [ const outputs = [ "docs_only", + "docs_changed", "source_changed", + "static_heavy_changed", "coverage_changed", "ui_changed", "advisory_ui_changed", @@ -27,9 +29,11 @@ const outputs = [ "container_changed", "rag_eval_changed", "workflow_changed", + "workflow_only", "codex_autofix_changed", "build_changed", "lockfile_changed", + "pr_policy_body_present", ]; function normalizePath(filePath) { @@ -242,20 +246,29 @@ const staticConfigPatterns = [ // Script-only `package.json` edits do not trip blocking audit (no lock churn). const lockfilePatterns = ["package-lock.json", ".npmrc"]; +/** Executable helpers under otherwise-light workflow/policy surfaces. */ +function isExecutableWorkflowSurfacePath(filePath) { + return /\.(?:mjs|cjs|js|ts|tsx|sh|bash|py)$/i.test(filePath); +} + +/** + * Recognised light paths may skip the heavy static/coverage route. Markdown and + * YAML/policy under workflow surfaces stay light; executable files there do not. + */ +function isRecognisedLightPath(filePath) { + if (pathMatches(filePath, docPatterns)) return true; + if (!pathMatches(filePath, workflowPatterns)) return false; + return !isExecutableWorkflowSurfacePath(filePath); +} + // The ledger read is injected so `classify` stays pure and the self-test can // drive both the empty and non-empty cases without touching the real file. const readFlakeLedger = () => readFileSync("tests/flake-ledger.json", "utf8"); -function classify(files, { readLedger = readFlakeLedger } = {}) { +function classify(files, { readLedger = readFlakeLedger, prPolicyBodyPresent = existsSync("PR_POLICY_BODY.md") } = {}) { const normalized = [...new Set(files.map(normalizePath).filter(Boolean))].sort(); + const docsChanged = normalized.some((file) => pathMatches(file, docPatterns)); const sourceChanged = normalized.some((file) => pathMatches(file, [...sourcePatterns, ...staticConfigPatterns])); - // Preserve the pre-consolidation unit gate for every non-documentation - // change. Narrower signals still scope build/UI/database work, but must not - // leave runtime, worker, workflow, or configuration changes without unit - // coverage. A workflow-only edit can change test setup or the - // coverage gate itself, so its ~4 minute proof is deliberate rather than an - // accidental over-trigger (#139). - const coverageChanged = normalized.some((file) => !pathMatches(file, docPatterns)); const uiChanged = normalized.some((file) => isUiChangedPath(file)); const advisoryUiChanged = normalized.some((file) => pathMatches(file, mockupPatterns)) || quarantineLedgerHasEntries(readLedger); @@ -266,16 +279,32 @@ function classify(files, { readLedger = readFlakeLedger } = {}) { const codexAutofixChanged = normalized.some((file) => pathMatches(file, codexAutofixPatterns)); const lockfileChanged = normalized.some((file) => pathMatches(file, lockfilePatterns)); const buildChanged = normalized.some((file) => pathMatches(file, buildPatterns)) || containerChanged; + // Only two categories are allowed to take the lightweight path: recognised + // documentation and recognised non-executable workflow/policy surfaces. + // Unknown non-doc files fail closed to the heavy plan. Executable files that + // also match a workflow pattern (for example `.agents/skills/**/scripts/*.mjs` + // or a workflow helper under `scripts/`) remain heavy even when the directory + // is otherwise treated as a light policy surface. + const hasUnknownNonLightPath = normalized.some((file) => !isRecognisedLightPath(file)); + const staticHeavyChanged = + sourceChanged || buildChanged || containerChanged || dbChanged || lockfileChanged || hasUnknownNonLightPath; + // Pure workflow YAML/policy changes use focused workflow-contract tests in + // static-pr. Product, test, build, database, dependency and unknown changes + // retain the complete coverage lane. + const coverageChanged = staticHeavyChanged; const docsOnly = normalized.length > 0 && normalized.every((file) => pathMatches(file, docPatterns)) && !sourceChanged && !workflowChanged; + const workflowOnly = workflowChanged && !staticHeavyChanged; return { files: normalized, docs_only: docsOnly, + docs_changed: docsChanged, source_changed: sourceChanged, + static_heavy_changed: staticHeavyChanged, coverage_changed: coverageChanged, ui_changed: uiChanged, advisory_ui_changed: advisoryUiChanged, @@ -283,9 +312,11 @@ function classify(files, { readLedger = readFlakeLedger } = {}) { container_changed: containerChanged, rag_eval_changed: ragEvalChanged, workflow_changed: workflowChanged, + workflow_only: workflowOnly, codex_autofix_changed: codexAutofixChanged, build_changed: buildChanged, lockfile_changed: lockfileChanged, + pr_policy_body_present: prPolicyBodyPresent, }; } @@ -442,8 +473,8 @@ function writeOutputs(result) { const emptyLedger = () => '{"flakes":[]}'; -function assertScope(name, files, expected, options = { readLedger: emptyLedger }) { - const result = classify(files, options); +function assertScope(name, files, expected, options = {}) { + const result = classify(files, { readLedger: emptyLedger, prPolicyBodyPresent: false, ...options }); for (const [key, value] of Object.entries(expected)) { if (result[key] !== value) { throw new Error(`${name}: expected ${key}=${value}, received ${result[key]} for ${files.join(", ")}`); @@ -550,13 +581,17 @@ function selfTest() { }, ); - assertScope("workflow-only-keeps-coverage", [".github/workflows/ci.yml"], { - coverage_changed: true, + assertScope("workflow-only-uses-focused-contracts", [".github/workflows/ci.yml"], { + coverage_changed: false, workflow_changed: true, + workflow_only: true, + static_heavy_changed: false, }); - assertScope("composite-action-only-keeps-coverage", [".github/actions/setup-ui-e2e/action.yml"], { - coverage_changed: true, + assertScope("composite-action-only-uses-focused-contracts", [".github/actions/setup-ui-e2e/action.yml"], { + coverage_changed: false, workflow_changed: true, + workflow_only: true, + static_heavy_changed: false, }); assertScope("runtime-config-keeps-coverage", ["lighthouse-budget.json"], { coverage_changed: true, @@ -568,7 +603,9 @@ function selfTest() { }); assertScope("docs-only", ["docs/process-note.md"], { docs_only: true, + docs_changed: true, source_changed: false, + static_heavy_changed: false, build_changed: false, lockfile_changed: false, }); @@ -576,6 +613,7 @@ function selfTest() { source_changed: true, coverage_changed: true, build_changed: false, + static_heavy_changed: true, }); assertScope("coverage-config", ["vitest.config.mts"], { source_changed: true, @@ -712,7 +750,9 @@ function selfTest() { ); assertScope("workflow", [".github/workflows/ci.yml", "docs/process-hardening.md"], { workflow_changed: true, - coverage_changed: true, + workflow_only: true, + coverage_changed: false, + static_heavy_changed: false, docs_only: false, build_changed: false, }); @@ -721,6 +761,7 @@ function selfTest() { source_changed: true, docs_only: false, build_changed: false, + static_heavy_changed: true, }); assertScope("repo-skill", [".agents/skills/database-flightplan/SKILL.md"], { workflow_changed: true, @@ -728,9 +769,24 @@ function selfTest() { // Skill Markdown is documentation-like: static policy checks still run, // but unit coverage has no executable product surface to measure. coverage_changed: false, + workflow_only: true, + static_heavy_changed: false, docs_only: false, build_changed: false, }); + assertScope( + "executable-skill-script-stays-heavy", + [".agents/skills/prompt-perfector/scripts/verify-repository-isolation.mjs"], + { + workflow_changed: true, + source_changed: false, + coverage_changed: true, + workflow_only: false, + static_heavy_changed: true, + docs_only: false, + build_changed: false, + }, + ); assertScope( "codex-autofix", [".github/workflows/codex-autofix-review-comments.yml", "AGENTS.md", "scripts/check-codex-autofix-workflow.mjs"], @@ -746,6 +802,7 @@ function selfTest() { container_changed: true, workflow_changed: false, build_changed: true, + static_heavy_changed: true, // Script/metadata edits to package.json alone do not introduce dependencies; // blocking audit still keys off package-lock.json / .npmrc. lockfile_changed: false, @@ -812,6 +869,7 @@ function selfTest() { }); assertScope("unknown-base-full-run", fullRunSentinelFiles, { source_changed: true, + static_heavy_changed: true, coverage_changed: true, ui_changed: true, db_changed: true, @@ -822,6 +880,18 @@ function selfTest() { build_changed: true, lockfile_changed: true, }); + assertScope("unknown-non-doc-fails-closed", ["custom.config"], { + docs_only: false, + workflow_only: false, + static_heavy_changed: true, + coverage_changed: true, + }); + assertScope( + "pr-policy-body-presence-is-routed-without-a-second-checkout", + ["PR_POLICY_BODY.md"], + { pr_policy_body_present: true }, + { prPolicyBodyPresent: true }, + ); console.log("CI change scope self-test passed."); } diff --git a/scripts/verify-pr-local.mjs b/scripts/verify-pr-local.mjs index 3a255a9733..5feb5e6c23 100644 --- a/scripts/verify-pr-local.mjs +++ b/scripts/verify-pr-local.mjs @@ -4,7 +4,28 @@ import { childProcessExitCode } from "./child-process-result.mjs"; const isWindows = process.platform === "win32"; // Live Supabase audits (check:locality-metadata) stay out of this unconditional gate. -const baseScripts = ["check:runtime", "check:installed-lock-parity", "format:changed", "lint", "typecheck", "test"]; +const commonScripts = ["check:runtime", "check:installed-lock-parity", "format:changed"]; +const docsScripts = [ + "sitemap:check", + "docs:check-index", + "docs:check-inventory", + "docs:check-scripts", + "docs:check-links", + "check:branch-review-ledger", + "check:outstanding-issues", +]; +const workflowScripts = [ + "check:github-actions", + "check:ci-scope", + "check:gitleaks-pinned", + "check:ci-triage", + "check:pr-policy", + "check:gate-manifest", + "check:pr-mergeability", + "check:verification-plan", + "test:ci-workflows", +]; +const staticHeavyScripts = ["lint", "typecheck", "test"]; function parseArgs(args) { const options = { dryRun: false, extended: false, files: undefined }; @@ -65,16 +86,60 @@ function readScope(files) { } function selectedScripts(scope, extended) { - const scripts = [...baseScripts]; + const scripts = []; + const add = (...items) => { + for (const item of items) if (!scripts.includes(item)) scripts.push(item); + }; + + add(...commonScripts); + if (scope.docs_changed) add(...docsScripts); + if (scope.workflow_changed) add(...workflowScripts); + if (scope.codex_autofix_changed) add("check:codex-autofix-workflow"); + if (scope.static_heavy_changed) add(...staticHeavyScripts); if (scope.build_changed) scripts.push("build"); - // Fixtures for every non-docs change; full offline RAG contracts when - // retrieval/answer surfaces are in scope (eval:rag:offline includes fixtures). - if (scope.rag_eval_changed) scripts.push("eval:rag:offline"); - else if (!scope.docs_only) scripts.push("check:rag:fixtures"); - if (extended && scope.ui_changed) scripts.push("verify:ui"); + // Full offline RAG contracts remain mandatory for retrieval/answer surfaces. + // Other executable changes retain the cheap fixture-integrity guard, while + // recognised docs and workflow-only changes avoid an unrelated RAG scan. + if (scope.rag_eval_changed) add("eval:rag:offline"); + else if (scope.static_heavy_changed) add("check:rag:fixtures"); + if (extended && scope.ui_changed) add("verify:ui"); return scripts; } +function assertPlan(name, scope, expected, extended = false) { + const actual = selectedScripts(scope, extended); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error(`${name}: expected ${expected.join(", ")}; received ${actual.join(", ")}`); + } +} + +function selfTest() { + assertPlan("docs-only", { docs_changed: true }, [...commonScripts, ...docsScripts]); + assertPlan("workflow-only", { workflow_changed: true }, [...commonScripts, ...workflowScripts]); + assertPlan("unknown-or-product-change-fails-heavy", { static_heavy_changed: true }, [ + ...commonScripts, + ...staticHeavyScripts, + "check:rag:fixtures", + ]); + assertPlan("rag-change", { static_heavy_changed: true, rag_eval_changed: true }, [ + ...commonScripts, + ...staticHeavyScripts, + "eval:rag:offline", + ]); + assertPlan( + "ui-extended", + { static_heavy_changed: true, ui_changed: true }, + [...commonScripts, ...staticHeavyScripts, "check:rag:fixtures", "verify:ui"], + true, + ); + console.log("PR-local verification plan self-test passed."); +} + +if (process.argv.includes("--self-test")) { + selfTest(); + process.exit(0); +} + const options = parseArgs(process.argv.slice(2)); const scope = readScope(options.files); const scripts = selectedScripts(scope, options.extended); @@ -83,8 +148,10 @@ console.log(`Changed files: ${scope.files.length > 0 ? scope.files.join(", ") : if (options.dryRun) { console.log("\nPR-local verification plan (dry run):"); for (const script of scripts) console.log(`- npm run ${script}`); + if (!scope.static_heavy_changed) + console.log("- lint, typecheck, full unit suite and RAG fixture scan skipped: recognised low-risk scope"); if (!scope.build_changed) console.log("- build skipped: no build-affecting changes detected"); - if (scope.docs_only) console.log("- offline RAG checks skipped: docs-only change"); + if (!scope.static_heavy_changed) console.log("- offline RAG checks skipped: no executable product scope"); else if (!scope.rag_eval_changed) console.log("- offline RAG production contracts skipped: no RAG-scoped changes (fixtures still selected)"); if (options.extended && !scope.ui_changed) diff --git a/tests/ci-cache-safety.test.ts b/tests/ci-cache-safety.test.ts index 037d54cea1..a192d7460f 100644 --- a/tests/ci-cache-safety.test.ts +++ b/tests/ci-cache-safety.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; const nodeSetup = readFileSync(new URL("../.github/actions/setup-node-cached/action.yml", import.meta.url), "utf8"); const uiSetup = readFileSync(new URL("../.github/actions/setup-ui-e2e/action.yml", import.meta.url), "utf8"); const workflow = readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"); +const opsDigestWorkflow = readFileSync(new URL("../.github/workflows/ops-digest.yml", import.meta.url), "utf8"); describe("CI cache safety", () => { it("uses npm's download cache but recreates node_modules on every job", () => { @@ -28,6 +29,75 @@ describe("CI cache safety", () => { expect(uiSetup).toMatch(/cache-hit.*?install-deps chromium.*?install chromium/s); expect(workflow).toMatch(/cache-hit.*?install-deps\n\s+npx playwright install/s); }); + + it("routes recognised workflow-only changes through focused contracts", () => { + expect(workflow).toContain("static_heavy_changed: ${{ steps.scope.outputs.static_heavy_changed }}"); + expect(workflow).toContain("workflow_changed: ${{ steps.scope.outputs.workflow_changed }}"); + expect(workflow).toContain("if: needs.changes.outputs.static_heavy_changed == 'true'"); + expect(workflow).toContain("run: npm run test:ci-workflows"); + expect(workflow).toContain("run: npm run check:verification-plan"); + }); + + it("keeps every workflow-reading unit contract in the focused suite", () => { + const packageJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")); + const focusedScript = packageJson.scripts["test:ci-workflows"] ?? ""; + const testsDirectory = new URL("./", import.meta.url); + const readers: string[] = []; + for (const name of readdirSync(testsDirectory).filter((entry) => entry.endsWith(".test.ts"))) { + const text = readFileSync(new URL(name, testsDirectory), "utf8"); + // Only count suites that load a committed workflow file — not incidental + // string mentions such as mock run paths in sync helpers. + const loadsWorkflow = + /new URL\(\s*["']\.\.\/\.github\/workflows\//.test(text) || + /read(?:FileSync)?\(\s*["']\.github\/workflows\//.test(text) || + /path\.resolve\(\s*["']\.github\/workflows\//.test(text) || + (/\.github["']\s*,\s*["']workflows["']/.test(text) && /readFileSync\(/.test(text)); + if (!loadsWorkflow) continue; + readers.push(`tests/${name}`); + } + const missing = readers.filter((file) => !focusedScript.includes(file)); + expect(missing, `add workflow-reading suites to test:ci-workflows: ${missing.join(", ")}`).toEqual([]); + }); + + it("runs ledger integrity checks when docs or heavy static scope changes", () => { + expect(workflow).toContain( + "if: needs.changes.outputs.docs_changed == 'true' || needs.changes.outputs.static_heavy_changed == 'true'", + ); + expect(workflow).toMatch( + /name: Branch review ledger integrity\n\s+(?:#[^\n]*\n\s+)*if: needs\.changes\.outputs\.docs_changed == 'true' \|\| needs\.changes\.outputs\.static_heavy_changed == 'true'/, + ); + expect(workflow).toMatch( + /name: Outstanding-issues ledger integrity\n\s+if: needs\.changes\.outputs\.docs_changed == 'true' \|\| needs\.changes\.outputs\.static_heavy_changed == 'true'/, + ); + }); + + it("forwards stale eval-canary status into the Ops Digest alert path", () => { + expect(opsDigestWorkflow).toContain("id: liveness"); + expect(opsDigestWorkflow).toContain('core.setOutput("stale"'); + expect(opsDigestWorkflow).toContain("needs: eval-canary-liveness"); + expect(opsDigestWorkflow).toContain("EVAL_CANARY_STALE"); + expect(opsDigestWorkflow).toContain("canaryStale"); + }); + + it("avoids unrelated network and checkout work on ordinary pull requests", () => { + expect(workflow).not.toContain("Dependency audit (advisory)"); + expect(workflow).not.toContain("github.rest.actions.listWorkflowRuns"); + expect(opsDigestWorkflow).toContain("eval-canary-liveness:"); + expect(opsDigestWorkflow).toContain("github.rest.actions.listWorkflowRuns"); + expect(workflow).toContain( + "if: github.event_name == 'pull_request' && needs.changes.outputs.pr_policy_body_present == 'true'", + ); + }); + + it("checks formatting only on the changed range in pull-request CI", () => { + expect(workflow).toContain("name: Changed-file format check"); + expect(workflow).toContain("if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch'"); + expect(workflow).toContain("run: npm run format:changed"); + expect(workflow).toContain("BASE_SHA: ${{ github.event.pull_request.base.sha"); + expect(workflow).toMatch( + /name: Scheduled full-tree format drift\s+if: github\.event_name == 'schedule' \|\| github\.event_name == 'workflow_dispatch'\s+run: npm run format:check/, + ); + }); }); /* @@ -61,7 +131,7 @@ describe("PR required aggregate — cancelled vs failed (#095)", () => { })(); const allGreen = { - DOCS_ONLY: "false", + STATIC_HEAVY_CHANGED: "false", COVERAGE_CHANGED: "false", UI_CHANGED: "false", DB_CHANGED: "false", @@ -70,8 +140,8 @@ describe("PR required aggregate — cancelled vs failed (#095)", () => { EVENT_NAME: "pull_request", CHANGES_RESULT: "success", STATIC_RESULT: "success", - // `safety` is required whenever DOCS_ONLY is false, so the green baseline must run it. - SAFETY_RESULT: "success", + // Recognised documentation/workflow-only scopes skip the heavy safety job. + SAFETY_RESULT: "skipped", COVERAGE_RESULT: "skipped", BUILD_RESULT: "skipped", CONTAINER_RESULT: "skipped", @@ -112,6 +182,12 @@ describe("PR required aggregate — cancelled vs failed (#095)", () => { expect(runAggregate().status).toBe(0); }); + it("requires safety for heavy scope and accepts a skip only for recognised light scope", () => { + expect(runAggregate({ STATIC_HEAVY_CHANGED: "true", SAFETY_RESULT: "success" }).status).toBe(0); + expect(runAggregate({ STATIC_HEAVY_CHANGED: "true", SAFETY_RESULT: "skipped" }).status).not.toBe(0); + expect(runAggregate({ STATIC_HEAVY_CHANGED: "false", SAFETY_RESULT: "skipped" }).status).toBe(0); + }); + it("reports a superseded run as CANCELLED rather than describing a failure", () => { // A supersession cancels the upstream jobs, so this is what a real one looks like. const { status, output } = runAggregate({ CHANGES_RESULT: "cancelled", STATIC_RESULT: "cancelled" }); @@ -150,6 +226,7 @@ describe("PR required aggregate — cancelled vs failed (#095)", () => { * failures must win, and a concurrent cancellation may only appear as context. */ const { status, output } = runAggregate({ + STATIC_HEAVY_CHANGED: "true", SAFETY_RESULT: "cancelled", BUILD_CHANGED: "true", BUILD_RESULT: "failure", @@ -183,7 +260,7 @@ describe("PR required aggregate — cancelled vs failed (#095)", () => { for (const key of ["CHANGES_RESULT", "STATIC_RESULT"]) { expect(runAggregate({ [key]: "cancelled" }).status).not.toBe(0); } - expect(runAggregate({ DOCS_ONLY: "false", SAFETY_RESULT: "cancelled" }).status).not.toBe(0); + expect(runAggregate({ STATIC_HEAVY_CHANGED: "true", SAFETY_RESULT: "cancelled" }).status).not.toBe(0); expect(runAggregate({ COVERAGE_CHANGED: "true", COVERAGE_RESULT: "cancelled" }).status).not.toBe(0); expect( runAggregate({