Harden the eval-canary safeguards: per-case rr diff, dispatch failure issues, contract pins

.github/workflows/eval-canary.yml

@@ -197,20 +197,34 @@ jobs:
           retention-days: 30
           if-no-files-found: ignore
 
+      # Dispatch failures matter as much as scheduled ones — a red dispatch is usually one
+      # half of a canary pair gating a behaviour change, so it must not fail silently.
       - name: Open or update canary failure issue
-        if: failure() && github.event_name == 'schedule'
+        if: failure() && (github.event_name == 'schedule' || github.event_name == 'repository_dispatch')
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
             const fs = require("node:fs");
             const path = require("node:path");
             const { pathToFileURL } = require("node:url");
-            const { analyzeFailureText } = await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/scripts/productivity-core.mjs`).href);
+            // If checkout failed, the analyzer module is absent; the issue must still open,
+            // so fall back to a fixed diagnosis instead of letting the import throw.
+            let analyzeFailureText = () => ({
+              category: "unclassified",
+              confidence: "none",
+              reason: "failure analyzer unavailable (checkout likely failed before scripts/ existed); inspect step outcomes",
+            });
+            try {
+              ({ analyzeFailureText } = await import(pathToFileURL(`${process.env.GITHUB_WORKSPACE}/scripts/productivity-core.mjs`).href));
+            } catch (error) {
+              core.warning(`analyzeFailureText unavailable, using fallback diagnosis: ${error.message}`);
+            }
             const label = "eval-canary";
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
             const logPaths = [
               ".local/eval-canary/golden-retrieval.log",
               ".local/eval-canary/answer-quality.log",
+              ".local/eval-canary/answer-targeting.log",
             ].map((item) => path.join(process.env.GITHUB_WORKSPACE, item));
             const failureText = logPaths
               .filter((item) => fs.existsSync(item))
@@ -222,12 +236,15 @@ jobs:
               preflight: "${{ steps.preflight.outcome }}",
               setup_node: "${{ steps.setup_node.outcome }}",
               install: "${{ steps.install.outcome }}",
+              validate_override: "${{ steps.validate_override.outcome }}",
               project_guard: "${{ steps.project_guard.outcome }}",
               golden_retrieval: "${{ steps.golden_retrieval.outcome }}",
               answer_quality: "${{ steps.answer_quality.outcome }}",
+              answer_targeting: "${{ steps.answer_targeting.outcome }}",
             };
+            const trigger = context.eventName === "schedule" ? "Weekly (scheduled)" : "Dispatched";
             const body = [
-              `Weekly eval canary failed on ${new Date().toISOString()}.`,
+              `${trigger} eval canary failed on ${new Date().toISOString()}.`,
               "",
               `Run: ${runUrl}`,
               "",
@@ -257,7 +274,7 @@ jobs:
               await github.rest.issues.create({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                title: "Eval canary failure: weekly evaluation did not complete",
+                title: "Eval canary failure: evaluation did not complete",
                 labels: [label],
                 body,
               });

docs/branch-review-ledger.md

@@ -899,4 +899,5 @@ Records before 2026-07-28 were written by hand and had drifted: 146 lines carrie
 | 2026-08-11 | claude/spacing-icon-design-review-rxwh28 | 5b96281ee7da817d5ce7f1102004ebe6f861b920 | pr-1815 heavy review-and-fix | remote already merged main (shadow-tight Switch kept); cherry-picked privacy -mb-4 reclaim + calculators dock cancel; removed duplicate UniversalSearchAlsoMatches; rail-aware section-sheet focus restore; dispositioned CodeRabbit docs/ledger/gates nits and outdated Sentry skeleton gap | verify:cheap PASS prior tip; verify:pr-local PASS prior tip; vitest privacy+in-page-nav 28 passed on cherry-pick; merge-tree clean vs origin/main |
 | 2026-08-12 | PR #1815 / claude/spacing-icon-design-review-rxwh28 | 9f266210f02081be54d407c70a85f52fed436128 | babysit | no remaining actionable findings; one pre-existing thread resolved as no-change (Dockerfile.worker follow-up needed) | required checks: Gitleaks PR policy PR required (all pass); targeted vitest passed: tests/document-frame-contract.test.ts + tests/in-page-nav-header.dom.test.tsx |
 | 2026-08-12 | 1815 | 27ce96e1755055ceee2eeae02d6efdf11259fcde | babysit | fixed | Unit coverage: targeted vitest passed: tests/shared-home-empty-state.dom.test.tsx (17 passed). PR required still blocked on pre-existing check failure at old remote head before sync. |
+| 2026-08-12 | claude/rag-canary-test-review-seprbt | bcf357a96fde74d39fc4726ffabb5079a744ef28 | eval-canary review: workflow, compare tooling, snapshot builder, alias tiering, rag-behaviour docs | PR #1843 opened; no retrieval behaviour change; snapshot refresh handed off as /issues #304 | verify:pr-local (green except env-only #296), eval:rag:offline 574/574, focused suites 40/40 |
 | 2026-08-12 | codex/pr-workflow-safety-230-296 | bc0a491fdf4146775629f9b2b03e2a2cc61bd7cb | pr-1830 unblock | unblocked: merged origin/main (outstanding-issues conflict), PR body RAG impact + governance, resolved Copilot thread; merge-tree clean; required CI in progress | check:outstanding-issues pass; evaluatePullRequestPolicy ok; merge-tree clean; PR policy/mergeability/Change scope in progress |

docs/outstanding-issues.md

@@ -163,7 +163,7 @@ removed after current-main verification; it is not missing recommended work.
 | 108 | `#289` | A2 | High — auth/identity | Next auth module touch | 1–2 hours | Export a named helper (e.g. `authorizationIdentity(headers)`) from the auth module and use it at every property-access call site; consider a lint rule or branded type so `.Authorization` stops type-checking at all. **Stop:** do not change `authorizationHeadersForAccessToken` to emit uppercase — lowercase is the correct Fetch/Headers convention and callers that pass the object wholesale to `fetch` depend on it. |
 
 
-<!-- issues:next-id=304 -->
+<!-- issues:next-id=308 -->
 ## Open items
 
 > **Merged-main canary update (2026-07-23, run `30018289898`):** the new structured report correctly recorded evaluated tree `c24f2e8f2d30d0c59fc1eba025d3dcd63478137e`, run/attempt identity and `cross-region-runner` latency context. Golden retrieval remained 36/36 with document/content recall 1.0 and no failed cases. The 44-case answer gate had grounded-supported and unsupported-correct rates of 1.0, but failed because `neuroleptic-side-effect-escalation` again returned one citation where two are required (citation-failure rate 0.0227). `admission-discharge-comparison` again omitted the specific AKG admission document after `comparison_source_extractive_fallback`; `admission-discharge-coverage-paraphrase` was advisory-only at 24,870 ms. Answer cost was reported as `$0.234736`. Do not retry immediately: retain this as the first structured datapoint, compare it with the scheduled 2026-07-26 report, and keep retrieval/ranking unchanged.
@@ -320,7 +320,10 @@ removed after current-main verification; it is not missing recommended work.
 | #301 | P3 | issue | Two sessions built #262 part 3 in parallel because the GATES.md row understated what had shipped | On 2026-08-09 two branches implemented the same raw-value ratchet independently. PR #1780 landed rawPaddingLiterals/rawRadiusLiterals/rawLineHeightLiterals; a concurrent session built arbitraryPadding/arbitraryGap/arbitraryRadius/arbitraryLeading against the same four files and discovered the collision only when syncing before PR. The duplicate was dropped and only the uncovered gap family was rebuilt on #1780's predicate (rawGapLiterals, 34 sites). Root cause is the same failure this document keeps producing: the §3 row read 'Contract ratchet \| implemented-partial (colour/shadow/tap literals only)' and named none of the metrics #1780 had just shipped, so the row still advertised the work as unstarted. Identical to the 2026-08-09 finding that four of #264's six prohibitions were already gated while their rows read 'planned'. Both rows are corrected now. Next action: when a gate lands, update its §3 row IN THE SAME COMMIT - a row that understates shipped work is not a stale doc, it is a duplicate-work generator. Consider asserting in a test that every metric key in design-system-contract-baseline.json appears somewhere in GATES.md. Stop: do not rely on the ledger alone to prevent this - both sessions had ledger access. | session 2026-08-09 M4; PR #1780 collision | 2026-08-09 |
 | #302 | P3 | rec | Design-system contract ratchets re-accumulate slack because paying debt down does not re-pin the ceiling | On 2026-08-10 the legacyShadowAliases ceiling in scripts/design-system-contract-baseline.json read 220 against a measured 217, so three files could each have gained an alias without failing. Ledger #264 corrected exactly this on 2026-08-09 (edgeOwnershipConflicts 28 to 27, legacyShadowAliases 231 to 224) and it had already re-accumulated one day later. The mechanism is structural, not a one-off: a metric only moves when someone hand-edits the baseline, so every paydown that forgets to re-pin leaves headroom, and nothing in the gate output shows the gap - the check prints the measured value and passes silently while under the ceiling. Per-path pins limit the blast radius but do not close it, since a path whose measured count fell below its pin still carries per-file headroom. Next action: have check:design-system-contract print measured-vs-baseline and the resulting slack per metric, and consider failing when total slack crosses a small threshold, so a forgotten re-pin is visible in the gate rather than found by the next person who measures. Stop: do not auto-write the baseline from measured inside the check - that direction silently absorbs a real regression instead of reporting it. | session 2026-08-10 shadow-tight retirement (PR #1803) | 2026-08-10 |
 | #303 | P3 | issue | ledger:append rejects any flag value that starts with a double-dash token, which is every design-token name | npm run ledger:append -- --scope "--shadow-tight migration onto the elevation ladder" fails with 'missing required flag(s): --scope'. The arg parser reads the token after --scope, sees it begin with a double dash, and treats it as the next flag rather than the value, so the required flag reads as absent. Hit on 2026-08-10 recording PR #1803; the workaround was to rewrite the prose so no value begins with a token name, which drops the exact CSS custom-property identifier from the permanent record - the one thing a token-retirement row most needs to name. Every future design-token ledger row hits this, and the error message points at the wrong cause (it reads as a forgotten flag, not a swallowed value). Next action: take the argv entry immediately after a required flag verbatim, or accept the --scope=<value> spelling; check whether scripts/outstanding-issues.mjs shares the parser before fixing only one. Stop: do not settle for a docs note telling authors to avoid leading token names - that is what already costs the identifier. | session 2026-08-10 shadow-tight retirement (PR #1803) | 2026-08-10 |
-
+| #306 | P3 | issue | `tests/helpers/style-contracts.ts` contains escaped line-break artifacts in the exemption map | `smart-search-phone-ticker*` entries were merged with literal backtick-`r`n escapes, which makes the style-exemptions object invalid for the required parse and blocks local checks. Cleanly split each ticker exemption to one line and keep the same reason text so the exception intent is preserved. | PR #1815 unblock follow-up (`tests/helpers/style-contracts.ts`) | 2026-08-11 |
+| #307 | P3 | task | `issues:next-id` is out of sync with declared rows | The outstanding-issues marker is `issues:next-id=302` with no `#302`/`#303` rows in either open or resolved tables, which `check:outstanding-issues` flags as missing-issue failures. Add both rows and bump marker to `304` to keep the ledger monotonic. | `docs/outstanding-issues.md` | 2026-08-11 |
+| #304 | P2 | task | Canary snapshot refresh blocked in cloud sessions: dispatch needs operator gh identity | The ranking snapshot (scripts/fixtures/rag-ranking-candidate-snapshot.v1.json, generatedAt 2026-07-20, sourceRunId 29763761133) trips the 30-day freshness gate in tests/ranking-tuning.test.ts on ~2026-08-19, going red across the whole unit suite. Refresh requires an eval-canary repository dispatch (approved by the user 2026-08-12), but eval-canary.yml deliberately accepts only repository_dispatch and the cloud GitHub MCP toolset has no repository-dispatch call, so the dispatch must come from an operator identity: gh api repos/bigsimmo/database/dispatches -f event_type=eval-canary. After the run is green, download the eval-canary-output artifact and run npm run build:ranking-snapshot -- --input <golden-retrieval.json> --output scripts/fixtures/rag-ranking-candidate-snapshot.v1.json --source-run-id <run-id>, then confirm tests/ranking-tuning.test.ts passes. Provider-backed (~$1-2), already user-approved for this refresh. | session 2026-08-12 RAG canary review | 2026-08-12 |
+| #305 | P3 | rec | Canary has no latency-mode coverage and its cost readout is a known lower bound | Two informational gaps from the 2026-08-12 canary review, deferred by scope decision. (1) eval:retrieval:latency (p90 20s gate) is never wired into eval-canary.yml, so live retrieval latency regressions are invisible to the weekly canary while the answer step relaxes its own gates via EVAL_LATENCY_CONTEXT=cross-region-runner. (2) estimated_cost_usd applies one rate set (gpt-5.6-terra) to all usage including 2x-priced strong-model retries, so any cost trend understates strong-retry runs — the workflow comments say so, but eval:trend consumers may not read them. Also noted: the workflow-wide concurrency group (eval-canary, cancel-in-progress false) can queue a dispatched pair run behind a scheduled run, interleaving pair evidence; and fixture coverage gaps tracked in #018 remain uncatchable by the canary. Next: decide whether a monthly latency-mode dispatch is worth the spend; add a strong-usage split to the estimator if cost trends start driving decisions. | session 2026-08-12 RAG canary review | 2026-08-12 |
 
 ## Resolved / archive
 

docs/rag-behaviour/README.md

@@ -18,8 +18,10 @@ Standing rules (mirrored in `AGENTS.md` so every agent session inherits them):
    `scripts/pr-policy.mjs` — the check fails without it).
 2. **Canary for behaviour changes.** Any retrieval/ranking/ordering behaviour change requires a
    live eval-canary pair — baseline + post — with doc/content recall pinned at 1.0 and zero
-   per-case regressions, before the change is trusted. `workflow_dispatch` on
-   `eval-canary.yml` (provider-backed → explicit user approval per run).
+   per-case regressions, before the change is trusted. Runs fire via an `eval-canary`
+   repository dispatch (`eval-canary.yml` deliberately has no `workflow_dispatch` and no
+   `ref` input — it always loads from the default branch; provider-backed → explicit user
+   approval per run).
 3. **Offline green is necessary, never sufficient.** The Phase C regression passed 121/121
    offline tests and an adversarial code review, then failed 3/36 live within one run. The live
    corpus is the only authority on ordering behaviour.

docs/rag-behaviour/safeguards.md

@@ -64,8 +64,19 @@ keys above relevance, read this folder first. **Also:** do not add `review_due`
 refuted (`refuted-approaches.md` § Refutation 3; ledger `#032`). The eval-canary pair protocol:
 
 1. Baseline: latest green canary on current main (or one dispatch).
-2. Change merges (or runs from a branch via the `ref` dispatch input).
-3. Post: one dispatch; gates = recall 1.0/1.0, zero per-case rr regressions.
+2. Change merges to main. (Post runs are post-merge only: the workflow accepts only the
+   `eval-canary` repository dispatch and always loads code from the default branch — no
+   `workflow_dispatch`, no `ref` input; both are asserted absent by
+   `tests/eval-canary-workflow.test.ts`.)
+3. Post: one dispatch; gates = recall 1.0/1.0, zero per-case rr regressions. Compare the
+   pair's `--json-out` artifacts — the command diffs per-case `rr@10`/`contentRR@10` and
+   exits non-zero on any per-case regression, a non-identical case set, or an unavailable
+   rank metric:
+
+   ```bash
+   npm run eval:retrieval:compare -- <baseline.json> <post.json> --fail-on-regression
+   ```
+
 4. Regression → immediate single-commit revert + one confirmation dispatch.
 
 Provider-backed dispatches always need explicit user approval (~$1–2 each).

scripts/build-ranking-snapshot.ts

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { readFileSync, writeFileSync } from "node:fs";
 import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
 import type { RagQueryClass } from "../src/lib/types";
 import {
   candidateFeatures,
@@ -78,7 +79,7 @@ const hardNegativeTemplates: Array<{
   features: RankingCandidateFeatures;
 }>;
 
-function convertArtifact(artifact: RetrievalArtifact, sourceRunId?: string): RankingSnapshot {
+export function convertArtifact(artifact: RetrievalArtifact, sourceRunId?: string): RankingSnapshot {
   // Floor, not an exact pin: the golden fixture only ever grows, and a short artifact means a
   // truncated or filtered eval run that must not silently become the tuner's ground truth.
   if (!Array.isArray(artifact.results) || artifact.results.length < 36) {
@@ -88,6 +89,18 @@ function convertArtifact(artifact: RetrievalArtifact, sourceRunId?: string): Ran
       })`,
     );
   }
+  // Templates attach by caseId filter, so a renamed golden case would silently drop its
+  // hard negatives and erode the below-threshold protection the tuner's floor depends on.
+  const artifactCaseIds = new Set(artifact.results.map((testCase) => testCase.id));
+  const orphanTemplateCaseIds = [
+    ...new Set(hardNegativeTemplates.map((item) => item.caseId).filter((caseId) => !artifactCaseIds.has(caseId))),
+  ];
+  if (orphanTemplateCaseIds.length > 0) {
+    throw new Error(
+      `Hard-negative template caseId(s) match no artifact case: ${orphanTemplateCaseIds.join(", ")}. ` +
+        "A renamed or removed golden case must update hardNegativeTemplates in the same change.",
+    );
+  }
   const cases = artifact.results.map((testCase) => {
     const expectedDocuments = testCase.expectedDocumentSubstrings ?? [];
     const expectedContent = testCase.expectedContentTerms ?? [];
@@ -156,4 +169,7 @@ function main() {
   console.log(JSON.stringify({ output: resolve(output), cases: snapshot.cases.length, version: snapshot.version }));
 }
 
-main();
+// Guarded so unit tests can import convertArtifact without executing the CLI.
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  main();
+}