diff --git a/.github/workflows/live-drift.yml b/.github/workflows/live-drift.yml index 56247f5967..b696115962 100644 --- a/.github/workflows/live-drift.yml +++ b/.github/workflows/live-drift.yml @@ -1,12 +1,20 @@ name: Live drift check # Live Supabase drift detection is useful, but it must not slow PR iteration or -# touch live services on every branch. Keep it scheduled/manual only. +# touch live services on every branch. Keep it off pull requests entirely: it runs +# on a schedule, on demand, and once a schema change actually reaches main. on: workflow_dispatch: schedule: # Weekly, aligned with the existing Sunday off-peak CI cadence. - cron: "30 18 * * 0" + push: + # A migration landing on main is the event most likely to cause drift, so + # check within minutes of it rather than waiting up to a week for the cron. + branches: [main] + paths: + - "supabase/migrations/**" + - "supabase/schema.sql" concurrency: group: live-drift-check @@ -15,17 +23,20 @@ concurrency: permissions: contents: read -env: - NEXT_PUBLIC_SUPABASE_URL: https://sjrfecxgysukkwxsowpy.supabase.co - SUPABASE_PROJECT_REF: sjrfecxgysukkwxsowpy - SUPABASE_PROJECT_NAME: Clinical KB Database - NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key - SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} - jobs: live-drift: runs-on: ubuntu-24.04 timeout-minutes: 20 + env: + NEXT_PUBLIC_SUPABASE_URL: https://sjrfecxgysukkwxsowpy.supabase.co + SUPABASE_PROJECT_REF: sjrfecxgysukkwxsowpy + SUPABASE_PROJECT_NAME: Clinical KB Database + NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} + outputs: + # Consumed by drift-routing below. Job outputs survive a failed job as long + # as the step that set them ran, which is why the capture step is always(). + findings: ${{ steps.findings.outputs.findings }} steps: - name: Checkout @@ -56,7 +67,126 @@ jobs: run: npm run check:supabase-project - name: Compare live schema drift - run: npm run check:drift + id: drift + run: | + # The default step shell is `bash -e`, which has no pipefail, so a + # failing check:drift would be masked by tee's exit code without this. + set -o pipefail + npm run check:drift 2>&1 | tee drift-output.txt + + - name: Capture drift findings + id: findings + if: always() + run: | + # Line shapes come from scripts/check-drift.ts: the manifest/compared + # context lines, the "! [category] kind key" unexpected-drift findings, + # and the "? [category]" stale allowlist entries. Absent when the run + # died before the comparison (missing secret, identity guard); the + # routing job says so rather than implying a clean result. + { + echo "findings<> "$GITHUB_OUTPUT" - name: Align migration history for Supabase Preview run: npm run check:migration-history + + # Split out so issues: write is scoped to a job that never installs or executes + # repository code — the drift job above keeps contents: read only. + drift-routing: + needs: live-drift + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + issues: write + + steps: + - name: Route drift status to the pinned issue + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + DRIFT_RESULT: ${{ needs.live-drift.result }} + DRIFT_FINDINGS: ${{ needs.live-drift.outputs.findings }} + with: + script: | + const title = "Live drift check failing"; + const label = "live-drift-failure"; + const owner = context.repo.owner; + const repo = context.repo.repo; + const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; + const result = process.env.DRIFT_RESULT || "unknown"; + const findings = (process.env.DRIFT_FINDINGS || "").trim(); + const now = new Date().toISOString(); + + // One pinned issue, found by label so a retitle by a human does not + // orphan it and start a second thread. + const { data: open } = await github.rest.issues.listForRepo({ + owner, + repo, + state: "open", + labels: label, + }); + const existing = open.find((issue) => issue.title === title) ?? open[0]; + + if (result === "success") { + if (!existing) { + core.info("Live drift check is green and no failure issue is open."); + return; + } + await github.rest.issues.createComment({ + owner, + repo, + issue_number: existing.number, + body: `Resolved — the live drift check is green again as of ${now}.\n\nRun: ${runUrl}`, + }); + await github.rest.issues.update({ + owner, + repo, + issue_number: existing.number, + state: "closed", + state_reason: "completed", + }); + core.info(`Closed #${existing.number} after a green live drift run.`); + return; + } + + const body = [ + "The live Supabase drift check is failing. This issue is updated in place by", + "`.github/workflows/live-drift.yml`; it is closed automatically on the next green run.", + "", + `- Latest failing run: ${runUrl}`, + `- Job result: \`${result}\``, + `- Trigger: \`${context.eventName}\``, + `- Last updated: ${now}`, + "", + findings + ? `### Drift findings\n\n\`\`\`\n${findings}\n\`\`\`` + : "### Drift findings\n\nNone captured — the run failed before the drift comparison ran, so this is **not** evidence of a clean schema. Read the run log linked above.", + "", + "Remediation context: `docs/database-remediation-plan.md`, `docs/database-drift-detection.md`.", + "Never fix drift with raw SQL against live — codify it (migration + `schema.sql` mirror + regenerated manifest).", + ].join("\n"); + + if (existing) { + await github.rest.issues.update({ owner, repo, issue_number: existing.number, body }); + await github.rest.issues.createComment({ + owner, + repo, + issue_number: existing.number, + body: `Still failing as of ${now} — ${runUrl}`, + }); + core.warning(`Live drift check failing; updated #${existing.number}.`); + } else { + const created = await github.rest.issues.create({ + owner, + repo, + title, + labels: [label], + body, + }); + core.warning(`Live drift check failing; opened #${created.data.number}.`); + } diff --git a/docs/audit/live-drift-forensics-2026-08.md b/docs/audit/live-drift-forensics-2026-08.md index 1e8e5464ff..19e0317ca4 100644 --- a/docs/audit/live-drift-forensics-2026-08.md +++ b/docs/audit/live-drift-forensics-2026-08.md @@ -3,3 +3,87 @@ Evidence record for the phased database remediation plan and playbook. No hosted reads or mutations were performed while creating this file. Add dated, source-linked evidence here as each approved phase completes. + +**Tracking anchor:** ledger `#316` — "Live DB is missing 21 repo-defined indexes and 10 retrieval +RPC bodies diverge; weekly live-drift has been red since 2026-07-26 with no routing". Update it via +`npm run issues:update` at the end of every phase; never hand-edit `docs/outstanding-issues.md`. + +**Plan of record:** [`docs/database-remediation-plan.md`](../database-remediation-plan.md) and +[`docs/database-remediation-playbook.md`](../database-remediation-playbook.md). Read both before +adding to this file. + +**How to use this file.** Every section below is deliberately empty until its phase runs inside an +approved hosted window. Record the decisive output — pasted lines, run IDs, dates — not a summary +and not an exit code. An empty section means the phase has not run; it never means the phase found +nothing. Leave a section empty rather than filling it from inference. + +## Phase 0 — Enablement (repo-side, no hosted access) + +_2026-08-14._ Drift-failure routing and the post-migration trigger landed in +`.github/workflows/live-drift.yml`: a failed run now creates or updates a single pinned issue +titled "Live drift check failing" (label `live-drift-failure`) carrying the captured finding lines +and the run URL, and the next green run comments the resolution and closes it. The workflow also +runs on pushes to `main` touching `supabase/migrations/**` or `supabase/schema.sql`. Schedule, +`workflow_dispatch`, the secret preflight, and `concurrency.cancel-in-progress: false` were kept +unchanged. No hosted Supabase call was made. + +Outstanding for the operator: dispatch `live-drift` once to confirm a real failure produces the +pinned issue (provider-backed — not run from the authoring session), and add +`SUPABASE_ACCESS_TOKEN` to environment secrets per plan step 0.3 and ledger `#183`. + +## Phase 1 — Read-only forensics + +_Not yet run. Requires an approved read-only production window._ + +### 1.1 Migration-history fingerprint + +_Pending._ Record every `statements IS NULL` version with its name, and state explicitly whether +`20260705180000` carries that signal — then pair it with audit history to distinguish a +mark-applied/repair history from indexes that were created and later dropped. Do not close `#248` +on the fingerprint alone. + +### 1.2 RPC divergence dossier + +_Pending._ One entry per mismatched `match_*` function, each classified **live-ahead**, +**repo-ahead**, **normalization noise**, or **UNCLASSIFIED**, quoting the decisive diff hunk. +Protected RAG surface: an ambiguous diff is recorded as UNCLASSIFIED and escalated, never guessed. + +### 1.3 Index inventory, sizing, and EXPLAIN baselines + +_Pending._ Owning-table `pg_relation_size` for the 21 missing and 2 unexpected indexes, plus +`EXPLAIN (ANALYZE, BUFFERS)` baselines for the `documents` title ILIKE query, the `document_chunks` +content search, and the `rag_retrieval_logs` miss scan. These are the before-measurements for +Phases 4 and 5. + +## Phase 2 — Staging parity rehearsal + +_Not yet run. Requires an approved staging window; production stays read-only._ + +_Pending._ Migration-replay tail, any migration that misbehaved on clean replay (a finding in its +own right), and the green `check:drift` output against staging. + +## Phase 3 — RPC reconciliation + +_Not yet run. Requires an approved production window, plus a separate canary approval per +repo-ahead RPC._ + +_Pending._ Per-RPC outcome against the Phase 1.2 classification, the migration that codified each +live-ahead body, and eval-canary evidence (36/36, recall 1.0, zero per-case rr regressions) for any +behaviour-changing deploy. + +## Phase 4 — Index restoration + +_Not yet run. Requires an approved off-peak production window._ + +_Pending._ PITR restore point, per-index `CREATE INDEX CONCURRENTLY` result with its +`indisvalid`/`indisready` verification, disposition of the 2 unexpected live indexes with reasons, +the guard migrations landed, and the green live-drift dispatch output. + +## Phase 5 — Measure and close the loop + +_Not yet run. Requires a read-only production window (plus eval approval only if Phase 3 changed +behaviour)._ + +_Pending._ Before/after `EXPLAIN` table against the Phase 1.3 baselines showing plan flips and +timings, the evidence-backed verdict on ledger `#231`'s 25 s fast-route budget, and the +`check:production-readiness` output. diff --git a/docs/outstanding-issues-inbox/17e1baf4-7d8f-4494-acd6-a845ade305ca.json b/docs/outstanding-issues-inbox/17e1baf4-7d8f-4494-acd6-a845ade305ca.json new file mode 100644 index 0000000000..76bf2e933d --- /dev/null +++ b/docs/outstanding-issues-inbox/17e1baf4-7d8f-4494-acd6-a845ade305ca.json @@ -0,0 +1,10 @@ +{ + "version": 1, + "id": "17e1baf4-7d8f-4494-acd6-a845ade305ca", + "createdOn": "2026-08-14", + "action": "update", + "payload": { + "id": "#316", + "detail": "Phase 0 delivered — drift routing + post-migration trigger + evidence scaffold, PR #1938. live-drift.yml now creates/updates one pinned issue 'Live drift check failing' (label live-drift-failure) with the captured finding lines and run URL on failure, and comments+closes it on the next green run; issues: write is scoped to a separate drift-routing job so the job running npm ci keeps contents: read. The workflow also runs on pushes to main touching supabase/migrations/** or supabase/schema.sql. docs/audit/live-drift-forensics-2026-08.md now carries dated empty Phase 1-5 evidence sections anchored here. Still outstanding: a forced workflow_dispatch failure to observe the pinned issue end-to-end (provider-backed, operator to run), SUPABASE_ACCESS_TOKEN per #183, and Phases 1-5, which all need approved hosted windows. Note: the Phase 0 task prompt named #312 as the anchor; that is the unrelated Playwright-browser P3, and the anchor was resolved to #316 by exact title per the playbook." + } +}