From 83f770ebc8c206fe3f9f2bf2b246bf50d7421405 Mon Sep 17 00:00:00 2001 From: Kiet Ho Date: Sat, 19 Sep 2026 13:16:33 -0700 Subject: [PATCH 1/2] fix(ci): stop the production migrate step reporting a failed migration as a success The retry loop pipes drizzle-kit through tee so it can grep the output for a lost lock race. GitHub runs a step that does not set `shell:` with `bash -e`, without pipefail, so the pipeline's status was tee's. On the first production run after #7671 the migration hit its lock timeout on automation_events, rolled back and exited 1, and the step reported success. The app deploys started against an unmigrated database; the run was cancelled before any of them reached its build step, so nothing shipped. The block now sets pipefail itself rather than depending on the runner's default. Run under `bash -e` against a Neon branch of production with 0119 pending: the old block exits 0 with nothing applied; the new one exits 1 after all eight attempts lose, exits 1 at once on bad credentials, and retries once then applies when the lock frees up. --- .github/workflows/deploy-production.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 5f3ecedbcb0..b2872367d60 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -46,6 +46,9 @@ jobs: PGOPTIONS: -c lock_timeout=5s -c statement_timeout=60s # Losing a lock race rolls back cleanly, so only that is retried. run: | + # GitHub's default shell is `bash -e` without pipefail, so without this + # the `| tee` below reports a failed migration as a success. + set -o pipefail for attempt in 1 2 3 4 5 6 7 8; do if bun drizzle-kit migrate 2>&1 | tee migrate.log; then exit 0; fi grep -qE "deadlock detected|lock timeout" migrate.log || exit 1 From a2fb18b0b0e8e74b735f8a5aa90a693f88c16fdb Mon Sep 17 00:00:00 2001 From: Kiet Ho Date: Sat, 19 Sep 2026 13:20:05 -0700 Subject: [PATCH 2/2] ci(db): verify the newest migration is recorded before the apps deploy The app deploys wait on the migrate job, so that job's verdict has to mean the schema is there. It did not on the first run after #7671, where a shell default turned a failed migration into a passing step. Fixing the exit code closes that one path; this closes the rest. After migrating, the job compares the newest migration recorded in the database with the newest entry in the repo's journal and fails if they differ, which is the check that caught the bad run by hand. The preview migrate step runs it too, so every pull request exercises it in GitHub's own environment before production depends on it. Checked both ways under `bash -e`: exits 0 against a database at 0119, exits 1 against one still at 0118. --- .github/workflows/deploy-preview.yml | 1 + .github/workflows/deploy-production.yml | 7 ++++ packages/db/src/verify-migrations-applied.ts | 34 ++++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 packages/db/src/verify-migrations-applied.ts diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml index 8ae87b2fa6d..da123b91c6e 100644 --- a/.github/workflows/deploy-preview.yml +++ b/.github/workflows/deploy-preview.yml @@ -61,6 +61,7 @@ jobs: run: | bun drizzle-kit check bun drizzle-kit migrate + bun run src/verify-migrations-applied.ts - name: Save database success status run: | diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index b2872367d60..74424e612ed 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -57,6 +57,13 @@ jobs: done exit 1 + - name: Verify migrations applied + working-directory: packages/db + env: + DATABASE_URL: ${{ secrets.DATABASE_URL_UNPOOLED }} + DATABASE_URL_UNPOOLED: ${{ secrets.DATABASE_URL_UNPOOLED }} + run: bun run src/verify-migrations-applied.ts + deploy-api: name: Deploy API to Vercel runs-on: ubuntu-latest diff --git a/packages/db/src/verify-migrations-applied.ts b/packages/db/src/verify-migrations-applied.ts new file mode 100644 index 00000000000..f8df08092a9 --- /dev/null +++ b/packages/db/src/verify-migrations-applied.ts @@ -0,0 +1,34 @@ +import { neon } from "@neondatabase/serverless"; +import journal from "../drizzle/meta/_journal.json"; +import { env } from "./env"; + +/** + * Fails unless the newest migration in the repo is the newest one recorded in + * the database. The deploy runs it after `drizzle-kit migrate` so a migration + * that did not apply cannot pass as one that did, whatever the migrate step's + * exit code said. + * + * Usage: bun run packages/db/src/verify-migrations-applied.ts + */ +async function verifyMigrationsApplied(): Promise { + const expected = journal.entries.at(-1); + if (!expected) throw new Error("The migration journal has no entries"); + + const sql = neon(env.DATABASE_URL_UNPOOLED); + const [row] = await sql` + select max(created_at)::text as latest from drizzle.__drizzle_migrations + `; + const latest = row?.latest ? Number(row.latest) : null; + + if (latest !== expected.when) { + throw new Error( + `Database is not at ${expected.tag}: newest applied migration is ${latest ?? "none"}, expected ${expected.when}`, + ); + } + console.log(`Database is at ${expected.tag}`); +} + +verifyMigrationsApplied().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exit(1); +});