diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 0ca6c53bc87e..02df09d6669d 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -55,12 +55,14 @@ import Migration0039 from "./Migrations/039_ProjectionProjectsDefaultThreadEnvMo import Migration0040 from "./Migrations/040_ProjectionProjectFaviconPath.ts"; import Migration0041 from "./Migrations/041_AuthSessionClientConnection.ts"; import Migration0042 from "./Migrations/042_ProjectionThreadLinkedPullRequest.ts"; -import Migration0043 from "./Migrations/043_ProjectionThreadAnnotation.ts"; -import Migration0044 from "./Migrations/044_UpdateDrain.ts"; -import Migration0045 from "./Migrations/045_UpdateDrainClaim.ts"; -import Migration0046 from "./Migrations/046_ProjectionTurnRequestCorrelations.ts"; -import Migration0047 from "./Migrations/047_ProjectionThreadWorktreeCleanup.ts"; -import Migration0048 from "./Migrations/048_ProjectionThreadLinkedPullRequest.ts"; +import Migration0043 from "./Migrations/043_ProjectionThreadsUnsettledAt.ts"; +import Migration0044 from "./Migrations/044_ProjectionThreadAnnotation.ts"; +import Migration0045 from "./Migrations/045_UpdateDrain.ts"; +import Migration0046 from "./Migrations/046_UpdateDrainClaim.ts"; +import Migration0047 from "./Migrations/047_ProjectionTurnRequestCorrelations.ts"; +import Migration0048 from "./Migrations/048_ProjectionThreadWorktreeCleanup.ts"; +import Migration0049 from "./Migrations/049_ProjectionThreadLinkedPullRequest.ts"; +import Migration0050 from "./Migrations/050_ProjectionThreadsUnsettledAt.ts"; /** * Migration loader with all migrations defined inline. @@ -115,12 +117,14 @@ export const migrationEntries = [ [40, "ProjectionProjectFaviconPath", Migration0040], [41, "AuthSessionClientConnection", Migration0041], [42, "ProjectionThreadLinkedPullRequest", Migration0042], - [43, "ProjectionThreadAnnotation", Migration0043], - [44, "UpdateDrain", Migration0044], - [45, "UpdateDrainClaim", Migration0045], - [46, "ProjectionTurnRequestCorrelations", Migration0046], - [47, "ProjectionThreadWorktreeCleanup", Migration0047], - [48, "ProjectionThreadLinkedPullRequest", Migration0048], + [43, "ProjectionThreadsUnsettledAt", Migration0043], + [44, "ProjectionThreadAnnotation", Migration0044], + [45, "UpdateDrain", Migration0045], + [46, "UpdateDrainClaim", Migration0046], + [47, "ProjectionTurnRequestCorrelations", Migration0047], + [48, "ProjectionThreadWorktreeCleanup", Migration0048], + [49, "ProjectionThreadLinkedPullRequest", Migration0049], + [50, "ProjectionThreadsUnsettledAt", Migration0050], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/043_ProjectionThreadsUnsettledAt.ts b/apps/server/src/persistence/Migrations/043_ProjectionThreadsUnsettledAt.ts new file mode 100644 index 000000000000..981d3c78f3a6 --- /dev/null +++ b/apps/server/src/persistence/Migrations/043_ProjectionThreadsUnsettledAt.ts @@ -0,0 +1,16 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + + if (!columns.some((column) => column.name === "unsettled_at")) { + yield* sql` + ALTER TABLE projection_threads + ADD COLUMN unsettled_at TEXT + `; + } +}); diff --git a/apps/server/src/persistence/Migrations/043_ProjectionThreadAnnotation.test.ts b/apps/server/src/persistence/Migrations/044_ProjectionThreadAnnotation.test.ts similarity index 96% rename from apps/server/src/persistence/Migrations/043_ProjectionThreadAnnotation.test.ts rename to apps/server/src/persistence/Migrations/044_ProjectionThreadAnnotation.test.ts index 98c3922d2451..25e828c130c2 100644 --- a/apps/server/src/persistence/Migrations/043_ProjectionThreadAnnotation.test.ts +++ b/apps/server/src/persistence/Migrations/044_ProjectionThreadAnnotation.test.ts @@ -8,12 +8,12 @@ import * as NodeSqliteClient from "../NodeSqliteClient.ts"; const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); -layer("043_ProjectionThreadAnnotation", (it) => { +layer("044_ProjectionThreadAnnotation", (it) => { it.effect("adds annotation and latest user marker fields to thread projections", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - yield* runMigrations({ toMigrationInclusive: 42 }); + yield* runMigrations({ toMigrationInclusive: 43 }); yield* sql` INSERT INTO projection_threads ( @@ -67,7 +67,7 @@ layer("043_ProjectionThreadAnnotation", (it) => { '2026-02-24T00:01:00.000Z' ) `; - yield* runMigrations({ toMigrationInclusive: 43 }); + yield* runMigrations({ toMigrationInclusive: 44 }); const columns = yield* sql<{ readonly name: string; readonly notnull: number }>` PRAGMA table_info(projection_threads) diff --git a/apps/server/src/persistence/Migrations/043_ProjectionThreadAnnotation.ts b/apps/server/src/persistence/Migrations/044_ProjectionThreadAnnotation.ts similarity index 100% rename from apps/server/src/persistence/Migrations/043_ProjectionThreadAnnotation.ts rename to apps/server/src/persistence/Migrations/044_ProjectionThreadAnnotation.ts diff --git a/apps/server/src/persistence/Migrations/044_UpdateDrain.test.ts b/apps/server/src/persistence/Migrations/045_UpdateDrain.test.ts similarity index 93% rename from apps/server/src/persistence/Migrations/044_UpdateDrain.test.ts rename to apps/server/src/persistence/Migrations/045_UpdateDrain.test.ts index eea6f6a58d7d..c3c099976a5f 100644 --- a/apps/server/src/persistence/Migrations/044_UpdateDrain.test.ts +++ b/apps/server/src/persistence/Migrations/045_UpdateDrain.test.ts @@ -8,13 +8,13 @@ import * as NodeSqliteClient from "../NodeSqliteClient.ts"; const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); -layer("044_UpdateDrain", (it) => { +layer("045_UpdateDrain", (it) => { it.effect("creates a narrow event stream and durable command receipts", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - yield* runMigrations({ toMigrationInclusive: 43 }); yield* runMigrations({ toMigrationInclusive: 44 }); + yield* runMigrations({ toMigrationInclusive: 45 }); const eventColumns = yield* sql<{ readonly name: string }>` PRAGMA table_info(update_drain_events) diff --git a/apps/server/src/persistence/Migrations/044_UpdateDrain.ts b/apps/server/src/persistence/Migrations/045_UpdateDrain.ts similarity index 100% rename from apps/server/src/persistence/Migrations/044_UpdateDrain.ts rename to apps/server/src/persistence/Migrations/045_UpdateDrain.ts diff --git a/apps/server/src/persistence/Migrations/045_UpdateDrainClaim.test.ts b/apps/server/src/persistence/Migrations/046_UpdateDrainClaim.test.ts similarity index 94% rename from apps/server/src/persistence/Migrations/045_UpdateDrainClaim.test.ts rename to apps/server/src/persistence/Migrations/046_UpdateDrainClaim.test.ts index 59e8a98ca5cd..679f537822bf 100644 --- a/apps/server/src/persistence/Migrations/045_UpdateDrainClaim.test.ts +++ b/apps/server/src/persistence/Migrations/046_UpdateDrainClaim.test.ts @@ -8,11 +8,11 @@ import * as NodeSqliteClient from "../NodeSqliteClient.ts"; const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); -layer("045_UpdateDrainClaim", (it) => { +layer("046_UpdateDrainClaim", (it) => { it.effect("preserves drain history and accepts one claimed transition", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - yield* runMigrations({ toMigrationInclusive: 44 }); + yield* runMigrations({ toMigrationInclusive: 45 }); yield* sql` INSERT INTO update_drain_events ( event_id, event_type, command_id, occurred_at, request_id, target_version, status @@ -21,7 +21,7 @@ layer("045_UpdateDrainClaim", (it) => { '2026-08-21T00:00:00.000Z', 'request-1', '1.2.3', 'draining' ) `; - yield* runMigrations({ toMigrationInclusive: 45 }); + yield* runMigrations({ toMigrationInclusive: 46 }); yield* sql` INSERT INTO update_drain_events ( event_id, event_type, command_id, occurred_at, request_id, target_version, status diff --git a/apps/server/src/persistence/Migrations/045_UpdateDrainClaim.ts b/apps/server/src/persistence/Migrations/046_UpdateDrainClaim.ts similarity index 100% rename from apps/server/src/persistence/Migrations/045_UpdateDrainClaim.ts rename to apps/server/src/persistence/Migrations/046_UpdateDrainClaim.ts diff --git a/apps/server/src/persistence/Migrations/046_ProjectionTurnRequestCorrelations.test.ts b/apps/server/src/persistence/Migrations/047_ProjectionTurnRequestCorrelations.test.ts similarity index 98% rename from apps/server/src/persistence/Migrations/046_ProjectionTurnRequestCorrelations.test.ts rename to apps/server/src/persistence/Migrations/047_ProjectionTurnRequestCorrelations.test.ts index 14ab26d4a201..402614f4c4de 100644 --- a/apps/server/src/persistence/Migrations/046_ProjectionTurnRequestCorrelations.test.ts +++ b/apps/server/src/persistence/Migrations/047_ProjectionTurnRequestCorrelations.test.ts @@ -15,7 +15,7 @@ const layer = it.layer( ), ); -layer("046_ProjectionTurnRequestCorrelations", (it) => { +layer("047_ProjectionTurnRequestCorrelations", (it) => { it.effect("inserts once, resolves once, and deletes by owning thread", () => Effect.gen(function* () { yield* runMigrations(); diff --git a/apps/server/src/persistence/Migrations/046_ProjectionTurnRequestCorrelations.ts b/apps/server/src/persistence/Migrations/047_ProjectionTurnRequestCorrelations.ts similarity index 100% rename from apps/server/src/persistence/Migrations/046_ProjectionTurnRequestCorrelations.ts rename to apps/server/src/persistence/Migrations/047_ProjectionTurnRequestCorrelations.ts diff --git a/apps/server/src/persistence/Migrations/047_ProjectionThreadWorktreeCleanup.test.ts b/apps/server/src/persistence/Migrations/048_ProjectionThreadWorktreeCleanup.test.ts similarity index 93% rename from apps/server/src/persistence/Migrations/047_ProjectionThreadWorktreeCleanup.test.ts rename to apps/server/src/persistence/Migrations/048_ProjectionThreadWorktreeCleanup.test.ts index 1f6a8913f158..75b9e7b77ece 100644 --- a/apps/server/src/persistence/Migrations/047_ProjectionThreadWorktreeCleanup.test.ts +++ b/apps/server/src/persistence/Migrations/048_ProjectionThreadWorktreeCleanup.test.ts @@ -8,11 +8,11 @@ import * as NodeSqliteClient from "../NodeSqliteClient.ts"; const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); -layer("047_ProjectionThreadWorktreeCleanup", (it) => { +layer("048_ProjectionThreadWorktreeCleanup", (it) => { it.effect("adds nullable cleanup state without changing existing rows", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; - yield* runMigrations({ toMigrationInclusive: 46 }); + yield* runMigrations({ toMigrationInclusive: 47 }); yield* sql` INSERT INTO projection_threads ( thread_id, @@ -36,7 +36,7 @@ layer("047_ProjectionThreadWorktreeCleanup", (it) => { ) `; - yield* runMigrations({ toMigrationInclusive: 47 }); + yield* runMigrations({ toMigrationInclusive: 48 }); const columns = yield* sql<{ readonly name: string; readonly notnull: number }>` PRAGMA table_info(projection_threads) diff --git a/apps/server/src/persistence/Migrations/047_ProjectionThreadWorktreeCleanup.ts b/apps/server/src/persistence/Migrations/048_ProjectionThreadWorktreeCleanup.ts similarity index 100% rename from apps/server/src/persistence/Migrations/047_ProjectionThreadWorktreeCleanup.ts rename to apps/server/src/persistence/Migrations/048_ProjectionThreadWorktreeCleanup.ts diff --git a/apps/server/src/persistence/Migrations/048_ProjectionThreadLinkedPullRequest.test.ts b/apps/server/src/persistence/Migrations/049_ProjectionThreadLinkedPullRequest.test.ts similarity index 69% rename from apps/server/src/persistence/Migrations/048_ProjectionThreadLinkedPullRequest.test.ts rename to apps/server/src/persistence/Migrations/049_ProjectionThreadLinkedPullRequest.test.ts index effa2ecfd68c..f50c4e41efd4 100644 --- a/apps/server/src/persistence/Migrations/048_ProjectionThreadLinkedPullRequest.test.ts +++ b/apps/server/src/persistence/Migrations/049_ProjectionThreadLinkedPullRequest.test.ts @@ -5,51 +5,52 @@ import * as SqlClient from "effect/unstable/sql/SqlClient"; import { runMigrations } from "../Migrations.ts"; import * as NodeSqliteClient from "../NodeSqliteClient.ts"; -import Migration0043 from "./043_ProjectionThreadAnnotation.ts"; -import Migration0044 from "./044_UpdateDrain.ts"; -import Migration0045 from "./045_UpdateDrainClaim.ts"; -import Migration0046 from "./046_ProjectionTurnRequestCorrelations.ts"; -import Migration0047 from "./047_ProjectionThreadWorktreeCleanup.ts"; -import Migration0048 from "./048_ProjectionThreadLinkedPullRequest.ts"; +import Migration0044 from "./044_ProjectionThreadAnnotation.ts"; +import Migration0045 from "./045_UpdateDrain.ts"; +import Migration0046 from "./046_UpdateDrainClaim.ts"; +import Migration0047 from "./047_ProjectionTurnRequestCorrelations.ts"; +import Migration0048 from "./048_ProjectionThreadWorktreeCleanup.ts"; +import Migration0049 from "./049_ProjectionThreadLinkedPullRequest.ts"; const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); -layer("048_ProjectionThreadLinkedPullRequest", (it) => { +layer("049_ProjectionThreadLinkedPullRequest", (it) => { it.effect("bridges databases that recorded the previous LastCode migration numbers", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; yield* runMigrations({ toMigrationInclusive: 41 }); - yield* Migration0043; yield* Migration0044; yield* Migration0045; yield* Migration0046; yield* Migration0047; + yield* Migration0048; yield* sql` - INSERT INTO effect_sql_migrations (migration_id, name) - VALUES - (42, 'ProjectionThreadAnnotation'), - (43, 'UpdateDrain'), - (44, 'UpdateDrainClaim'), - (45, 'ProjectionTurnRequestCorrelations'), - (46, 'ProjectionThreadWorktreeCleanup') - `; + INSERT INTO effect_sql_migrations (migration_id, name) + VALUES + (42, 'ProjectionThreadAnnotation'), + (43, 'UpdateDrain'), + (44, 'UpdateDrainClaim'), + (45, 'ProjectionTurnRequestCorrelations'), + (46, 'ProjectionThreadWorktreeCleanup') + `; const before = yield* sql<{ readonly name: string }>` - PRAGMA table_info(projection_threads) - `; + PRAGMA table_info(projection_threads) + `; assert.isFalse(before.some((column) => column.name === "linked_pull_request_json")); - const executed = yield* runMigrations({ toMigrationInclusive: 48 }); + const executed = yield* runMigrations({ toMigrationInclusive: 49 }); assert.deepStrictEqual(executed, [ - [47, "ProjectionThreadWorktreeCleanup"], - [48, "ProjectionThreadLinkedPullRequest"], + [47, "ProjectionTurnRequestCorrelations"], + [48, "ProjectionThreadWorktreeCleanup"], + [49, "ProjectionThreadLinkedPullRequest"], ]); - yield* Migration0048; + yield* Migration0049; const after = yield* sql<{ readonly name: string }>` - PRAGMA table_info(projection_threads) - `; + PRAGMA table_info(projection_threads) + `; assert.equal(after.filter((column) => column.name === "linked_pull_request_json").length, 1); }), ); @@ -57,15 +58,15 @@ layer("048_ProjectionThreadLinkedPullRequest", (it) => { const partialUpgradeLayer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); -partialUpgradeLayer("048_ProjectionThreadLinkedPullRequest partial upgrades", (it) => { - it.effect("preserves update drain data when upgrading from the previous migration 44", () => +partialUpgradeLayer("049_ProjectionThreadLinkedPullRequest partial upgrades", (it) => { + it.effect("preserves update drain data recorded with the previous migration numbers", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; yield* runMigrations({ toMigrationInclusive: 41 }); - yield* Migration0043; yield* Migration0044; yield* Migration0045; + yield* Migration0046; yield* sql` INSERT INTO update_drain_events ( event_id, event_type, command_id, occurred_at, request_id, target_version, status @@ -91,12 +92,13 @@ partialUpgradeLayer("048_ProjectionThreadLinkedPullRequest partial upgrades", (i (44, 'UpdateDrainClaim') `; - const executed = yield* runMigrations({ toMigrationInclusive: 48 }); + const executed = yield* runMigrations({ toMigrationInclusive: 49 }); assert.deepStrictEqual(executed, [ - [45, "UpdateDrainClaim"], - [46, "ProjectionTurnRequestCorrelations"], - [47, "ProjectionThreadWorktreeCleanup"], - [48, "ProjectionThreadLinkedPullRequest"], + [45, "UpdateDrain"], + [46, "UpdateDrainClaim"], + [47, "ProjectionTurnRequestCorrelations"], + [48, "ProjectionThreadWorktreeCleanup"], + [49, "ProjectionThreadLinkedPullRequest"], ]); const events = yield* sql<{ diff --git a/apps/server/src/persistence/Migrations/048_ProjectionThreadLinkedPullRequest.ts b/apps/server/src/persistence/Migrations/049_ProjectionThreadLinkedPullRequest.ts similarity index 100% rename from apps/server/src/persistence/Migrations/048_ProjectionThreadLinkedPullRequest.ts rename to apps/server/src/persistence/Migrations/049_ProjectionThreadLinkedPullRequest.ts diff --git a/apps/server/src/persistence/Migrations/050_ProjectionThreadsUnsettledAt.test.ts b/apps/server/src/persistence/Migrations/050_ProjectionThreadsUnsettledAt.test.ts new file mode 100644 index 000000000000..da181095169b --- /dev/null +++ b/apps/server/src/persistence/Migrations/050_ProjectionThreadsUnsettledAt.test.ts @@ -0,0 +1,59 @@ +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { runMigrations } from "../Migrations.ts"; +import * as NodeSqliteClient from "../NodeSqliteClient.ts"; +import Migration0044 from "./044_ProjectionThreadAnnotation.ts"; +import Migration0045 from "./045_UpdateDrain.ts"; +import Migration0046 from "./046_UpdateDrainClaim.ts"; +import Migration0047 from "./047_ProjectionTurnRequestCorrelations.ts"; +import Migration0048 from "./048_ProjectionThreadWorktreeCleanup.ts"; +import Migration0049 from "./049_ProjectionThreadLinkedPullRequest.ts"; + +const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory())); + +layer("050_ProjectionThreadsUnsettledAt", (it) => { + it.effect("bridges databases whose old ledger would skip the new upstream migration", () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + yield* runMigrations({ toMigrationInclusive: 41 }); + yield* Migration0044; + yield* Migration0045; + yield* Migration0046; + yield* Migration0047; + yield* Migration0048; + yield* Migration0049; + yield* sql` + INSERT INTO effect_sql_migrations (migration_id, name) + VALUES + (42, 'ProjectionThreadAnnotation'), + (43, 'ProjectionThreadAnnotation'), + (44, 'UpdateDrain'), + (45, 'UpdateDrainClaim'), + (46, 'ProjectionTurnRequestCorrelations'), + (47, 'ProjectionThreadWorktreeCleanup'), + (48, 'ProjectionThreadLinkedPullRequest') + `; + + const before = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + assert.isFalse(before.some((column) => column.name === "unsettled_at")); + + const executed = yield* runMigrations({ toMigrationInclusive: 50 }); + assert.deepStrictEqual(executed, [ + [49, "ProjectionThreadLinkedPullRequest"], + [50, "ProjectionThreadsUnsettledAt"], + ]); + + const after = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + assert.equal(after.filter((column) => column.name === "linked_pull_request_json").length, 1); + assert.equal(after.filter((column) => column.name === "unsettled_at").length, 1); + }), + ); +}); diff --git a/apps/server/src/persistence/Migrations/050_ProjectionThreadsUnsettledAt.ts b/apps/server/src/persistence/Migrations/050_ProjectionThreadsUnsettledAt.ts new file mode 100644 index 000000000000..981d3c78f3a6 --- /dev/null +++ b/apps/server/src/persistence/Migrations/050_ProjectionThreadsUnsettledAt.ts @@ -0,0 +1,16 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const columns = yield* sql<{ readonly name: string }>` + PRAGMA table_info(projection_threads) + `; + + if (!columns.some((column) => column.name === "unsettled_at")) { + yield* sql` + ALTER TABLE projection_threads + ADD COLUMN unsettled_at TEXT + `; + } +}); diff --git a/docs/lastcode/nightly-workflow.md b/docs/lastcode/nightly-workflow.md index fd37820ac0e7..e35c6ed58d1b 100644 --- a/docs/lastcode/nightly-workflow.md +++ b/docs/lastcode/nightly-workflow.md @@ -144,11 +144,14 @@ retains both: - the corresponding `sync/nightly/...` or `sync/revision/...` branch; and - the printed `lastcode-nightly-sync` worktree path. -It also posts a macOS notification. Resolve the rebase or failure in that -worktree, then release the retained worktree and generated recovery branch so -the daemon can retry. `lastcode-checkpoints` prints the exact commands for the -current state: it distinguishes an in-progress rebase, a rebase already -completed by the operator, and a smoke-gate failure that must first be fixed on +It also posts a macOS notification. When the service is configured with a +maintenance thread, its supervisor durably sends one turn to that same thread +for each distinct blocker and retries delivery if the LastCode server is +temporarily unavailable. Resolve the rebase or failure in the retained +worktree, then release the worktree and generated recovery branch so the daemon +can retry. `lastcode-checkpoints` prints the exact commands for the current +state: it distinguishes an in-progress rebase, a rebase already completed by +the operator, and a smoke-gate failure that must first be fixed on `lastcode/main`. The final printed command runs the daemon immediately. The next automated run refuses to replace an existing recovery worktree until those cleanup steps are complete. @@ -180,11 +183,36 @@ Install the per-user launch agent: pnpm lastcode:checkpoint:service install ``` +For unattended recovery, dedicate one durable LastCode thread to checkpoint +maintenance and configure its thread ID once: + +```bash +pnpm lastcode:checkpoint:service install --recovery-thread +``` + +The service reuses that thread instead of creating a new thread per failure. A +standalone supervisor covers fetch, checkout, dependency setup, and checkpoint +execution; every run writes terminal state to +`~/.lastcode/automation/checkpoint-service-state.json`. A failed alert remains +pending until LastCode accepts it, and the same incident is not sent again after +acknowledgement. When a later run succeeds, the supervisor sends one closure to +the maintenance thread. The launch agent starts the pinned Node runtime without +a login shell; the supervisor passes only a narrow allowlist to child processes, +including the launchd SSH agent socket needed for Git fetches. The macOS +notification remains a secondary signal. + +Disable maintenance-thread delivery explicitly (uninstalling the service also +clears the saved destination): + +```bash +pnpm lastcode:checkpoint:service install --no-recovery-thread +``` + The job runs at login and hourly while the Mac is awake. Missed intervals do not matter: every run discovers all uncheckpointed tags and catches up oldest-first. When the latest checkpoint is already promoted, the job exits without running the local push gate or pushing an unchanged branch. -The job executes: +The supervisor executes the equivalent of: ```bash pnpm run lastcode:checkpoint -- \ diff --git a/scripts/lastcode-checkpoint-supervisor.mjs b/scripts/lastcode-checkpoint-supervisor.mjs new file mode 100644 index 000000000000..64afd1be500f --- /dev/null +++ b/scripts/lastcode-checkpoint-supervisor.mjs @@ -0,0 +1,683 @@ +#!/usr/bin/env node + +// Standalone launchd supervisor for the LastCode checkpoint job. This file is +// copied outside the automation checkout so it can report checkout/setup +// failures as well as failures from the checkpoint command itself. +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeURL from "node:url"; +import * as NodeUtil from "node:util"; + +const CHECKPOINT_ARGS = [ + "scripts/lastcode-checkpoint.ts", + "--push-tags", + "--promote-if-no-open-prs", + "--mirror-upstream-main", +]; +const DIAGNOSTIC_MAX_CHARACTERS = 1_200; + +function parseJson(raw, label) { + try { + return JSON.parse(raw); + } catch (error) { + throw new Error( + `Could not parse ${label}: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } +} + +function readJson(path, label) { + try { + return parseJson(NodeFS.readFileSync(path, "utf8"), label); + } catch (error) { + if (error && typeof error === "object" && error.code === "ENOENT") return null; + throw error; + } +} + +function writeJsonAtomic(path, value) { + NodeFS.mkdirSync(NodePath.dirname(path), { recursive: true, mode: 0o700 }); + const temporaryPath = `${path}.tmp`; + NodeFS.writeFileSync(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); + NodeFS.renameSync(temporaryPath, path); +} + +export function supervisorPaths(home = NodeOS.homedir()) { + const rootDirectory = NodePath.join(home, ".lastcode", "automation"); + return { + configPath: NodePath.join(rootDirectory, "checkpoint-supervisor.json"), + runHistoryPath: NodePath.join(rootDirectory, "checkpoint-runs.jsonl"), + statePath: NodePath.join(rootDirectory, "checkpoint-service-state.json"), + threadToolPath: NodePath.join(home, ".lastcode", "userdata", "bin", "lastcode-thread"), + }; +} + +export function checkpointEnvironment( + source = process.env, + home = NodeOS.homedir(), + nodePath = process.execPath, +) { + const environment = { + HOME: home, + PATH: `${NodePath.dirname(nodePath)}:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin`, + }; + for (const name of ["LANG", "LC_ALL", "LOGNAME", "SHELL", "SSH_AUTH_SOCK", "TMPDIR", "USER"]) { + const value = source[name]; + if (value) environment[name] = value; + } + return environment; +} + +class CommandFailure extends Error { + constructor(phase, command, status, diagnostic) { + super(`${command} failed with exit code ${status ?? "unknown"}.`); + this.diagnostic = diagnostic; + this.phase = phase; + this.status = status; + } +} + +export function boundedCommandDiagnostic(raw) { + const redacted = NodeUtil.stripVTControlCharacters(raw) + .replaceAll(/(https?:\/\/)[^/@\s]+@/giu, "$1@") + .replaceAll(/\b(?:github_pat_|gh[pousr]_|ctx7sk-|sk-)[A-Za-z0-9_-]{8,}\b/gu, "") + .replaceAll(/\b((?:api[_-]?key|password|secret|token)\s*[=:]\s*)\S+/giu, "$1") + .split("") + .map((character) => { + const code = character.codePointAt(0) ?? 0; + return (code < 32 && character !== "\n" && character !== "\r" && character !== "\t") || + code === 127 + ? " " + : character; + }) + .join("") + .trim(); + return redacted.length > DIAGNOSTIC_MAX_CHARACTERS + ? `...${redacted.slice(-DIAGNOSTIC_MAX_CHARACTERS)}` + : redacted; +} + +function runCommand(phase, cwd, command, args, environment, options = {}) { + if (options.capture) { + const result = NodeChildProcess.spawnSync(command, args, { + cwd, + encoding: "utf8", + env: environment, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new CommandFailure( + phase, + command, + result.status, + boundedCommandDiagnostic(result.stderr), + ); + } + return result.stdout.trim(); + } + + const temporaryDirectory = NodeFS.mkdtempSync( + NodePath.join(NodeOS.tmpdir(), "lastcode-checkpoint-command-"), + ); + const stderrPath = NodePath.join(temporaryDirectory, "stderr.log"); + let stderrDescriptor; + try { + stderrDescriptor = NodeFS.openSync(stderrPath, "w", 0o600); + const result = NodeChildProcess.spawnSync(command, args, { + cwd, + env: environment, + stdio: ["ignore", "inherit", stderrDescriptor], + }); + NodeFS.closeSync(stderrDescriptor); + stderrDescriptor = undefined; + const stderr = NodeFS.readFileSync(stderrPath, "utf8"); + if (stderr) process.stderr.write(stderr); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new CommandFailure(phase, command, result.status, boundedCommandDiagnostic(stderr)); + } + return ""; + } finally { + if (stderrDescriptor !== undefined) NodeFS.closeSync(stderrDescriptor); + NodeFS.rmSync(temporaryDirectory, { force: true, recursive: true }); + } +} + +export function latestFailedCheckpointRun(path) { + try { + const lines = NodeFS.readFileSync(path, "utf8").trim().split(/\r?\n/u); + for (let index = lines.length - 1; index >= 0; index -= 1) { + const line = lines[index]; + if (!line) continue; + let record; + try { + record = parseJson(line, "checkpoint run history"); + } catch { + console.error( + "[lastcode:checkpoint-supervisor] Ignoring an incomplete checkpoint history record.", + ); + continue; + } + if (!record || typeof record !== "object" || Array.isArray(record)) continue; + if (record.status === "failed") return record; + if (record.status === "success") return null; + } + return null; + } catch (error) { + if (error && typeof error === "object" && error.code === "ENOENT") return null; + throw error; + } +} + +function sameCheckpointRun(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +function retainedRecoveryMatches(repoRoot, checkpointRun, environment) { + if (!checkpointRun?.recoveryBranch) return false; + const worktree = NodePath.join(NodePath.dirname(repoRoot), "lastcode-nightly-sync"); + if (!NodeFS.existsSync(worktree)) return false; + try { + return ( + runCommand( + "history-correlation", + repoRoot, + "git", + ["-C", worktree, "branch", "--show-current"], + environment, + { capture: true }, + ) === checkpointRun.recoveryBranch + ); + } catch { + return false; + } +} + +export function refreshInstalledSupervisor(repoRoot, installedPath) { + const sourcePath = NodePath.join(repoRoot, "scripts", "lastcode-checkpoint-supervisor.mjs"); + if (NodePath.resolve(sourcePath) === NodePath.resolve(installedPath)) return; + if (!NodeFS.existsSync(sourcePath)) return; + const source = NodeFS.readFileSync(sourcePath); + const installed = NodeFS.existsSync(installedPath) ? NodeFS.readFileSync(installedPath) : null; + if (installed?.equals(source)) return; + const temporaryPath = `${installedPath}.tmp`; + NodeFS.writeFileSync(temporaryPath, source, { mode: 0o700 }); + NodeFS.renameSync(temporaryPath, installedPath); +} + +export function checkpointIncidentFingerprint(failure) { + return NodeCrypto.createHash("sha256") + .update( + JSON.stringify({ + error: failure.error, + checkpointError: failure.checkpointRun?.error ?? null, + diagnostic: failure.diagnostic ?? null, + failurePhase: failure.checkpointRun?.failurePhase ?? null, + phase: failure.phase, + recoveryBranch: failure.checkpointRun?.recoveryBranch ?? null, + upstreamTag: failure.checkpointRun?.upstreamTag ?? null, + }), + ) + .digest("hex"); +} + +export function checkpointFailureMessage(failure, fingerprint) { + const nightly = failure.checkpointRun?.upstreamTag + ? ` for ${failure.checkpointRun.upstreamTag}` + : ""; + const recovery = failure.checkpointRun?.recoveryBranch + ? ` Recovery branch: ${failure.checkpointRun.recoveryBranch}.` + : ""; + const diagnostic = failure.diagnostic ? `Diagnostic: ${failure.diagnostic}` : null; + return [ + `Automated LastCode checkpoint maintenance alert ${fingerprint.slice(0, 12)}.`, + `The hourly service failed during ${failure.phase}${nightly}.${recovery}`, + diagnostic, + "Run lastcode-checkpoints --verbose, inspect the live daemon and retained recovery state, and address the concrete problem.", + "The bounded service logs are ~/.lastcode/automation/nightly-checkpoint.stdout.log and nightly-checkpoint.stderr.log.", + "Use this thread for the recovery; do not create a new maintenance thread and do not only report the alert.", + ] + .filter(Boolean) + .join("\n"); +} + +export function checkpointResolvedMessage(incident) { + const nightly = incident.failure.checkpointRun?.upstreamTag + ? ` through ${incident.failure.checkpointRun.upstreamTag}` + : ""; + return `Automated LastCode checkpoint maintenance resolved alert ${incident.fingerprint.slice(0, 12)}${nightly}. The hourly service completed successfully again.`; +} + +function sendThread(threadToolPath, threadId, message, environment) { + runCommand( + "alert-delivery", + NodePath.dirname(threadToolPath), + threadToolPath, + ["send", threadId, "--message", message, "--json"], + environment, + { capture: true }, + ); +} + +function notify(title, message, environment) { + const result = NodeChildProcess.spawnSync( + "osascript", + [ + "-e", + "on run argv", + "-e", + "display notification (item 2 of argv) with title (item 1 of argv)", + "-e", + "end run", + title, + message, + ], + { env: environment, stdio: "inherit" }, + ); + return result.status === 0; +} + +function isDurableIncident(value) { + return ( + value !== null && + typeof value === "object" && + !Array.isArray(value) && + typeof value.fingerprint === "string" && + value.fingerprint.length > 0 && + value.failure !== null && + typeof value.failure === "object" && + !Array.isArray(value.failure) + ); +} + +function durableIncident(value) { + return isDurableIncident(value) ? value : null; +} + +function durableIncidentList(state, key) { + return Array.isArray(state?.[key]) ? state[key].filter(isDurableIncident) : []; +} + +function attemptDelivery(state, config, dependencies) { + if (!config?.recoveryThreadId) return state; + const recoveryThreadId = config.recoveryThreadId; + let nextState = state; + let pendingIncidents = [...durableIncidentList(state, "pendingIncidents")]; + while (pendingIncidents.length > 0) { + const [incident, ...remaining] = pendingIncidents; + if (!incident) break; + const updatedIncident = { + ...incident, + alertDelivery: "pending", + deliveryAttempts: (incident.deliveryAttempts ?? 0) + 1, + lastDeliveryAttemptAt: dependencies.now(), + }; + nextState = Object.assign({}, nextState, { + pendingIncidents: [updatedIncident, ...remaining], + }); + dependencies.writeState(nextState); + try { + dependencies.sendThread( + recoveryThreadId, + checkpointFailureMessage(updatedIncident.failure, updatedIncident.fingerprint), + ); + pendingIncidents = remaining; + const currentIncident = nextState.incident; + const deliveredIncident = { + ...updatedIncident, + alertDelivery: "sent", + deliveryThreadId: recoveryThreadId, + }; + const pendingResolutions = [...durableIncidentList(nextState, "pendingResolutions")]; + if (currentIncident?.fingerprint !== updatedIncident.fingerprint) { + const pendingResolutionIndex = pendingResolutions.findIndex( + (pendingIncident) => pendingIncident.fingerprint === updatedIncident.fingerprint, + ); + if (pendingResolutionIndex >= 0) { + pendingResolutions[pendingResolutionIndex] = { + ...pendingResolutions[pendingResolutionIndex], + ...deliveredIncident, + }; + } else { + pendingResolutions.push(deliveredIncident); + } + } + nextState = Object.assign( + {}, + nextState, + pendingIncidents.length > 0 ? { pendingIncidents } : { pendingIncidents: undefined }, + pendingResolutions.length > 0 ? { pendingResolutions } : {}, + currentIncident?.fingerprint === updatedIncident.fingerprint + ? { + incident: { ...currentIncident, ...deliveredIncident }, + } + : {}, + ); + dependencies.writeState(nextState); + } catch (error) { + console.error( + `[lastcode:checkpoint-supervisor] Could not deliver a pending alert to the maintenance thread: ${error instanceof Error ? error.message : String(error)}`, + ); + return nextState; + } + } + + if (nextState.status === "success") { + let pendingResolutions = [...durableIncidentList(nextState, "pendingResolutions")]; + while (pendingResolutions.length > 0) { + const [incident, ...remaining] = pendingResolutions; + if (!incident) break; + const updatedIncident = { + ...incident, + resolutionDelivery: "pending", + deliveryAttempts: (incident.deliveryAttempts ?? 0) + 1, + lastDeliveryAttemptAt: dependencies.now(), + }; + nextState = Object.assign({}, nextState, { + pendingResolutions: [updatedIncident, ...remaining], + }); + dependencies.writeState(nextState); + try { + dependencies.sendThread( + updatedIncident.deliveryThreadId ?? recoveryThreadId, + checkpointResolvedMessage(updatedIncident), + ); + pendingResolutions = remaining; + nextState = Object.assign( + {}, + nextState, + pendingResolutions.length > 0 + ? { pendingResolutions } + : { pendingResolutions: undefined }, + ); + dependencies.writeState(nextState); + } catch (error) { + console.error( + `[lastcode:checkpoint-supervisor] Could not deliver a pending resolution to the maintenance thread: ${error instanceof Error ? error.message : String(error)}`, + ); + return nextState; + } + } + } + + const incident = nextState.incident; + if (!incident) return nextState; + const resolving = nextState.status === "success"; + const key = resolving ? "resolutionDelivery" : "alertDelivery"; + if (incident[key] === "sent" || incident[key] === "not-needed") return nextState; + const message = resolving + ? checkpointResolvedMessage(incident) + : checkpointFailureMessage(incident.failure, incident.fingerprint); + const deliveryThreadId = resolving + ? (incident.deliveryThreadId ?? recoveryThreadId) + : recoveryThreadId; + const nextIncident = { + ...incident, + [key]: "pending", + deliveryAttempts: (incident.deliveryAttempts ?? 0) + 1, + lastDeliveryAttemptAt: dependencies.now(), + }; + nextState = { ...nextState, incident: nextIncident }; + dependencies.writeState(nextState); + try { + dependencies.sendThread(deliveryThreadId, message); + nextState = { + ...nextState, + incident: { + ...nextIncident, + [key]: "sent", + ...(!resolving ? { deliveryThreadId } : {}), + }, + }; + dependencies.writeState(nextState); + } catch (error) { + console.error( + `[lastcode:checkpoint-supervisor] Could not deliver ${resolving ? "resolution" : "alert"} to the maintenance thread: ${error instanceof Error ? error.message : String(error)}`, + ); + } + return nextState; +} + +export function runCheckpointSupervisor(options = {}, overrides = {}) { + const home = options.home ?? NodeOS.homedir(); + const repoRoot = options.repoRoot ?? process.cwd(); + const paths = supervisorPaths(home); + const environment = checkpointEnvironment(options.environment ?? process.env, home); + const installedSupervisorPath = NodeURL.fileURLToPath(import.meta.url); + const dependencies = { + latestFailedCheckpointRun: () => latestFailedCheckpointRun(paths.runHistoryPath), + loadConfig: () => readJson(paths.configPath, "checkpoint supervisor config"), + loadState: () => readJson(paths.statePath, "checkpoint supervisor state"), + now: () => new Date().toISOString(), + notify: (title, message) => notify(title, message, environment), + retainedRecoveryMatches: (checkpointRun) => + retainedRecoveryMatches(repoRoot, checkpointRun, environment), + refreshSupervisor: () => refreshInstalledSupervisor(repoRoot, installedSupervisorPath), + runPhase: (phase, command, args) => runCommand(phase, repoRoot, command, args, environment), + sendThread: (threadId, message) => + sendThread(paths.threadToolPath, threadId, message, environment), + writeState: (state) => writeJsonAtomic(paths.statePath, state), + ...overrides, + }; + const startedAt = dependencies.now(); + let previous = null; + let config = null; + let preflightError = null; + let phase = "supervisor-state"; + let checkpointHistoryBeforeRun = null; + let checkpointHistoryBaselineAvailable = false; + try { + previous = dependencies.loadState(); + } catch (error) { + preflightError = error; + } + try { + config = dependencies.loadConfig(); + } catch (error) { + if (!preflightError) { + preflightError = error; + phase = "supervisor-config"; + } + } + try { + if (preflightError) throw preflightError; + phase = "fetch"; + dependencies.runPhase("fetch", "git", [ + "fetch", + "origin", + "+refs/heads/lastcode/main:refs/remotes/origin/lastcode/main", + ]); + phase = "checkout"; + dependencies.runPhase("checkout", "git", [ + "checkout", + "--detach", + "--force", + "refs/remotes/origin/lastcode/main", + ]); + phase = "supervisor-refresh"; + dependencies.refreshSupervisor(); + phase = "dependencies"; + dependencies.runPhase("dependencies", "./node_modules/.bin/vp", [ + "install", + "--frozen-lockfile", + ]); + phase = "checkpoint"; + try { + checkpointHistoryBeforeRun = dependencies.latestFailedCheckpointRun(); + checkpointHistoryBaselineAvailable = true; + } catch (historyError) { + console.error( + `[lastcode:checkpoint-supervisor] Could not snapshot checkpoint history before the run: ${historyError instanceof Error ? historyError.message : String(historyError)}`, + ); + } + dependencies.runPhase("checkpoint", process.execPath, CHECKPOINT_ARGS); + } catch (error) { + const finishedAt = dependencies.now(); + let checkpointRun = null; + if (phase === "checkpoint") { + try { + const latestCheckpointRun = dependencies.latestFailedCheckpointRun(); + const writtenByCurrentInvocation = + checkpointHistoryBaselineAvailable && + !sameCheckpointRun(checkpointHistoryBeforeRun, latestCheckpointRun); + if ( + latestCheckpointRun && + (writtenByCurrentInvocation || dependencies.retainedRecoveryMatches(latestCheckpointRun)) + ) { + checkpointRun = latestCheckpointRun; + } + } catch (historyError) { + console.error( + `[lastcode:checkpoint-supervisor] Could not enrich the incident from checkpoint history: ${historyError instanceof Error ? historyError.message : String(historyError)}`, + ); + } + } + const failure = { + checkpointRun, + ...(error instanceof CommandFailure && error.diagnostic + ? { diagnostic: error.diagnostic } + : {}), + error: error instanceof Error ? error.message : String(error), + phase: error instanceof CommandFailure ? error.phase : phase, + }; + const fingerprint = checkpointIncidentFingerprint(failure); + const pendingIncidents = [...durableIncidentList(previous, "pendingIncidents")]; + const pendingResolutions = [...durableIncidentList(previous, "pendingResolutions")]; + const previousIncident = durableIncident(previous?.incident); + if ( + previous?.status === "success" && + previousIncident && + previousIncident.resolutionDelivery === "pending" && + !pendingResolutions.some( + (pendingIncident) => pendingIncident.fingerprint === previousIncident.fingerprint, + ) + ) { + pendingResolutions.push(previousIncident); + } + const sameOpenIncident = + previous?.status === "failed" && previousIncident?.fingerprint === fingerprint; + const pendingIncidentIndex = pendingIncidents.findIndex( + (pendingIncident) => pendingIncident.fingerprint === fingerprint, + ); + const pendingResolutionIndex = pendingResolutions.findIndex( + (pendingIncident) => pendingIncident.fingerprint === fingerprint, + ); + const restoredIncident = + (pendingResolutionIndex >= 0 + ? pendingResolutions.splice(pendingResolutionIndex, 1)[0] + : null) ?? + (pendingIncidentIndex >= 0 ? pendingIncidents.splice(pendingIncidentIndex, 1)[0] : null); + const knownIncident = sameOpenIncident || Boolean(restoredIncident); + const incident = (sameOpenIncident ? previousIncident : restoredIncident) ?? { + alertDelivery: "pending", + deliveryAttempts: 0, + failure, + fingerprint, + openedAt: finishedAt, + }; + if (!sameOpenIncident && previous?.status === "failed" && previousIncident) { + const destination = + previousIncident.alertDelivery === "sent" ? pendingResolutions : pendingIncidents; + if ( + !destination.some( + (pendingIncident) => pendingIncident.fingerprint === previousIncident.fingerprint, + ) + ) { + destination.push(previousIncident); + } + } + let state = { + schemaVersion: 1, + status: "failed", + phase: failure.phase, + startedAt, + finishedAt, + ...(previous?.lastSuccessAt ? { lastSuccessAt: previous.lastSuccessAt } : {}), + ...(pendingIncidents.length > 0 ? { pendingIncidents } : {}), + ...(pendingResolutions.length > 0 ? { pendingResolutions } : {}), + incident, + }; + dependencies.writeState(state); + state = attemptDelivery(state, config, dependencies); + if (!knownIncident) { + const delivered = state.incident.alertDelivery === "sent"; + dependencies.notify( + "LastCode checkpoint automation needs attention", + delivered + ? `${failure.phase} failed; the maintenance thread was notified.` + : `${failure.phase} failed; thread delivery is pending. Run lastcode-checkpoints --verbose.`, + ); + } + throw error; + } + + const finishedAt = dependencies.now(); + const deliveryConfigured = Boolean(config?.recoveryThreadId); + let incident = durableIncident(previous?.incident); + const pendingIncidents = deliveryConfigured + ? [...durableIncidentList(previous, "pendingIncidents")] + : []; + const pendingResolutions = deliveryConfigured + ? durableIncidentList(previous, "pendingResolutions").map((pendingIncident) => ({ + ...pendingIncident, + resolvedAt: pendingIncident.resolvedAt ?? finishedAt, + resolutionDelivery: "pending", + })) + : []; + if (previous?.status === "failed" && incident) { + if ( + deliveryConfigured && + incident.alertDelivery !== "sent" && + !pendingIncidents.some( + (pendingIncident) => pendingIncident.fingerprint === incident.fingerprint, + ) + ) { + pendingIncidents.push(incident); + } + incident = { + ...incident, + resolvedAt: finishedAt, + resolutionDelivery: deliveryConfigured ? "pending" : "not-needed", + ...(!deliveryConfigured && incident.alertDelivery !== "sent" + ? { alertDelivery: "not-needed" } + : {}), + }; + } else if (!deliveryConfigured && incident) { + incident = { ...incident, resolutionDelivery: "not-needed" }; + } + let state = { + schemaVersion: 1, + status: "success", + phase: "complete", + startedAt, + finishedAt, + lastSuccessAt: finishedAt, + ...(pendingIncidents.length > 0 ? { pendingIncidents } : {}), + ...(pendingResolutions.length > 0 ? { pendingResolutions } : {}), + ...(incident ? { incident } : {}), + }; + dependencies.writeState(state); + state = attemptDelivery(state, config, dependencies); + return state; +} + +if (import.meta.main) { + if (process.argv.length !== 3 || process.argv[2] !== "run") { + console.error("Usage: lastcode-checkpoint-supervisor.mjs run"); + process.exitCode = 64; + } else { + try { + runCheckpointSupervisor(); + } catch (error) { + console.error( + `[lastcode:checkpoint-supervisor] ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + } + } +} diff --git a/scripts/lastcode-checkpoint-supervisor.test.mjs b/scripts/lastcode-checkpoint-supervisor.test.mjs new file mode 100644 index 000000000000..0d1447bc5ca6 --- /dev/null +++ b/scripts/lastcode-checkpoint-supervisor.test.mjs @@ -0,0 +1,700 @@ +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import { describe, expect, it, vi } from "vite-plus/test"; + +import { + boundedCommandDiagnostic, + checkpointEnvironment, + checkpointFailureMessage, + checkpointIncidentFingerprint, + latestFailedCheckpointRun, + refreshInstalledSupervisor, + runCheckpointSupervisor, +} from "./lastcode-checkpoint-supervisor.mjs"; + +function fixture(overrides = {}) { + const states = []; + const messages = []; + let state = overrides.state ?? null; + let now = 0; + const dependencies = { + latestFailedCheckpointRun: () => ({ + status: "failed", + upstreamTag: "v0.0.35-nightly.20260826.1195", + failurePhase: "rebase", + recoveryBranch: "sync/nightly/v0.0.35-nightly.20260826.1195", + error: "rebase failed", + }), + loadConfig: () => ({ schemaVersion: 1, recoveryThreadId: "thread-maintenance" }), + loadState: () => state, + now: () => `2026-08-26T00:00:0${now++}.000Z`, + notify: vi.fn(), + refreshSupervisor: vi.fn(), + runPhase: vi.fn(), + sendThread: (threadId, message) => messages.push({ message, threadId }), + writeState: (next) => { + state = next; + states.push(next); + }, + ...overrides.dependencies, + }; + return { + dependencies, + get state() { + return state; + }, + messages, + states, + }; +} + +describe("LastCode checkpoint supervisor", () => { + it("runs the complete checkpoint pipeline and records terminal success", () => { + const test = fixture(); + + expect(runCheckpointSupervisor({}, test.dependencies)).toMatchObject({ + status: "success", + phase: "complete", + }); + expect(test.dependencies.runPhase.mock.calls.map(([phase]) => phase)).toEqual([ + "fetch", + "checkout", + "dependencies", + "checkpoint", + ]); + expect(test.dependencies.refreshSupervisor).toHaveBeenCalledOnce(); + expect(test.messages).toEqual([]); + expect(test.states.at(-1)).toMatchObject({ status: "success", phase: "complete" }); + }); + + it("persists a new checkpoint blocker before alerting one maintenance thread", () => { + const test = fixture({ + dependencies: { + runPhase: vi.fn((phase) => { + if (phase === "checkpoint") throw new Error("checkpoint failed"); + }), + }, + }); + + expect(() => runCheckpointSupervisor({}, test.dependencies)).toThrow("checkpoint failed"); + expect(test.states[0]).toMatchObject({ + status: "failed", + phase: "checkpoint", + incident: { alertDelivery: "pending", deliveryAttempts: 0 }, + }); + expect(test.state).toMatchObject({ + status: "failed", + incident: { alertDelivery: "sent", deliveryAttempts: 1 }, + }); + expect(test.messages).toHaveLength(1); + expect(test.messages[0]).toMatchObject({ threadId: "thread-maintenance" }); + expect(test.messages[0]?.message).toContain("Use this thread for the recovery"); + }); + + it("does not redeliver the same blocker after acknowledgement", () => { + const failedRun = () => { + throw new Error("fetch failed"); + }; + const first = fixture({ dependencies: { runPhase: failedRun } }); + expect(() => runCheckpointSupervisor({}, first.dependencies)).toThrow("fetch failed"); + + const second = fixture({ + state: first.state, + dependencies: { runPhase: failedRun }, + }); + expect(() => runCheckpointSupervisor({}, second.dependencies)).toThrow("fetch failed"); + expect(second.messages).toEqual([]); + expect(second.state.incident.deliveryAttempts).toBe(1); + }); + + it("keeps failed delivery pending and retries it on the next run", () => { + const runPhase = () => { + throw new Error("fetch failed"); + }; + const first = fixture({ + dependencies: { + runPhase, + sendThread: () => { + throw new Error("server unavailable"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, first.dependencies)).toThrow("fetch failed"); + expect(first.state.incident.alertDelivery).toBe("pending"); + + const second = fixture({ state: first.state, dependencies: { runPhase } }); + expect(() => runCheckpointSupervisor({}, second.dependencies)).toThrow("fetch failed"); + expect(second.messages).toHaveLength(1); + expect(second.state.incident.alertDelivery).toBe("sent"); + expect(second.state.incident.deliveryAttempts).toBe(2); + }); + + it("delivers every distinct blocker after thread delivery recovers", () => { + const first = fixture({ + dependencies: { + runPhase: () => { + throw new Error("first fetch blocker"); + }, + sendThread: () => { + throw new Error("server unavailable"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, first.dependencies)).toThrow("first fetch blocker"); + + const second = fixture({ + state: first.state, + dependencies: { + runPhase: () => { + throw new Error("second fetch blocker"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, second.dependencies)).toThrow("second fetch blocker"); + + expect(second.messages).toHaveLength(2); + expect(second.messages[0]?.message).not.toBe(second.messages[1]?.message); + expect(second.state.pendingIncidents).toBeUndefined(); + expect(second.state.pendingResolutions).toHaveLength(1); + expect(second.state.incident).toMatchObject({ alertDelivery: "sent" }); + + const recovered = fixture({ state: second.state }); + runCheckpointSupervisor({}, recovered.dependencies); + expect(recovered.messages).toHaveLength(2); + expect(recovered.messages.every(({ message }) => message.includes("resolved alert"))).toBe( + true, + ); + expect(recovered.state.pendingResolutions).toBeUndefined(); + }); + + it("closes every delivered blocker after distinct failures recover", () => { + const first = fixture({ + dependencies: { + runPhase: () => { + throw new Error("first fetch blocker"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, first.dependencies)).toThrow("first fetch blocker"); + + const second = fixture({ + state: first.state, + dependencies: { + runPhase: () => { + throw new Error("second fetch blocker"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, second.dependencies)).toThrow("second fetch blocker"); + expect(second.state.pendingResolutions).toHaveLength(1); + + const recovered = fixture({ state: second.state }); + runCheckpointSupervisor({}, recovered.dependencies); + + expect(recovered.messages).toHaveLength(2); + expect(recovered.messages.every(({ message }) => message.includes("resolved alert"))).toBe( + true, + ); + expect(recovered.messages[0]?.message).not.toBe(recovered.messages[1]?.message); + expect(recovered.state.pendingResolutions).toBeUndefined(); + expect(recovered.state.incident).toMatchObject({ resolutionDelivery: "sent" }); + }); + + it("reuses a queued incident when alternating blockers return", () => { + const failWith = (message) => () => { + throw new Error(message); + }; + const first = fixture({ dependencies: { runPhase: failWith("blocker A") } }); + expect(() => runCheckpointSupervisor({}, first.dependencies)).toThrow("blocker A"); + + const second = fixture({ + state: first.state, + dependencies: { runPhase: failWith("blocker B") }, + }); + expect(() => runCheckpointSupervisor({}, second.dependencies)).toThrow("blocker B"); + + const third = fixture({ + state: second.state, + dependencies: { runPhase: failWith("blocker A") }, + }); + expect(() => runCheckpointSupervisor({}, third.dependencies)).toThrow("blocker A"); + expect(third.messages).toEqual([]); + expect(third.state.pendingResolutions).toHaveLength(1); + + const recovered = fixture({ state: third.state }); + runCheckpointSupervisor({}, recovered.dependencies); + expect(recovered.messages).toHaveLength(2); + expect(recovered.messages[0]?.message).not.toBe(recovered.messages[1]?.message); + expect(recovered.state.pendingResolutions).toBeUndefined(); + }); + + it("retains an undelivered closure when a later run fails", () => { + const failed = fixture({ + dependencies: { + runPhase: () => { + throw new Error("blocker A"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, failed.dependencies)).toThrow("blocker A"); + + const recoveredWithoutDelivery = fixture({ + state: failed.state, + dependencies: { + sendThread: () => { + throw new Error("server unavailable"); + }, + }, + }); + runCheckpointSupervisor({}, recoveredWithoutDelivery.dependencies); + expect(recoveredWithoutDelivery.state).toMatchObject({ + status: "success", + incident: { resolutionDelivery: "pending" }, + }); + + const failedAgain = fixture({ + state: recoveredWithoutDelivery.state, + dependencies: { + runPhase: () => { + throw new Error("blocker B"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, failedAgain.dependencies)).toThrow("blocker B"); + expect(failedAgain.messages).toHaveLength(1); + expect(failedAgain.state.pendingResolutions).toHaveLength(1); + + const finallyRecovered = fixture({ state: failedAgain.state }); + runCheckpointSupervisor({}, finallyRecovered.dependencies); + expect(finallyRecovered.messages).toHaveLength(2); + expect( + finallyRecovered.messages.every(({ message }) => message.includes("resolved alert")), + ).toBe(true); + expect(finallyRecovered.state.pendingResolutions).toBeUndefined(); + }); + + it("reopens recurring incidents when no recovery thread is configured", () => { + const runPhase = () => { + throw new Error("fetch failed"); + }; + const first = fixture({ + dependencies: { loadConfig: () => null, runPhase }, + }); + expect(() => runCheckpointSupervisor({}, first.dependencies)).toThrow("fetch failed"); + expect(first.dependencies.notify).toHaveBeenCalledOnce(); + + const recovered = fixture({ + state: first.state, + dependencies: { loadConfig: () => null }, + }); + runCheckpointSupervisor({}, recovered.dependencies); + expect(recovered.state).toMatchObject({ + status: "success", + incident: { alertDelivery: "not-needed", resolutionDelivery: "not-needed" }, + }); + expect(recovered.state.pendingIncidents).toBeUndefined(); + + const recurring = fixture({ + state: recovered.state, + dependencies: { loadConfig: () => null, runPhase }, + }); + expect(() => runCheckpointSupervisor({}, recurring.dependencies)).toThrow("fetch failed"); + expect(recurring.dependencies.notify).toHaveBeenCalledOnce(); + expect(recurring.state.incident).toMatchObject({ alertDelivery: "pending" }); + }); + + it("updates a queued closure with the delayed alert destination", () => { + const failed = fixture({ + dependencies: { + runPhase: () => { + throw new Error("blocker A"); + }, + sendThread: () => { + throw new Error("server unavailable"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, failed.dependencies)).toThrow("blocker A"); + + const recoveredWithoutDelivery = fixture({ + state: failed.state, + dependencies: { + sendThread: () => { + throw new Error("server unavailable"); + }, + }, + }); + runCheckpointSupervisor({}, recoveredWithoutDelivery.dependencies); + + const failedAgain = fixture({ + state: recoveredWithoutDelivery.state, + dependencies: { + loadConfig: () => ({ schemaVersion: 1, recoveryThreadId: "thread-delivery" }), + runPhase: () => { + throw new Error("blocker B"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, failedAgain.dependencies)).toThrow("blocker B"); + expect(failedAgain.state.pendingResolutions).toHaveLength(1); + expect(failedAgain.state.pendingResolutions[0]).toMatchObject({ + deliveryThreadId: "thread-delivery", + }); + + const finallyRecovered = fixture({ + state: failedAgain.state, + dependencies: { + loadConfig: () => ({ schemaVersion: 1, recoveryThreadId: "thread-replacement" }), + }, + }); + runCheckpointSupervisor({}, finallyRecovered.dependencies); + expect(finallyRecovered.messages).toHaveLength(2); + expect(finallyRecovered.messages.every(({ threadId }) => threadId === "thread-delivery")).toBe( + true, + ); + }); + + it("sends one closure after a failed incident recovers", () => { + const previous = fixture({ + dependencies: { + runPhase: () => { + throw new Error("fetch failed"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, previous.dependencies)).toThrow("fetch failed"); + + const recovered = fixture({ state: previous.state }); + expect(runCheckpointSupervisor({}, recovered.dependencies)).toMatchObject({ + status: "success", + incident: { resolutionDelivery: "sent" }, + }); + expect(recovered.messages).toHaveLength(1); + expect(recovered.messages[0]?.message).toContain("maintenance resolved alert"); + + const stillHealthy = fixture({ state: recovered.state }); + runCheckpointSupervisor({}, stillHealthy.dependencies); + expect(stillHealthy.messages).toEqual([]); + }); + + it("sends an incident closure to the thread that received its alert", () => { + const previous = fixture({ + dependencies: { + loadConfig: () => ({ schemaVersion: 1, recoveryThreadId: "thread-original" }), + runPhase: () => { + throw new Error("fetch failed"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, previous.dependencies)).toThrow("fetch failed"); + expect(previous.state.incident).toMatchObject({ + alertDelivery: "sent", + deliveryThreadId: "thread-original", + }); + + const recovered = fixture({ + state: previous.state, + dependencies: { + loadConfig: () => ({ schemaVersion: 1, recoveryThreadId: "thread-replacement" }), + }, + }); + runCheckpointSupervisor({}, recovered.dependencies); + + expect(recovered.messages).toHaveLength(1); + expect(recovered.messages[0]).toMatchObject({ threadId: "thread-original" }); + expect(recovered.messages[0]?.message).toContain("maintenance resolved alert"); + }); + + it("delivers the alert and closure when recovery precedes the alert retry", () => { + const previous = fixture({ + dependencies: { + runPhase: () => { + throw new Error("fetch failed"); + }, + sendThread: () => { + throw new Error("server unavailable"); + }, + }, + }); + expect(() => runCheckpointSupervisor({}, previous.dependencies)).toThrow("fetch failed"); + + const recovered = fixture({ state: previous.state }); + expect(runCheckpointSupervisor({}, recovered.dependencies)).toMatchObject({ + status: "success", + incident: { resolutionDelivery: "sent" }, + }); + expect(recovered.messages).toHaveLength(2); + expect(recovered.messages[0]?.message).toContain("maintenance alert"); + expect(recovered.messages[1]?.message).toContain("maintenance resolved alert"); + expect(recovered.state.pendingIncidents).toBeUndefined(); + }); + + it("ignores a malformed durable incident backlog", () => { + const test = fixture({ + state: { schemaVersion: 1, status: "success", pendingIncidents: "invalid" }, + dependencies: { + runPhase: () => { + throw new Error("fetch failed"); + }, + }, + }); + + expect(() => runCheckpointSupervisor({}, test.dependencies)).toThrow("fetch failed"); + expect(test.messages).toHaveLength(1); + expect(test.state).toMatchObject({ + status: "failed", + incident: { alertDelivery: "sent" }, + }); + }); + + it("discards malformed records inside durable incident state", () => { + const test = fixture({ + state: { + schemaVersion: 1, + status: "success", + pendingIncidents: ["invalid", { fingerprint: "missing-failure" }], + pendingResolutions: [null, 42], + incident: "invalid", + }, + dependencies: { + runPhase: () => { + throw new Error("fetch failed"); + }, + }, + }); + + expect(() => runCheckpointSupervisor({}, test.dependencies)).toThrow("fetch failed"); + expect(test.messages).toHaveLength(1); + expect(test.state.pendingIncidents).toBeUndefined(); + expect(test.state.pendingResolutions).toBeUndefined(); + expect(test.state).toMatchObject({ + status: "failed", + incident: { alertDelivery: "sent" }, + }); + }); + + it("does not inherit unrelated or credential-bearing session variables", () => { + expect( + checkpointEnvironment( + { + HOME: "/wrong", + PATH: "/wrong", + SSH_AUTH_SOCK: "/tmp/agent.sock", + ELECTRON_RUN_AS_NODE: "1", + SECRET_TOKEN: "do-not-copy", + USER: "lasto", + }, + "/Users/lasto", + "/opt/pinned-node/bin/node", + ), + ).toEqual({ + HOME: "/Users/lasto", + PATH: "/opt/pinned-node/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin", + SSH_AUTH_SOCK: "/tmp/agent.sock", + USER: "lasto", + }); + }); + + it("keeps recovery instructions bounded and tied to the retained branch", () => { + const message = checkpointFailureMessage( + { + phase: "checkpoint", + error: "rebase failed", + checkpointRun: { + upstreamTag: "nightly-1", + recoveryBranch: "sync/nightly/nightly-1", + }, + }, + "abcdef1234567890", + ); + expect(message).toContain("abcdef123456"); + expect(message).toContain("sync/nightly/nightly-1"); + expect(message).toContain("lastcode-checkpoints --verbose"); + expect(message).toContain("nightly-checkpoint.stderr.log"); + }); + + it("distinguishes different recorded checkpoint blockers behind the same wrapper error", () => { + const baseFailure = { + phase: "checkpoint", + error: "node failed with exit code 1", + checkpointRun: { + upstreamTag: "nightly-1", + failurePhase: "smoke", + recoveryBranch: "sync/nightly/nightly-1", + }, + }; + + expect( + checkpointIncidentFingerprint({ + ...baseFailure, + checkpointRun: { ...baseFailure.checkpointRun, error: "migration smoke failed" }, + }), + ).not.toBe( + checkpointIncidentFingerprint({ + ...baseFailure, + checkpointRun: { ...baseFailure.checkpointRun, error: "server typecheck failed" }, + }), + ); + }); + + it("distinguishes pre-checkpoint blockers with the same command exit", () => { + const baseFailure = { + phase: "fetch", + error: "git failed with exit code 128", + checkpointRun: null, + }; + + expect( + checkpointIncidentFingerprint({ ...baseFailure, diagnostic: "host key verification failed" }), + ).not.toBe( + checkpointIncidentFingerprint({ ...baseFailure, diagnostic: "repository not found" }), + ); + }); + + it("bounds and redacts command diagnostics before persistence or delivery", () => { + const diagnostic = boundedCommandDiagnostic( + `https://user:password@example.com/repo token=secret-value github_pat_abcdefghijklmnopqrstuvwxyz ${"x".repeat(2_000)}`, + ); + + expect(diagnostic.length).toBeLessThanOrEqual(1_203); + expect(diagnostic).not.toContain("user:password"); + expect(diagnostic).not.toContain("secret-value"); + expect(diagnostic).not.toContain("github_pat_"); + expect(diagnostic.endsWith("x".repeat(1_000))).toBe(true); + }); + + it("ignores invalid history shapes while finding the latest valid failure", () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "lastcode-history-")); + const historyPath = NodePath.join(directory, "checkpoint-runs.jsonl"); + NodeFS.writeFileSync( + historyPath, + `${JSON.stringify({ status: "failed", error: "original failure" })}\nnull\n`, + ); + + expect(latestFailedCheckpointRun(historyPath)).toMatchObject({ + status: "failed", + error: "original failure", + }); + NodeFS.rmSync(directory, { recursive: true }); + }); + + it("preserves and reports the original failure when history enrichment fails", () => { + const test = fixture({ + dependencies: { + latestFailedCheckpointRun: () => { + throw new Error("history unreadable"); + }, + runPhase: (phase) => { + if (phase === "checkpoint") throw new Error("original checkpoint failure"); + }, + }, + }); + + expect(() => runCheckpointSupervisor({}, test.dependencies)).toThrow( + "original checkpoint failure", + ); + expect(test.state).toMatchObject({ + status: "failed", + incident: { + alertDelivery: "sent", + failure: { error: "original checkpoint failure", checkpointRun: null }, + }, + }); + expect(test.messages).toHaveLength(1); + }); + + it("does not enrich a checkpoint failure with stale history", () => { + const stale = { + status: "failed", + upstreamTag: "v0.0.35-nightly.stale", + recoveryBranch: "sync/nightly/v0.0.35-nightly.stale", + error: "old smoke failure", + }; + const test = fixture({ + dependencies: { + latestFailedCheckpointRun: () => stale, + retainedRecoveryMatches: () => false, + runPhase: (phase) => { + if (phase === "checkpoint") throw new Error("current planning failure"); + }, + }, + }); + + expect(() => runCheckpointSupervisor({}, test.dependencies)).toThrow( + "current planning failure", + ); + expect(test.state.incident.failure).toMatchObject({ checkpointRun: null }); + expect(test.messages[0]?.message).not.toContain("v0.0.35-nightly.stale"); + }); + + it("uses a checkpoint history record written by the current invocation", () => { + const stale = { status: "failed", upstreamTag: "stale", error: "old failure" }; + const current = { status: "failed", upstreamTag: "current", error: "current failure" }; + const latestFailedCheckpointRun = vi + .fn() + .mockReturnValueOnce(stale) + .mockReturnValueOnce(current); + const test = fixture({ + dependencies: { + latestFailedCheckpointRun, + retainedRecoveryMatches: () => false, + runPhase: (phase) => { + if (phase === "checkpoint") throw new Error("checkpoint failed"); + }, + }, + }); + + expect(() => runCheckpointSupervisor({}, test.dependencies)).toThrow("checkpoint failed"); + expect(test.state.incident.failure.checkpointRun).toEqual(current); + }); + + it("uses unchanged history only for the matching retained recovery", () => { + const retained = { + status: "failed", + upstreamTag: "v0.0.35-nightly.retained", + recoveryBranch: "sync/nightly/v0.0.35-nightly.retained", + error: "retained smoke failure", + }; + const test = fixture({ + dependencies: { + latestFailedCheckpointRun: () => retained, + retainedRecoveryMatches: () => true, + runPhase: (phase) => { + if (phase === "checkpoint") throw new Error("retained worktree blocks retry"); + }, + }, + }); + + expect(() => runCheckpointSupervisor({}, test.dependencies)).toThrow( + "retained worktree blocks retry", + ); + expect(test.state.incident.failure.checkpointRun).toEqual(retained); + }); + + it("turns corrupt durable state into a reportable supervisor incident", () => { + const test = fixture({ + dependencies: { + loadState: () => { + throw new Error("invalid state json"); + }, + }, + }); + + expect(() => runCheckpointSupervisor({}, test.dependencies)).toThrow("invalid state json"); + expect(test.state).toMatchObject({ status: "failed", phase: "supervisor-state" }); + expect(test.messages).toHaveLength(1); + }); + + it("keeps the installed supervisor until the source version lands", () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "lastcode-supervisor-")); + const installedPath = NodePath.join(directory, "installed.mjs"); + NodeFS.writeFileSync(installedPath, "installed\n"); + + expect(() => refreshInstalledSupervisor(directory, installedPath)).not.toThrow(); + expect(NodeFS.readFileSync(installedPath, "utf8")).toBe("installed\n"); + + NodeFS.rmSync(directory, { recursive: true }); + }); +}); diff --git a/scripts/lastcode-checkpoint.ts b/scripts/lastcode-checkpoint.ts index 1bdda81c5158..c5b1e53c16a6 100644 --- a/scripts/lastcode-checkpoint.ts +++ b/scripts/lastcode-checkpoint.ts @@ -679,7 +679,8 @@ function runSmokeGate(repoRoot: string, worktree: string): void { "scripts/lastcode-local-ci.test.ts", "scripts/build-desktop-artifact.test.ts", "apps/desktop/src/electron/ElectronProtocol.test.ts", - "apps/server/src/persistence/Migrations/048_ProjectionThreadLinkedPullRequest.test.ts", + "apps/server/src/persistence/Migrations/049_ProjectionThreadLinkedPullRequest.test.ts", + "apps/server/src/persistence/Migrations/050_ProjectionThreadsUnsettledAt.test.ts", ], { environment }, ); diff --git a/scripts/lastcode-nightly-service.test.ts b/scripts/lastcode-nightly-service.test.ts index 6dda85da5318..a8e7fe3c3e07 100644 --- a/scripts/lastcode-nightly-service.test.ts +++ b/scripts/lastcode-nightly-service.test.ts @@ -1,22 +1,64 @@ import { expect, it } from "vite-plus/test"; -import { renderLaunchAgentPlist, runNowArguments } from "./lastcode-nightly-service.ts"; +import { + parseNightlyServiceArgs, + renderLaunchAgentPlist, + runNowArguments, +} from "./lastcode-nightly-service.ts"; it("renders an hourly source-only launch agent with escaped durable paths", () => { const plist = renderLaunchAgentPlist({ repoRoot: "/Users/example/LastCode & experiments", logDirectory: "/Users/example/.lastcode/automation", + nodePath: "/Users/example/.local/share/mise/installs/node/24.13.1/bin/node", + supervisorPath: "/Users/example/.lastcode/automation/bin/checkpoint supervisor.mjs", }); expect(plist).toContain("3600"); - expect(plist).toContain("--push-tags --promote-if-no-open-prs --mirror-upstream-main"); - expect(plist).toContain("git checkout --detach --force refs/remotes/origin/lastcode/main"); - expect(plist).toContain("./node_modules/.bin/vp install --frozen-lockfile"); + expect(plist).toContain("node/24.13.1/bin/node"); + expect(plist).toContain("checkpoint supervisor.mjs"); + expect(plist).not.toContain("/bin/zsh"); + expect(plist).not.toContain("/usr/bin/env"); + expect(plist).not.toContain("mise exec"); + expect(plist).not.toContain("git checkout"); + expect(plist).not.toContain("vp install"); expect(plist).not.toContain("lastcode-build"); expect(plist).toContain("LastCode & experiments"); expect(plist).toContain("nightly-checkpoint.stderr.log"); }); +it("configures one durable recovery thread only during installation", () => { + expect( + parseNightlyServiceArgs([ + "install", + "--recovery-thread", + "e01b7101-0713-4df7-9b6b-5c46f9d507db", + ]), + ).toEqual({ + command: "install", + recoveryThreadId: "e01b7101-0713-4df7-9b6b-5c46f9d507db", + }); + expect(parseNightlyServiceArgs(["install"])).toEqual({ command: "install" }); + expect(parseNightlyServiceArgs(["install", "--no-recovery-thread"])).toEqual({ + command: "install", + clearRecoveryThread: true, + }); + expect(() => parseNightlyServiceArgs(["run-now", "--recovery-thread", "thread-1"])).toThrow( + "accepted only", + ); + expect(() => parseNightlyServiceArgs(["install", "--recovery-thread", "short"])).toThrow( + "valid thread ID", + ); + expect(() => + parseNightlyServiceArgs([ + "install", + "--no-recovery-thread", + "--recovery-thread", + "thread-maintenance", + ]), + ).toThrow("accepts only"); +}); + it("requests an idle service run without terminating an active checkpoint", () => { expect(runNowArguments("gui/501/codes.lastobelus.lastcode-nightly-checkpoint")).toEqual([ "kickstart", diff --git a/scripts/lastcode-nightly-service.ts b/scripts/lastcode-nightly-service.ts index c0e9357f441d..11f3a06b06c7 100644 --- a/scripts/lastcode-nightly-service.ts +++ b/scripts/lastcode-nightly-service.ts @@ -11,6 +11,7 @@ import * as Effect from "effect/Effect"; const LABEL = "codes.lastobelus.lastcode-nightly-checkpoint"; const INTERVAL_SECONDS = 60 * 60; +const SUPERVISOR_FILE = "lastcode-checkpoint-supervisor.mjs"; function xml(value: string): string { return value @@ -23,18 +24,10 @@ function xml(value: string): string { export function renderLaunchAgentPlist(input: { readonly logDirectory: string; + readonly nodePath: string; readonly repoRoot: string; + readonly supervisorPath: string; }): string { - const command = [ - "git fetch origin +refs/heads/lastcode/main:refs/remotes/origin/lastcode/main", - "&& git checkout --detach --force refs/remotes/origin/lastcode/main", - "&& ./node_modules/.bin/vp install --frozen-lockfile", - "&&", - "mise exec node@24.13.1 -- node scripts/lastcode-checkpoint.ts", - "--push-tags", - "--promote-if-no-open-prs", - "--mirror-upstream-main", - ].join(" "); return ` @@ -43,9 +36,9 @@ export function renderLaunchAgentPlist(input: { ${LABEL} ProgramArguments - /bin/zsh - -lc - ${command} + ${xml(input.nodePath)} + ${xml(input.supervisorPath)} + run WorkingDirectory ${xml(input.repoRoot)} @@ -69,6 +62,44 @@ export function renderLaunchAgentPlist(input: { `; } +export function parseNightlyServiceArgs(argv: ReadonlyArray): { + readonly command: "install" | "run-now" | "status" | "uninstall"; + readonly clearRecoveryThread?: boolean; + readonly recoveryThreadId?: string; +} { + const command = argv[0]; + if ( + command !== "install" && + command !== "run-now" && + command !== "status" && + command !== "uninstall" + ) { + throw new Error( + "Usage: pnpm lastcode:checkpoint:service [--recovery-thread | --no-recovery-thread]", + ); + } + if (command !== "install") { + if (argv.length !== 1) { + throw new Error("--recovery-thread is accepted only by the install command."); + } + return { command } as const; + } + if (argv.length === 1) return { command } as const; + if (argv.length === 2 && argv[1] === "--no-recovery-thread") { + return { command, clearRecoveryThread: true } as const; + } + const recoveryThreadId = argv[2]; + if ( + argv.length !== 3 || + argv[1] !== "--recovery-thread" || + !recoveryThreadId || + !/^[A-Za-z0-9-]{8,}$/u.test(recoveryThreadId) + ) { + throw new Error("install accepts only --recovery-thread followed by a valid thread ID."); + } + return { command, recoveryThreadId } as const; +} + export function runNowArguments(service: string): ReadonlyArray { return ["kickstart", service]; } @@ -91,14 +122,7 @@ function run( function main(argv: ReadonlyArray): void { if (Effect.runSync(HostProcessPlatform) !== "darwin") throw new Error("LastCode nightly scheduling requires macOS launchd."); - const command = argv[0]; - if ( - !command || - argv.length !== 1 || - !["install", "run-now", "status", "uninstall"].includes(command) - ) { - throw new Error("Usage: pnpm lastcode:checkpoint:service "); - } + const { clearRecoveryThread, command, recoveryThreadId } = parseNightlyServiceArgs(argv); const repoRoot = NodeChildProcess.execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd: process.cwd(), @@ -121,6 +145,9 @@ function main(argv: ReadonlyArray): void { const home = NodeOS.homedir(); const plistPath = NodePath.join(home, "Library", "LaunchAgents", `${LABEL}.plist`); const logDirectory = NodePath.join(home, ".lastcode", "automation"); + const supervisorDirectory = NodePath.join(logDirectory, "bin"); + const supervisorPath = NodePath.join(supervisorDirectory, SUPERVISOR_FILE); + const supervisorConfigPath = NodePath.join(logDirectory, "checkpoint-supervisor.json"); const getuid = process.getuid; if (!getuid) throw new Error("Could not resolve the current macOS user ID."); const domain = `gui/${getuid()}`; @@ -141,6 +168,7 @@ function main(argv: ReadonlyArray): void { NodeFS.renameSync(plistPath, backupPath); console.log(`[lastcode:service] Disabled plist retained at ${backupPath}.`); } + if (NodeFS.existsSync(supervisorConfigPath)) NodeFS.rmSync(supervisorConfigPath); return; } @@ -169,15 +197,41 @@ function main(argv: ReadonlyArray): void { }); NodeFS.mkdirSync(NodePath.dirname(plistPath), { recursive: true }); NodeFS.mkdirSync(logDirectory, { recursive: true }); + NodeFS.mkdirSync(supervisorDirectory, { recursive: true, mode: 0o700 }); + NodeFS.copyFileSync(NodePath.join(repoRoot, "scripts", SUPERVISOR_FILE), supervisorPath); + NodeFS.chmodSync(supervisorPath, 0o700); + if (clearRecoveryThread) { + if (NodeFS.existsSync(supervisorConfigPath)) NodeFS.rmSync(supervisorConfigPath); + } else if (recoveryThreadId) { + const temporaryConfigPath = `${supervisorConfigPath}.tmp`; + NodeFS.writeFileSync( + temporaryConfigPath, + `${JSON.stringify({ schemaVersion: 1, recoveryThreadId }, null, 2)}\n`, + { mode: 0o600 }, + ); + NodeFS.renameSync(temporaryConfigPath, supervisorConfigPath); + } NodeFS.writeFileSync( plistPath, - renderLaunchAgentPlist({ logDirectory, repoRoot: automationWorktree }), + renderLaunchAgentPlist({ + logDirectory, + nodePath: process.execPath, + repoRoot: automationWorktree, + supervisorPath, + }), ); run("plutil", ["-lint", plistPath]); run("launchctl", ["bootout", service], { allowFailure: true }); run("launchctl", ["bootstrap", domain, plistPath]); run("launchctl", ["kickstart", service]); console.log(`[lastcode:service] Installed ${LABEL}; it runs at login and hourly.`); + if (recoveryThreadId) { + console.log( + "[lastcode:service] Automatic recovery alerts use the configured maintenance thread.", + ); + } else if (clearRecoveryThread) { + console.log("[lastcode:service] Automatic recovery thread delivery is disabled."); + } } if (import.meta.main) {