fix(server): wait for a complete Bun before restarting onto a replaced package tree

src/lib/package-tree-integrity.ts

@@ -1,6 +1,7 @@
 import { readFileSync, statSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { isRealBunBinary } from "./bun-binary-validator.mjs";
 import { isStandaloneBinary } from "./standalone";
 
 export interface PackageTreeObservation {
@@ -56,6 +57,14 @@ export interface PackageTreeIntegrityOptions {
   schedule?: (callback: () => void, delayMs: number) => (() => void) | void;
   /** Test seam for `installedVersion()`; production reads the package manifest. */
   readInstalledVersion?: () => string | undefined;
+  /**
+   * Whether the runtime a restart would spawn is complete. An in-place npm install writes
+   * package.json and extracts the `bun` package's small placeholder before its postinstall
+   * swaps the real binary in, and the replacement spawns `process.execPath`, that same path.
+   * While this answers false the replacement is treated like an unreadable tree: no restart,
+   * then a fresh full debounce once the runtime is ready.
+   */
+  runtimeReady?: () => boolean;
 }
 
 export type ObservePackageTree = () => PackageTreeObservation | null;
@@ -142,6 +151,12 @@ export function sameObservation(left: PackageTreeObservation, right: PackageTree
  */
 const PACKAGE_TREE_RECHECK_MS = 1_000;
 
+/**
+ * Fences a live process whose package manifest was replaced, and once the replacement is readable,
+ * stable for the debounce interval and any configured `runtimeReady` check passes, hands it to
+ * `onReplaced`. If the callback throws, the guard rechecks the tree and may call it again after
+ * another debounce interval while the replacement persists.
+ */
 export function createPackageTreeIntegrityGuard(
   observe: ObservePackageTree = observePackageManifest,
   now: () => number = Date.now,
@@ -154,6 +169,8 @@ export function createPackageTreeIntegrityGuard(
   let timerScheduled = false;
   let cancelScheduled: (() => void) | null = null;
   let waitingForReadableTree = false;
+  /** Logged once per wait, so a postinstall that never runs is visible rather than silent. */
+  let waitingForRuntime = false;
   let replacementCandidate: PackageTreeObservation | null = null;
   /** The replacement identity that survived a full stability interval (see installedVersion). */
   let settledReplacement: PackageTreeObservation | null = null;
@@ -192,9 +209,24 @@ export function createPackageTreeIntegrityGuard(
       if (sameObservation(boot, current)) {
         resetRestartTimer();
         waitingForReadableTree = false;
+        waitingForRuntime = false;
         replacementCandidate = null;
         return;
       }
+      if (options.runtimeReady && !options.runtimeReady()) {
+        if (!waitingForRuntime) {
+          waitingForRuntime = true;
+          console.warn("Package tree replaced; waiting for its Bun runtime to finish installing before restarting");
+        }
+        // A replacement settled before a failed admission is no longer restart-ready: withdraw
+        // it so installedVersion() stays silent until the fresh debounce settles it again.
+        settledReplacement = null;
+        resetRestartTimer();
+        waitingForReadableTree = true;
+        armRestartTimer(PACKAGE_TREE_RECHECK_MS);
+        return;
+      }
+      waitingForRuntime = false;
       if (waitingForReadableTree) {
         waitingForReadableTree = false;
         replacementCandidate = current;
@@ -239,6 +271,7 @@ export function createPackageTreeIntegrityGuard(
   return {
     installedVersion: () => {
       if (settledReplacement === null) return undefined;
+      if (options.runtimeReady && !options.runtimeReady()) return undefined;
       const current = observe();
       if (current === null || !sameObservation(settledReplacement, current)) return undefined;
       return readInstalledVersion();
@@ -271,6 +304,7 @@ export function createPackageTreeIntegrityGuard(
       lastOkAt = at;
       resetRestartTimer();
       waitingForReadableTree = false;
+      waitingForRuntime = false;
       replacementCandidate = null;
       return { ok: true };
     },
@@ -292,5 +326,12 @@ export function createRuntimePackageTreeIntegrityGuard(
     if (installer === "source" || isStandaloneBinary()) {
       return { status: () => ({ ok: true }), dispose: () => {} };
     }
-    return createPackageTreeIntegrityGuard(observe, now, options);
+    // The restart replacement spawns process.execPath, the bundled Bun in an npm install. Read it
+    // now: Linux Bun resolves it lazily, and after npm swaps the package directory a first read
+    // reports the retired binary as "(deleted)"; reading it once pins the boot path.
+    const runtimePath = process.execPath;
+    return createPackageTreeIntegrityGuard(observe, now, {
+      ...options,
+      runtimeReady: options.runtimeReady ?? (() => isRealBunBinary(runtimePath)),
+    });
   }

structure/ops/docs-and-release.md

@@ -366,7 +366,7 @@ observed at startup. Replacing that manifest under a live process fences `/healt
 stability timer; if the same new manifest identity remains readable and distinct, the timer enters the existing
 drain-and-restart handoff without waiting for another request. A temporarily unreadable manifest
 is polled until readable and then receives a fresh full stability interval, while a return to the
-startup identity cancels the pending restart. Failed restart admission retries after the same
+startup identity cancels the pending restart. The replacement spawns `process.execPath`, which an in-place npm install leaves as the `bun` package's small placeholder until its postinstall runs, so while that path fails the `REAL_BUN_MIN_BYTES` gate the restart waits the same way and then debounces afresh; meanwhile `installedVersion` stays unreported, so a manual restart is refused as unsettled. Failed restart admission retries after the same
 bounded delay. Stopping the server before the accepted restart begins vetoes it, and a service child
 restarts only while it still owns the service home. Source checkouts and standalone binaries remain outside this fence.
 

tests/ci-workflows/package-tree-integrity.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { chmodSync, existsSync, mkdirSync, renameSync, statSync, writeFileSync } from "node:fs";
+import { chmodSync, copyFileSync, existsSync, mkdirSync, renameSync, statSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { saveConfig } from "../../src/config";
 import {
@@ -204,9 +204,12 @@ describe("package tree integrity", () => {
     };
     const createScheduler = () => {
       const pending: Array<() => void> = [];
+      const delays: number[] = [];
       return {
         pending,
-        schedule: (callback: () => void) => {
+        delays,
+        schedule: (callback: () => void, delayMs: number) => {
+          delays.push(delayMs);
           pending.push(callback);
           return () => {
             const index = pending.indexOf(callback);
@@ -254,6 +257,79 @@ describe("package tree integrity", () => {
       expect(calls).toBe(1);
     });
 
+    test("waits for the runtime a restart would spawn, then debounces afresh", async () => {
+      let observation: PackageTreeObservation | null = base;
+      let clock = 0;
+      let calls = 0;
+      let runtimeReady = false;
+      const scheduler = createScheduler();
+      const guard = createPackageTreeIntegrityGuard(
+        () => observation,
+        () => clock,
+        {
+          onReplaced: () => { calls += 1; },
+          replacedRestartDelayMs: 5_000,
+          schedule: scheduler.schedule,
+          runtimeReady: () => runtimeReady,
+        },
+      );
+
+      expect(guard.status()).toEqual({ ok: true });
+      // npm wrote the new package.json; bun's postinstall has not replaced its placeholder yet.
+      observation = { ...base, inode: 11n, contentTimeNs: 200n };
+      clock += 2_000;
+      expect(guard.status()).toEqual({ ok: false, reason: "package_tree_replaced" });
+      await scheduler.runNext();
+      await scheduler.runNext();
+      expect(calls).toBe(0);
+      expect(scheduler.pending).toHaveLength(1);
+
+      runtimeReady = true;
+      await scheduler.runNext();
+      expect(calls).toBe(0);
+      await scheduler.runNext();
+      expect(calls).toBe(1);
+      // Debounce, two 1s runtime polls, then a fresh full debounce once the runtime is ready.
+      expect(scheduler.delays).toEqual([5_000, 1_000, 1_000, 5_000]);
+    });
+
+    test("the installed guard reads process.execPath as the runtime", async () => {
+      // A child Bun runs from a disposable copy, then that path is swapped for a placeholder the
+      // way an in-place npm install leaves node_modules/bun/bin/bun.exe before its postinstall.
+      const dir = join(TEST_DIR, "runtime-placeholder");
+      mkdirSync(dir, { recursive: true });
+      const runtime = join(dir, process.platform === "win32" ? "bun.exe" : "bun");
+      copyFileSync(process.execPath, runtime);
+      chmodSync(runtime, 0o755);
+      const script = join(dir, "probe.ts");
+      const guardModule = join(import.meta.dir, "../../src/lib/package-tree-integrity.ts").replaceAll("\\", "/");
+      writeFileSync(script, `
+import { renameSync, writeFileSync } from "node:fs";
+import { createRuntimePackageTreeIntegrityGuard } from ${JSON.stringify(guardModule)};
+const base = { device: 1n, inode: 10n, contentTimeNs: 100n, size: 500n };
+let observation = base;
+const pending = [];
+let calls = 0;
+let clock = 0;
+const guard = createRuntimePackageTreeIntegrityGuard("npm", () => observation, () => clock, {
+  onReplaced: () => { calls += 1; },
+  replacedRestartDelayMs: 5_000,
+  schedule: callback => { pending.push(callback); },
+});
+guard.status();
+renameSync(process.execPath, process.execPath + ".old");
+writeFileSync(process.execPath, "placeholder replaced by the bun package postinstall\\n");
+observation = { ...base, inode: 11n };
+clock += 2_000;
+if (guard.status().ok) throw new Error("replacement not detected");
+for (let k = 0; k < 3 && pending.length; k += 1) { pending.shift()(); await Promise.resolve(); await Promise.resolve(); }
+console.log(JSON.stringify({ calls }));
+`);
+      const child = Bun.spawnSync([runtime, script], { stdout: "pipe", stderr: "pipe" });
+      expect(child.exitCode).toBe(0);
+      expect(JSON.parse(child.stdout.toString().trim())).toEqual({ calls: 0 });
+    });
+
     test("retries when restart acceptance throws", async () => {
       let observation: PackageTreeObservation | null = base;
       let clock = 0;
@@ -284,6 +360,51 @@ describe("package tree integrity", () => {
       expect(scheduler.pending).toHaveLength(0);
     });
 
+    test("a runtime that stops being ready withdraws a settled replacement until it settles again", async () => {
+      let observation: PackageTreeObservation | null = base;
+      let clock = 0;
+      let attempts = 0;
+      let runtimeReady = true;
+      const scheduler = createScheduler();
+      const guard = createPackageTreeIntegrityGuard(
+        () => observation,
+        () => clock,
+        {
+          onReplaced: () => {
+            attempts += 1;
+            if (attempts === 1) throw new Error("restart unavailable");
+          },
+          replacedRestartDelayMs: 5_000,
+          schedule: scheduler.schedule,
+          readInstalledVersion: () => "9.9.9",
+          runtimeReady: () => runtimeReady,
+        },
+      );
+
+      expect(guard.status()).toEqual({ ok: true });
+      observation = { ...base, inode: 11n };
+      clock += 2_000;
+      expect(guard.status()).toEqual({ ok: false, reason: "package_tree_replaced" });
+      await scheduler.runNext();
+      expect(attempts).toBe(1);
+      expect(guard.installedVersion()).toBe("9.9.9");
+
+      // A second install puts the bun placeholder back before the admission retry.
+      runtimeReady = false;
+      expect(guard.installedVersion()).toBeUndefined();
+      await scheduler.runNext();
+      expect(attempts).toBe(1);
+      runtimeReady = true;
+      expect(guard.installedVersion()).toBeUndefined();
+
+      await scheduler.runNext();
+      expect(attempts).toBe(1);
+      expect(guard.installedVersion()).toBeUndefined();
+      await scheduler.runNext();
+      expect(attempts).toBe(2);
+      expect(guard.installedVersion()).toBe("9.9.9");
+    });
+
     test("baseline recovery cancels the old timer and starts a fresh debounce", async () => {
       let observation: PackageTreeObservation | null = base;
       let clock = 0;