feat(observability): write a server heap snapshot on SIGUSR2

apps/server/src/observability/HeapSnapshot.test.ts

@@ -0,0 +1,36 @@
+// @effect-diagnostics nodeBuiltinImport:off - tests fake a failed write at the native v8 boundary.
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import * as NodeFS from "node:fs";
+import * as NodePath from "node:path";
+import * as NodeV8 from "node:v8";
+import { assert, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import { vi } from "vite-plus/test";
+
+import { writeHeapSnapshot } from "./HeapSnapshot.ts";
+
+vi.mock("node:v8", async (importOriginal) => {
+  const actual = await importOriginal<typeof NodeV8>();
+  return { ...actual, writeHeapSnapshot: vi.fn(actual.writeHeapSnapshot) };
+});
+
+it.layer(NodeServices.layer)("writeHeapSnapshot", (it) => {
+  it.effect("removes the partial file when the write fails", () =>
+    Effect.gen(function* () {
+      const fs = yield* FileSystem.FileSystem;
+      const logsDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-heap-snapshot-test-" });
+      let partialPath: string | undefined;
+      vi.mocked(NodeV8.writeHeapSnapshot).mockImplementationOnce((path) => {
+        partialPath = path;
+        if (path) NodeFS.writeFileSync(path, "partial");
+        throw new Error("ENOSPC: no space left on device");
+      });
+
+      yield* writeHeapSnapshot(logsDir);
+
+      assert.strictEqual(NodePath.dirname(partialPath ?? ""), logsDir);
+      assert.deepEqual(yield* fs.readDirectory(logsDir), []);
+    }),
+  );
+});

apps/server/src/observability/HeapSnapshot.ts

@@ -0,0 +1,52 @@
+// @effect-diagnostics nodeBuiltinImport:off - v8.writeHeapSnapshot has no Effect equivalent.
+import * as NodePath from "node:path";
+import * as NodeV8 from "node:v8";
+
+import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
+import * as DateTime from "effect/DateTime";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+
+import * as ServerConfig from "../config.ts";
+
+/**
+ * Writes one V8 heap snapshot into `logsDir` and logs its path. A failed write
+ * logs a warning and removes any partial file, because that file can hold
+ * secrets and the failure is often a full disk.
+ */
+export const writeHeapSnapshot = Effect.fn("server.heapSnapshot", { root: true })(
+  function* (logsDir: string) {
+    const fs = yield* FileSystem.FileSystem;
+    const timestamp = DateTime.formatIso(yield* DateTime.now).replaceAll(":", "-");
+    const path = NodePath.join(logsDir, `server-${process.pid}-${timestamp}.heapsnapshot`);
+    yield* Effect.annotateCurrentSpan({ path });
+    yield* Effect.try(() => NodeV8.writeHeapSnapshot(path)).pipe(
+      Effect.tapError(() => fs.remove(path, { force: true }).pipe(Effect.ignore)),
+    );
+    yield* Effect.logInfo("Wrote heap snapshot.", { path });
+  },
+  Effect.catch((cause) => Effect.logWarning("Failed to write heap snapshot.", { cause })),
+);
+
+/**
+ * Writes a heap snapshot when the process gets SIGUSR2 (`kill -USR2 <pid>`),
+ * so a maintainer can see what a long-running server holds. See "Heap
+ * Snapshots" in docs/operations/observability.md.
+ *
+ * The write blocks the event loop, so two snapshots never overlap: a signal
+ * sent during a write waits until it finishes. Windows has no SIGUSR2, so the
+ * layer does nothing there.
+ */
+export const layer = Layer.effectDiscard(
+  Effect.gen(function* () {
+    if ((yield* HostProcessPlatform) === "win32") return;
+    const { logsDir } = yield* ServerConfig.ServerConfig;
+    const runFork = Effect.runForkWith(yield* Effect.context<FileSystem.FileSystem>());
+    const onSignal = () => void runFork(writeHeapSnapshot(logsDir));
+    yield* Effect.acquireRelease(
+      Effect.sync(() => process.on("SIGUSR2", onSignal)),
+      () => Effect.sync(() => process.off("SIGUSR2", onSignal)),
+    );
+  }),
+);

apps/server/src/server.ts

@@ -119,6 +119,7 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe
 import * as ProjectSetupScriptRunner from "./project/ProjectSetupScriptRunner.ts";
 import * as WorktreeSetupTracker from "./project/WorktreeSetupTracker.ts";
 import { ObservabilityLive } from "./observability/Layers/Observability.ts";
+import * as HeapSnapshot from "./observability/HeapSnapshot.ts";
 import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
 import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";
 import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts";
@@ -957,6 +958,7 @@ const makeServerLayer = Layer.unwrap(
       runtimeStateLayer.pipe(Layer.provide(launcherLayer)),
       tailscaleServeLayer,
       cloudDesiredLinkReconcileLayer,
+      HeapSnapshot.layer,
     );
 
     return serverApplicationLayer.pipe(

docs/operations/observability.md

@@ -618,3 +618,40 @@ Current high-value span and metric boundaries include:
 - logs outside spans are not persisted in the trace file; SSH-managed launch stdout/stderr is still
   captured in its launcher log
 - metrics are not snapshotted locally
+
+## Heap Snapshots
+
+To see what a long-running server holds in memory, send it `SIGUSR2`. The server writes a V8 heap
+snapshot to its logs dir and logs the path. This works for desktop, `npx t3`, and service installs
+on macOS and Linux. Windows has no `SIGUSR2`.
+
+Send the signal to the server pid in `server-runtime.json`, which sits in the server's state dir
+next to the `logs` dir. For a dev server or a `--home-dir` launch, use that server's state dir from
+[Traces](#traces). Do not send it to the desktop app or the service launcher: a process without the
+handler exits on `SIGUSR2`. After a crash the file can keep a stale pid that now belongs to a
+different process, so check the pid first.
+
+```bash
+pid="$(jq .pid "${T3CODE_HOME:-$HOME/.t3}/userdata/server-runtime.json")"
+ps -p "$pid" -o command=
+```
+
+If `ps` shows the T3 Code server, send the signal:
+
+```bash
+kill -USR2 "$pid"
+```
+
+The file is `<logsDir>/server-<pid>-<timestamp>.heapsnapshot`, next to `server.trace.ndjson`. To
+open it, use the Memory tab in Chrome DevTools and select Load.
+
+Before you take one:
+
+- The server stops while it writes the file. For a large heap this can take a minute or more.
+  Connected clients can reconnect during the pause, and an event loop monitor, if the server has
+  one, records the pause as a stall. Send the signal once. A second signal sent during a write
+  takes another snapshot after the first one finishes.
+- The write needs about as much free memory as the heap uses. On a machine that is already
+  swapping, it can make the problem worse or crash the server.
+- The file contains everything in server memory, including tokens, secrets, and thread content. Do
+  not share it publicly. Delete it when you are done, because storage cleanup does not remove it.