diff --git a/README.md b/README.md index 74a2f5f..64a2d62 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ The CLI is `wtd`: a daemon per host owns the PTYs and git watches, and a local h Three running components, all started from the one `wtd` binary: -| Component | Runs | Owns | -| --- | --- | --- | -| Daemon (`wtd daemon`) | One per host, independent of any page or hub | The PTYs and their screen mirrors, git worktree watches, and `~/.local/state/worktree-term/state.json`: each worktree's terminal tabs, splits, active tab and checkbox | -| Hub (`wtd hub`, started by `wtd ui`) | One on the machine you browse from | `~/.config/worktree-term/config.json` (repos, roots, presets, hosts and their hub ports); nothing else | -| Page | The browser | Rendering (xterm.js) and view preferences in `localStorage`: selected repo, selected worktree per repo, "checked only" filter, name search, sidebar width | +| Component | Runs | Owns | +| ------------------------------------ | -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Daemon (`wtd daemon`) | One per host, independent of any page or hub | The PTYs and their screen mirrors, git worktree watches, and `~/.local/state/worktree-term/state.json`: each worktree's terminal tabs, splits, active tab and checkbox | +| Hub (`wtd hub`, started by `wtd ui`) | One on the machine you browse from | `~/.config/worktree-term/config.json` (repos, roots, presets, hosts and their hub ports); nothing else | +| Page | The browser | Rendering (xterm.js) and view preferences in `localStorage`: selected repo, selected worktree per repo, "checked only" filter, name search, sidebar width | - **The browser talks only to the hub**: the hub serves the page and one WebSocket per page, on `127.0.0.1` only. - **The daemon manages the terminals.** A terminal lives until it is closed, its worktree is removed, or its program exits cleanly; one that fails stays open, marked, until closed. Closing the page or restarting the hub does not end it. diff --git a/architecture/model.c4 b/architecture/model.c4 index 242e00e..0ca0fd1 100644 --- a/architecture/model.c4 +++ b/architecture/model.c4 @@ -23,6 +23,7 @@ model { files = component 'Files' dialer = component 'Dialer' install = component 'Install' + memory = component 'Memory' } daemon = node 'Daemon' { @@ -71,6 +72,7 @@ model { platform.dialer -> platform.files platform.install -> platform.files platform.install -> platform.dialer + platform.memory -> platform.files daemon -> protocol daemon -> platform.files @@ -78,6 +80,7 @@ model { daemon.main -> daemon.worktrees daemon.main -> daemon.terminals daemon.main -> daemon.state + daemon.main -> platform.memory daemon.server -> daemon.worktrees daemon.server -> daemon.terminals daemon.server -> daemon.state @@ -91,6 +94,7 @@ model { hub.main -> hub.router hub.main -> hub.links hub.main -> hub.config + hub.main -> platform.memory hub.server -> hub.router hub.router -> hub.links hub.router -> hub.config diff --git a/architecture/system.arc42.md b/architecture/system.arc42.md index daaa432..f3abee7 100644 --- a/architecture/system.arc42.md +++ b/architecture/system.arc42.md @@ -17,6 +17,7 @@ flowchart TD platform__files["Files"] platform__dialer["Dialer"] platform__install["Install"] + platform__memory["Memory"] end subgraph daemon["Daemon"] daemon__worktrees["Worktrees"] @@ -43,12 +44,14 @@ flowchart TD platform__dialer --> platform__files platform__install --> platform__files platform__install --> platform__dialer + platform__memory --> platform__files daemon --> protocol daemon --> platform__files daemon__main --> daemon__server daemon__main --> daemon__worktrees daemon__main --> daemon__terminals daemon__main --> daemon__state + daemon__main --> platform__memory daemon__server --> daemon__worktrees daemon__server --> daemon__terminals daemon__server --> daemon__state @@ -61,6 +64,7 @@ flowchart TD hub__main --> hub__router hub__main --> hub__links hub__main --> hub__config + hub__main --> platform__memory hub__server --> hub__router hub__router --> hub__links hub__router --> hub__config @@ -79,7 +83,7 @@ flowchart TD web__ui --> web__layout web__terminals --> web__client classDef leaf fill:none; - class protocol,platform__files,platform__dialer,platform__install,daemon__worktrees,daemon__terminals,daemon__state,daemon__server,daemon__main,hub__server,hub__links,hub__config,hub__router,hub__main,cli,web__client,web__terminals,web__layout,web__ui,web__main leaf; + class protocol,platform__files,platform__dialer,platform__install,platform__memory,daemon__worktrees,daemon__terminals,daemon__state,daemon__server,daemon__main,hub__server,hub__links,hub__config,hub__router,hub__main,cli,web__client,web__terminals,web__layout,web__ui,web__main leaf; ``` diff --git a/architecture/typescript.daemon.arc42.md b/architecture/typescript.daemon.arc42.md index 920a220..0db93dd 100644 --- a/architecture/typescript.daemon.arc42.md +++ b/architecture/typescript.daemon.arc42.md @@ -15,6 +15,7 @@ flowchart TD protocol["Protocol"] subgraph platform["Platform"] platform__files["Files"] + platform__memory["Memory"] end subgraph daemon["Daemon"] daemon__worktrees["Worktrees"] @@ -30,12 +31,13 @@ flowchart TD daemon__main --> daemon__worktrees daemon__main --> daemon__terminals daemon__main --> daemon__state + daemon__main --> platform__memory daemon__server --> daemon__worktrees daemon__server --> daemon__terminals daemon__server --> daemon__state cli --> daemon__main classDef leaf fill:none; - class protocol,platform__files,daemon__worktrees,daemon__terminals,daemon__state,daemon__server,daemon__main,cli leaf; + class protocol,platform__files,platform__memory,daemon__worktrees,daemon__terminals,daemon__state,daemon__server,daemon__main,cli leaf; ``` diff --git a/architecture/typescript.hub.arc42.md b/architecture/typescript.hub.arc42.md index fd33fdd..766beb6 100644 --- a/architecture/typescript.hub.arc42.md +++ b/architecture/typescript.hub.arc42.md @@ -17,6 +17,7 @@ flowchart TD platform__files["Files"] platform__dialer["Dialer"] platform__install["Install"] + platform__memory["Memory"] end subgraph hub["Hub"] hub__server["Server"] @@ -35,12 +36,13 @@ flowchart TD hub__main --> hub__router hub__main --> hub__links hub__main --> hub__config + hub__main --> platform__memory hub__server --> hub__router hub__router --> hub__links hub__router --> hub__config cli --> hub__main classDef leaf fill:none; - class protocol,platform__files,platform__dialer,platform__install,hub__server,hub__links,hub__config,hub__router,hub__main,cli leaf; + class protocol,platform__files,platform__dialer,platform__install,platform__memory,hub__server,hub__links,hub__config,hub__router,hub__main,cli leaf; ``` diff --git a/architecture/typescript.platform.arc42.md b/architecture/typescript.platform.arc42.md index f207ca0..4ba5924 100644 --- a/architecture/typescript.platform.arc42.md +++ b/architecture/typescript.platform.arc42.md @@ -2,7 +2,7 @@ ## Purpose & context -Node-only infrastructure shared by daemon, hub and CLI: XDG paths and file I/O (`files`); reaching a host's daemon over its socket, with auto-start, or over `ssh` (`dialer`); installing the bundle locally or over `ssh` (`install`). +Node-only infrastructure shared by daemon, hub and CLI: XDG paths and file I/O (`files`); reaching a host's daemon over its socket, with auto-start, or over `ssh` (`dialer`); installing the bundle locally or over `ssh` (`install`); sampling process, cgroup and system memory into a rotating log (`memory`). ## Building blocks @@ -16,27 +16,34 @@ flowchart TD platform__files["Files"] platform__dialer["Dialer"] platform__install["Install"] + platform__memory["Memory"] + end + subgraph daemon["Daemon"] + daemon__main["Main"] end - daemon["Daemon"] subgraph hub["Hub"] hub__server["Server"] hub__links["Links"] hub__config["Config"] + hub__main["Main"] end cli["CLI"] platform__dialer --> platform__files platform__install --> platform__files platform__install --> platform__dialer + platform__memory --> platform__files daemon --> platform__files + daemon__main --> platform__memory hub__server --> platform__files hub__config --> platform__files hub__links --> platform__dialer hub__links --> platform__install + hub__main --> platform__memory cli --> platform__files cli --> platform__dialer cli --> platform__install classDef leaf fill:none; - class platform__files,platform__dialer,platform__install,daemon,hub__server,hub__links,hub__config,cli leaf; + class platform__files,platform__dialer,platform__install,platform__memory,daemon__main,hub__server,hub__links,hub__config,hub__main,cli leaf; ``` @@ -45,4 +52,4 @@ flowchart TD 1. Knows nothing of protocol, daemon, hub or web. [enforced: arch_check:model-truth] 2. Only `dialer` connects to or spawns a daemon. [enforced: test:test/architecture/dependencies.test.ts] 3. Files and directories it creates are owner-only. [enforced: test:src/platform/files/files.test.ts] -4. File writes are atomic. [enforced: test:src/platform/files/files.test.ts] +4. File writes are atomic; log appends write whole lines. [enforced: test:src/platform/files/files.test.ts] diff --git a/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/.openspec.yaml b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/.openspec.yaml new file mode 100644 index 0000000..1bb6456 --- /dev/null +++ b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-10-28 diff --git a/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/design.md b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/design.md new file mode 100644 index 0000000..f7a97eb --- /dev/null +++ b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/design.md @@ -0,0 +1,85 @@ +## Context + +See proposal.md for the incident. Constraints found while planning: + +- systemd-oomd killed the `vte-spawn-…` scope because the pressure of its monitored ancestor `user@1000.service` was 79.97% > 50% for > 20 s; the root slice also has `ManagedOOMSwap=auto`. Pressure and swap, not RSS alone, decide a kill. +- The daemon is spawned detached (own session) by whoever dials first, so it lives in that process's cgroup: on the dev box the hub and daemon sit in a GitKraken app scope with 141 processes. The cgroup is shared with unrelated apps. +- `TerminalProcess.close()` sends `SIGHUP`/`SIGKILL` to the shell's process group only. An interactive shell puts each job in its own process group, so jobs that survive the hangup (an older terminal's `claude`, `tsserver`, `codex` on the dev box) outlive the terminal; `setsid` children (a leaked test daemon) leave the session altogether and get reparented to `systemd --user`. +- The hub is a separate process and the parent of the daemon it starts; only it can report its own heap. +- Principles: system 1/3/5/6/8; daemon 1 and 3; platform 1, 3 and 4 (amended, D10). + +## Goals / Non-Goals + +**Goals:** +- Name the processes, terminals and escaped leftovers holding memory in the minutes before a kill, and the pressure/swap figures oomd acted on. +- Attribute leftovers to the terminal they came from, across daemon restarts and cgroups. + +**Non-Goals:** +- Preventing the kill (e.g. a systemd scope per terminal) or killing escaped processes. +- Showing memory in the UI; any protocol change. +- Native macOS sources (`ps`, `task_info`): without `/proc` only `self` and `mirrors` are logged. + +## Decisions + +### D1 — Terminals are tagged; cgroup leftovers are a separate bucket (user) +Every terminal's environment gets `WTD_TERMINAL=//`; samples scan all of the user's processes, so escapes from earlier daemons in other cgroups are found too. The rest of the daemon's cgroup is logged as `others`. +- Alternative: "cgroup members under no live terminal" as the issue words it — rejected by the user: on a shared app scope it lists GitKraken and editors and cannot tell which processes wtd leaked. + +### D2 — Attribution rules (default; rule 3 and closed-terminal expiry from codex) +In order: (1) live member = session id equals the shell pid, or descends from the shell by ppid; (2) tag names this daemon (pid and instance) and a live terminal → that terminal's `detached`; (3) tag names another pid whose command line is a running `wtd … daemon` (last argument `daemon`, an argument whose basename starts with `wtd`) → that daemon's, not escaped, whatever its ancestry — that daemon logs its own escapes; (4) any other tagged process, or an untagged process in the session of one of the last 256 terminals this daemon closed → escaped, grouped by tag; (5) other cgroup members → `others`. The daemon itself is only in `self`. A remembered closed terminal is forgotten as soon as a sample finds none of its processes (tagged or in its session), which bounds session-id reuse to the life of the leftovers. +- Codex: rule 3 originally required the process to still descend from the foreign daemon, which misfiled double-forked children of its live terminals as escaped (folded). Remembered sessions are exposed to pid reuse; the dead shell's start time cannot be checked, so expiry on the first empty sample is used instead (folded, differently). + +### D3 — Terminal ids taken before spawning (default) +The tag needs the id in the environment, so `createTerm` takes `nextTermId++` before calling `spawnTerminal`; a failed spawn leaves a gap. The spec only requires ids to count up and never be reused. + +### D4 — Hub logs itself to `hub-memory.log` (user) +`self` from its own `process.memoryUsage()`; `children` are all ppid-descendants of the hub except the subtree rooted at a process whose command line equals the hub's daemon command (`[...wtd, 'daemon']`), which the daemon logs; its `cgroup` has `others` too, which covers a daemon in its own systemd unit cgroup. +- Alternatives: the daemon reading the hub's `/proc` entry (no heap breakdown; the hub record parser would have to leave `hub.server`) or one shared file (two writers racing on rotation) — rejected by the user. +- Codex: limiting children to the hub's session missed detached descendants (folded). + +### D5 — Fields beyond the issue (user) +Own cgroup `memory.pressure`, `memory.swap.current`, `memory.max`, `memory.high`; `/proc/meminfo` `MemTotal`, `MemAvailable`, `SwapTotal`, `SwapFree`. They are what oomd's pressure and swap policies act on. `RssAnon` is logged next to `VmRSS` because sums of `VmRSS` count shared pages (the node binary) repeatedly (default). + +### D6 — Cadence (default; 5 s check from codex) +Pressure (system and cgroup PSI) and meminfo are checked every 5 s (2–3 small reads); a full sample is taken when the base interval (60 s, `WTD_MEMORY_SAMPLE_MS` override) has passed or memory is tight (`some avg10` ≥ 10 or `MemAvailable` < 10% of `MemTotal`). The check period drops to the base interval when that is below 5 s, for tests. One loop, single-flight: the next check is scheduled after the previous check/sample (and its write) finished; the timer is `unref`ed; `stop()` cancels the timer and suppresses the write of a sample in flight. +- Codex: a pure 60 s timer would notice rising pressure up to a minute late, inside oomd's 20 s window (folded); overlapping samples could race rotation (folded). + +### D7 — Bounded records (default; caps from codex) +Top 3 processes per terminal member/detached group, top 5 elsewhere, commands cut to 200 characters; at most 20 escaped groups (largest `rss`), the rest summed in `escapedOmitted`. Terminals are all listed (about 1 KB each). Estimated mirror bytes = (normal + alternate buffer lines) × cols × 12 (xterm stores 3 × 32-bit words per cell). +- Codex: the escaped list was unbounded, so one line could exceed the rotation size (folded). + +### D8 — Rotating line log (default) +Before each append, if size + line > 16 MiB: rename `.2`→`.3`, `.1`→`.2`, log→`.1` (missing skipped, `.3` replaced), then append. One `appendFile` per line (O_APPEND, mode 0600). A failed rename is reported, the line is still appended, and rotation is retried on the next append. A line over 16 MiB is written after rotating, so a log never exceeds the limit by more than one line. At ~10 KB per daemon sample (10 terminals) that is ~14 MB/day, so 4–5 days of history; the hub's lines are a few hundred bytes. + +### D9 — Placement (default) +- New `platform.memory` (allowlist `fs`): readers for the process table (`stat`, `status`, lazily `environ` and `cmdline`), the own cgroup and the system, all taking `/proc` and `/sys/fs/cgroup` roots so unit tests use fake trees; text is validated and parsed at this boundary into typed values or null (system principle 5); the sampling loop of D6; group summarising (totals + top N). It writes through `platform.files`. +- `platform.files`: the rotating appender of D8; `HostPaths.memoryLog` and `hubMemoryLog`. +- `daemon.terminals`: `TerminalProcess.pid` and `mirrorStats()`. +- `daemon.server`: tagging (D3), the closed-terminal list, and `Daemon.memoryView()` returning plain data in its own types (no platform import). +- `daemon.main`: a pure attribution module (D2) and the sampler wiring; `hub.main`: the hub sampler (D4). `cli` passes `WTD_MEMORY_SAMPLE_MS` through `DaemonRun`/`HubRun` env. +- Model arrows: `platform.memory -> platform.files`, `daemon.main -> platform.memory`, `hub.main -> platform.memory`. + +### D10 — Architecture edits (user, approved wording) +- `architecture/model.c4`: inside `platform`, `memory = component 'Memory'`; arrows `platform.memory -> platform.files`, `daemon.main -> platform.memory`, `hub.main -> platform.memory`. Diagrams regenerated with `la-arch-diagrams`. +- `architecture/typescript.platform.arc42.md` Purpose (user, approved wording): "Node-only infrastructure shared by daemon, hub and CLI: XDG paths and file I/O (`files`); reaching a host's daemon over its socket, with auto-start, or over `ssh` (`dialer`); installing the bundle locally or over `ssh` (`install`); sampling process, cgroup and system memory into a rotating log (`memory`)." +- Platform principle 4 (user, approved wording): "4. File writes are atomic; log appends write whole lines. [enforced: test:src/platform/files/files.test.ts]" +- `test/architecture/dependencies.ts`: `'platform.memory': ['fs']`. +- rejected (codex): rewording principle 4 as "File replacements are atomic; logs are appended as one whole-line write" — the user had approved the wording above, whose second clause already scopes the exception, and "replacements" would weaken the existing guarantee. + +### D11 — Failure handling (default) +`ENOENT`/`EACCES` on a source → that section null, silently (absence is the expected state on macOS or without cgroup v2). Malformed content or any other error → reported through the process's `report` (`daemon.log` / `hub.log`), section null, sample still written. A vanished process (`ENOENT`/`ESRCH` mid-read) is left out. A failed write is reported; the loop continues (system principle 6). + +### D12 — Test coverage (default; rotation cases from codex) +Unit: parsers against fake trees including absent and malformed files; attribution rules 1–5 including a running foreign daemon, a double-forked foreign child and an env-scrubbed process in a closed session, closed-terminal expiry, the escaped cap; cadence with a fake clock (calm, tight, back to calm, slow sample, stop in flight); rotation (sequence, existing `.1`–`.3`, failure at each rename and recovery, oversized line, mode). Process: daemon terminal tree, `setsid` escape, `SIGHUP`-ignoring job escape, detached child of a live terminal, tag in the environment, hub sample with an SSH child and without the daemon, nothing written after shutdown; `test/process/rss.test.ts` still passes with sampling on. + +## Risks / Trade-offs + +- [Scanning every user process each minute] → `environ` is read only outside live sessions and `cmdline` only for top-list entries; at 5 s cadence under pressure the scan adds I/O when the host is already struggling — acceptable for ~1k small procfs reads. +- [A process that scrubs its environment and calls `setsid`] → not attributable; it shows only under `others` if it is in the daemon's cgroup. +- [Hub and daemon samples in two files] → lined up by `ts`. +- [Session-id reuse for remembered closed terminals] → bounded by forgetting a terminal on its first empty sample (D2). +- [Logs up to 64 MiB per file set] → owner-only, in the state directory. + +## Migration Plan + +None: the logs appear after the daemon and hub are restarted on the new version; no protocol change. Rollback is reverting; leftover log files are inert. diff --git a/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/proposal.md b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/proposal.md new file mode 100644 index 0000000..132e229 --- /dev/null +++ b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/proposal.md @@ -0,0 +1,30 @@ +## Why + +On 2026-10-28 systemd-oomd killed the GNOME Terminal scope holding the hub, the daemon and every wtd terminal (79.7 GB at the kill, 91.8 GB peak, about 50 GB held for days), because the pressure on `user@1000.service` stayed above 50% for 20 s. Nothing recorded which process held the memory. Processes that outlive a closed terminal (jobs in their own process groups, `setsid` children, leaked test daemons) are the prime suspects but unproven; the next kill must be explainable from a log. + +## What Changes + +- The daemon appends one JSON line per sample to `memory.log` in the state directory: its own `process.memoryUsage()`, the size of its headless terminal mirrors, per live terminal the memory of its process tree with its largest processes, processes that escaped closed terminals, the rest of its cgroup, its cgroup's memory counters and pressure, and system memory pressure and headroom. +- Every terminal's environment carries `WTD_TERMINAL=//`, so its processes stay attributable after they leave its session or outlive it; terminal ids are allocated before the process is spawned. +- The hub appends its own samples to `hub-memory.log`: its `process.memoryUsage()`, its child processes other than the daemon, its cgroup and system memory. +- Samples are taken every 60 s, and every 5 s while memory pressure is high or available memory is low; pressure is checked every 5 s. Linux-only sources are null where absent. +- Both logs rotate at 16 MiB, keeping three old files, written owner-only and one whole line per append. +- `WTD_MEMORY_SAMPLE_MS` overrides the 60 s base interval. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `platform`: host paths gain `memory.log` and `hub-memory.log`; new memory sampling (process table, cgroup, system) and rotating line log requirements. +- `daemon`: terminals are tagged in their environment and ids are allocated before spawning; new daemon memory log requirement. +- `hub`: new hub memory log requirement. + +## Impact + +- `src/platform/files` (rotating appender, new paths), new `src/platform/memory` component, `src/daemon/terminals` (pid, mirror stats), `src/daemon/server` (tagging, closed-terminal list, memory view), `src/daemon/main` (attribution, sampler wiring), `src/hub/main` (hub sampler), `src/cli` (passes `WTD_MEMORY_SAMPLE_MS`). +- `architecture/model.c4`: new `platform.memory` component and its arrows; `architecture/typescript.platform.arc42.md` purpose and principle 4; regenerated diagrams; `test/architecture/dependencies.ts` allowlist. +- No protocol, wire version or UI change. Up to 64 MiB of logs per process kind in the state directory. diff --git a/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/daemon/spec.md b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/daemon/spec.md new file mode 100644 index 0000000..014b0e3 --- /dev/null +++ b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/daemon/spec.md @@ -0,0 +1,84 @@ +## MODIFIED Requirements + +### Requirement: Terminal creation +`createTerm` SHALL fail with `unknown-worktree` unless the worktree is in the current list of a repo this connection watches, and with `busy` if that repo already has 1024 live terminals. Otherwise the daemon SHALL start `$SHELL -l -i` when `command` is null, or `$SHELL -l -i -c ` otherwise (`/bin/sh` when `SHELL` is unset or not absolute), in the worktree directory, with `TERM=xterm-256color`, `COLORTERM=truecolor` and `WTD_TERMINAL=//`, at the requested size. A failure to start SHALL fail with `spawn-failed`. Terminal ids SHALL count up from 1 and never be reused within a daemon instance; an id SHALL be taken before the process is started, so a failed start leaves its id unused. `preset` SHALL be stored and reported as given. The requester SHALL receive `termCreated` with its `req`; every other watcher of the repo SHALL receive `termCreated` with `req` null. Creating a terminal SHALL NOT attach it. + +#### Scenario: Shell in the worktree +- **WHEN** a client creates a terminal in a worktree, attaches, and types `pwd` +- **THEN** the output shows the worktree path + +#### Scenario: Terminal tagged +- **WHEN** a client creates a terminal, attaches, and types `echo $WTD_TERMINAL` +- **THEN** the output is the daemon's pid, its instance id and the terminal's id, separated by `/` + +#### Scenario: Failed start leaves a gap +- **WHEN** a terminal is created after an attempt that failed with `spawn-failed`, both after terminal 1 +- **THEN** the new terminal's id is 3 + +#### Scenario: Other watchers learn of the terminal +- **WHEN** client A creates a terminal in a repo that client B also watches +- **THEN** A receives `termCreated` with its `req` and B receives `termCreated` with `req` null + +#### Scenario: Unwatched worktree +- **WHEN** a client creates a terminal in a worktree of a repo it does not watch +- **THEN** the request fails with `unknown-worktree` + +## ADDED Requirements + +### Requirement: Memory log +While it runs, the daemon SHALL sample memory into `memory.log` as a rotating line log, at the memory sampling cadence, stopping before its shutdown completes. Each line SHALL be a JSON object with: +- `ts` (ISO 8601), `source` `"daemon"`, `pid`, `instance`, `version`, `intervalMs` (the base interval); +- `self`: the daemon's `rss`, `heapTotal`, `heapUsed`, `external` and `arrayBuffers` in bytes; +- `mirrors`: the number of live terminal screen mirrors and their total estimated `bytes`, a mirror's estimate being its normal and alternate buffer lines times its columns times 12; +- `terminals`: per live terminal its `termId`, `repo`, `worktree`, `preset`, shell `pid`, `mirror` (`lines`, `cols`, `bytes`), and its members' and its detached processes' each as a group; +- `escaped`: the 20 escaped groups with the largest `rss`, each with its `tag`, `termId`, and `worktree` and `preset` when this daemon instance closed that terminal (null otherwise), and `escapedOmitted` with the `groups`, `processes`, `rss` and `anon` of the rest; +- `cgroup`: the own cgroup's figures and, as a group, its `others`; null without cgroup v2; +- `system`: the system pressure and meminfo figures, each null when absent. +A group SHALL give `processes`, total `rss` and `anon`, and its largest processes by `rss` (3 for a terminal's members and detached processes, 5 otherwise) each with `pid`, `rss`, `anon` and `command`. Without a process table, the groups SHALL be null and `self` and `mirrors` SHALL still be logged. +Processes of the current user other than the daemon SHALL be placed by the first rule that applies: +1. a **member** of a live terminal is in its session (session id = shell pid) or descends from its shell; +2. a process whose `WTD_TERMINAL` names this daemon's pid and instance and one of its live terminals is **detached** under that terminal; +3. a process whose `WTD_TERMINAL` names another pid that is a running `wtd` daemon belongs to that daemon and is not escaped; +4. any other process with a `WTD_TERMINAL`, and a process in the session of one of the last 256 terminals this daemon closed, is **escaped**, grouped by tag (by the closed terminal's tag for an untagged one); +5. any other member of the daemon's cgroup is among **others**. +A closed terminal SHALL be forgotten once a sample finds none of its processes. + +#### Scenario: Terminal tree logged +- **WHEN** a terminal runs `sleep 1000` and a sample is taken +- **THEN** that terminal's record has its `termId`, `worktree` and `preset`, and its members include the `sleep` process with its `rss` + +#### Scenario: setsid child of a closed terminal +- **WHEN** a terminal runs `setsid sleep 1000 &`, is closed, and a sample is taken +- **THEN** the `sleep` is in an `escaped` group with that terminal's `termId` and `worktree` + +#### Scenario: Job surviving closeTerm +- **WHEN** a terminal starts a job that ignores `SIGHUP` in its own process group, is closed, and a sample is taken +- **THEN** the job's process is in an `escaped` group with that terminal's `termId` + +#### Scenario: Detached child of a live terminal +- **WHEN** a live terminal runs `setsid sleep 1000 &` and a sample is taken +- **THEN** the `sleep` is among that terminal's detached processes and in no `escaped` group + +#### Scenario: Escape from an earlier daemon +- **WHEN** a process carries a `WTD_TERMINAL` naming a pid that is no running `wtd` daemon +- **THEN** it is in an `escaped` group with that tag and null `worktree` + +#### Scenario: Another running daemon's terminal +- **WHEN** a process carries a `WTD_TERMINAL` naming another running `wtd` daemon and no longer descends from it +- **THEN** it is in no `escaped` group + +#### Scenario: Untagged process in a closed terminal's session +- **WHEN** a process without `WTD_TERMINAL` is in the session of a terminal this daemon closed +- **THEN** it is in an `escaped` group with that terminal's tag + +#### Scenario: Escaped groups capped +- **WHEN** 25 escaped groups exist +- **THEN** `escaped` holds the 20 with the largest `rss` and `escapedOmitted` counts the other 5 + +#### Scenario: No process table +- **WHEN** the process table is unavailable +- **THEN** the sample still has `self` and `mirrors`, and its groups are null + +#### Scenario: Sampling stops with the daemon +- **WHEN** the daemon shuts down +- **THEN** nothing is appended to `memory.log` after it exits diff --git a/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/hub/spec.md b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/hub/spec.md new file mode 100644 index 0000000..8488733 --- /dev/null +++ b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/hub/spec.md @@ -0,0 +1,26 @@ +## ADDED Requirements + +### Requirement: Hub memory log +While it runs, the hub SHALL sample memory into `hub-memory.log` as a rotating line log, at the memory sampling cadence, stopping before its shutdown completes. Each line SHALL be a JSON object with: +- `ts` (ISO 8601), `source` `"hub"`, `pid`, `instance`, `version`, `intervalMs` (the base interval); +- `self`: the hub's `rss`, `heapTotal`, `heapUsed`, `external` and `arrayBuffers` in bytes; +- `children`: as a group, every process descending from the hub except a process running the hub's daemon command and its descendants; +- `cgroup`: the own cgroup's figures and, as a group, its `others` (members other than the hub and its `children`); null without cgroup v2; +- `system`: the system pressure and meminfo figures, each null when absent. +A group SHALL give `processes`, total `rss` and `anon`, and its 5 largest processes by `rss`, each with `pid`, `rss`, `anon` and `command`; without a process table the groups SHALL be null and `self` SHALL still be logged. + +#### Scenario: Hub sample +- **WHEN** the hub has run long enough for one sample +- **THEN** `hub-memory.log` holds a line with `source` `"hub"`, the hub's `pid` and a `self.heapUsed` above zero + +#### Scenario: SSH link counted as a child +- **WHEN** the hub holds an SSH link to a remote host and a sample is taken +- **THEN** the SSH process is among the hub's `children` + +#### Scenario: Daemon not counted as a child +- **WHEN** the hub started the local daemon and a sample is taken +- **THEN** neither the daemon nor its terminals' processes are among the hub's `children` + +#### Scenario: Sampling stops with the hub +- **WHEN** the hub shuts down +- **THEN** nothing is appended to `hub-memory.log` after it exits diff --git a/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/platform/spec.md b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/platform/spec.md new file mode 100644 index 0000000..87673d6 --- /dev/null +++ b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/specs/platform/spec.md @@ -0,0 +1,114 @@ +## MODIFIED Requirements + +### Requirement: Host paths +The state directory SHALL be `$XDG_STATE_HOME/worktree-term` when `XDG_STATE_HOME` is an absolute path, and `~/.local/state/worktree-term` otherwise. Under it, `run/.sock` SHALL be the daemon socket, `run/.lock` the start lock, `state.json` the persisted daemon state, `daemon.log` the log of an auto-started daemon, and `memory.log` the daemon's memory log. `` SHALL be the host name with every character outside `[A-Za-z0-9._-]` replaced by `_`. A socket path whose UTF-8 encoding exceeds 107 bytes SHALL be refused with an error naming the path. + +#### Scenario: XDG_STATE_HOME honoured +- **WHEN** `XDG_STATE_HOME` is `/tmp/x` and the host name is `box` +- **THEN** the socket path is `/tmp/x/worktree-term/run/box.sock` and the memory log is `/tmp/x/worktree-term/memory.log` + +#### Scenario: Relative XDG_STATE_HOME ignored +- **WHEN** `XDG_STATE_HOME` is `rel/dir` +- **THEN** the state directory is `~/.local/state/worktree-term` + +#### Scenario: Host name sanitised +- **WHEN** the host name is `my box/1` +- **THEN** the socket file name is `my_box_1.sock` + +#### Scenario: Over-long socket path refused +- **WHEN** the socket path encodes to 108 bytes +- **THEN** resolving the socket path fails with an error naming the path + +### Requirement: Hub and configuration paths +The configuration directory SHALL be `$XDG_CONFIG_HOME/worktree-term` when `XDG_CONFIG_HOME` is an absolute path, and `~/.config/worktree-term` otherwise, with `config.json` the hub configuration. The daemon's systemd user unit file SHALL be `systemd/user/worktree-term-daemon.service` under `$XDG_CONFIG_HOME` when it is an absolute path, and under `~/.config` otherwise. Under the state directory, `hub-token` SHALL be the hub token, `hub.log` the log of a detached hub, `hub-memory.log` the hub's memory log, `run/hub.lock` and `run/hub.json` the hub start lock and hub record, and `run/ssh-%C` the SSH control path, `%C` standing for the 40 hexadecimal digits SSH substitutes. A control path whose UTF-8 encoding, with `%C` substituted, exceeds 107 bytes SHALL be refused with an error naming the path. Files and directories created for them SHALL be owner-only, as for the daemon's. + +#### Scenario: XDG_CONFIG_HOME honoured +- **WHEN** `XDG_CONFIG_HOME` is `/tmp/c` +- **THEN** the configuration file is `/tmp/c/worktree-term/config.json` and the unit file is `/tmp/c/systemd/user/worktree-term-daemon.service` + +#### Scenario: Relative XDG_CONFIG_HOME ignored +- **WHEN** `XDG_CONFIG_HOME` is `rel/dir` +- **THEN** the configuration file is `~/.config/worktree-term/config.json` + +#### Scenario: Hub memory log path +- **WHEN** `XDG_STATE_HOME` is `/tmp/x` +- **THEN** the hub memory log is `/tmp/x/worktree-term/hub-memory.log` + +#### Scenario: Over-long control path refused +- **WHEN** the state directory is so long that the substituted control path encodes to 108 bytes +- **THEN** resolving the control path fails with an error naming the path + +## ADDED Requirements + +### Requirement: Memory sources +Memory figures SHALL be read as follows, each source independently and each yielding null when its files are absent or unreadable (`ENOENT`, `EACCES`), so that a host without `/proc` or cgroup v2 still yields the others: +- the **process table**: every process of the current user, with its pid, parent pid, session id, resident memory (`VmRSS`) and anonymous resident memory (`RssAnon`) in bytes; on request, a process's `WTD_TERMINAL` environment value and its command line (arguments joined by spaces, at most 200 characters). A process that vanishes while being read SHALL be left out without error. +- the **own cgroup** (cgroup v2): its path, `memory.current`, `memory.peak`, `memory.max`, `memory.high` and `memory.swap.current` (null each when absent, `max` as null), `memory.pressure` `some` and `full` `avg10`, `avg60` and `avg300`, and its member pids. +- the **system**: `/proc/pressure/memory` `some` and `full` `avg10`, `avg60` and `avg300`, and `MemTotal`, `MemAvailable`, `SwapTotal` and `SwapFree` from `/proc/meminfo` in bytes. +Malformed content SHALL be a reported failure of that source, never a coerced value. + +#### Scenario: Process table read +- **WHEN** the process table is read while a child of the reader runs `sleep` +- **THEN** it contains the child with its parent pid, session id, and a resident memory above zero + +#### Scenario: Process vanishing mid-read +- **WHEN** a process's `stat` exists but its `status` is gone by the time it is read +- **THEN** the table omits that process and reading succeeds + +#### Scenario: No cgroup v2 +- **WHEN** the reader's cgroup has no `memory.current` +- **THEN** the own-cgroup source yields null and the system source is still read + +#### Scenario: No /proc +- **WHEN** `/proc` is absent +- **THEN** the process table and system sources yield null + +#### Scenario: Malformed pressure file +- **WHEN** `/proc/pressure/memory` lacks the `some` line +- **THEN** the system pressure is null and the failure is reported + +### Requirement: Memory sampling cadence +A memory sampler SHALL take its first sample when it starts. Every 5 s it SHALL check system and own-cgroup memory pressure and system meminfo, and SHALL take a sample when the base interval (60 s, or `WTD_MEMORY_SAMPLE_MS` when that is a positive integer) has passed since the last sample, or when memory is tight: system or own-cgroup `some avg10` at least 10, or `MemAvailable` below 10% of `MemTotal`. The check period SHALL be the base interval when that is shorter than 5 s. A check or sample SHALL start only after the previous one has finished. Its timer SHALL NOT keep the process alive, and once the sampler is stopped it SHALL write nothing more. A failed sample or write SHALL be reported and sampling SHALL continue. + +#### Scenario: Calm cadence +- **WHEN** memory is not tight for 3 minutes +- **THEN** samples are taken at start and then 60 s apart + +#### Scenario: Tight memory samples often +- **WHEN** system `some avg10` rises to 25 between two calm samples +- **THEN** a sample is taken within 5 s, and further samples every 5 s while it stays at or above 10 + +#### Scenario: Back to calm +- **WHEN** memory stops being tight +- **THEN** the next sample is taken 60 s after the previous one + +#### Scenario: Slow sample +- **WHEN** a sample takes longer than the check period +- **THEN** no second sample starts before it finishes + +#### Scenario: Stopped sampler +- **WHEN** the sampler is stopped while a sample is being taken +- **THEN** that sample is not written and nothing is written afterwards + +#### Scenario: Write failure +- **WHEN** appending a sample fails +- **THEN** the failure is reported and the next sample is still taken + +### Requirement: Rotating line log +A memory log SHALL be appended to one JSON object per line, each line written by a single append of the whole line, to a file created with mode 0600. Before an append that would make the log exceed 16 MiB, the log SHALL be rotated: `.2` renamed to `.3`, `.1` to `.2` and `` to `.1`, missing files skipped and an existing `.3` replaced. A line longer than 16 MiB SHALL still be written, after rotating. A failed rotation SHALL be reported, the line SHALL still be appended, and the rotation SHALL be retried before the next append. + +#### Scenario: Rotation at the limit +- **WHEN** a log of 16 MiB less 10 bytes gets a 100-byte line +- **THEN** the previous content is in `.1` and `` holds only the new line + +#### Scenario: Three old files kept +- **WHEN** ``, `.1`, `.2` and `.3` exist and the log rotates +- **THEN** the old `.3` is gone, the old `.2` is `.3`, and so on + +#### Scenario: Owner-only log +- **WHEN** a memory log is created +- **THEN** it has mode 0600 + +#### Scenario: Rotation failure recovers +- **WHEN** renaming `` to `.1` fails once +- **THEN** the failure is reported, the line is appended to ``, and the next append rotates and writes diff --git a/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/tasks.md b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/tasks.md new file mode 100644 index 0000000..bf47b71 --- /dev/null +++ b/openspec/changes/archive/2026-10-29-dev-2161-worktree-term-log-memory-use-per-component-so-an-oom-kill/tasks.md @@ -0,0 +1,32 @@ +## 1. Architecture (approved wording, design D10) + +- [x] 1.1 Add `memory = component 'Memory'` inside `platform` in `architecture/model.c4` with arrows `platform.memory -> platform.files`, `daemon.main -> platform.memory`, `hub.main -> platform.memory`; regenerate diagrams with `la-arch-diagrams`; verify `la-arch-check` passes +- [x] 1.2 Apply the approved Purpose and principle 4 wording to `architecture/typescript.platform.arc42.md` verbatim from design D10; add `'platform.memory': ['fs']` to `test/architecture/dependencies.ts`; verify `test/architecture` passes + +## 2. Tests first (pr-tests) + +- [x] 2.1 `src/platform/files/files.test.ts`: `memoryLog` / `hubMemoryLog` paths; rotating appender — rotation at the limit, existing `.1`–`.3` shifted and `.3` replaced, failure at each rename reported with the line still appended and rotation retried, a line over 16 MiB written after rotating, mode 0600, one whole line per append; verify they fail before implementation +- [x] 2.2 `src/platform/memory/*.test.ts`: process table, cgroup and system readers against fake `/proc` and `/sys/fs/cgroup` trees (present, absent → null, malformed → reported + null, process vanishing mid-read, `max` → null, command cut to 200 characters); group summarising (totals, top N by rss); sampler cadence with a fake clock (first sample at start, calm 60 s, tight by system PSI / cgroup PSI / MemAvailable → within 5 s then every 5 s, back to calm, check period = base interval below 5 s, slow sample single-flight, stop with a sample in flight writes nothing, write failure reported and sampling continues, `WTD_MEMORY_SAMPLE_MS` parsing); verify they fail before implementation +- [x] 2.3 `src/daemon/main/*.test.ts` (attribution module): rules 1–5 of design D2 — session member, descendant outside the session, detached tagged child of a live terminal, running foreign daemon (including a child no longer descending from it), tag of a pid that is no running daemon, untagged process in a closed terminal's session, cgroup `others`, daemon excluded; closed terminal forgotten on its first empty sample and at most 256 kept; escaped capped at 20 with `escapedOmitted`; null groups without a process table; mirror byte estimate; verify they fail before implementation +- [x] 2.4 `src/daemon/server/daemon.test.ts`: terminal env carries `WTD_TERMINAL=//`; id taken before spawning (failed spawn leaves a gap); `memoryView()` lists live terminals (termId, repo, worktree, preset, pid, mirror) and closed ones; `src/daemon/terminals/terminal.test.ts`: `pid` and `mirrorStats()` (lines grow with output, alternate buffer counted); verify they fail before implementation +- [x] 2.5 Hub children selection unit test (descendants by ppid minus the daemon-command subtree, detached descendants included); verify it fails before implementation +- [x] 2.6 Process tests in `test/process/memory-log.test.ts` (with `WTD_MEMORY_SAMPLE_MS` short): daemon line shape; terminal running `sleep 1000` lists it; `setsid sleep 1000 &` then `closeTerm` → escaped with termId and worktree; a `SIGHUP`-ignoring job in its own process group survives `closeTerm` → escaped; live terminal's `setsid` child → detached, not escaped; `echo $WTD_TERMINAL` shows pid/instance/termId; nothing appended after daemon shutdown; hub writes `hub-memory.log` with `source` `"hub"`, an SSH link child (fake ssh) and without the daemon or its terminals; nothing appended after hub shutdown. Kill every escaped test process in teardown. Verify they fail before implementation +- [x] 2.7 Keep `test/process/rss.test.ts` unchanged; it must pass with sampling on after implementation + +## 3. Platform + +- [x] 3.1 `platform.files`: `HostPaths.memoryLog`, `hubMemoryLog`, and the rotating whole-line appender (design D8); verify 2.1 passes +- [x] 3.2 New `src/platform/memory/` (index.ts exposing only its surface): readers with configurable roots and boundary parsing, group summarising, the single-flight adaptive sampler (design D6, D9, D11); verify 2.2 passes + +## 4. Daemon + +- [x] 4.1 `TerminalProcess.pid` and `mirrorStats()`; `Daemon` tags terminal env, takes ids before spawning, keeps the closed-terminal list, exposes `memoryView()`; verify 2.4 and existing daemon tests pass +- [x] 4.2 `daemon.main`: attribution module (design D2, D7) and sampler wiring into `memory.log`, stopped before `daemon.shutdown()` resolves; `cli` passes `WTD_MEMORY_SAMPLE_MS`; verify 2.3 and the daemon cases of 2.6 pass + +## 5. Hub + +- [x] 5.1 `hub.main`: hub sampler into `hub-memory.log` with children selection (design D4), stopped before the hub exits; verify 2.5 and the hub cases of 2.6 pass + +## 6. Gates + +- [x] 6.1 Run `pnpm lint`, `tsc -b`, `la-arch-check` and the full `pnpm test`; all green, including `test/process/rss.test.ts` diff --git a/openspec/specs/daemon/spec.md b/openspec/specs/daemon/spec.md index 983039f..cfff550 100644 --- a/openspec/specs/daemon/spec.md +++ b/openspec/specs/daemon/spec.md @@ -138,12 +138,20 @@ While a repo is watched, the daemon SHALL send every watcher `worktreesChanged` - **THEN** the repo is reported by its real path, which `watchRepo` accepts ### Requirement: Terminal creation -`createTerm` SHALL fail with `unknown-worktree` unless the worktree is in the current list of a repo this connection watches, and with `busy` if that repo already has 1024 live terminals. Otherwise the daemon SHALL start `$SHELL -l -i` when `command` is null, or `$SHELL -l -i -c ` otherwise (`/bin/sh` when `SHELL` is unset or not absolute), in the worktree directory, with `TERM=xterm-256color` and `COLORTERM=truecolor`, at the requested size. A failure to start SHALL fail with `spawn-failed`. Terminal ids SHALL count up from 1 and never be reused within a daemon instance. `preset` SHALL be stored and reported as given. The requester SHALL receive `termCreated` with its `req`; every other watcher of the repo SHALL receive `termCreated` with `req` null. Creating a terminal SHALL NOT attach it. +`createTerm` SHALL fail with `unknown-worktree` unless the worktree is in the current list of a repo this connection watches, and with `busy` if that repo already has 1024 live terminals. Otherwise the daemon SHALL start `$SHELL -l -i` when `command` is null, or `$SHELL -l -i -c ` otherwise (`/bin/sh` when `SHELL` is unset or not absolute), in the worktree directory, with `TERM=xterm-256color`, `COLORTERM=truecolor` and `WTD_TERMINAL=//`, at the requested size. A failure to start SHALL fail with `spawn-failed`. Terminal ids SHALL count up from 1 and never be reused within a daemon instance; an id SHALL be taken before the process is started, so a failed start leaves its id unused. `preset` SHALL be stored and reported as given. The requester SHALL receive `termCreated` with its `req`; every other watcher of the repo SHALL receive `termCreated` with `req` null. Creating a terminal SHALL NOT attach it. #### Scenario: Shell in the worktree - **WHEN** a client creates a terminal in a worktree, attaches, and types `pwd` - **THEN** the output shows the worktree path +#### Scenario: Terminal tagged +- **WHEN** a client creates a terminal, attaches, and types `echo $WTD_TERMINAL` +- **THEN** the output is the daemon's pid, its instance id and the terminal's id, separated by `/` + +#### Scenario: Failed start leaves a gap +- **WHEN** a terminal is created after an attempt that failed with `spawn-failed`, both after terminal 1 +- **THEN** the new terminal's id is 3 + #### Scenario: Other watchers learn of the terminal - **WHEN** client A creates a terminal in a repo that client B also watches - **THEN** A receives `termCreated` with its `req` and B receives `termCreated` with `req` null @@ -444,3 +452,61 @@ Checked state and layouts SHALL be stored in `state.json` and survive daemon res #### Scenario: Main and locked worktrees refused - **WHEN** the main worktree or a locked worktree is removed, even with `force` - **THEN** the requester receives `error` with code `busy` and nothing changes + +### Requirement: Memory log +While it runs, the daemon SHALL sample memory into `memory.log` as a rotating line log, at the memory sampling cadence, stopping before its shutdown completes. Each line SHALL be a JSON object with: +- `ts` (ISO 8601), `source` `"daemon"`, `pid`, `instance`, `version`, `intervalMs` (the base interval); +- `self`: the daemon's `rss`, `heapTotal`, `heapUsed`, `external` and `arrayBuffers` in bytes; +- `mirrors`: the number of live terminal screen mirrors and their total estimated `bytes`, a mirror's estimate being its normal and alternate buffer lines times its columns times 12; +- `terminals`: per live terminal its `termId`, `repo`, `worktree`, `preset`, shell `pid`, `mirror` (`lines`, `cols`, `bytes`), and its members' and its detached processes' each as a group; +- `escaped`: the 20 escaped groups with the largest `rss`, each with its `tag`, `termId`, and `worktree` and `preset` when this daemon instance closed that terminal (null otherwise), and `escapedOmitted` with the `groups`, `processes`, `rss` and `anon` of the rest; +- `cgroup`: the own cgroup's figures and, as a group, its `others`; null without cgroup v2; +- `system`: the system pressure and meminfo figures, each null when absent. +A group SHALL give `processes`, total `rss` and `anon`, and its largest processes by `rss` (3 for a terminal's members and detached processes, 5 otherwise) each with `pid`, `rss`, `anon` and `command`. Without a process table, the groups SHALL be null and `self` and `mirrors` SHALL still be logged. +Processes of the current user other than the daemon SHALL be placed by the first rule that applies: +1. a **member** of a live terminal is in its session (session id = shell pid) or descends from its shell; +2. a process whose `WTD_TERMINAL` names this daemon's pid and instance and one of its live terminals is **detached** under that terminal; +3. a process whose `WTD_TERMINAL` names another pid that is a running `wtd` daemon belongs to that daemon and is not escaped; +4. any other process with a `WTD_TERMINAL`, and a process in the session of one of the last 256 terminals this daemon closed, is **escaped**, grouped by tag (by the closed terminal's tag for an untagged one); +5. any other member of the daemon's cgroup is among **others**. +A closed terminal SHALL be forgotten once a sample finds none of its processes. + +#### Scenario: Terminal tree logged +- **WHEN** a terminal runs `sleep 1000` and a sample is taken +- **THEN** that terminal's record has its `termId`, `worktree` and `preset`, and its members include the `sleep` process with its `rss` + +#### Scenario: setsid child of a closed terminal +- **WHEN** a terminal runs `setsid sleep 1000 &`, is closed, and a sample is taken +- **THEN** the `sleep` is in an `escaped` group with that terminal's `termId` and `worktree` + +#### Scenario: Job surviving closeTerm +- **WHEN** a terminal starts a job that ignores `SIGHUP` in its own process group, is closed, and a sample is taken +- **THEN** the job's process is in an `escaped` group with that terminal's `termId` + +#### Scenario: Detached child of a live terminal +- **WHEN** a live terminal runs `setsid sleep 1000 &` and a sample is taken +- **THEN** the `sleep` is among that terminal's detached processes and in no `escaped` group + +#### Scenario: Escape from an earlier daemon +- **WHEN** a process carries a `WTD_TERMINAL` naming a pid that is no running `wtd` daemon +- **THEN** it is in an `escaped` group with that tag and null `worktree` + +#### Scenario: Another running daemon's terminal +- **WHEN** a process carries a `WTD_TERMINAL` naming another running `wtd` daemon and no longer descends from it +- **THEN** it is in no `escaped` group + +#### Scenario: Untagged process in a closed terminal's session +- **WHEN** a process without `WTD_TERMINAL` is in the session of a terminal this daemon closed +- **THEN** it is in an `escaped` group with that terminal's tag + +#### Scenario: Escaped groups capped +- **WHEN** 25 escaped groups exist +- **THEN** `escaped` holds the 20 with the largest `rss` and `escapedOmitted` counts the other 5 + +#### Scenario: No process table +- **WHEN** the process table is unavailable +- **THEN** the sample still has `self` and `mirrors`, and its groups are null + +#### Scenario: Sampling stops with the daemon +- **WHEN** the daemon shuts down +- **THEN** nothing is appended to `memory.log` after it exits diff --git a/openspec/specs/hub/spec.md b/openspec/specs/hub/spec.md index db9c977..947b40b 100644 --- a/openspec/specs/hub/spec.md +++ b/openspec/specs/hub/spec.md @@ -554,3 +554,28 @@ A tunnel listener SHALL listen on `127.0.0.1` only. Each connection it accepts S #### Scenario: Restart waits for an install - **WHEN** an `installHub` for host 1 is running and a `restartDaemon` for host 1 arrives - **THEN** the restart starts only after the install ended + +### Requirement: Hub memory log +While it runs, the hub SHALL sample memory into `hub-memory.log` as a rotating line log, at the memory sampling cadence, stopping before its shutdown completes. Each line SHALL be a JSON object with: +- `ts` (ISO 8601), `source` `"hub"`, `pid`, `instance`, `version`, `intervalMs` (the base interval); +- `self`: the hub's `rss`, `heapTotal`, `heapUsed`, `external` and `arrayBuffers` in bytes; +- `children`: as a group, every process descending from the hub except a process running the hub's daemon command and its descendants; +- `cgroup`: the own cgroup's figures and, as a group, its `others` (members other than the hub and its `children`); null without cgroup v2; +- `system`: the system pressure and meminfo figures, each null when absent. +A group SHALL give `processes`, total `rss` and `anon`, and its 5 largest processes by `rss`, each with `pid`, `rss`, `anon` and `command`; without a process table the groups SHALL be null and `self` SHALL still be logged. + +#### Scenario: Hub sample +- **WHEN** the hub has run long enough for one sample +- **THEN** `hub-memory.log` holds a line with `source` `"hub"`, the hub's `pid` and a `self.heapUsed` above zero + +#### Scenario: SSH link counted as a child +- **WHEN** the hub holds an SSH link to a remote host and a sample is taken +- **THEN** the SSH process is among the hub's `children` + +#### Scenario: Daemon not counted as a child +- **WHEN** the hub started the local daemon and a sample is taken +- **THEN** neither the daemon nor its terminals' processes are among the hub's `children` + +#### Scenario: Sampling stops with the hub +- **WHEN** the hub shuts down +- **THEN** nothing is appended to `hub-memory.log` after it exits diff --git a/openspec/specs/platform/spec.md b/openspec/specs/platform/spec.md index 1b5452c..cc3309f 100644 --- a/openspec/specs/platform/spec.md +++ b/openspec/specs/platform/spec.md @@ -6,11 +6,11 @@ Host-level infrastructure shared by daemon, hub and CLI: where worktree-term kee ## Requirements ### Requirement: Host paths -The state directory SHALL be `$XDG_STATE_HOME/worktree-term` when `XDG_STATE_HOME` is an absolute path, and `~/.local/state/worktree-term` otherwise. Under it, `run/.sock` SHALL be the daemon socket, `run/.lock` the start lock, `state.json` the persisted daemon state, and `daemon.log` the log of an auto-started daemon. `` SHALL be the host name with every character outside `[A-Za-z0-9._-]` replaced by `_`. A socket path whose UTF-8 encoding exceeds 107 bytes SHALL be refused with an error naming the path. +The state directory SHALL be `$XDG_STATE_HOME/worktree-term` when `XDG_STATE_HOME` is an absolute path, and `~/.local/state/worktree-term` otherwise. Under it, `run/.sock` SHALL be the daemon socket, `run/.lock` the start lock, `state.json` the persisted daemon state, `daemon.log` the log of an auto-started daemon, and `memory.log` the daemon's memory log. `` SHALL be the host name with every character outside `[A-Za-z0-9._-]` replaced by `_`. A socket path whose UTF-8 encoding exceeds 107 bytes SHALL be refused with an error naming the path. #### Scenario: XDG_STATE_HOME honoured - **WHEN** `XDG_STATE_HOME` is `/tmp/x` and the host name is `box` -- **THEN** the socket path is `/tmp/x/worktree-term/run/box.sock` +- **THEN** the socket path is `/tmp/x/worktree-term/run/box.sock` and the memory log is `/tmp/x/worktree-term/memory.log` #### Scenario: Relative XDG_STATE_HOME ignored - **WHEN** `XDG_STATE_HOME` is `rel/dir` @@ -93,7 +93,7 @@ Dialing SHALL connect to this host's daemon socket. When the socket does not exi - **THEN** dialing fails after 5 s with a message naming the log path ### Requirement: Hub and configuration paths -The configuration directory SHALL be `$XDG_CONFIG_HOME/worktree-term` when `XDG_CONFIG_HOME` is an absolute path, and `~/.config/worktree-term` otherwise, with `config.json` the hub configuration. The daemon's systemd user unit file SHALL be `systemd/user/worktree-term-daemon.service` under `$XDG_CONFIG_HOME` when it is an absolute path, and under `~/.config` otherwise. Under the state directory, `hub-token` SHALL be the hub token, `hub.log` the log of a detached hub, `run/hub.lock` and `run/hub.json` the hub start lock and hub record, and `run/ssh-%C` the SSH control path, `%C` standing for the 40 hexadecimal digits SSH substitutes. A control path whose UTF-8 encoding, with `%C` substituted, exceeds 107 bytes SHALL be refused with an error naming the path. Files and directories created for them SHALL be owner-only, as for the daemon's. +The configuration directory SHALL be `$XDG_CONFIG_HOME/worktree-term` when `XDG_CONFIG_HOME` is an absolute path, and `~/.config/worktree-term` otherwise, with `config.json` the hub configuration. The daemon's systemd user unit file SHALL be `systemd/user/worktree-term-daemon.service` under `$XDG_CONFIG_HOME` when it is an absolute path, and under `~/.config` otherwise. Under the state directory, `hub-token` SHALL be the hub token, `hub.log` the log of a detached hub, `hub-memory.log` the hub's memory log, `run/hub.lock` and `run/hub.json` the hub start lock and hub record, and `run/ssh-%C` the SSH control path, `%C` standing for the 40 hexadecimal digits SSH substitutes. A control path whose UTF-8 encoding, with `%C` substituted, exceeds 107 bytes SHALL be refused with an error naming the path. Files and directories created for them SHALL be owner-only, as for the daemon's. #### Scenario: XDG_CONFIG_HOME honoured - **WHEN** `XDG_CONFIG_HOME` is `/tmp/c` @@ -103,6 +103,10 @@ The configuration directory SHALL be `$XDG_CONFIG_HOME/worktree-term` when `XDG_ - **WHEN** `XDG_CONFIG_HOME` is `rel/dir` - **THEN** the configuration file is `~/.config/worktree-term/config.json` +#### Scenario: Hub memory log path +- **WHEN** `XDG_STATE_HOME` is `/tmp/x` +- **THEN** the hub memory log is `/tmp/x/worktree-term/hub-memory.log` + #### Scenario: Over-long control path refused - **WHEN** the state directory is so long that the substituted control path encodes to 108 bytes - **THEN** resolving the control path fails with an error naming the path @@ -182,3 +186,76 @@ A remote installation SHALL send the bundle of the running `wtd` as one tar arch #### Scenario: Unreachable host reported - **WHEN** SSH cannot reach the alias - **THEN** the installation fails with SSH's last error line + +### Requirement: Memory sources +Memory figures SHALL be read as follows, each source independently and each yielding null when its files are absent or unreadable (`ENOENT`, `EACCES`), so that a host without `/proc` or cgroup v2 still yields the others: +- the **process table**: every process of the current user, with its pid, parent pid, session id, resident memory (`VmRSS`) and anonymous resident memory (`RssAnon`) in bytes; on request, a process's `WTD_TERMINAL` environment value and its command line (arguments joined by spaces, at most 200 characters). A process that vanishes while being read SHALL be left out without error. +- the **own cgroup** (cgroup v2): its path, `memory.current`, `memory.peak`, `memory.max`, `memory.high` and `memory.swap.current` (null each when absent, `max` as null), `memory.pressure` `some` and `full` `avg10`, `avg60` and `avg300`, and its member pids. +- the **system**: `/proc/pressure/memory` `some` and `full` `avg10`, `avg60` and `avg300`, and `MemTotal`, `MemAvailable`, `SwapTotal` and `SwapFree` from `/proc/meminfo` in bytes. +Malformed content SHALL be a reported failure of that source, never a coerced value. + +#### Scenario: Process table read +- **WHEN** the process table is read while a child of the reader runs `sleep` +- **THEN** it contains the child with its parent pid, session id, and a resident memory above zero + +#### Scenario: Process vanishing mid-read +- **WHEN** a process's `stat` exists but its `status` is gone by the time it is read +- **THEN** the table omits that process and reading succeeds + +#### Scenario: No cgroup v2 +- **WHEN** the reader's cgroup has no `memory.current` +- **THEN** the own-cgroup source yields null and the system source is still read + +#### Scenario: No /proc +- **WHEN** `/proc` is absent +- **THEN** the process table and system sources yield null + +#### Scenario: Malformed pressure file +- **WHEN** `/proc/pressure/memory` lacks the `some` line +- **THEN** the system pressure is null and the failure is reported + +### Requirement: Memory sampling cadence +A memory sampler SHALL take its first sample when it starts. Every 5 s it SHALL check system and own-cgroup memory pressure and system meminfo, and SHALL take a sample when the base interval (60 s, or `WTD_MEMORY_SAMPLE_MS` when that is a positive integer) has passed since the last sample, or when memory is tight: system or own-cgroup `some avg10` at least 10, or `MemAvailable` below 10% of `MemTotal`. The check period SHALL be the base interval when that is shorter than 5 s. A check or sample SHALL start only after the previous one has finished. Its timer SHALL NOT keep the process alive, and once the sampler is stopped it SHALL write nothing more. A failed sample or write SHALL be reported and sampling SHALL continue. + +#### Scenario: Calm cadence +- **WHEN** memory is not tight for 3 minutes +- **THEN** samples are taken at start and then 60 s apart + +#### Scenario: Tight memory samples often +- **WHEN** system `some avg10` rises to 25 between two calm samples +- **THEN** a sample is taken within 5 s, and further samples every 5 s while it stays at or above 10 + +#### Scenario: Back to calm +- **WHEN** memory stops being tight +- **THEN** the next sample is taken 60 s after the previous one + +#### Scenario: Slow sample +- **WHEN** a sample takes longer than the check period +- **THEN** no second sample starts before it finishes + +#### Scenario: Stopped sampler +- **WHEN** the sampler is stopped while a sample is being taken +- **THEN** that sample is not written and nothing is written afterwards + +#### Scenario: Write failure +- **WHEN** appending a sample fails +- **THEN** the failure is reported and the next sample is still taken + +### Requirement: Rotating line log +A memory log SHALL be appended to one JSON object per line, each line written by a single append of the whole line, to a file created with mode 0600. Before an append that would make the log exceed 16 MiB, the log SHALL be rotated: `.2` renamed to `.3`, `.1` to `.2` and `` to `.1`, missing files skipped and an existing `.3` replaced. A line longer than 16 MiB SHALL still be written, after rotating. A failed rotation SHALL be reported, the line SHALL still be appended, and the rotation SHALL be retried before the next append. + +#### Scenario: Rotation at the limit +- **WHEN** a log of 16 MiB less 10 bytes gets a 100-byte line +- **THEN** the previous content is in `.1` and `` holds only the new line + +#### Scenario: Three old files kept +- **WHEN** ``, `.1`, `.2` and `.3` exist and the log rotates +- **THEN** the old `.3` is gone, the old `.2` is `.3`, and so on + +#### Scenario: Owner-only log +- **WHEN** a memory log is created +- **THEN** it has mode 0600 + +#### Scenario: Rotation failure recovers +- **WHEN** renaming `` to `.1` fails once +- **THEN** the failure is reported, the line is appended to ``, and the next append rotates and writes diff --git a/src/daemon/main/index.ts b/src/daemon/main/index.ts index 9db8e89..1a9f0c9 100644 --- a/src/daemon/main/index.ts +++ b/src/daemon/main/index.ts @@ -1,9 +1,21 @@ import { randomBytes } from 'node:crypto'; import type { HostPaths } from '../../platform/files/index.js'; +import { + hostRoots, + logWriter, + memoryTight, + processLookup, + readMemory, + sampleIntervalMs, + selfMemory, + startSampler, + type MemorySampler, +} from '../../platform/memory/index.js'; import { bindSocket, Daemon } from '../server/index.js'; import { StateStore } from '../state/index.js'; import { TerminalProcess } from '../terminals/index.js'; import { discoverRepos, listWorktrees, removeWorktree, watchWorktrees, worktreeRisks } from '../worktrees/index.js'; +import { daemonRecord } from './memory.js'; export { AlreadyRunningError } from '../server/index.js'; @@ -11,7 +23,7 @@ export interface DaemonRun { /** The package version announced in `hello`. */ version: string; paths: HostPaths; - /** Environment of terminal processes. */ + /** Environment of terminal processes; read for `WTD_MEMORY_SAMPLE_MS`, the memory log's base interval. */ env: Readonly>; } @@ -19,11 +31,39 @@ const STOP_SIGNALS = ['SIGTERM', 'SIGINT'] as const; const ignore = (): void => undefined; +/** Samples the daemon's memory into `memory.log` until stopped. */ +const sampleMemory = ( + daemon: Daemon, + { version, paths, env }: DaemonRun, + instance: string, + report: (error: unknown) => void, +): MemorySampler => { + const roots = hostRoots(); + const lookup = processLookup(roots); + const intervalMs = sampleIntervalMs(env['WTD_MEMORY_SAMPLE_MS']); + return startSampler({ + intervalMs, + tight: () => memoryTight(roots, report), + take: async () => { + const ts = new Date().toISOString(); + const self = selfMemory(); + const view = daemon.memoryView(); + const reading = await readMemory(roots, report); + const { record, gone } = await daemonRecord({ ts, pid: process.pid, instance, version, intervalMs, self, view, ...reading, lookup }); + daemon.forgetClosed(gone); + return record; + }, + write: logWriter(paths.memoryLog, report), + report, + }); +}; + /** * Runs this host's daemon until `shutdown`, SIGTERM or SIGINT; rejects with AlreadyRunningError * when another daemon serves the socket. */ -export const runDaemon = async ({ version, paths, env }: DaemonRun): Promise => { +export const runDaemon = async (run: DaemonRun): Promise => { + const { version, paths, env } = run; let stop = (): void => undefined; const stopped = new Promise((resolve) => { stop = resolve; @@ -38,6 +78,7 @@ export const runDaemon = async ({ version, paths, env }: DaemonRun): Promise TerminalProcess.spawn(spec, events), }, version, - instance: randomBytes(12).toString('base64url'), + instance, store, env, shutdown: stop, @@ -57,7 +98,9 @@ export const runDaemon = async ({ version, paths, env }: DaemonRun): Promise { daemon.accept(socket); }); + const sampler = sampleMemory(daemon, run, instance, report); await stopped; + sampler.stop(); await daemon.shutdown(); } finally { await listener.close(); diff --git a/src/daemon/main/memory.test.ts b/src/daemon/main/memory.test.ts new file mode 100644 index 0000000..1cbd848 --- /dev/null +++ b/src/daemon/main/memory.test.ts @@ -0,0 +1,307 @@ +import { describe, expect, it } from 'vitest'; +import type { OwnCgroup, ProcessInfo, ProcessLookup, SystemMemory } from '../../platform/memory/index.js'; +import type { MemoryView } from '../server/index.js'; +import { daemonRecord, type DaemonSampleInput } from './memory.js'; + +const DAEMON = 100; +const INSTANCE = 'inst'; +const WTD_DAEMON = ['/usr/bin/node', '/opt/wtd/dist/wtd.mjs', 'daemon']; + +/** A process whose resident memory defaults to a distinct amount per pid. */ +const p = (pid: number, ppid: number, sid: number, rss = pid * 1000): ProcessInfo => ({ pid, ppid, sid, rss, anon: rss / 2 }); + +interface Fakes { + tags?: Readonly>; + args?: Readonly>; +} + +/** A lookup over `fakes`; records which pids had their tag read. */ +const lookupOf = ({ tags = {}, args = {} }: Fakes): ProcessLookup & { tagged: number[] } => { + const tagged: number[] = []; + return { + tagged, + tag: (pid) => { + tagged.push(pid); + return Promise.resolve(tags[pid] ?? null); + }, + args: (pid) => Promise.resolve(args[pid] ?? null), + command: (pid) => Promise.resolve(args[pid]?.join(' ') ?? `cmd${String(pid)}`), + }; +}; + +const VIEW: MemoryView = { + terminals: [ + { termId: 1, repo: '/r', worktree: '/r/w1', preset: 'shell', pid: 200, mirror: { lines: 100, cols: 80 } }, + { termId: 5, repo: '/r', worktree: '/r/w5', preset: 'claude', pid: 250, mirror: { lines: 24, cols: 120 } }, + ], + closed: [ + { termId: 2, pid: 500, worktree: '/r/w2', preset: 'codex' }, + { termId: 3, pid: 700, worktree: '/r/w3', preset: 'shell' }, + ], +}; + +const PRESSURE = { some: { avg10: 1, avg60: 2, avg300: 3 }, full: { avg10: 0, avg60: 0, avg300: 0 } }; +const SYSTEM: SystemMemory = { pressure: PRESSURE, meminfo: { total: 1000, available: 500, swapTotal: 0, swapFree: 0 } }; +const cgroupOf = (pids: readonly number[]): OwnCgroup => ({ + path: '/app.slice/x.scope', + current: 10, + peak: 20, + max: null, + high: null, + swapCurrent: 0, + pressure: PRESSURE, + pids: [...pids], +}); + +const input = (table: readonly ProcessInfo[] | null, fakes: Fakes = {}, over: Partial = {}): DaemonSampleInput => ({ + ts: '2026-10-28T10:00:00.000Z', + pid: DAEMON, + instance: INSTANCE, + version: '1.2.3', + intervalMs: 60_000, + self: { rss: 1, heapTotal: 2, heapUsed: 3, external: 4, arrayBuffers: 5 }, + view: VIEW, + table, + cgroup: null, + system: SYSTEM, + lookup: lookupOf(fakes), + ...over, +}); + +/** The daemon and both live shells; every table includes them. */ +const BASE = [p(DAEMON, 1, DAEMON), p(200, DAEMON, 200), p(250, DAEMON, 250)]; + +const terminal = async (termId: number, table: readonly ProcessInfo[], fakes: Fakes = {}) => { + const { record } = await daemonRecord(input(table, fakes)); + const found = record.terminals.find((t) => t.termId === termId); + if (found === undefined) throw new Error(`no terminal ${String(termId)}`); + return found; +}; + +const escapedOf = async (table: readonly ProcessInfo[], fakes: Fakes = {}) => (await daemonRecord(input(table, fakes))).record.escaped; + +describe('the record', () => { + it('has the identity, own memory, mirrors, terminals, cgroup and system figures', async () => { + const { record, gone } = await daemonRecord(input(BASE, {}, { cgroup: cgroupOf([DAEMON]) })); + expect(record).toEqual({ + ts: '2026-10-28T10:00:00.000Z', + source: 'daemon', + pid: DAEMON, + instance: INSTANCE, + version: '1.2.3', + intervalMs: 60_000, + self: { rss: 1, heapTotal: 2, heapUsed: 3, external: 4, arrayBuffers: 5 }, + mirrors: { count: 2, bytes: 100 * 80 * 12 + 24 * 120 * 12 }, + terminals: [ + { + termId: 1, + repo: '/r', + worktree: '/r/w1', + preset: 'shell', + pid: 200, + mirror: { lines: 100, cols: 80, bytes: 100 * 80 * 12 }, + members: { processes: 1, rss: 200_000, anon: 100_000, top: [{ pid: 200, rss: 200_000, anon: 100_000, command: 'cmd200' }] }, + detached: { processes: 0, rss: 0, anon: 0, top: [] }, + }, + { + termId: 5, + repo: '/r', + worktree: '/r/w5', + preset: 'claude', + pid: 250, + mirror: { lines: 24, cols: 120, bytes: 24 * 120 * 12 }, + members: { processes: 1, rss: 250_000, anon: 125_000, top: [{ pid: 250, rss: 250_000, anon: 125_000, command: 'cmd250' }] }, + detached: { processes: 0, rss: 0, anon: 0, top: [] }, + }, + ], + escaped: [], + escapedOmitted: { groups: 0, processes: 0, rss: 0, anon: 0 }, + cgroup: { + path: '/app.slice/x.scope', + current: 10, + peak: 20, + max: null, + high: null, + swapCurrent: 0, + pressure: PRESSURE, + others: { processes: 0, rss: 0, anon: 0, top: [] }, + }, + system: SYSTEM, + }); + expect([...gone].sort((a, b) => a - b)).toEqual([2, 3]); + }); + + it('keeps own memory and mirrors but gives null groups without a process table, forgetting no closed terminal', async () => { + const { record, gone } = await daemonRecord(input(null, {}, { cgroup: cgroupOf([DAEMON]) })); + expect(record.self).toEqual({ rss: 1, heapTotal: 2, heapUsed: 3, external: 4, arrayBuffers: 5 }); + expect(record.mirrors).toEqual({ count: 2, bytes: 100 * 80 * 12 + 24 * 120 * 12 }); + expect(record.terminals.map((t) => [t.termId, t.members, t.detached])).toEqual([ + [1, null, null], + [5, null, null], + ]); + expect(record.escaped).toBeNull(); + expect(record.escapedOmitted).toBeNull(); + expect(record.cgroup).toMatchObject({ current: 10, others: null }); + expect(gone).toEqual([]); + }); + + it('gives a null cgroup without one', async () => { + expect((await daemonRecord(input(BASE))).record.cgroup).toBeNull(); + }); +}); + +describe('rule 1: members of a live terminal', () => { + it('are in its session or descend from its shell, the 3 largest listed', async () => { + const table = [...BASE, p(201, 200, 200), p(202, 201, 202), p(210, 1, 200), p(211, 1, 211)]; + const term = await terminal(1, table); + expect(term.members).toMatchObject({ processes: 4, rss: 813_000, anon: 406_500 }); + expect(term.members?.top.map((t) => t.pid)).toEqual([210, 202, 201]); + }); + + it('come first, whatever their tag, and their tags are not read', async () => { + const fakes = { tags: { 201: '999/gone/9', 202: '300/other/4' }, args: { 300: WTD_DAEMON } }; + const lookup = lookupOf(fakes); + const table = [...BASE, p(201, 200, 200), p(202, 201, 202), p(300, 1, 300)]; + const { record } = await daemonRecord(input(table, fakes, { lookup })); + expect(record.terminals.find((t) => t.termId === 1)?.members?.processes).toBe(3); + expect(record.escaped).toEqual([]); + expect(lookup.tagged).not.toContain(200); + expect(lookup.tagged).not.toContain(201); + expect(lookup.tagged).not.toContain(202); + }); + + it('leave the daemon itself out', async () => { + const { record } = await daemonRecord(input(BASE, {}, { cgroup: cgroupOf([DAEMON, 200]) })); + expect(record.terminals.flatMap((t) => t.members?.top.map((m) => m.pid) ?? [])).not.toContain(DAEMON); + expect(record.cgroup?.others?.processes).toBe(0); + }); +}); + +describe('rule 2: detached processes of a live terminal', () => { + it('carry its tag outside its session and tree', async () => { + const table = [...BASE, p(203, 1, 203), p(204, 203, 203)]; + const term = await terminal(1, table, { tags: { 203: '100/inst/1', 204: '100/inst/1' } }); + expect(term.detached).toMatchObject({ processes: 2, rss: 407_000 }); + expect(term.members?.processes).toBe(1); + expect(await escapedOf(table, { tags: { 203: '100/inst/1', 204: '100/inst/1' } })).toEqual([]); + }); +}); + +describe("rule 3: another running daemon's processes", () => { + it('are not escaped, even when they no longer descend from it, nor among others', async () => { + const table = [...BASE, p(300, 1, 300), p(301, 1, 301), p(302, 301, 301)]; + const fakes = { tags: { 301: '300/other/4', 302: '300/other/4' }, args: { 300: WTD_DAEMON } }; + const { record } = await daemonRecord(input(table, fakes, { cgroup: cgroupOf([DAEMON, 301, 302]) })); + expect(record.escaped).toEqual([]); + expect(record.cgroup?.others?.processes).toBe(0); + }); +}); + +describe('rule 4: escaped processes', () => { + it.each([ + ['is not running', {}], + ['runs something else', { 999: ['/bin/sleep', '100'] }], + ['runs a wtd hub', { 999: ['/usr/bin/node', '/opt/wtd/dist/wtd.mjs', 'hub'] }], + ['runs a daemon that is not wtd', { 999: ['/usr/bin/node', '/opt/other.mjs', 'daemon'] }], + ])('carry a tag of a pid that %s, with null worktree and preset', async (_name, args) => { + const table = [...BASE, p(400, 1, 400)]; + expect(await escapedOf(table, { tags: { 400: '999/gone/7' }, args })).toEqual([ + { + tag: '999/gone/7', + termId: 7, + worktree: null, + preset: null, + processes: 1, + rss: 400_000, + anon: 200_000, + top: [{ pid: 400, rss: 400_000, anon: 200_000, command: 'cmd400' }], + }, + ]); + }); + + it('carry a tag of this pid and another instance', async () => { + const table = [...BASE, p(600, 1, 600)]; + expect(await escapedOf(table, { tags: { 600: '100/old/1' } })).toMatchObject([ + { tag: '100/old/1', termId: 1, worktree: null, preset: null }, + ]); + }); + + it('carry a tag of a terminal of this instance that is not live, with its worktree and preset when this instance closed it', async () => { + const table = [...BASE, p(502, 1, 502), p(800, 1, 800)]; + const escaped = await escapedOf(table, { tags: { 502: '100/inst/2', 800: '100/inst/9' } }); + expect(escaped).toHaveLength(2); + expect(escaped).toEqual( + expect.arrayContaining([ + expect.objectContaining({ tag: '100/inst/9', termId: 9, worktree: null, preset: null, processes: 1 }), + expect.objectContaining({ tag: '100/inst/2', termId: 2, worktree: '/r/w2', preset: 'codex', processes: 1 }), + ]), + ); + }); + + it("are untagged processes in a closed terminal's session, grouped with its tagged ones", async () => { + const table = [...BASE, p(501, 1, 500), p(502, 1, 502)]; + expect(await escapedOf(table, { tags: { 502: '100/inst/2' } })).toEqual([ + { + tag: '100/inst/2', + termId: 2, + worktree: '/r/w2', + preset: 'codex', + processes: 2, + rss: 1_003_000, + anon: 501_500, + top: [ + { pid: 502, rss: 502_000, anon: 251_000, command: 'cmd502' }, + { pid: 501, rss: 501_000, anon: 250_500, command: 'cmd501' }, + ], + }, + ]); + }); + + it('carry a malformed tag, grouped by it with a null terminal id', async () => { + const table = [...BASE, p(601, 1, 601)]; + expect(await escapedOf(table, { tags: { 601: 'junk' } })).toMatchObject([{ tag: 'junk', termId: null, worktree: null, preset: null }]); + }); + + it('are the 20 groups with the largest resident memory, the rest counted as omitted', async () => { + const pids = Array.from({ length: 25 }, (_, i) => 1000 + i); + const table = [...BASE, ...pids.map((pid) => p(pid, 1, pid))]; + const tags = Object.fromEntries(pids.map((pid) => [pid, `9${String(pid)}/x/1`])); + const { record } = await daemonRecord(input(table, { tags })); + expect(record.escaped?.map((g) => g.tag).sort()).toEqual(pids.slice(5).map((pid) => `9${String(pid)}/x/1`)); + const omitted = pids.slice(0, 5).reduce((sum, pid) => sum + pid * 1000, 0); + expect(record.escapedOmitted).toEqual({ groups: 5, processes: 5, rss: omitted, anon: omitted / 2 }); + }); + + it('list the 5 largest processes of a group', async () => { + const pids = [1001, 1002, 1003, 1004, 1005, 1006]; + const table = [...BASE, ...pids.map((pid) => p(pid, 1, pid))]; + const tags = Object.fromEntries(pids.map((pid) => [pid, '999/gone/7'])); + const [group] = (await escapedOf(table, { tags })) ?? []; + expect(group?.processes).toBe(6); + expect(group?.top.map((t) => t.pid)).toEqual([1006, 1005, 1004, 1003, 1002]); + }); +}); + +describe('rule 5: others', () => { + it('are the remaining members of the cgroup that are in the table', async () => { + const table = [...BASE, p(203, 1, 203), p(800, 1, 800), p(900, 1, 900)]; + const { record } = await daemonRecord( + input(table, { tags: { 203: '100/inst/1' } }, { cgroup: cgroupOf([DAEMON, 200, 203, 800, 801]) }), + ); + expect(record.cgroup?.others).toEqual({ + processes: 1, + rss: 800_000, + anon: 400_000, + top: [{ pid: 800, rss: 800_000, anon: 400_000, command: 'cmd800' }], + }); + }); +}); + +describe('closed terminals', () => { + it('are gone once a sample finds none of their processes, tagged or in their session', async () => { + const inSession = await daemonRecord(input([...BASE, p(501, 1, 500)])); + expect(inSession.gone).toEqual([3]); + const tagged = await daemonRecord(input([...BASE, p(701, 1, 701)], { tags: { 701: '100/inst/3' } })); + expect(tagged.gone).toEqual([2]); + }); +}); diff --git a/src/daemon/main/memory.ts b/src/daemon/main/memory.ts new file mode 100644 index 0000000..1c14e0c --- /dev/null +++ b/src/daemon/main/memory.ts @@ -0,0 +1,279 @@ +import { + summarise, + type OwnCgroup, + type ProcessGroup, + type ProcessInfo, + type ProcessLookup, + type SelfMemory, + type SystemMemory, +} from '../../platform/memory/index.js'; +import type { ClosedTerminalView, LiveTerminalView, MemoryView } from '../server/index.js'; + +/** Bytes per mirror cell: xterm stores three 32-bit words. */ +const CELL_BYTES = 12; +const TERMINAL_TOP = 3; +const GROUP_TOP = 5; +const MAX_ESCAPED = 20; + +export interface DaemonSampleInput { + ts: string; + pid: number; + instance: string; + version: string; + intervalMs: number; + self: SelfMemory; + view: MemoryView; + /** The current user's processes; null without a process table. */ + table: readonly ProcessInfo[] | null; + cgroup: OwnCgroup | null; + system: SystemMemory; + lookup: ProcessLookup; +} + +export interface TerminalRecord { + termId: number; + repo: string; + worktree: string; + preset: string; + pid: number; + mirror: { lines: number; cols: number; bytes: number }; + members: ProcessGroup | null; + detached: ProcessGroup | null; +} + +export interface EscapedGroup extends ProcessGroup { + tag: string; + termId: number | null; + /** Set when this daemon instance closed the terminal. */ + worktree: string | null; + preset: string | null; +} + +export interface DaemonRecord { + ts: string; + source: 'daemon'; + pid: number; + instance: string; + version: string; + intervalMs: number; + self: SelfMemory; + mirrors: { count: number; bytes: number }; + terminals: TerminalRecord[]; + escaped: EscapedGroup[] | null; + escapedOmitted: { groups: number; processes: number; rss: number; anon: number } | null; + cgroup: (Omit & { others: ProcessGroup | null }) | null; + system: SystemMemory; +} + +interface ParsedTag { + pid: number; + instance: string; + termId: number; +} + +const parseTag = (tag: string): ParsedTag | null => { + const match = /^(\d+)\/([^/]+)\/(\d+)$/.exec(tag); + if (match?.[1] === undefined || match[2] === undefined || match[3] === undefined) return null; + return { pid: Number(match[1]), instance: match[2], termId: Number(match[3]) }; +}; + +/** Whether `args` run a wtd daemon: last argument `daemon`, an argument whose basename starts with `wtd`. */ +const isWtdDaemon = (args: readonly string[] | null): boolean => + args !== null && args.at(-1) === 'daemon' && args.some((arg) => (arg.split('/').at(-1) ?? '').startsWith('wtd')); + +const mirrorBytes = ({ lines, cols }: { lines: number; cols: number }): number => lines * cols * CELL_BYTES; + +/** Pids of `table` descending, by parent pid, from `root`. */ +const descendants = (table: readonly ProcessInfo[], root: number): Set => { + const children = new Map(); + for (const p of table) children.set(p.ppid, [...(children.get(p.ppid) ?? []), p.pid]); + const found = new Set(); + const pending = [root]; + for (let pid = pending.pop(); pid !== undefined; pid = pending.pop()) { + for (const child of children.get(pid) ?? []) { + if (found.has(child)) continue; + found.add(child); + pending.push(child); + } + } + return found; +}; + +interface Placement { + members: Map; + detached: Map; + escaped: Map; + /** Pids placed by any rule, foreign daemons' included. */ + placed: Set; + /** Closed terminals some process was attributed to. */ + found: Set; +} + +/** Where one process belongs; `closed` is this daemon's closed terminal it is attributed to. */ +type Slot = + | { kind: 'member' | 'detached'; termId: number } + | { kind: 'foreign' } + | { kind: 'escaped'; tag: string; closed: number | null }; + +interface PlaceContext { + self: number; + instance: string; + lookup: ProcessLookup; + live: Set; + closedBySession: Map; + trees: { termId: number; shell: number; tree: Set }[]; + /** Memoised per pid. */ + daemons: Map>; +} + +const runsDaemon = (ctx: PlaceContext, pid: number): Promise => { + let known = ctx.daemons.get(pid); + if (known === undefined) { + known = ctx.lookup.args(pid).then(isWtdDaemon); + ctx.daemons.set(pid, known); + } + return known; +}; + +/** The first placement rule that applies to `p`; see the daemon memory log spec. */ +const slotOf = async (ctx: PlaceContext, p: ProcessInfo): Promise => { + const member = ctx.trees.find((t) => p.sid === t.shell || p.pid === t.shell || t.tree.has(p.pid)); + if (member !== undefined) return { kind: 'member', termId: member.termId }; + const tag = await ctx.lookup.tag(p.pid); + const parsed = tag === null ? null : parseTag(tag); + const ours = parsed !== null && parsed.pid === ctx.self && parsed.instance === ctx.instance ? parsed.termId : null; + if (ours !== null && ctx.live.has(ours)) return { kind: 'detached', termId: ours }; + if (parsed !== null && parsed.pid !== ctx.self && (await runsDaemon(ctx, parsed.pid))) return { kind: 'foreign' }; + if (tag !== null) return { kind: 'escaped', tag, closed: ours }; + const session = ctx.closedBySession.get(p.sid); + if (session === undefined) return null; + return { kind: 'escaped', tag: `${String(ctx.self)}/${ctx.instance}/${String(session.termId)}`, closed: session.termId }; +}; + +/** Places each process of `table` by {@link slotOf}. */ +const place = async (input: DaemonSampleInput, table: readonly ProcessInfo[]): Promise => { + const { view } = input; + const ctx: PlaceContext = { + self: input.pid, + instance: input.instance, + lookup: input.lookup, + live: new Set(view.terminals.map((t) => t.termId)), + closedBySession: new Map(view.closed.map((t) => [t.pid, t])), + trees: view.terminals.map((t) => ({ termId: t.termId, shell: t.pid, tree: descendants(table, t.pid) })), + daemons: new Map(), + }; + const result: Placement = { + members: new Map(view.terminals.map((t) => [t.termId, []])), + detached: new Map(view.terminals.map((t) => [t.termId, []])), + escaped: new Map(), + placed: new Set(), + found: new Set(), + }; + for (const p of table) { + if (p.pid === ctx.self) continue; + const slot = await slotOf(ctx, p); // NOSONAR(S9382) — one environ read at a time + if (slot === null) continue; + result.placed.add(p.pid); + if (slot.kind === 'member') result.members.get(slot.termId)?.push(p); + else if (slot.kind === 'detached') result.detached.get(slot.termId)?.push(p); + else if (slot.kind === 'escaped') { + if (slot.closed !== null) result.found.add(slot.closed); + result.escaped.set(slot.tag, [...(result.escaped.get(slot.tag) ?? []), p]); + } + } + const closedIds = new Set(view.closed.map((t) => t.termId)); + for (const termId of result.found) if (!closedIds.has(termId)) result.found.delete(termId); + return result; +}; + +const terminalRecord = (t: LiveTerminalView, members: ProcessGroup | null, detached: ProcessGroup | null): TerminalRecord => ({ + termId: t.termId, + repo: t.repo, + worktree: t.worktree, + preset: t.preset, + pid: t.pid, + mirror: { ...t.mirror, bytes: mirrorBytes(t.mirror) }, + members, + detached, +}); + +const escapedGroups = async ( + input: DaemonSampleInput, + escaped: Map, +): Promise<{ groups: EscapedGroup[]; omitted: NonNullable }> => { + const closed = new Map(input.view.closed.map((t) => [t.termId, t])); + const ranked = [...escaped] + .map(([tag, members]) => ({ tag, members, rss: members.reduce((sum, p) => sum + p.rss, 0) })) + .sort((a, b) => b.rss - a.rss); + const kept = ranked.slice(0, MAX_ESCAPED); + const rest = ranked.slice(MAX_ESCAPED).flatMap((g) => g.members); + const groups = await Promise.all( + kept.map(async ({ tag, members }): Promise => { + const parsed = parseTag(tag); + const ours = + parsed !== null && parsed.pid === input.pid && parsed.instance === input.instance ? closed.get(parsed.termId) : undefined; + return { + tag, + termId: parsed?.termId ?? null, + worktree: ours?.worktree ?? null, + preset: ours?.preset ?? null, + ...(await summarise(members, GROUP_TOP, input.lookup.command)), + }; + }), + ); + return { + groups, + omitted: { + groups: ranked.length - kept.length, + processes: rest.length, + rss: rest.reduce((sum, p) => sum + p.rss, 0), + anon: rest.reduce((sum, p) => sum + p.anon, 0), + }, + }; +}; + +/** The daemon's memory log record, and the closed terminals none of whose processes remain. */ +export const daemonRecord = async (input: DaemonSampleInput): Promise<{ record: DaemonRecord; gone: number[] }> => { + const { view, table, cgroup, lookup } = input; + const placement = table === null ? null : await place(input, table); + const group = (members: readonly ProcessInfo[] | undefined, n: number): Promise => + members === undefined ? Promise.resolve(null) : summarise(members, n, lookup.command); + const terminals = await Promise.all( + view.terminals.map(async (t) => + terminalRecord( + t, + await group(placement?.members.get(t.termId), TERMINAL_TOP), + await group(placement?.detached.get(t.termId), TERMINAL_TOP), + ), + ), + ); + const escaped = placement === null ? null : await escapedGroups(input, placement.escaped); + let cgroupRecord: DaemonRecord['cgroup'] = null; + if (cgroup !== null) { + const { pids, ...figures } = cgroup; + let others: ProcessGroup | null = null; + if (table !== null && placement !== null) { + const members = new Set(pids); + const rest = table.filter((p) => members.has(p.pid) && p.pid !== input.pid && !placement.placed.has(p.pid)); + others = await summarise(rest, GROUP_TOP, lookup.command); + } + cgroupRecord = { ...figures, others }; + } + const record: DaemonRecord = { + ts: input.ts, + source: 'daemon', + pid: input.pid, + instance: input.instance, + version: input.version, + intervalMs: input.intervalMs, + self: input.self, + mirrors: { count: view.terminals.length, bytes: view.terminals.reduce((sum, t) => sum + mirrorBytes(t.mirror), 0) }, + terminals, + escaped: escaped?.groups ?? null, + escapedOmitted: escaped?.omitted ?? null, + cgroup: cgroupRecord, + system: input.system, + }; + const gone = placement === null ? [] : view.closed.map((t) => t.termId).filter((termId) => !placement.found.has(termId)); + return { record, gone }; +}; diff --git a/src/daemon/server/daemon.test.ts b/src/daemon/server/daemon.test.ts index f25bdb4..33e96fe 100644 --- a/src/daemon/server/daemon.test.ts +++ b/src/daemon/server/daemon.test.ts @@ -266,6 +266,137 @@ it('neither counts commits nor fetches for a forced removal', async () => { expect(removed).toEqual([WT]); }); +describe('terminal tags and memory view', () => { + afterEach(async () => { + await daemon.shutdown(); + }); + + /** Creates a terminal in `worktree` whose spawn runs `command`; resolves with its id and process. */ + const launch = async ( + client: DaemonClient, + command: string, + { worktree = WT, preset = 'shell', shell = '/bin/bash' }: { worktree?: string; preset?: string; shell?: string } = {}, + ): Promise<{ termId: number; process: TerminalProcess }> => { + const count = spawns.length; + const reply = client.request({ t: 'createTerm', worktree, preset, command: null, cols: 80, rows: 24 }); + await vi.waitFor(() => { + expect(spawns).toHaveLength(count + 1); + }); + const pending = spawns[count]; + if (pending === undefined) throw new Error('no spawn'); + const env = { ...process.env, HOME: dir, SHELL: shell }; + const spawned = await TerminalProcess.spawn({ ...pending.spec, cwd: dir, command, env }, pending.events); + pending.resolve(spawned); + const created = await reply; + if (created.t !== 'termCreated') throw new Error(`createTerm failed: ${JSON.stringify(created)}`); + return { termId: created.term.termId, process: spawned }; + }; + + it("starts each terminal with WTD_TERMINAL naming the daemon's pid, its instance and the terminal id", async () => { + const client = await connected(); + await client.watch(REPO); + const { termId } = await launch(client, 'exec sleep 60'); + expect(spawns[0]?.spec.env).toMatchObject({ + PATH: process.env['PATH'], + WTD_TERMINAL: `${String(process.pid)}/test_daemon/${String(termId)}`, + }); + }); + + it('takes the id before starting, so a failed start leaves a gap', async () => { + const client = await connected(); + await client.watch(REPO); + expect((await launch(client, 'exec sleep 60')).termId).toBe(1); + create(client, 2); + await vi.waitFor(() => { + expect(spawns).toHaveLength(2); + }); + const tag = spawns[1]?.spec.env['WTD_TERMINAL']; + spawns[1]?.reject(new SpawnError('no shell')); + expect(tag).toBe(`${String(process.pid)}/test_daemon/2`); + expect(await client.waitFor('error', (m) => m.req === 2)).toMatchObject({ code: 'spawn-failed' }); + expect((await launch(client, 'exec sleep 60')).termId).toBe(3); + expect(spawns[2]?.spec.env['WTD_TERMINAL']).toBe(`${String(process.pid)}/test_daemon/3`); + }); + + it('lists live terminals with their shell pid and mirror, and closed ones with their shell pid', async () => { + const client = await connected(); + await client.watch(REPO); + const kept = await launch(client, 'exec sleep 60', { preset: 'claude' }); + const closed = await launch(client, 'exec sleep 60', { worktree: REPO }); + expect(daemon.memoryView()).toEqual({ + terminals: [ + { termId: kept.termId, repo: REPO, worktree: WT, preset: 'claude', pid: kept.process.pid, mirror: kept.process.mirrorStats() }, + { + termId: closed.termId, + repo: REPO, + worktree: REPO, + preset: 'shell', + pid: closed.process.pid, + mirror: closed.process.mirrorStats(), + }, + ], + closed: [], + }); + await client.ok({ t: 'closeTerm', termId: closed.termId }); + expect(daemon.memoryView()).toEqual({ + terminals: [expect.objectContaining({ termId: kept.termId })], + closed: [{ termId: closed.termId, pid: closed.process.pid, worktree: REPO, preset: 'shell' }], + }); + }); + + it('remembers a cleanly exited terminal as closed, and forgets it when asked', async () => { + const client = await connected(); + await client.watch(REPO); + const from = client.mark(); + const leaving = await launch(client, 'read -r _; exit 0'); + client.sendInput(leaving.termId, '\r'); + await client.waitFor('termClosed', (m) => m.termId === leaving.termId, { from, timeout: 3000 }); + expect(daemon.memoryView().closed).toEqual([{ termId: leaving.termId, pid: leaving.process.pid, worktree: WT, preset: 'shell' }]); + daemon.forgetClosed([leaving.termId, 99]); + expect(daemon.memoryView().closed).toEqual([]); + }); + + it('remembers the last 256 closed terminals', async () => { + const client = await connected(); + await client.watch(REPO); + for (let req = 1; req <= 257; req++) create(client, req); + await vi.waitFor(() => { + expect(spawns).toHaveLength(257); + }); + const env = { ...process.env, HOME: dir, SHELL: '/bin/sh' }; + /** Starts the shells of `pending`, lets them exit cleanly and waits until their terminals closed; resolves with their ids. */ + const closeAll = async (pending: readonly PendingSpawn[]): Promise => { + const mark = client.mark(); + await Promise.all( + pending.map(async (p) => { + p.resolve(await TerminalProcess.spawn({ ...p.spec, cwd: dir, command: 'read -r _; exit 0', env }, p.events)); + }), + ); + const ids = await vi.waitFor(() => { + const created = client.messages.slice(mark).flatMap((m) => (m.t === 'termCreated' ? [m.term.termId] : [])); + expect(created).toHaveLength(pending.length); + return created; + }); + for (const termId of ids) client.sendInput(termId, '\r'); + await vi.waitFor(() => { + expect(client.messages.slice(mark).filter((m) => m.t === 'termClosed')).toHaveLength(pending.length); + }); + return ids; + }; + // The first closes alone, so it is the oldest; at most 32 shells run at a time. + await closeAll(spawns.slice(0, 1)); + const rest: number[] = []; + for (let i = 1; i < spawns.length; i += 32) rest.push(...(await closeAll(spawns.slice(i, i + 32)))); // NOSONAR(S9382) — batches in order + rest.sort((a, b) => a - b); + expect( + daemon + .memoryView() + .closed.map((t) => t.termId) + .sort((a, b) => a - b), + ).toEqual(rest); + }, 60_000); +}); + describe('ending terminals', () => { afterEach(async () => { await daemon.shutdown(); diff --git a/src/daemon/server/daemon.ts b/src/daemon/server/daemon.ts index 49883ce..f8513b9 100644 --- a/src/daemon/server/daemon.ts +++ b/src/daemon/server/daemon.ts @@ -1,7 +1,14 @@ import type { Socket } from 'node:net'; import { FLOW_HIGH, DAEMON_PROTOCOL_VERSION, type Layout, type Terminal, type Worktree } from '../../protocol/index.js'; import { pruneLayout, type StateStore } from '../state/index.js'; -import { SpawnError, type OutputSink, type TerminalEvents, type TerminalProcess, type TerminalSpec } from '../terminals/index.js'; +import { + SpawnError, + type MirrorStats, + type OutputSink, + type TerminalEvents, + type TerminalProcess, + type TerminalSpec, +} from '../terminals/index.js'; import { NotARepoError, type discoverRepos, @@ -16,6 +23,8 @@ import { Peer, type Event } from './peer.js'; /** Most live terminals per repo. */ const MAX_TERMINALS = 1024; +/** Most closed terminals remembered for the memory log. */ +const MAX_CLOSED = 256; type ErrorCode = Extract['code']; type Correlated = Extract; @@ -45,6 +54,30 @@ interface TermEntry { process: TerminalProcess; } +/** A live terminal as the memory log sees it. */ +export interface LiveTerminalView { + termId: number; + repo: string; + worktree: string; + preset: string; + /** The shell's pid, which leads its session. */ + pid: number; + mirror: MirrorStats; +} + +/** A closed terminal whose leftover processes the memory log attributes to it. */ +export interface ClosedTerminalView { + termId: number; + pid: number; + worktree: string; + preset: string; +} + +export interface MemoryView { + terminals: LiveTerminalView[]; + closed: ClosedTerminalView[]; +} + /** What the daemon composes: worktree watching and listing, discovery, and terminals. */ export interface DaemonServices { watchWorktrees: typeof watchWorktrees; @@ -83,6 +116,8 @@ export class Daemon { private readonly spawnsIn = new Map>>(); /** Worktrees being removed; no terminal starts in them meanwhile. */ private readonly removing = new Set(); + /** The last MAX_CLOSED terminals closed, oldest first. */ + private readonly closed = new Map(); /** Terminals closed but not yet reaped. */ private readonly reaping = new Set>(); private stopping = false; @@ -120,6 +155,26 @@ export class Daemon { for (const peer of this.peers) peer.end(); } + /** Live terminals with their shell pid and mirror, and remembered closed ones. */ + memoryView(): MemoryView { + return { + terminals: [...this.terminals.values()].map((t) => ({ + termId: t.termId, + repo: t.repo, + worktree: t.worktree, + preset: t.preset, + pid: t.process.pid, + mirror: t.process.mirrorStats(), + })), + closed: [...this.closed.values()], + }; + } + + /** Stops remembering the closed terminals `termIds`. */ + forgetClosed(termIds: readonly number[]): void { + for (const termId of termIds) this.closed.delete(termId); + } + private receive(peer: Peer, outcome: GateOutcome): void { switch (outcome.kind) { case 'hello': @@ -359,9 +414,10 @@ export class Daemon { if (this.terminalsOf(repo).length + (this.spawning.get(repo) ?? 0) >= MAX_TERMINALS) { throw new RequestError('busy', `${repo} has ${String(MAX_TERMINALS)} terminals`); } - let termId = 0; + const termId = this.nextTermId++; + const env = { ...this.options.env, WTD_TERMINAL: `${String(process.pid)}/${this.options.instance}/${String(termId)}` }; const spawn = this.options.services.spawnTerminal( - { cwd: m.worktree, command: m.command, cols: m.cols, rows: m.rows, env: this.options.env }, + { cwd: m.worktree, command: m.command, cols: m.cols, rows: m.rows, env }, { activity: (unseen, state) => { this.broadcast(repo, { t: 'activity', termId, unseen, state }); @@ -392,7 +448,6 @@ export class Daemon { inWorktree.delete(spawn); if (inWorktree.size === 0) this.spawnsIn.delete(m.worktree); } - termId = this.nextTermId++; const term: TermEntry = { termId, repo, worktree: m.worktree, preset: m.preset, process: spawned }; this.terminals.set(termId, term); peer.send({ t: 'termCreated', req: m.req, term: describe(term) }); @@ -510,6 +565,11 @@ export class Daemon { /** Removes `term` and ends its process. */ private closeTerminal(term: TermEntry): void { this.terminals.delete(term.termId); + this.closed.set(term.termId, { termId: term.termId, pid: term.process.pid, worktree: term.worktree, preset: term.preset }); + for (const oldest of this.closed.keys()) { + if (this.closed.size <= MAX_CLOSED) break; + this.closed.delete(oldest); + } const reaped = term.process.close().finally(() => this.reaping.delete(reaped)); this.reaping.add(reaped); for (const peer of this.peers) if (peer.visible.has(term.termId)) this.showOnly(peer, [...peer.visible]); diff --git a/src/daemon/server/index.ts b/src/daemon/server/index.ts index bb46773..66bc011 100644 --- a/src/daemon/server/index.ts +++ b/src/daemon/server/index.ts @@ -1,2 +1,9 @@ export { AlreadyRunningError, bindSocket, type Listener } from './bind.js'; -export { Daemon, type DaemonOptions, type DaemonServices } from './daemon.js'; +export { + Daemon, + type ClosedTerminalView, + type DaemonOptions, + type DaemonServices, + type LiveTerminalView, + type MemoryView, +} from './daemon.js'; diff --git a/src/daemon/terminals/index.ts b/src/daemon/terminals/index.ts index fdf69b0..2b88850 100644 --- a/src/daemon/terminals/index.ts +++ b/src/daemon/terminals/index.ts @@ -1,3 +1,11 @@ export type { AttentionState } from './attention.js'; export type { AckResult } from './flow.js'; -export { SpawnError, TerminalProcess, type ExitStatus, type OutputSink, type TerminalEvents, type TerminalSpec } from './terminal.js'; +export { + SpawnError, + TerminalProcess, + type ExitStatus, + type MirrorStats, + type OutputSink, + type TerminalEvents, + type TerminalSpec, +} from './terminal.js'; diff --git a/src/daemon/terminals/terminal.test.ts b/src/daemon/terminals/terminal.test.ts index 2d7c46e..9aa9421 100644 --- a/src/daemon/terminals/terminal.test.ts +++ b/src/daemon/terminals/terminal.test.ts @@ -1,8 +1,8 @@ import { spawn, type ChildProcess } from 'node:child_process'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { cpus, tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { AttentionState } from './attention.js'; import { TerminalProcess } from './terminal.js'; @@ -192,3 +192,48 @@ describe('attention signals in the output', () => { expect(events.slice(before)).toEqual([]); }); }); + +describe('process and mirror figures', () => { + let home: string; + let term: TerminalProcess | null; + + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'wtd-figures-')); + term = null; + }); + + afterEach(async () => { + await term?.close(); + rmSync(home, { recursive: true, force: true }); + }); + + const run = async (command: string): Promise => { + term = await TerminalProcess.spawn( + { cwd: home, command, cols: 80, rows: 24, env: { ...process.env, HOME: home, SHELL: '/bin/bash' } }, + { activity: () => undefined, exited: () => undefined }, + ); + return term; + }; + + it("gives the shell's pid", async () => { + const started = await run('echo $$ > pid; exec sleep 60'); + const file = join(home, 'pid'); + await vi.waitFor(() => { + expect(readFileSync(file, 'utf8')).toMatch(/\n$/); + }); + expect(started.pid).toBe(Number(readFileSync(file, 'utf8'))); + }); + + it('counts the lines of the normal and the alternate buffer, and the columns', async () => { + const started = await run(String.raw`seq 1 300; read -r _; printf '\033[?1049hALT'; exec sleep 60`); + await vi.waitFor(() => { + expect(started.mirrorStats().lines).toBeGreaterThanOrEqual(300); + }); + const normal = started.mirrorStats().lines; + expect(started.mirrorStats().cols).toBe(80); + started.write(Buffer.from('\r')); + await vi.waitFor(() => { + expect(started.mirrorStats().lines).toBeGreaterThanOrEqual(normal + 24); + }); + }); +}); diff --git a/src/daemon/terminals/terminal.ts b/src/daemon/terminals/terminal.ts index a7d4442..0305cf2 100644 --- a/src/daemon/terminals/terminal.ts +++ b/src/daemon/terminals/terminal.ts @@ -53,6 +53,12 @@ interface Consumer { pending: { offset: number; data: Uint8Array }[]; } +/** Lines of the screen mirror's normal and alternate buffers, and its columns. */ +export interface MirrorStats { + lines: number; + cols: number; +} + export interface ExitStatus { code: number; signal: string | null; @@ -200,6 +206,16 @@ export class TerminalProcess { } } + /** The shell's pid, which leads its session. */ + get pid(): number { + return this.pty.pid; + } + + mirrorStats(): MirrorStats { + const { normal, alternate } = this.mirror.buffer; + return { lines: normal.length + alternate.length, cols: this.mirror.cols }; + } + get exit(): ExitStatus | null { return this.exitStatus; } diff --git a/src/hub/main/index.ts b/src/hub/main/index.ts index 6a71797..4b6be6c 100644 --- a/src/hub/main/index.ts +++ b/src/hub/main/index.ts @@ -1,5 +1,16 @@ import { randomBytes } from 'node:crypto'; import { hostname } from 'node:os'; +import { + hostRoots, + logWriter, + memoryTight, + processLookup, + readMemory, + sampleIntervalMs, + selfMemory, + startSampler, + type MemorySampler, +} from '../../platform/memory/index.js'; import { encodeHomeFragment, encodePrintCode } from '../../protocol/index.js'; import { loadConfig, type HubConfig } from '../config/index.js'; import { forwardToHub, startDetached, type DaemonPaths } from '../links/index.js'; @@ -16,12 +27,13 @@ import { TunnelListeners, type HubPaths, } from '../server/index.js'; +import { hubRecord } from './memory.js'; export { ConfigError } from '../config/index.js'; export { HubAlreadyRunningError, PortInUseError } from '../server/index.js'; /** Host files the hub uses. */ -export type HubHostPaths = HubPaths & DaemonPaths & { config: string }; +export type HubHostPaths = HubPaths & DaemonPaths & { config: string; hubMemoryLog: string }; export interface HubRun { /** The package version. */ @@ -32,7 +44,10 @@ export interface HubRun { /** The web bundle directory. */ webDir: string; home: string; - /** Read for `WTD_HUB_OPEN_MS`, `WTD_HUB_PROBE_MS` and `WTD_HUB_INSTALL_MS`, which shorten the hub's time limits. */ + /** + * Read for `WTD_HUB_OPEN_MS`, `WTD_HUB_PROBE_MS` and `WTD_HUB_INSTALL_MS`, which shorten the hub's time limits, and + * `WTD_MEMORY_SAMPLE_MS`, the memory log's base interval. + */ env: Readonly>; } @@ -68,6 +83,31 @@ const budget = (value: string | undefined): number | undefined => { const maps = (config: HubConfig, { host, alias, port }: { host: string; alias: string; port: number }): boolean => config.hosts.some((h) => h.name === host && h.ssh === alias && h.port === port); +interface HubIdentity { + version: string; + instance: string; + daemonCommand: readonly string[]; +} + +/** Samples the hub's memory into `hub-memory.log` until stopped. */ +const sampleMemory = (path: string, env: HubRun['env'], identity: HubIdentity, report: (error: unknown) => void): MemorySampler => { + const roots = hostRoots(); + const lookup = processLookup(roots); + const intervalMs = sampleIntervalMs(env['WTD_MEMORY_SAMPLE_MS']); + return startSampler({ + intervalMs, + tight: () => memoryTight(roots, report), + take: async () => { + const ts = new Date().toISOString(); + const self = selfMemory(); + const reading = await readMemory(roots, report); + return hubRecord({ ts, pid: process.pid, ...identity, intervalMs, self, ...reading, lookup }); + }, + write: logWriter(path, report), + report, + }); +}; + /** Runs the hub until SIGTERM, SIGINT or an authenticated shutdown request. */ export const runHub = async ({ version, paths, wtd, webDir, home, env }: HubRun): Promise => { let stop = (): void => undefined; @@ -84,9 +124,13 @@ export const runHub = async ({ version, paths, wtd, webDir, home, env }: HubRun) current: (target) => maps(latest, target), }); const probeMs = budget(env['WTD_HUB_PROBE_MS']) ?? PROBE_MS; + const daemonCommand = [...wtd, 'daemon']; + const report = (error: unknown): void => { + console.error(error); + }; const router = new Router({ paths, - daemonCommand: [...wtd, 'daemon'], + daemonCommand, configPath: paths.config, home, config, @@ -94,9 +138,7 @@ export const runHub = async ({ version, paths, wtd, webDir, home, env }: HubRun) instance, hostName: hostname(), bundle: wtd[1] ?? '', - report: (error) => { - console.error(error); - }, + report, tunnels: { ensure: (target, signal) => tunnels.ensure(target, signal), probe: (port, signal) => tunnels.probe(port, probeMs, signal), @@ -115,7 +157,9 @@ export const runHub = async ({ version, paths, wtd, webDir, home, env }: HubRun) openSession: (channel) => router.open(channel), shutdown: stop, }); + const sampler = sampleMemory(paths.hubMemoryLog, env, { version, instance, daemonCommand }, report); await stopped; + sampler.stop(); await Promise.all([server.close(), tunnels.close()]); } finally { for (const signal of STOP_SIGNALS) process.off(signal, stop); diff --git a/src/hub/main/memory.test.ts b/src/hub/main/memory.test.ts new file mode 100644 index 0000000..564c515 --- /dev/null +++ b/src/hub/main/memory.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from 'vitest'; +import type { OwnCgroup, ProcessInfo, ProcessLookup, SystemMemory } from '../../platform/memory/index.js'; +import { hubRecord, type HubSampleInput } from './memory.js'; + +const HUB = 100; +const WTD = ['/usr/bin/node', '/opt/wtd/dist/wtd.mjs']; +const DAEMON_COMMAND = [...WTD, 'daemon']; + +const p = (pid: number, ppid: number, sid = pid): ProcessInfo => ({ pid, ppid, sid, rss: pid * 1000, anon: pid * 500 }); + +const lookupOf = (args: Readonly>): ProcessLookup => ({ + tag: () => Promise.resolve(null), + args: (pid) => Promise.resolve(args[pid] ?? [`cmd${String(pid)}`]), + command: (pid) => Promise.resolve((args[pid] ?? [`cmd${String(pid)}`]).join(' ')), +}); + +const PRESSURE = { some: { avg10: 1, avg60: 2, avg300: 3 }, full: { avg10: 0, avg60: 0, avg300: 0 } }; +const SYSTEM: SystemMemory = { pressure: PRESSURE, meminfo: null }; +const cgroupOf = (pids: readonly number[]): OwnCgroup => ({ + path: '/app.slice/x.scope', + current: 10, + peak: 20, + max: 30, + high: null, + swapCurrent: null, + pressure: null, + pids: [...pids], +}); + +/** + * The hub (100) with an SSH child (101) that has a detached child (105), a hub-run helper (108), and the daemon + * (102) with a shell (103) and its child (104); 106 is unrelated and 107 runs the daemon command elsewhere. + */ +const TABLE = [ + p(HUB, 1), + p(101, HUB), + p(105, 101), + p(108, HUB), + p(102, HUB, 102), + p(103, 102, 103), + p(104, 103, 103), + p(106, 1), + p(107, 1), +]; +const ARGS = { 101: ['ssh', 'box', 'wtd connect'], 102: DAEMON_COMMAND, 107: DAEMON_COMMAND, 108: [...WTD, 'hub'] }; + +const input = (over: Partial = {}): HubSampleInput => ({ + ts: '2026-10-28T10:00:00.000Z', + pid: HUB, + instance: 'hubinst', + version: '1.2.3', + intervalMs: 60_000, + self: { rss: 1, heapTotal: 2, heapUsed: 3, external: 4, arrayBuffers: 5 }, + daemonCommand: DAEMON_COMMAND, + table: TABLE, + cgroup: null, + system: SYSTEM, + lookup: lookupOf(ARGS), + ...over, +}); + +describe('hub record', () => { + it('has the identity, own memory, children, cgroup and system figures', async () => { + expect(await hubRecord(input({ cgroup: cgroupOf([HUB, 101, 102, 103, 106, 999]) }))).toEqual({ + ts: '2026-10-28T10:00:00.000Z', + source: 'hub', + pid: HUB, + instance: 'hubinst', + version: '1.2.3', + intervalMs: 60_000, + self: { rss: 1, heapTotal: 2, heapUsed: 3, external: 4, arrayBuffers: 5 }, + children: { + processes: 3, + rss: 314_000, + anon: 157_000, + top: [ + { pid: 108, rss: 108_000, anon: 54_000, command: `${WTD.join(' ')} hub` }, + { pid: 105, rss: 105_000, anon: 52_500, command: 'cmd105' }, + { pid: 101, rss: 101_000, anon: 50_500, command: 'ssh box wtd connect' }, + ], + }, + cgroup: { + path: '/app.slice/x.scope', + current: 10, + peak: 20, + max: 30, + high: null, + swapCurrent: null, + pressure: null, + others: { + processes: 3, + rss: 311_000, + anon: 155_500, + top: [ + { pid: 106, rss: 106_000, anon: 53_000, command: 'cmd106' }, + { pid: 103, rss: 103_000, anon: 51_500, command: 'cmd103' }, + { pid: 102, rss: 102_000, anon: 51_000, command: DAEMON_COMMAND.join(' ') }, + ], + }, + }, + system: SYSTEM, + }); + }); + + it('lists the 5 largest children', async () => { + const many = Array.from({ length: 7 }, (_, i) => p(200 + i, HUB)); + const record = await hubRecord(input({ table: [p(HUB, 1), ...many] })); + expect(record.children?.processes).toBe(7); + expect(record.children?.top.map((t) => t.pid)).toEqual([206, 205, 204, 203, 202]); + }); + + it('counts a process running another command than the daemon command as a child', async () => { + const record = await hubRecord(input({ lookup: lookupOf({ ...ARGS, 102: [...WTD, 'daemon', '--x'] }) })); + expect(record.children?.processes).toBe(6); + }); + + it('keeps own memory but gives null groups without a process table', async () => { + const record = await hubRecord(input({ table: null, cgroup: cgroupOf([HUB]) })); + expect(record.self).toEqual({ rss: 1, heapTotal: 2, heapUsed: 3, external: 4, arrayBuffers: 5 }); + expect(record.children).toBeNull(); + expect(record.cgroup).toMatchObject({ current: 10, others: null }); + }); + + it('gives a null cgroup without one', async () => { + expect((await hubRecord(input())).cgroup).toBeNull(); + }); +}); diff --git a/src/hub/main/memory.ts b/src/hub/main/memory.ts new file mode 100644 index 0000000..f4e1e7a --- /dev/null +++ b/src/hub/main/memory.ts @@ -0,0 +1,95 @@ +import { + summarise, + type OwnCgroup, + type ProcessGroup, + type ProcessInfo, + type ProcessLookup, + type SelfMemory, + type SystemMemory, +} from '../../platform/memory/index.js'; + +const GROUP_TOP = 5; + +export interface HubSampleInput { + ts: string; + pid: number; + instance: string; + version: string; + intervalMs: number; + self: SelfMemory; + /** The command the hub starts its daemon with; that process and its descendants are the daemon's to log. */ + daemonCommand: readonly string[]; + /** The current user's processes; null without a process table. */ + table: readonly ProcessInfo[] | null; + cgroup: OwnCgroup | null; + system: SystemMemory; + lookup: ProcessLookup; +} + +export interface HubRecord { + ts: string; + source: 'hub'; + pid: number; + instance: string; + version: string; + intervalMs: number; + self: SelfMemory; + children: ProcessGroup | null; + cgroup: (Omit & { others: ProcessGroup | null }) | null; + system: SystemMemory; +} + +const sameArgs = (a: readonly string[] | null, b: readonly string[]): boolean => + a !== null && a.length === b.length && a.every((arg, i) => arg === b[i]); + +/** Descendants of the hub by parent pid, without the subtree of a process running the daemon command. */ +const childrenOf = async (input: HubSampleInput, table: readonly ProcessInfo[]): Promise => { + const byParent = new Map(); + for (const p of table) byParent.set(p.ppid, [...(byParent.get(p.ppid) ?? []), p]); + const children: ProcessInfo[] = []; + const seen = new Set([input.pid]); + const pending = [input.pid]; + for (let pid = pending.pop(); pid !== undefined; pid = pending.pop()) { + for (const child of byParent.get(pid) ?? []) { + if (seen.has(child.pid)) continue; + seen.add(child.pid); + if (sameArgs(await input.lookup.args(child.pid), input.daemonCommand)) continue; // NOSONAR(S9382) — walks the tree in order + children.push(child); + pending.push(child.pid); + } + } + return children; +}; + +/** The hub's memory log record. */ +export const hubRecord = async (input: HubSampleInput): Promise => { + const { table, cgroup, lookup } = input; + const children = table === null ? null : await childrenOf(input, table); + let cgroupRecord: HubRecord['cgroup'] = null; + if (cgroup !== null) { + const { pids, ...figures } = cgroup; + let others: ProcessGroup | null = null; + if (table !== null && children !== null) { + const members = new Set(pids); + const excluded = new Set([input.pid, ...children.map((p) => p.pid)]); + others = await summarise( + table.filter((p) => members.has(p.pid) && !excluded.has(p.pid)), + GROUP_TOP, + lookup.command, + ); + } + cgroupRecord = { ...figures, others }; + } + return { + ts: input.ts, + source: 'hub', + pid: input.pid, + instance: input.instance, + version: input.version, + intervalMs: input.intervalMs, + self: input.self, + children: children === null ? null : await summarise(children, GROUP_TOP, lookup.command), + cgroup: cgroupRecord, + system: input.system, + }; +}; diff --git a/src/platform/files/files.test.ts b/src/platform/files/files.test.ts index f808586..5cd61e7 100644 --- a/src/platform/files/files.test.ts +++ b/src/platform/files/files.test.ts @@ -8,6 +8,7 @@ import { rmSync, statSync, symlinkSync, + truncateSync, writeFileSync, writeSync, type BigIntStats, @@ -21,6 +22,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; import { acquireStartLock, + appendLogLine, currentHostPaths, fileIdentity, hostPaths, @@ -52,6 +54,8 @@ interface Hooks { failSyncOf: string | null; /** The first unlink or rename away of this path waits 100 ms. */ delayRemovalOf: string | null; + /** The next rename away of this path fails with EBUSY. */ + failRenameOf: string | null; calls: string[]; } @@ -61,6 +65,7 @@ const hooks = vi.hoisted((): Hooks => ({ failUnlinkOf: null, failSyncOf: null, delayRemovalOf: null, + failRenameOf: null, calls: [], })); @@ -82,6 +87,12 @@ vi.mock('node:fs/promises', async (importOriginal) => { await target.sync(); }; } + if (key === 'appendFile') { + return async (data: string | Uint8Array): Promise => { + hooks.calls.push(`append ${path} ${String(Buffer.byteLength(data))}`); + await target.appendFile(data); + }; + } if ((key === 'write' || key === 'writeFile') && failing(path)) { return async (data: string | Uint8Array): Promise => { await target.write(half(data)); @@ -118,6 +129,10 @@ vi.mock('node:fs/promises', async (importOriginal) => { }, rename: async (from: PathLike, to: PathLike): Promise => { hooks.calls.push(`rename ${String(from)} ${String(to)}`); + if (String(from) === hooks.failRenameOf) { + hooks.failRenameOf = null; + throw Object.assign(new Error('EBUSY: resource busy, rename'), { code: 'EBUSY' }); + } await delayedRemoval(from); await real.rename(from, to); }, @@ -147,6 +162,7 @@ beforeEach(() => { hooks.failUnlinkOf = null; hooks.failSyncOf = null; hooks.delayRemovalOf = null; + hooks.failRenameOf = null; hooks.calls.length = 0; }); @@ -169,10 +185,12 @@ describe('host paths', () => { lock: '/tmp/x/worktree-term/run/box.lock', state: '/tmp/x/worktree-term/state.json', log: '/tmp/x/worktree-term/daemon.log', + memoryLog: '/tmp/x/worktree-term/memory.log', configDir: '/home/u/.config/worktree-term', config: '/home/u/.config/worktree-term/config.json', hubToken: '/tmp/x/worktree-term/hub-token', hubLog: '/tmp/x/worktree-term/hub.log', + hubMemoryLog: '/tmp/x/worktree-term/hub-memory.log', hubLock: '/tmp/x/worktree-term/run/hub.lock', hubRecord: '/tmp/x/worktree-term/run/hub.json', unitFile: '/home/u/.config/systemd/user/worktree-term-daemon.service', @@ -198,9 +216,10 @@ describe('host paths', () => { it('keeps hub files under the state directory whatever the configuration directory', () => { const paths = hostPaths({ env: { XDG_STATE_HOME: '/s', XDG_CONFIG_HOME: '/c' }, home: '/home/u', host: 'box' }); - expect([paths.hubToken, paths.hubLog, paths.hubLock, paths.hubRecord]).toEqual([ + expect([paths.hubToken, paths.hubLog, paths.hubMemoryLog, paths.hubLock, paths.hubRecord]).toEqual([ '/s/worktree-term/hub-token', '/s/worktree-term/hub.log', + '/s/worktree-term/hub-memory.log', '/s/worktree-term/run/hub.lock', '/s/worktree-term/run/hub.json', ]); @@ -523,6 +542,108 @@ describe('daemon log', () => { }); }); +describe('memory log lines', () => { + const LIMIT = 16 * MiB; + let reported: unknown[]; + const report = (error: unknown): void => { + reported.push(error); + }; + const log = (): string => join(dir, 'memory.log'); + /** A log of `bytes` bytes; its content does not matter. */ + const sized = (path: string, bytes: number): void => { + writeFileSync(path, ''); + truncateSync(path, bytes); + }; + + beforeEach(() => { + reported = []; + }); + + it('appends each line whole, newline included, in one append to a log created owner-only', async () => { + await appendLogLine(log(), '{"a":1}', report); + await appendLogLine(log(), '{"b":22}', report); + expect(readFileSync(log(), 'utf8')).toBe('{"a":1}\n{"b":22}\n'); + expect(modeOf(log())).toBe(0o600); + expect(hooks.calls.filter((call) => call.startsWith('append '))).toEqual([`append ${log()} 8`, `append ${log()} 9`]); + expect(reported).toEqual([]); + }); + + it('makes an existing log owner-only before appending', async () => { + writeFileSync(log(), 'old\n', { mode: 0o644 }); + await appendLogLine(log(), 'new', report); + expect(modeOf(log())).toBe(0o600); + }); + + it('keeps appending while the log stays within 16 MiB', async () => { + sized(log(), LIMIT - 100); + await appendLogLine(log(), 'y'.repeat(99), report); + expect(statSync(log()).size).toBe(LIMIT); + expect(readdirSync(dir)).toEqual(['memory.log']); + }); + + it('moves a log the line would take over 16 MiB to .1 and writes the line to a fresh log', async () => { + sized(log(), LIMIT - 10); + await appendLogLine(log(), 'y'.repeat(99), report); + expect(statSync(`${log()}.1`).size).toBe(LIMIT - 10); + expect(readFileSync(log(), 'utf8')).toBe(`${'y'.repeat(99)}\n`); + expect(reported).toEqual([]); + }); + + it('shifts .1 and .2 up and replaces .3 when rotating', async () => { + sized(log(), LIMIT); + writeFileSync(`${log()}.1`, 'one'); + writeFileSync(`${log()}.2`, 'two'); + writeFileSync(`${log()}.3`, 'three'); + await appendLogLine(log(), 'new', report); + expect(readFileSync(`${log()}.3`, 'utf8')).toBe('two'); + expect(readFileSync(`${log()}.2`, 'utf8')).toBe('one'); + expect(statSync(`${log()}.1`).size).toBe(LIMIT); + expect(readFileSync(log(), 'utf8')).toBe('new\n'); + expect(readdirSync(dir).sort()).toEqual(['memory.log', 'memory.log.1', 'memory.log.2', 'memory.log.3']); + }); + + it('skips missing old files when rotating', async () => { + sized(log(), LIMIT); + writeFileSync(`${log()}.2`, 'two'); + await appendLogLine(log(), 'new', report); + expect(readFileSync(`${log()}.3`, 'utf8')).toBe('two'); + expect(statSync(`${log()}.1`).size).toBe(LIMIT); + expect(readdirSync(dir).sort()).toEqual(['memory.log', 'memory.log.1', 'memory.log.3']); + expect(reported).toEqual([]); + }); + + it('writes a line over 16 MiB after rotating', async () => { + writeFileSync(log(), 'old\n'); + const line = 'z'.repeat(LIMIT + 1); + await appendLogLine(log(), line, report); + expect(readFileSync(`${log()}.1`, 'utf8')).toBe('old\n'); + expect(statSync(log()).size).toBe(LIMIT + 2); + }); + + it.each([ + ['.2 to .3', '.2'], + ['.1 to .2', '.1'], + ['the log to .1', ''], + ])('reports a failed rename of %s, still appends the line, and rotates on the next append', async (_name, suffix) => { + sized(log(), LIMIT); + writeFileSync(`${log()}.1`, 'one'); + writeFileSync(`${log()}.2`, 'two'); + hooks.failRenameOf = `${log()}${suffix}`; + await appendLogLine(log(), 'first', report); + expect(reported).toEqual([expect.objectContaining({ code: 'EBUSY' })]); + expect(statSync(log()).size).toBe(LIMIT + 'first\n'.length); + await appendLogLine(log(), 'second', report); + expect(reported).toHaveLength(1); + expect(readFileSync(log(), 'utf8')).toBe('second\n'); + expect(statSync(`${log()}.1`).size).toBe(LIMIT + 'first\n'.length); + }); + + it('fails when the line cannot be written', async () => { + const path = join(dir, 'missing', 'memory.log'); + await expect(appendLogLine(path, 'x', report)).rejects.toMatchObject({ code: 'ENOENT' }); + }); +}); + describe('start lock', () => { const lockFile = (): string => join(dir, 'box.lock'); const holderSchema = z.strictObject({ pid: z.int(), nonce: z.string() }); diff --git a/src/platform/files/index.ts b/src/platform/files/index.ts index 326835a..97dc78c 100644 --- a/src/platform/files/index.ts +++ b/src/platform/files/index.ts @@ -12,10 +12,12 @@ export interface HostPaths { lock: string; state: string; log: string; + memoryLog: string; configDir: string; config: string; hubToken: string; hubLog: string; + hubMemoryLog: string; hubLock: string; hubRecord: string; unitFile: string; @@ -33,6 +35,8 @@ export const UNIT_NAME = 'worktree-term-daemon.service'; /** Longest unix socket path, in bytes, that fits `sockaddr_un`. */ const MAX_SOCKET_PATH = 107; const LOG_ROTATE_BYTES = 1024 * 1024; +const LINE_LOG_ROTATE_BYTES = 16 * 1024 * 1024; +const LINE_LOG_KEPT = 3; const LOCK_RETRY_MS = 50; const LOCK_WAIT_MS = 5000; @@ -58,10 +62,12 @@ export const hostPaths = ({ env, home, host }: HostIdentity): HostPaths => { lock: join(runDir, `${name}.lock`), state: join(stateDir, 'state.json'), log: join(stateDir, 'daemon.log'), + memoryLog: join(stateDir, 'memory.log'), configDir, config: join(configDir, 'config.json'), hubToken: join(stateDir, 'hub-token'), hubLog: join(stateDir, 'hub.log'), + hubMemoryLog: join(stateDir, 'hub-memory.log'), hubLock: join(runDir, 'hub.lock'), hubRecord: join(runDir, 'hub.json'), unitFile: join(configHome, 'systemd', 'user', UNIT_NAME), @@ -281,6 +287,48 @@ export const openLog = async (path: string): Promise => { return handle; }; +const sizeOf = async (path: string): Promise => { + try { + return (await stat(path)).size; + } catch (error) { + if (hasCode(error, 'ENOENT') || hasCode(error, 'ENOTDIR')) return 0; + throw error; + } +}; + +/** Shifts `.2` to `.3`, `.1` to `.2` and `path` to `.1`, skipping missing files. */ +const rotate = async (path: string): Promise => { + for (let n = LINE_LOG_KEPT - 1; n >= 0; n--) { + const from = n === 0 ? path : `${path}.${String(n)}`; + try { + await rename(from, `${path}.${String(n + 1)}`); // NOSONAR(S9382) — renames must run in order + } catch (error) { + if (!hasCode(error, 'ENOENT')) throw error; + } + } +}; + +/** + * Appends `line` and a newline to the owner-only log at `path` in one write, first rotating a log the + * line would take over 16 MiB; a failed rotation is reported and retried on the next append. + */ +export const appendLogLine = async (path: string, line: string, report: (error: unknown) => void): Promise => { + const data = `${line}\n`; + try { + const size = await sizeOf(path); + if (size > 0 && size + Buffer.byteLength(data) > LINE_LOG_ROTATE_BYTES) await rotate(path); + } catch (error) { + report(error); + } + const handle = await open(path, 'a', 0o600); + try { + await handle.chmod(0o600); + await handle.appendFile(data); + } finally { + await handle.close(); + } +}; + export interface StartLock { release: () => Promise; } diff --git a/src/platform/memory/index.ts b/src/platform/memory/index.ts new file mode 100644 index 0000000..4baaa5b --- /dev/null +++ b/src/platform/memory/index.ts @@ -0,0 +1,26 @@ +export { logWriter, sampleIntervalMs, startSampler, type MemorySampler, type SamplerOptions } from './sampler.js'; +export { + hostRoots, + isTight, + memoryTight, + processLookup, + readMemory, + readOwnCgroup, + readProcessTable, + readSystemMemory, + selfMemory, + summarise, + type Meminfo, + type MemoryReading, + type MemoryRoots, + type OwnCgroup, + type Pressure, + type PressureFigures, + type ProcessGroup, + type ProcessInfo, + type ProcessLookup, + type Report, + type SelfMemory, + type SystemMemory, + type TopProcess, +} from './sources.js'; diff --git a/src/platform/memory/sampler.test.ts b/src/platform/memory/sampler.test.ts new file mode 100644 index 0000000..a7e9ba5 --- /dev/null +++ b/src/platform/memory/sampler.test.ts @@ -0,0 +1,250 @@ +import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { logWriter, sampleIntervalMs, startSampler, type MemorySampler } from './index.js'; + +const MINUTE = 60_000; + +describe('sampling cadence', () => { + let tight: boolean; + let takes: number[]; + let checks: number[]; + let lines: string[]; + let reported: unknown[]; + let sampler: MemorySampler | null; + /** Replaces the next take, when set. */ + let nextTake: (() => Promise) | null; + /** Replaces the next write, when set. */ + let nextWrite: (() => Promise) | null; + let running: number; + let mostRunning: number; + /** Fails the next check, when set. */ + let failCheck: Error | null; + + const start = (intervalMs = MINUTE): void => { + sampler = startSampler({ + intervalMs, + tight: async () => { + checks.push(Date.now()); + running++; + mostRunning = Math.max(mostRunning, running); + await Promise.resolve(); + running--; + const failure = failCheck; + failCheck = null; + if (failure !== null) throw failure; + return tight; + }, + take: async () => { + takes.push(Date.now()); + running++; + mostRunning = Math.max(mostRunning, running); + try { + const replaced = nextTake; + nextTake = null; + return replaced === null ? { n: takes.length } : await replaced(); + } finally { + running--; + } + }, + write: (line) => { + const replaced = nextWrite; + nextWrite = null; + if (replaced !== null) return replaced(); + lines.push(line); + return Promise.resolve(); + }, + report: (error) => { + reported.push(error); + }, + }); + }; + + /** Advances the fake clock to `ms`. */ + const at = async (ms: number): Promise => { + await vi.advanceTimersByTimeAsync(ms - Date.now()); + }; + + beforeEach(() => { + vi.useFakeTimers({ now: 0 }); + tight = false; + takes = []; + checks = []; + lines = []; + reported = []; + sampler = null; + nextTake = null; + nextWrite = null; + running = 0; + mostRunning = 0; + failCheck = null; + }); + + afterEach(() => { + sampler?.stop(); + vi.useRealTimers(); + }); + + it('samples at start and then every 60 s while memory is calm, writing each sample as a JSON line', async () => { + start(); + await at(3 * MINUTE + 1000); + expect(takes).toEqual([0, MINUTE, 2 * MINUTE, 3 * MINUTE]); + expect(lines).toEqual(['{"n":1}', '{"n":2}', '{"n":3}', '{"n":4}']); + expect(reported).toEqual([]); + }); + + it('checks memory every 5 s between samples', async () => { + start(); + await at(MINUTE - 1000); + expect(checks).toEqual([5000, 10_000, 15_000, 20_000, 25_000, 30_000, 35_000, 40_000, 45_000, 50_000, 55_000]); + }); + + it('samples within 5 s once memory is tight, every 5 s while it stays tight, and 60 s after the last sample once calm', async () => { + start(); + await at(32_500); + tight = true; + await at(50_000); + expect(takes).toEqual([0, 35_000, 40_000, 45_000, 50_000]); + tight = false; + await at(115_000); + expect(takes).toEqual([0, 35_000, 40_000, 45_000, 50_000, 50_000 + MINUTE]); + }); + + it('checks and samples at the base interval when it is shorter than 5 s, tight or not', async () => { + tight = true; + start(1000); + await at(3500); + expect(takes).toEqual([0, 1000, 2000, 3000]); + }); + + it('starts no check or sample before a slow sample finished', async () => { + tight = true; + start(); + await at(4000); + nextTake = () => + new Promise((resolve) => { + setTimeout(() => { + resolve({ slow: true }); + }, 12_000); + }); + await at(23_000); + expect(takes).toEqual([0, 5000, 22_000]); + expect(checks.filter((t) => t > 5000 && t < 17_000)).toEqual([]); + expect(mostRunning).toBe(1); + expect(lines).toContain('{"slow":true}'); + }); + + it('writes nothing once stopped, not even the sample being taken', async () => { + let finish = (): void => undefined; + nextTake = () => + new Promise((resolve) => { + finish = () => { + resolve({ late: true }); + }; + }); + start(); + await at(0); + expect(takes).toEqual([0]); + sampler?.stop(); + finish(); + await at(10 * MINUTE); + expect(takes).toEqual([0]); + expect(checks).toEqual([]); + expect(lines).toEqual([]); + }); + + it('reports a failed write and takes the next sample', async () => { + const failure = new Error('disk full'); + nextWrite = () => Promise.reject(failure); + start(); + await at(MINUTE + 1000); + expect(reported).toEqual([failure]); + expect(takes).toEqual([0, MINUTE]); + expect(lines).toEqual(['{"n":2}']); + }); + + it('reports a failed sample and takes the next one', async () => { + const failure = new Error('no proc'); + nextTake = () => Promise.reject(failure); + start(); + await at(MINUTE + 1000); + expect(reported).toEqual([failure]); + expect(lines).toEqual(['{"n":2}']); + }); + + it('reports a failed check, counting memory as calm, and keeps checking', async () => { + const failure = new Error('no pressure'); + failCheck = failure; + tight = true; + start(); + await at(12_000); + expect(reported).toEqual([failure]); + expect(takes).toEqual([0, 10_000]); + }); +}); + +describe('sampler timers', () => { + it('never keep the process alive', async () => { + const timers: NodeJS.Timeout[] = []; + const real = globalThis.setTimeout; + const spy = vi.spyOn(globalThis, 'setTimeout').mockImplementation((fn: () => void, ms?: number) => { + const timer = real(fn, ms); + timers.push(timer); + return timer; + }); + const sampler = startSampler({ + intervalMs: 20, + tight: () => Promise.resolve(false), + take: () => Promise.resolve({}), + write: () => Promise.resolve(), + report: () => undefined, + }); + try { + await new Promise((resolve) => real(resolve, 200)); + } finally { + sampler.stop(); + spy.mockRestore(); + } + expect(timers.length).toBeGreaterThanOrEqual(3); + expect(timers.filter((t) => t.hasRef())).toEqual([]); + }); +}); + +describe('base interval', () => { + it.each([ + [undefined, MINUTE], + ['250', 250], + ['1', 1], + ['0', MINUTE], + ['-5', MINUTE], + ['1.5', MINUTE], + ['9'.repeat(400), MINUTE], + ['9007199254740993', MINUTE], + ['abc', MINUTE], + ['', MINUTE], + ])('WTD_MEMORY_SAMPLE_MS %j gives %d ms', (value, ms) => { + expect(sampleIntervalMs(value)).toBe(ms); + }); +}); + +describe('log writer', () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'wtd-memory-log-')); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it('appends lines to an owner-only rotating log', async () => { + const path = join(dir, 'memory.log'); + const write = logWriter(path, () => undefined); + await write('{"a":1}'); + await write('{"b":2}'); + expect(readFileSync(path, 'utf8')).toBe('{"a":1}\n{"b":2}\n'); + expect(statSync(path).mode & 0o077).toBe(0); + }); +}); diff --git a/src/platform/memory/sampler.ts b/src/platform/memory/sampler.ts new file mode 100644 index 0000000..66cbf96 --- /dev/null +++ b/src/platform/memory/sampler.ts @@ -0,0 +1,93 @@ +import { appendLogLine } from '../files/index.js'; +import type { Report } from './sources.js'; + +const DEFAULT_INTERVAL_MS = 60_000; +const CHECK_MS = 5000; + +export interface SamplerOptions { + /** The base interval between samples. */ + intervalMs: number; + /** Whether memory is tight, which samples at every check. */ + tight: () => Promise; + /** Takes a sample, written as one JSON line. */ + take: () => Promise; + write: (line: string) => Promise; + report: Report; +} + +export interface MemorySampler { + /** Cancels the next check and drops a sample in flight. */ + stop: () => void; +} + +/** The base interval: `WTD_MEMORY_SAMPLE_MS` when a positive safe integer, else 60 s. */ +export const sampleIntervalMs = (value: string | undefined): number => { + const ms = value !== undefined && /^\d+$/.test(value) ? Number(value) : 0; + return Number.isSafeInteger(ms) && ms > 0 ? ms : DEFAULT_INTERVAL_MS; +}; + +/** Appends lines to the rotating owner-only log at `path`. */ +export const logWriter = + (path: string, report: Report): ((line: string) => Promise) => + (line) => + appendLogLine(path, line, report); + +/** + * Samples at start, then checks every 5 s (the base interval when shorter) and samples when the base + * interval has passed or memory is tight. One check or sample runs at a time; timers are unref'd. + */ +export const startSampler = ({ intervalMs, tight, take, write, report }: SamplerOptions): MemorySampler => { + const checkMs = Math.min(CHECK_MS, intervalMs); + let stopped = false; + let timer: NodeJS.Timeout | null = null; + let lastSample = Date.now(); + + const sample = async (): Promise => { + lastSample = Date.now(); + let line: string; + try { + line = JSON.stringify(await take()); + } catch (error) { + if (!stopped) report(error); + return; + } + if (stopped) return; + try { + await write(line); + } catch (error) { + report(error); + } + }; + + const due = async (): Promise => { + if (Date.now() - lastSample >= intervalMs) return true; + try { + return await tight(); + } catch (error) { + report(error); + return false; + } + }; + + const schedule = (): void => { + if (stopped) return; + timer = setTimeout(() => { + timer = null; + void check(); + }, checkMs); + timer.unref(); + }; + + const check = async (): Promise => { + if ((await due()) && !stopped) await sample(); + schedule(); + }; + + void sample().then(schedule); + return { + stop: () => { + stopped = true; + if (timer !== null) clearTimeout(timer); + }, + }; +}; diff --git a/src/platform/memory/sources.test.ts b/src/platform/memory/sources.test.ts new file mode 100644 index 0000000..95f74c1 --- /dev/null +++ b/src/platform/memory/sources.test.ts @@ -0,0 +1,436 @@ +import { spawn } from 'node:child_process'; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, type PathLike } from 'node:fs'; +import type * as FsPromises from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + hostRoots, + isTight, + processLookup, + readOwnCgroup, + readProcessTable, + readSystemMemory, + summarise, + type MemoryRoots, + type OwnCgroup, + type ProcessInfo, + type SystemMemory, +} from './index.js'; + +// platform.memory reads through node:fs/promises; these hooks make reads of one path fail with a code. +const hooks = vi.hoisted((): { failing: Map } => ({ failing: new Map() })); + +vi.mock('node:fs/promises', async (importOriginal) => { + const real = await importOriginal(); + const fail = (path: PathLike): void => { + const code = hooks.failing.get(String(path)); + if (code !== undefined) throw Object.assign(new Error(`${code}: ${String(path)}`), { code }); + }; + return { + ...real, + readFile: async (path: PathLike, options?: object) => { + fail(path); + return real.readFile(path, options); + }, + readdir: async (path: PathLike, options?: object) => { + fail(path); + return real.readdir(path, options); + }, + }; +}); + +const UID = 4242; +const KiB = 1024; +const PSI = 'some avg10=1.50 avg60=2.25 avg300=3.00 total=123\nfull avg10=0.50 avg60=0.75 avg300=1.00 total=45\n'; +const PRESSURE = { some: { avg10: 1.5, avg60: 2.25, avg300: 3 }, full: { avg10: 0.5, avg60: 0.75, avg300: 1 } }; +const MEMINFO = + 'MemTotal: 1000 kB\nMemFree: 100 kB\nMemAvailable: 400 kB\nSwapTotal: 200 kB\nSwapFree: 150 kB\n'; + +let root: string; +let roots: MemoryRoots; +let reported: unknown[]; +const report = (error: unknown): void => { + reported.push(error); +}; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'wtd-memory-')); + roots = { proc: join(root, 'proc'), cgroup: join(root, 'cgroup'), uid: UID }; + mkdirSync(roots.proc); + mkdirSync(roots.cgroup); + reported = []; + hooks.failing.clear(); +}); + +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +interface FakeProcess { + name?: string; + state?: string; + ppid?: number; + sid?: number; + uid?: number; + /** VmRSS in kB; null leaves the line out. */ + rss?: number | null; + /** RssAnon in kB; null leaves the line out. */ + anon?: number | null; + args?: readonly string[]; + env?: Readonly>; +} + +/** Writes `/proc/` of the fake tree. */ +const fakeProcess = (pid: number, p: FakeProcess = {}): string => { + const { name = 'sh', state = 'S', ppid = 1, sid = pid, uid = UID, rss = 1000, anon = 600, args = [name], env = {} } = p; + const dir = join(roots.proc, String(pid)); + mkdirSync(dir, { recursive: true }); + const rest = Array.from({ length: 45 }, () => '0').join(' '); + writeFileSync(join(dir, 'stat'), `${String(pid)} (${name}) ${state} ${String(ppid)} ${String(pid)} ${String(sid)} 0 -1 ${rest}\n`); + const lines = [ + `Name:\t${name}`, + `State:\t${state} (sleeping)`, + `PPid:\t${String(ppid)}`, + `Uid:\t${String(uid)}\t${String(uid)}\t${String(uid)}\t${String(uid)}`, + ]; + if (rss !== null) lines.push(`VmRSS:\t${String(rss).padStart(8)} kB`); + if (anon !== null) lines.push(`RssAnon:\t${String(anon).padStart(8)} kB`); + writeFileSync(join(dir, 'status'), `${lines.join('\n')}\n`); + writeFileSync(join(dir, 'cmdline'), args.map((a) => `${a}\0`).join('')); + writeFileSync( + join(dir, 'environ'), + Object.entries(env) + .map(([k, v]) => `${k}=${v}\0`) + .join(''), + ); + return dir; +}; + +/** Writes a fake file at `relative` under the root. */ +const fakeFile = (relative: string, text: string): void => { + const path = join(root, relative); + mkdirSync(join(path, '..'), { recursive: true }); + writeFileSync(path, text); +}; + +/** A cgroup v2 own cgroup at `/app.slice/x.scope` with every file present. */ +const fakeCgroup = (): string => { + fakeFile('proc/self/cgroup', '0::/app.slice/x.scope\n'); + const dir = 'cgroup/app.slice/x.scope'; + fakeFile(`${dir}/memory.current`, '123456\n'); + fakeFile(`${dir}/memory.peak`, '234567\n'); + fakeFile(`${dir}/memory.max`, 'max\n'); + fakeFile(`${dir}/memory.high`, '1000000\n'); + fakeFile(`${dir}/memory.swap.current`, '4096\n'); + fakeFile(`${dir}/memory.pressure`, PSI); + fakeFile(`${dir}/cgroup.procs`, '10\n11\n12\n'); + return join(root, dir); +}; + +const byPid = (table: readonly ProcessInfo[] | null): ProcessInfo[] => [...(table ?? [])].sort((a, b) => a.pid - b.pid); + +describe('process table', () => { + it('lists the processes of the user with parent, session and resident memory in bytes', async () => { + fakeProcess(10, { ppid: 1, sid: 10, rss: 2000, anon: 1500 }); + fakeProcess(11, { ppid: 10, sid: 10, rss: 300, anon: 100 }); + fakeProcess(12, { uid: 0 }); + fakeFile('proc/meminfo', MEMINFO); + mkdirSync(join(roots.proc, 'self'), { recursive: true }); + expect(byPid(await readProcessTable(roots, report))).toEqual([ + { pid: 10, ppid: 1, sid: 10, rss: 2000 * KiB, anon: 1500 * KiB }, + { pid: 11, ppid: 10, sid: 10, rss: 300 * KiB, anon: 100 * KiB }, + ]); + expect(reported).toEqual([]); + }); + + it('reads parent and session after a command name holding spaces and parentheses', async () => { + fakeProcess(20, { name: 'a) (b c', ppid: 7, sid: 8 }); + expect(await readProcessTable(roots, report)).toEqual([expect.objectContaining({ pid: 20, ppid: 7, sid: 8 })]); + }); + + it('counts a zombie, which has no resident memory lines, with none', async () => { + fakeProcess(30, { state: 'Z', rss: null, anon: null }); + expect(await readProcessTable(roots, report)).toEqual([{ pid: 30, ppid: 1, sid: 30, rss: 0, anon: 0 }]); + expect(reported).toEqual([]); + }); + + it('leaves out a process whose status is gone by the time it is read', async () => { + fakeProcess(40); + const vanishing = fakeProcess(41); + rmSync(join(vanishing, 'status')); + expect((await readProcessTable(roots, report))?.map((p) => p.pid)).toEqual([40]); + expect(reported).toEqual([]); + }); + + it.each(['ESRCH', 'ENOENT'])('leaves out a process whose files fail with %s mid-read', async (code) => { + fakeProcess(40); + const vanishing = fakeProcess(41); + hooks.failing.set(join(vanishing, 'status'), code); + expect((await readProcessTable(roots, report))?.map((p) => p.pid)).toEqual([40]); + expect(reported).toEqual([]); + }); + + it('yields null without /proc, reporting nothing', async () => { + rmSync(roots.proc, { recursive: true }); + expect(await readProcessTable(roots, report)).toBeNull(); + expect(reported).toEqual([]); + }); + + it('yields null when /proc cannot be read, reporting nothing', async () => { + fakeProcess(10); + hooks.failing.set(roots.proc, 'EACCES'); + expect(await readProcessTable(roots, report)).toBeNull(); + expect(reported).toEqual([]); + }); + + it('yields null and reports a malformed status', async () => { + const dir = fakeProcess(50); + writeFileSync( + join(dir, 'status'), + `Name:\tsh\nPPid:\t1\nUid:\t${String(UID)}\t${String(UID)}\t${String(UID)}\t${String(UID)}\nVmRSS:\tlots kB\n`, + ); + expect(await readProcessTable(roots, report)).toBeNull(); + expect(reported).toHaveLength(1); + }); + + it.runIf(existsSync('/proc/self/stat'))('contains a child of the reader running sleep, from the real /proc', async () => { + const child = spawn('sleep', ['30'], { stdio: 'ignore', env: { ...process.env, WTD_TERMINAL: '1/real/2' } }); + try { + const pid = child.pid ?? 0; + const stat = readFileSync('/proc/self/stat', 'utf8'); + const ownSession = Number(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[3]); + const real = hostRoots(); + let entry: ProcessInfo | undefined; + for (let attempt = 0; attempt < 50 && entry === undefined; attempt++) { + entry = (await readProcessTable(real, report))?.find((p) => p.pid === pid); // NOSONAR(S9382) — polls until the child has exec'd + if (entry === undefined) await new Promise((resolve) => setTimeout(resolve, 20)); // NOSONAR(S9382) — polling loop + } + expect(entry).toMatchObject({ pid, ppid: process.pid, sid: ownSession }); + expect(entry?.rss).toBeGreaterThan(0); + const lookup = processLookup(real); + expect(await lookup.tag(pid)).toBe('1/real/2'); + expect(await lookup.args(pid)).toEqual(['sleep', '30']); + expect(reported).toEqual([]); + } finally { + child.kill('SIGKILL'); + } + }); +}); + +describe('process lookup', () => { + it("reads a process's WTD_TERMINAL, null when it has none, is gone or cannot be read", async () => { + fakeProcess(10, { env: { HOME: '/h', WTD_TERMINAL: '100/inst/3', WTD_TERMINAL_X: 'no' } }); + fakeProcess(11, { env: { HOME: '/h' } }); + const hidden = fakeProcess(12, { env: { WTD_TERMINAL: '1/a/1' } }); + chmodSync(join(hidden, 'environ'), 0o000); + const lookup = processLookup(roots); + expect(await lookup.tag(10)).toBe('100/inst/3'); + expect(await lookup.tag(11)).toBeNull(); + expect(await lookup.tag(99)).toBeNull(); + expect(await lookup.tag(12)).toBeNull(); + }); + + it('reads the arguments and the command line, cut to 200 characters', async () => { + fakeProcess(10, { args: ['node', '/opt/wtd.mjs', 'daemon'] }); + fakeProcess(11, { args: ['x', 'y'.repeat(300)] }); + const lookup = processLookup(roots); + expect(await lookup.args(10)).toEqual(['node', '/opt/wtd.mjs', 'daemon']); + expect(await lookup.command(10)).toBe('node /opt/wtd.mjs daemon'); + expect(await lookup.command(11)).toBe(`x ${'y'.repeat(198)}`); + expect(await lookup.args(99)).toBeNull(); + expect(await lookup.command(99)).toBeNull(); + }); +}); + +describe('own cgroup', () => { + it('reads the cgroup v2 figures, pressure and member pids', async () => { + fakeCgroup(); + expect(await readOwnCgroup(roots, report)).toEqual({ + path: '/app.slice/x.scope', + current: 123456, + peak: 234567, + max: null, + high: 1000000, + swapCurrent: 4096, + pressure: PRESSURE, + pids: [10, 11, 12], + }); + expect(reported).toEqual([]); + }); + + it('gives null for each optional file that is absent', async () => { + const dir = fakeCgroup(); + for (const name of ['memory.peak', 'memory.swap.current', 'memory.pressure', 'memory.high', 'memory.max']) rmSync(join(dir, name)); + expect(await readOwnCgroup(roots, report)).toMatchObject({ + current: 123456, + peak: null, + max: null, + high: null, + swapCurrent: null, + pressure: null, + }); + expect(reported).toEqual([]); + }); + + it('gives null pressure with PSI disabled, keeping the other figures', async () => { + const dir = fakeCgroup(); + hooks.failing.set(join(dir, 'memory.pressure'), 'EOPNOTSUPP'); + expect(await readOwnCgroup(roots, report)).toMatchObject({ current: 123456, pressure: null, pids: [10, 11, 12] }); + expect(reported).toEqual([]); + }); + + it('yields null and reports EOPNOTSUPP on a file other than memory.pressure', async () => { + hooks.failing.set(join(fakeCgroup(), 'memory.peak'), 'EOPNOTSUPP'); + expect(await readOwnCgroup(roots, report)).toBeNull(); + expect(reported).toEqual([expect.objectContaining({ code: 'EOPNOTSUPP' })]); + }); + + it('yields null without memory.current while the system source is still read', async () => { + rmSync(join(fakeCgroup(), 'memory.current')); + fakeFile('proc/meminfo', MEMINFO); + fakeFile('proc/pressure/memory', PSI); + expect(await readOwnCgroup(roots, report)).toBeNull(); + expect((await readSystemMemory(roots, report)).meminfo).not.toBeNull(); + expect(reported).toEqual([]); + }); + + it.each([ + ['cgroup v1 only', '12:memory:/user.slice\n1:name=systemd:/user.slice\n'], + ['no cgroup file', null], + ])('yields null with %s', async (_name, text) => { + if (text !== null) fakeFile('proc/self/cgroup', text); + expect(await readOwnCgroup(roots, report)).toBeNull(); + expect(reported).toEqual([]); + }); + + it.each(['proc/self/cgroup', 'cgroup/app.slice/x.scope/memory.current'])( + 'yields null when %s cannot be read, reporting nothing', + async (file) => { + fakeCgroup(); + hooks.failing.set(join(root, file), 'EACCES'); + expect(await readOwnCgroup(roots, report)).toBeNull(); + expect(reported).toEqual([]); + }, + ); + + it('yields null and reports a malformed memory.current', async () => { + writeFileSync(join(fakeCgroup(), 'memory.current'), 'lots\n'); + expect(await readOwnCgroup(roots, report)).toBeNull(); + expect(reported).toHaveLength(1); + }); +}); + +describe('system memory', () => { + it('reads system pressure and meminfo in bytes', async () => { + fakeFile('proc/pressure/memory', PSI); + fakeFile('proc/meminfo', MEMINFO); + expect(await readSystemMemory(roots, report)).toEqual({ + pressure: PRESSURE, + meminfo: { total: 1000 * KiB, available: 400 * KiB, swapTotal: 200 * KiB, swapFree: 150 * KiB }, + }); + expect(reported).toEqual([]); + }); + + it('yields null figures without /proc, reporting nothing', async () => { + rmSync(roots.proc, { recursive: true }); + expect(await readSystemMemory(roots, report)).toEqual({ pressure: null, meminfo: null }); + expect(reported).toEqual([]); + }); + + it('gives null figures for files that cannot be read, reporting nothing', async () => { + fakeFile('proc/pressure/memory', PSI); + fakeFile('proc/meminfo', MEMINFO); + hooks.failing.set(join(roots.proc, 'pressure', 'memory'), 'EACCES'); + hooks.failing.set(join(roots.proc, 'meminfo'), 'EACCES'); + expect(await readSystemMemory(roots, report)).toEqual({ pressure: null, meminfo: null }); + expect(reported).toEqual([]); + }); + + it('gives null pressure without PSI, still reading meminfo', async () => { + fakeFile('proc/meminfo', MEMINFO); + const meminfo: unknown = expect.objectContaining({ total: 1000 * KiB }); + expect(await readSystemMemory(roots, report)).toEqual({ pressure: null, meminfo }); + expect(reported).toEqual([]); + }); + + it('gives null pressure with PSI disabled, reporting nothing', async () => { + fakeFile('proc/pressure/memory', PSI); + fakeFile('proc/meminfo', MEMINFO); + hooks.failing.set(join(roots.proc, 'pressure', 'memory'), 'EOPNOTSUPP'); + const meminfo: unknown = expect.objectContaining({ total: 1000 * KiB }); + expect(await readSystemMemory(roots, report)).toEqual({ pressure: null, meminfo }); + expect(reported).toEqual([]); + }); + + it('gives null pressure and reports a pressure file without its some line', async () => { + fakeFile('proc/pressure/memory', 'full avg10=0.50 avg60=0.75 avg300=1.00 total=45\n'); + fakeFile('proc/meminfo', MEMINFO); + const system = await readSystemMemory(roots, report); + expect(system.pressure).toBeNull(); + expect(system.meminfo).not.toBeNull(); + expect(reported).toHaveLength(1); + expect(String(reported[0])).toContain(join(roots.proc, 'pressure', 'memory')); + }); + + it('gives null meminfo and reports one without MemAvailable', async () => { + fakeFile('proc/meminfo', 'MemTotal: 1000 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n'); + expect((await readSystemMemory(roots, report)).meminfo).toBeNull(); + expect(reported).toHaveLength(1); + }); +}); + +describe('tight memory', () => { + const figures = (avg10: number) => ({ some: { avg10, avg60: 0, avg300: 0 }, full: { avg10: 0, avg60: 0, avg300: 0 } }); + const system = (avg10: number, available = 500): SystemMemory => ({ + pressure: figures(avg10), + meminfo: { total: 1000, available, swapTotal: 0, swapFree: 0 }, + }); + const cgroup = (avg10: number): OwnCgroup => ({ + path: '/x', + current: 1, + peak: null, + max: null, + high: null, + swapCurrent: null, + pressure: figures(avg10), + pids: [], + }); + + it.each([ + ['calm', system(9.99), cgroup(9.99), false], + ['system some avg10 at 10', system(10), cgroup(0), true], + ['cgroup some avg10 at 10', system(0), cgroup(10), true], + ['MemAvailable below 10% of MemTotal', system(0, 99), null, true], + ['MemAvailable at 10% of MemTotal', system(0, 100), null, false], + ['no figures', { pressure: null, meminfo: null }, null, false], + ] as const)('is %s → %s', (_name, sys, group, tight) => { + expect(isTight(sys, group)).toBe(tight); + }); +}); + +describe('groups', () => { + const p = (pid: number, rss: number): ProcessInfo => ({ pid, ppid: 1, sid: pid, rss, anon: rss / 2 }); + + it('sums the members and lists the largest by resident memory with their commands, looked up for those only', async () => { + const looked: number[] = []; + const command = (pid: number): Promise => { + looked.push(pid); + return Promise.resolve(pid === 3 ? null : `cmd ${String(pid)}`); + }; + expect(await summarise([p(1, 10), p(2, 40), p(3, 30), p(4, 20)], 2, command)).toEqual({ + processes: 4, + rss: 100, + anon: 50, + top: [ + { pid: 2, rss: 40, anon: 20, command: 'cmd 2' }, + { pid: 3, rss: 30, anon: 15, command: null }, + ], + }); + expect(looked.sort()).toEqual([2, 3]); + }); + + it('summarises no members as empty', async () => { + expect(await summarise([], 5, () => Promise.resolve(''))).toEqual({ processes: 0, rss: 0, anon: 0, top: [] }); + }); +}); diff --git a/src/platform/memory/sources.ts b/src/platform/memory/sources.ts new file mode 100644 index 0000000..dc71b48 --- /dev/null +++ b/src/platform/memory/sources.ts @@ -0,0 +1,348 @@ +import { readdir, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +/** Where the memory sources are read from, and whose processes count. */ +export interface MemoryRoots { + proc: string; + cgroup: string; + uid: number; +} + +export interface ProcessInfo { + pid: number; + ppid: number; + sid: number; + /** `VmRSS` in bytes. */ + rss: number; + /** `RssAnon` in bytes. */ + anon: number; +} + +export interface PressureFigures { + avg10: number; + avg60: number; + avg300: number; +} + +export interface Pressure { + some: PressureFigures; + full: PressureFigures; +} + +/** `/proc/meminfo` figures in bytes. */ +export interface Meminfo { + total: number; + available: number; + swapTotal: number; + swapFree: number; +} + +export interface SystemMemory { + pressure: Pressure | null; + meminfo: Meminfo | null; +} + +/** The own cgroup v2's figures in bytes; null where a file is absent or `max`. */ +export interface OwnCgroup { + path: string; + current: number; + peak: number | null; + max: number | null; + high: number | null; + swapCurrent: number | null; + pressure: Pressure | null; + pids: number[]; +} + +/** Per-process details read on request; null when the process is gone or unreadable. */ +export interface ProcessLookup { + /** Its `WTD_TERMINAL` environment value. */ + tag: (pid: number) => Promise; + args: (pid: number) => Promise; + /** Its arguments joined by spaces, at most 200 characters. */ + command: (pid: number) => Promise; +} + +export type Report = (error: unknown) => void; + +const KiB = 1024; +const COMMAND_CHARS = 200; +/** Processes read at a time; each procfs read takes a 64 KiB buffer, so this bounds a sample's footprint. */ +const READ_BATCH = 8; +const TAG = 'WTD_TERMINAL='; + +/** Content that does not parse; a failure of its source. */ +class MalformedError extends Error { + override readonly name = 'MalformedError'; + + constructor(path: string, what: string) { + super(`malformed ${path}: ${what}`); + } +} + +const codeOf = (error: unknown): unknown => (error instanceof Error && 'code' in error ? error.code : undefined); + +/** Absent or unreadable: the expected state where a source does not exist. */ +const isAbsent = (error: unknown): boolean => codeOf(error) === 'ENOENT' || codeOf(error) === 'EACCES' || codeOf(error) === 'ENOTDIR'; + +/** Gone while being read. */ +const isGone = (error: unknown): boolean => isAbsent(error) || codeOf(error) === 'ESRCH'; + +/** The roots of this host and user. */ +export const hostRoots = (): MemoryRoots => ({ proc: '/proc', cgroup: '/sys/fs/cgroup', uid: process.getuid?.() ?? -1 }); + +/** Runs a source: null silently when absent, null and reported on any other failure. */ +const source = async (read: () => Promise, report: Report): Promise => { + try { + return await read(); + } catch (error) { + if (!isAbsent(error)) report(error); + return null; + } +}; + +/** The content of `path`, or null when absent. */ +const readOptional = async (path: string): Promise => { + try { + return await readFile(path, 'utf8'); + } catch (error) { + if (isAbsent(error)) return null; + throw error; + } +}; + +const integer = (text: string, path: string, what: string): number => { + if (!/^\d+$/.test(text)) throw new MalformedError(path, what); + return Number(text); +}; + +/** The kB figure on `key`'s line of a `status` or `meminfo` text in bytes; null without the line. */ +const kibLine = (text: string, key: string, path: string): number | null => { + const line = text.split('\n').find((l) => l.startsWith(`${key}:`)); + if (line === undefined) return null; + const match = /^\S+:\s+(\d+) kB$/.exec(line); + if (match?.[1] === undefined) throw new MalformedError(path, key); + return Number(match[1]) * KiB; +}; + +const parseStatus = (text: string, path: string): { uid: number; rss: number; anon: number } => { + const uid = /^Uid:\t(\d+)\t/m.exec(text)?.[1]; + if (uid === undefined) throw new MalformedError(path, 'Uid'); + return { uid: Number(uid), rss: kibLine(text, 'VmRSS', path) ?? 0, anon: kibLine(text, 'RssAnon', path) ?? 0 }; +}; + +/** Parent pid and session id from a `stat` text, read after the command name. */ +const parseStat = (text: string, path: string): { ppid: number; sid: number } => { + const fields = text.slice(text.lastIndexOf(')') + 2).split(' '); + return { ppid: integer(fields[1] ?? '', path, 'ppid'), sid: integer(fields[3] ?? '', path, 'session') }; +}; + +/** One process of the user, or null when it is another user's or gone. */ +const readProcess = async (roots: MemoryRoots, pid: number): Promise => { + const dir = join(roots.proc, String(pid)); + try { + const statusPath = join(dir, 'status'); + const status = parseStatus(await readFile(statusPath, 'utf8'), statusPath); + if (status.uid !== roots.uid) return null; + const statPath = join(dir, 'stat'); + const { ppid, sid } = parseStat(await readFile(statPath, 'utf8'), statPath); + return { pid, ppid, sid, rss: status.rss, anon: status.anon }; + } catch (error) { + if (isGone(error)) return null; + throw error; + } +}; + +/** Every process of `roots.uid`; null without a readable `/proc`. */ +export const readProcessTable = (roots: MemoryRoots, report: Report): Promise => + source(async () => { + const pids = (await readdir(roots.proc)).filter((name) => /^\d+$/.test(name)).map(Number); + const table: ProcessInfo[] = []; + for (let i = 0; i < pids.length; i += READ_BATCH) { + const batch = await Promise.all(pids.slice(i, i + READ_BATCH).map((pid) => readProcess(roots, pid))); // NOSONAR(S9382) — bounded batches + for (const entry of batch) if (entry !== null) table.push(entry); + } + return table; + }, report); + +/** Reads of single processes' environment and command line. */ +export const processLookup = (roots: MemoryRoots): ProcessLookup => { + const read = async (pid: number, name: string): Promise => { + try { + return await readFile(join(roots.proc, String(pid), name), 'utf8'); + } catch (error) { + if (isGone(error)) return null; + throw error; + } + }; + const args = async (pid: number): Promise => { + const text = await read(pid, 'cmdline'); + if (text === null) return null; + const parts = text.split('\0'); + if (parts.at(-1) === '') parts.pop(); + return parts; + }; + return { + tag: async (pid) => { + const text = await read(pid, 'environ'); + const entry = text?.split('\0').find((e) => e.startsWith(TAG)); + return entry === undefined ? null : entry.slice(TAG.length); + }, + args, + command: async (pid) => (await args(pid))?.join(' ').slice(0, COMMAND_CHARS) ?? null, + }; +}; + +const parsePressure = (text: string, path: string): Pressure => { + const figures = (kind: string): PressureFigures => { + const match = new RegExp(String.raw`^${kind} avg10=(\d+\.\d+) avg60=(\d+\.\d+) avg300=(\d+\.\d+) total=\d+$`, 'm').exec(text); + if (match === null) throw new MalformedError(path, `no ${kind} line`); + return { avg10: Number(match[1]), avg60: Number(match[2]), avg300: Number(match[3]) }; + }; + return { some: figures('some'), full: figures('full') }; +}; + +/** Null when absent or with PSI disabled, where reading fails with `EOPNOTSUPP`. */ +const readPressure = async (path: string): Promise => { + let text: string | null; + try { + text = await readOptional(path); + } catch (error) { + if (codeOf(error) === 'EOPNOTSUPP') return null; + throw error; + } + return text === null ? null : parsePressure(text, path); +}; + +const parseMeminfo = (text: string, path: string): Meminfo => { + const required = (key: string): number => { + const value = kibLine(text, key, path); + if (value === null) throw new MalformedError(path, `no ${key}`); + return value; + }; + return { + total: required('MemTotal'), + available: required('MemAvailable'), + swapTotal: required('SwapTotal'), + swapFree: required('SwapFree'), + }; +}; + +/** System pressure and meminfo, each null when absent or malformed. */ +export const readSystemMemory = async (roots: MemoryRoots, report: Report): Promise => { + const pressure = source(() => readPressure(join(roots.proc, 'pressure', 'memory')), report); + const meminfoPath = join(roots.proc, 'meminfo'); + const meminfo = source(async () => parseMeminfo(await readFile(meminfoPath, 'utf8'), meminfoPath), report); + return { pressure: await pressure, meminfo: await meminfo }; +}; + +/** A byte count file's value; null when absent or `max`. */ +const readBytes = async (path: string): Promise => { + const text = (await readOptional(path))?.trim() ?? null; + return text === null || text === 'max' ? null : integer(text, path, 'bytes'); +}; + +const readPids = async (path: string): Promise => { + const text = await readOptional(path); + if (text === null) return []; + return text + .split('\n') + .filter((line) => line !== '') + .map((line) => integer(line, path, 'pid')); +}; + +/** The own cgroup v2; null without one or without its `memory.current`. */ +export const readOwnCgroup = (roots: MemoryRoots, report: Report): Promise => + source(async () => { + const membership = await readFile(join(roots.proc, 'self', 'cgroup'), 'utf8'); + const path = /^0::(\/.*)$/m.exec(membership)?.[1]; + if (path === undefined) return null; + const dir = join(roots.cgroup, path); + const currentPath = join(dir, 'memory.current'); + const current = integer((await readFile(currentPath, 'utf8')).trim(), currentPath, 'bytes'); + const [peak, max, high, swapCurrent, pressure, pids] = await Promise.all([ + readBytes(join(dir, 'memory.peak')), + readBytes(join(dir, 'memory.max')), + readBytes(join(dir, 'memory.high')), + readBytes(join(dir, 'memory.swap.current')), + readPressure(join(dir, 'memory.pressure')), + readPids(join(dir, 'cgroup.procs')), + ]); + return { path, current, peak, max, high, swapCurrent, pressure, pids }; + }, report); + +const TIGHT_PRESSURE = 10; +const TIGHT_AVAILABLE = 0.1; + +/** Whether memory is tight: `some avg10` at least 10, or MemAvailable below 10% of MemTotal. */ +export const isTight = (system: SystemMemory, cgroup: OwnCgroup | null): boolean => { + const pressed = (pressure: Pressure | null | undefined): boolean => (pressure?.some.avg10 ?? 0) >= TIGHT_PRESSURE; + const { meminfo } = system; + return pressed(system.pressure) || pressed(cgroup?.pressure) || (meminfo !== null && meminfo.available < meminfo.total * TIGHT_AVAILABLE); +}; + +/** Whether memory is tight now, by system and own-cgroup figures. */ +export const memoryTight = async (roots: MemoryRoots, report: Report): Promise => { + const [system, cgroup] = await Promise.all([readSystemMemory(roots, report), readOwnCgroup(roots, report)]); + return isTight(system, cgroup); +}; + +export interface MemoryReading { + table: ProcessInfo[] | null; + cgroup: OwnCgroup | null; + system: SystemMemory; +} + +/** The process table, own cgroup and system memory. */ +export const readMemory = async (roots: MemoryRoots, report: Report): Promise => { + const [table, cgroup, system] = await Promise.all([ + readProcessTable(roots, report), + readOwnCgroup(roots, report), + readSystemMemory(roots, report), + ]); + return { table, cgroup, system }; +}; + +export interface TopProcess { + pid: number; + rss: number; + anon: number; + command: string | null; +} + +/** Totals of a set of processes and its largest by resident memory. */ +export interface ProcessGroup { + processes: number; + rss: number; + anon: number; + top: TopProcess[]; +} + +/** Sums `members` and lists the `n` largest by rss, looking up commands for those only. */ +export const summarise = async ( + members: readonly ProcessInfo[], + n: number, + command: (pid: number) => Promise, +): Promise => { + const largest = [...members].sort((a, b) => b.rss - a.rss).slice(0, n); + return { + processes: members.length, + rss: members.reduce((sum, p) => sum + p.rss, 0), + anon: members.reduce((sum, p) => sum + p.anon, 0), + top: await Promise.all(largest.map(async (p) => ({ pid: p.pid, rss: p.rss, anon: p.anon, command: await command(p.pid) }))), + }; +}; + +/** The process's own memory figures in bytes. */ +export interface SelfMemory { + rss: number; + heapTotal: number; + heapUsed: number; + external: number; + arrayBuffers: number; +} + +export const selfMemory = (): SelfMemory => { + const { rss, heapTotal, heapUsed, external, arrayBuffers } = process.memoryUsage(); + return { rss, heapTotal, heapUsed, external, arrayBuffers }; +}; diff --git a/src/web/client/connection.ts b/src/web/client/connection.ts index b95f645..b45b594 100644 --- a/src/web/client/connection.ts +++ b/src/web/client/connection.ts @@ -368,7 +368,10 @@ export class HubClient { return; case 'error': // A superseded opening's error, sent before the hub ended it, is no notice. - if (m.req === null || (!this.pending.resolve(m.req, { ok: false, code: m.code, message: m.message }) && !this.openings.isSuperseded(m.req))) { + if ( + m.req === null || + (!this.pending.resolve(m.req, { ok: false, code: m.code, message: m.message }) && !this.openings.isSuperseded(m.req)) + ) { this.store.notice.value = `${m.code}: ${m.message}`; } if (m.req !== null) this.opening(m); diff --git a/test/architecture/dependencies.ts b/test/architecture/dependencies.ts index 2c0f17d..8f5d535 100644 --- a/test/architecture/dependencies.ts +++ b/test/architecture/dependencies.ts @@ -25,6 +25,7 @@ export const ALLOWLIST: Readonly> = { 'platform.files': ['fs'], 'platform.dialer': ['net', 'child_process'], 'platform.install': ['fs', 'child_process'], + 'platform.memory': ['fs'], 'daemon.worktrees': ['fs', 'child_process'], 'daemon.terminals': ['@homebridge/node-pty-prebuilt-multiarch', '@xterm/headless', '@xterm/addon-serialize', 'fs'], 'daemon.state': ['zod'], diff --git a/test/architecture/structure.test.ts b/test/architecture/structure.test.ts index 1c9c191..2572c58 100644 --- a/test/architecture/structure.test.ts +++ b/test/architecture/structure.test.ts @@ -25,6 +25,7 @@ describe('model', () => { ['platform.files', 'src/platform/files', true], ['platform.dialer', 'src/platform/dialer', true], ['platform.install', 'src/platform/install', true], + ['platform.memory', 'src/platform/memory', true], ['daemon.worktrees', 'src/daemon/worktrees', true], ['daemon.terminals', 'src/daemon/terminals', true], ['daemon.state', 'src/daemon/state', true], @@ -47,7 +48,8 @@ describe('model', () => { expect(model.arrows).toContainEqual({ from: 'cli', to: 'daemon.main' }); expect(model.arrows).toContainEqual({ from: 'daemon', to: 'protocol' }); expect(model.arrows).toContainEqual({ from: 'platform.install', to: 'platform.dialer' }); - expect(model.arrows).toHaveLength(38); + expect(model.arrows).toContainEqual({ from: 'daemon.main', to: 'platform.memory' }); + expect(model.arrows).toHaveLength(41); }); }); diff --git a/test/process/memory-log.test.ts b/test/process/memory-log.test.ts new file mode 100644 index 0000000..ffd33c0 --- /dev/null +++ b/test/process/memory-log.test.ts @@ -0,0 +1,344 @@ +import { spawn, spawnSync } from 'node:child_process'; +import { existsSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { z } from 'zod'; +import type { Terminal } from '../../src/protocol/index.js'; +import type { DaemonClient } from '../support/daemon-client.js'; +import { addWorktree, DaemonHost, livePidsWhere, makeRepo, sessionOf, sleep, waitUntil, type WtdProcess } from '../support/daemon-host.js'; +import { FakeSsh } from '../support/fake-ssh.js'; +import { HubHost } from '../support/hub-host.js'; +import { packageVersion } from '../support/hub-router.js'; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 30_000 }); + +const SAMPLE_MS = 200; +const ENV = { WTD_MEMORY_SAMPLE_MS: String(SAMPLE_MS) }; +const MiB = 1024 * 1024; + +const top = z.object({ pid: z.number(), rss: z.number(), anon: z.number(), command: z.string().nullable() }); +const group = z.object({ processes: z.number(), rss: z.number(), anon: z.number(), top: z.array(top) }); +const self = z.object({ rss: z.number(), heapTotal: z.number(), heapUsed: z.number(), external: z.number(), arrayBuffers: z.number() }); +const system = z.object({ pressure: z.unknown(), meminfo: z.unknown() }); +const daemonLine = z.object({ + ts: z.string(), + source: z.literal('daemon'), + pid: z.number(), + instance: z.string(), + version: z.string(), + intervalMs: z.number(), + self, + mirrors: z.object({ count: z.number(), bytes: z.number() }), + terminals: z.array( + z.object({ + termId: z.number(), + repo: z.string(), + worktree: z.string(), + preset: z.string(), + pid: z.number(), + mirror: z.object({ lines: z.number(), cols: z.number(), bytes: z.number() }), + members: group.nullable(), + detached: group.nullable(), + }), + ), + escaped: z + .array(group.extend({ tag: z.string(), termId: z.number().nullable(), worktree: z.string().nullable(), preset: z.string().nullable() })) + .nullable(), + escapedOmitted: z.object({ groups: z.number(), processes: z.number(), rss: z.number(), anon: z.number() }).nullable(), + cgroup: z.object({ current: z.number(), others: group.nullable() }).nullable(), + system, +}); +const hubLine = z.object({ + ts: z.string(), + source: z.literal('hub'), + pid: z.number(), + instance: z.string(), + version: z.string(), + intervalMs: z.number(), + self, + children: group.nullable(), + cgroup: z.object({ current: z.number(), others: group.nullable() }).nullable(), + system, +}); + +/** The complete lines of `path`, parsed as JSON. */ +const linesOf = (path: string): unknown[] => + existsSync(path) + ? readFileSync(path, 'utf8') + .split('\n') + .slice(0, -1) + .map((line): unknown => JSON.parse(line)) + : []; + +/** The first line of `path` from index `from` on that `schema` parses and `matches` accepts. */ +const sampleWhere = (path: string, schema: z.ZodType, matches: (line: T) => boolean, from = 0): Promise => + waitUntil( + () => + linesOf(path) + .slice(from) + .map((line) => schema.parse(line)) + .find(matches), + `a matching sample in ${path}`, + 15_000, + ); + +/** Holds 32 MiB, ignores SIGHUP and runs until killed; `marker` names it. */ +const HOG_SCRIPT = `process.on('SIGHUP',()=>{});globalThis.b=Buffer.alloc(${String(32 * MiB)},1);setInterval(()=>{},1e6)`; +const hog = (marker: string): string => `${process.execPath} -e "${HOG_SCRIPT}" ${marker}`; + +/** Whether `pid` has installed a SIGHUP handler; node resets inherited dispositions at startup, so until then a hangup kills it. */ +const catchesHup = (pid: number): boolean => { + const status = `/proc/${String(pid)}/status`; + const mask = existsSync(status) ? /^SigCgt:\t([0-9a-f]+)$/m.exec(readFileSync(status, 'utf8'))?.[1] : undefined; + return mask !== undefined && (BigInt(`0x${mask}`) & 1n) === 1n; +}; + +/** Parent pid of `pid`. */ +const parentOf = (pid: number): number => { + const stat = readFileSync(`/proc/${String(pid)}/stat`, 'utf8'); + return Number(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[1]); +}; + +/** Waits for the bytes of `path` to stay unchanged for 1 s, well over the sample interval. */ +const expectNoMoreLines = async (path: string): Promise => { + const size = statSync(path).size; + await sleep(5 * SAMPLE_MS); + expect(statSync(path).size).toBe(size); +}; + +describe('daemon memory log', () => { + let host: DaemonHost; + let daemon: WtdProcess; + let client: DaemonClient; + let instance: string; + let repo: string; + let wt: string; + + beforeEach(async () => { + host = DaemonHost.create(); + repo = makeRepo(join(host.dir, 'repo')); + wt = addWorktree(repo, join(host.dir, 'wt'), 'wt'); + daemon = await host.start(ENV); + client = await host.rawClient(); + instance = (await client.handshake()).instance; + await client.watch(repo); + }); + + afterEach(async () => { + // Also kills every process left behind with this host's HOME, escaped ones included. + await host.cleanup(); + }); + + const log = (): string => join(host.stateDir, 'memory.log'); + const tagOf = (termId: number): string => `${String(daemon.pid)}/${instance}/${String(termId)}`; + + /** An interactive shell in the worktree that has run a command. */ + const shell = async (): Promise => { + const term = await client.create(wt); + await client.attach(term.termId); + client.sendInput(term.termId, 'echo READY-$((6*7))\r'); + await client.waitOutput(term.termId, 'READY-42\r\n', 10_000); + return term; + }; + + /** The pid of the live process of this host whose argv is `argv`. */ + const pidOf = (argv: readonly string[]): Promise => + waitUntil( + () => + livePidsWhere( + (cmdline, environ) => cmdline.slice(0, -1).join('\0') === argv.join('\0') && environ.includes(`HOME=${host.home}`), + )[0], + argv.join(' '), + 10_000, + ); + + /** The pid of the hog named `marker`, once it ignores hangups. */ + const hogPid = (marker: string): Promise => + waitUntil( + () => + livePidsWhere( + (cmdline, environ) => cmdline[0] === process.execPath && cmdline.includes(marker) && environ.includes(`HOME=${host.home}`), + ).find(catchesHup), + marker, + 10_000, + ); + + it('appends samples of the daemon, with its identity and own memory, to an owner-only memory.log', async () => { + const line = await sampleWhere(log(), daemonLine, () => true); + expect(line).toMatchObject({ + source: 'daemon', + pid: daemon.pid, + instance, + version: packageVersion(), + intervalMs: SAMPLE_MS, + terminals: [], + }); + expect(line.mirrors).toEqual({ count: 0, bytes: 0 }); + expect(Number.isNaN(Date.parse(line.ts))).toBe(false); + expect(line.self.rss).toBeGreaterThan(0); + expect(line.self.heapUsed).toBeGreaterThan(0); + expect(statSync(log()).mode & 0o777).toBe(0o600); + }); + + it("tags a terminal's environment with the daemon's pid, its instance and the terminal id", async () => { + const term = await shell(); + client.sendInput(term.termId, 'echo "tag=$WTD_TERMINAL"\r'); + await expect(client.waitOutput(term.termId, `tag=${tagOf(term.termId)}\r\n`)).resolves.toBeUndefined(); + }); + + it("logs a terminal's tree with its members' memory", async () => { + const term = await shell(); + client.sendInput(term.termId, 'sleep 1001\r'); + const sleeping = await pidOf(['sleep', '1001']); + const line = await sampleWhere(log(), daemonLine, (l) => + l.terminals.some((t) => t.members?.top.some((m) => m.pid === sleeping) === true), + ); + const entry = line.terminals.find((t) => t.termId === term.termId); + expect(entry).toMatchObject({ + termId: term.termId, + repo, + worktree: wt, + preset: 'shell', + pid: sessionOf(sleeping), + mirror: { cols: 80 }, + }); + expect(entry?.members?.top.find((m) => m.pid === sleeping)?.rss).toBeGreaterThan(0); + expect(line.mirrors.count).toBe(1); + }); + + it('logs a setsid child of a closed terminal as escaped with its terminal id and worktree', async () => { + const term = await shell(); + client.sendInput(term.termId, `setsid ${hog('wtd-memory-setsid')} &\r`); + const escaped = await hogPid('wtd-memory-setsid'); + await client.ok({ t: 'closeTerm', termId: term.termId }); + const line = await sampleWhere(log(), daemonLine, (l) => l.escaped?.some((g) => g.top.some((t) => t.pid === escaped)) === true); + expect(line.escaped?.find((g) => g.top.some((t) => t.pid === escaped))).toMatchObject({ + tag: tagOf(term.termId), + termId: term.termId, + worktree: wt, + preset: 'shell', + }); + }); + + it('logs a job that ignores SIGHUP in its own process group and survives closeTerm as escaped', async () => { + const term = await shell(); + client.sendInput(term.termId, `(trap '' HUP; exec ${hog('wtd-memory-nohup')}) &\r`); + const job = await hogPid('wtd-memory-nohup'); + await client.ok({ t: 'closeTerm', termId: term.termId }); + const line = await sampleWhere(log(), daemonLine, (l) => l.escaped?.some((g) => g.top.some((t) => t.pid === job)) === true); + expect(line.escaped?.find((g) => g.top.some((t) => t.pid === job))).toMatchObject({ tag: tagOf(term.termId), termId: term.termId }); + }); + + it('logs a setsid child of a live terminal as detached under it, not escaped', async () => { + const term = await shell(); + client.sendInput(term.termId, `setsid ${hog('wtd-memory-detached')} &\r`); + const detached = await hogPid('wtd-memory-detached'); + const line = await sampleWhere(log(), daemonLine, (l) => + l.terminals.some((t) => t.termId === term.termId && t.detached?.top.some((d) => d.pid === detached) === true), + ); + expect(line.escaped?.filter((g) => g.tag === tagOf(term.termId))).toEqual([]); + }); + + it('logs a process tagged by a daemon that is gone as escaped, with no worktree', async () => { + const dead = spawnSync('true').pid; + const tag = `${String(dead)}/gone/7`; + const child = spawn(process.execPath, ['-e', HOG_SCRIPT, 'wtd-memory-earlier'], { + env: { ...host.env(), WTD_TERMINAL: tag }, + stdio: 'ignore', + }); + try { + const line = await sampleWhere(log(), daemonLine, (l) => l.escaped?.some((g) => g.tag === tag) === true); + expect(line.escaped?.find((g) => g.tag === tag)).toMatchObject({ termId: 7, worktree: null, preset: null, processes: 1 }); + } finally { + child.kill('SIGKILL'); + } + }); + + it('appends nothing once the daemon has shut down', async () => { + await sampleWhere(log(), daemonLine, () => true); + client.send({ t: 'shutdown' }); + await daemon.exited; + await expectNoMoreLines(log()); + }); +}); + +describe('hub memory log', () => { + let host: HubHost; + let ssh: FakeSsh; + let repo: string; + + beforeEach(async () => { + host = await HubHost.createHub(); + ssh = new FakeSsh(join(host.dir, 'ssh')); + host.sshProgram = ssh.program; + repo = makeRepo(join(host.dir, 'repo')); + host.writeRepos([repo]); + }); + + afterEach(async () => { + for (const pid of host.hubPids()) process.kill(pid, 'SIGKILL'); + await waitUntil(() => host.hubPids().length === 0, 'hubs to exit').catch(() => undefined); + await ssh.cleanup(); + await host.cleanup(); + }); + + const log = (): string => join(host.stateDir, 'hub-memory.log'); + + it('appends samples of the hub, with its identity and own memory, to hub-memory.log', async () => { + const hub = await host.startHub(ENV); + const line = await sampleWhere(log(), hubLine, () => true); + expect(line).toMatchObject({ + source: 'hub', + pid: hub.pid, + instance: (await host.identity()).instance, + version: packageVersion(), + intervalMs: SAMPLE_MS, + }); + expect(line.self.heapUsed).toBeGreaterThan(0); + expect(statSync(log()).mode & 0o777).toBe(0o600); + }); + + it("counts an SSH link's process among the hub's children", async () => { + ssh.addHost('box'); + ssh.hangHost('box'); + host.writeConfig({ port: host.port, repos: [repo], hosts: [{ name: 'box', ssh: 'box' }] }); + await host.startHub(ENV); + await host.session(); + const link = await waitUntil(() => ssh.livePids('box')[0], 'the SSH link to box', 10_000); + const line = await sampleWhere(log(), hubLine, (l) => l.children?.top.some((t) => t.pid === link) === true); + expect(line.children?.top.find((t) => t.pid === link)?.rss).toBeGreaterThan(0); + }); + + it("counts neither the daemon it started nor its terminals' processes among the hub's children", async () => { + const hub = await host.startHub(ENV); + const client = await host.session(); + await client.waitHost(0, (h) => h.status === 'connected', { timeout: 10_000 }); + await client.watch(0, repo); + const term = await client.create(0, repo); + await client.attach(0, term.termId); + client.sendInput(0, term.termId, 'sleep 1006\r'); + const sleeping = await waitUntil( + () => + livePidsWhere((cmdline, environ) => cmdline.slice(0, -1).join(' ') === 'sleep 1006' && environ.includes(`HOME=${host.home}`))[0], + 'sleep 1006', + 10_000, + ); + const [daemon] = host.daemonPids(); + if (daemon === undefined) throw new Error('no daemon'); + expect(parentOf(daemon)).toBe(hub.pid); + const line = await sampleWhere(log(), hubLine, () => true, linesOf(log()).length); + const children = line.children?.top.map((t) => t.pid) ?? []; + expect(children).not.toContain(daemon); + expect(children).not.toContain(sessionOf(sleeping)); + expect(children).not.toContain(sleeping); + expect(line.children?.processes).toBe(line.children?.top.length); + }); + + it('appends nothing once the hub has shut down', async () => { + const hub = await host.startHub(ENV); + await sampleWhere(log(), hubLine, () => true); + hub.kill('SIGTERM'); + await hub.exited; + await expectNoMoreLines(log()); + }); +});