diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index e028f5d68..28f841dbf 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ "plugins": [ { "name": "mh", - "version": "1.1.393", + "version": "1.1.394", "source": ".", "description": "A Claude Code harness that composes mattpocock-skills instead of duplicating it. Nine computational deny/ask gates (irrecoverable commands, subagent git guard, subagent spawn guard, task-complete separation, test integrity, config-write guard, secret-content scan, codex-setup guard, routine-trigger guard) plus a SubagentStop verdict-check gate, a 4 KB METHODOLOGY injected at SessionStart, and memory-lint for the auto-memory store.", "author": { diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 44542299b..29950076b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "mh", "displayName": "Method Harness", - "version": "1.1.393", + "version": "1.1.394", "description": "A Claude Code harness that composes mattpocock-skills instead of duplicating it. Nine computational deny/ask gates (irrecoverable commands, subagent git guard, subagent spawn guard, task-complete separation, test integrity, config-write guard, secret-content scan, codex-setup guard, routine-trigger guard) plus a SubagentStop verdict-check gate, a 4 KB METHODOLOGY injected at SessionStart, and memory-lint for the auto-memory store.", "author": { "name": "wasikarn" diff --git a/docs/reference/env-vars.md b/docs/reference/env-vars.md index 7d79ac64e..78f01b1d5 100644 --- a/docs/reference/env-vars.md +++ b/docs/reference/env-vars.md @@ -24,6 +24,7 @@ under `env`, never in a committed file. | `GAUNTLET_JOBS` | `scripts/run-gauntlet.sh` | Test files run in parallel (default `4`, or `1` when `CI` is set). | | `GAUNTLET_SHARD` | `scripts/run-gauntlet.sh` | `i/N`: run only shard `i` of `N` (GH #400); anything else fails. Unset before child tests run. | | `GAUNTLET_LOCK_DIR`, `GAUNTLET_LOCK_WAIT_SECS`, `GAUNTLET_LOCK_POLL`, `GAUNTLET_LOCK_MAX_AGE`, `GAUNTLET_LOCK_NOPID_SECS` | `scripts/_lib/gauntlet-lock.sh` | Machine-wide gauntlet queue: lock dir (default `~/.cache/mh/gauntlet.lock`), max wait before running unlocked (`1800` s), poll (`5` s plus jitter), age past which a lock is stale (`3600` s), age a pid-less lock dir must reach to count as a crash leftover (`60` s). | +| `GAUNTLET_STRAY_MIN_SECS`, `GAUNTLET_STRAY_MIN_CPU` | `scripts/_lib/stray-procs.sh` (called by `scripts/run-gauntlet.sh`) | Thresholds for the warn-only stray-process report: your processes older than this many seconds (default `1800`) are grouped by command name, and a group whose %CPU sums to this or more (default `80`) is named before the layers run and again on failure. A non-numeric value falls back to the default. | | `MH_GAUNTLET_LOCK_OWNER` | `scripts/_lib/gauntlet-lock.sh` | Exported by the lock holder with its pid, so a child (a test that runs the pre-push hook) skips the queue instead of deadlocking. Not a user knob. | | `MERGE_PR_LOAD_MAX` | `scripts/merge-pr.sh` | Load average below which the script merges (default `4`); `tests/scripts/test-merge-pr.sh` sets `4` explicitly. | | `MERGE_PR_LOAD_WAIT_SECS` | `scripts/merge-pr.sh` | Seconds the script waits for load to drop before it fails with the load figure (default `900`); the test sets `0`. | diff --git a/docs/reference/repo-gotchas.md b/docs/reference/repo-gotchas.md index 744f747e0..c8b6a27fe 100644 --- a/docs/reference/repo-gotchas.md +++ b/docs/reference/repo-gotchas.md @@ -160,6 +160,15 @@ already set alone. names non-UUID ids but does not drop them). Probe with `MH_GATE_JOURNAL_PATH=/dev/null` (or a scratch file when you want to read the row back), and use `scripts/gate-differential.sh` for old-vs-new comparisons, which already isolates it. +- **Background load runs go in a bash file, with explicit PIDs.** The Bash tool's shell is zsh: + `wait $pids` does not word-split, and under `zsh -c` `kill $(jobs -p)` finds no jobs, so the + workers outlive the run. Twice this left a flood behind (289 processes, load ~248, 2026-10-05; + 4 `yes` at 100% CPU for ~4 h, 2026-10-25) that failed every session's timing rows. Write the + run as a file and start it with `bash `, collect `pids+=($!)`, `kill "${pids[@]}"` in a + `trap ... EXIT`, then confirm no worker is left and the load has dropped. `run-gauntlet.sh` now + names long-running CPU-heavy process groups of yours, a flood of low-CPU workers included + (`scripts/_lib/stray-procs.sh`, warn-only), but it can only catch a stray after the fact; killing + one that a peer session owns is the operator's call. ## Research: check qmd before web search diff --git a/scripts/_lib/stray-procs.sh b/scripts/_lib/stray-procs.sh new file mode 100644 index 000000000..10d7c0602 --- /dev/null +++ b/scripts/_lib/stray-procs.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# stray-procs.sh — name the operator's own long-running CPU-heavy process groups (GH #158). Two +# stress runs typed into the Bash tool's zsh left their workers behind: 289 processes at load ~248 +# (2026-10-05, `wait $pids` does not word-split in zsh) and 4 `yes` at 100% CPU for ~4 h +# (2026-10-25, `kill $(jobs -p)` found no jobs under `zsh -c`). Every push's timing rows failed +# until someone spotted them. This shortens detection to the next gauntlet run. Source it, then: +# stray_procs_report +# Your processes older than GAUNTLET_STRAY_MIN_SECS (1800) are grouped by command name, and a group +# whose %CPU sums to GAUNTLET_STRAY_MIN_CPU (80) or more is listed on stderr with the load average. +# Summing per group catches both shapes: one process at full CPU, and a flood of many processes at a +# few percent each; many idle copies (macOS keeps 40+ SetStoreUpdateService) sum to ~0 and stay quiet. +# It never kills anything (a process may belong to a peer session's live run; killing it is the +# operator's call) and never returns non-zero, so a `set -e` caller is never aborted; when ps fails it +# says so in one line and skips. Detection is by behaviour, not by name: a long build at full CPU is +# named too, which is the point of a warning. The 30 min default skips a busy headless `claude` run +# (named at 13 min on the first live run); the strays this targets ran for hours. Bash 3.2. + +# A numeric env setting, or its default when unset or not a whole number: a typo such as "1h" must +# not reach awk as a threshold (same rule as _gl_num in gauntlet-lock.sh). +_sp_num() { + case "${1:-}" in ''|*[!0-9]*) echo "$2" ;; *) echo "$1" ;; esac +} + +# Your own processes as "pid etime %cpu command" (etime: [[dd-]hh:]mm:ss; macOS ps has no etimes). +_sp_ps() { ps -U "$(id -u)" -o pid=,etime=,%cpu=,command= 2>/dev/null; } + +_sp_load() { + uptime 2>/dev/null | sed 's/.*load average[s]*: *//' | awk -F'[ ,]+' '{print $1}' +} + +stray_procs_report() { + local min_secs min_cpu rows found load + min_secs=$(_sp_num "${GAUNTLET_STRAY_MIN_SECS:-}" 1800) + min_cpu=$(_sp_num "${GAUNTLET_STRAY_MIN_CPU:-}" 80) + rows=$(_sp_ps) || { echo "gauntlet: stray-process check skipped (ps failed)" >&2; return 0; } + found=$(printf '%s\n' "$rows" | awk -v ms="$min_secs" -v mc="$min_cpu" ' + NF >= 4 { + e = $2; d = 0 + if (index(e, "-")) { d = substr(e, 1, index(e, "-") - 1); e = substr(e, index(e, "-") + 1) } + n = split(e, t, ":"); s = 0 + for (i = 1; i <= n; i++) s = s * 60 + t[i] + s += d * 86400 + if (s < ms) next + k = $4; sub(".*/", "", k) + if (!(k in cnt)) { keys[++nk] = k; cmd[k] = $4; for (i = 5; i <= NF; i++) cmd[k] = cmd[k] " " $i } + cnt[k]++; sum[k] += $3 + if (cnt[k] <= 8) pids[k] = pids[k] " " $1; else if (cnt[k] == 9) pids[k] = pids[k] " ..." + if (s > old[k]) { old[k] = s; oldest[k] = $2 } + } + END { + for (j = 1; j <= nk; j++) { + k = keys[j] + if (sum[k] < mc) continue + printf " %s: %d process%s at %.1f%% CPU, oldest %s, pids%s; %s\n", k, cnt[k], (cnt[k] == 1 ? "" : "es"), sum[k], oldest[k], pids[k], substr(cmd[k], 1, 100) + } + }') || { echo "gauntlet: stray-process check skipped (awk failed)" >&2; return 0; } + [ -n "$found" ] || return 0 + load=$(_sp_load) || load="" + { + echo "gauntlet: warning: load ${load:-unknown}; these process groups of yours have run ${min_secs}s+ at ${min_cpu}%+ CPU in total." + echo "gauntlet: a leftover stress run fails the timing rows; check them before re-running (this never kills anything):" + printf '%s\n' "$found" + } >&2 + return 0 +} diff --git a/scripts/run-gauntlet.sh b/scripts/run-gauntlet.sh index 15dcfb08a..cc695fb9b 100755 --- a/scripts/run-gauntlet.sh +++ b/scripts/run-gauntlet.sh @@ -187,6 +187,12 @@ run_hook_tests() { return "$rc" } +# GH #158: name leftover CPU-heavy processes (a stray stress run) before the timing rows run, and +# again on failure, so a load flake can be told from a regression. Warn-only, never fails the run. +# shellcheck source=_lib/stray-procs.sh +. "$ROOT/scripts/_lib/stray-procs.sh" +stray_procs_report + p1="" p2="" if [ -z "$shard" ] || [ "${shard%/*}" -eq 1 ]; then # check-observer-contract.sh (GH #443): the mh module stays a turn.complete observer. @@ -214,6 +220,7 @@ if [ "$fail" -eq 0 ]; then echo "gauntlet: all layers passed" else echo "gauntlet: FAILED (full logs kept at $LOG)" >&2 + stray_procs_report # before the lists: the last line stays the failing: list (GH #387) /usr/bin/grep '^failed-at:' "$LOG/tests" >&2 /usr/bin/grep '^failing:' "$LOG/tests" >&2 fi diff --git a/tests/scripts/test-gauntlet-failed-summary.sh b/tests/scripts/test-gauntlet-failed-summary.sh index a0787146f..4cded10ce 100644 --- a/tests/scripts/test-gauntlet-failed-summary.sh +++ b/tests/scripts/test-gauntlet-failed-summary.sh @@ -71,7 +71,15 @@ elif [ -z "$LOGDIR" ]; then bad "mktemp -d failed for the log dir" else printf -- '--- tests/x.sh\nnoise\nfailed-at: tests/scripts/test-red.sh: ❌ row\nfailing: tests/scripts/test-red.sh\n' >"$LOGDIR/tests" - out=$(bash -c "LOG='$LOGDIR'; fail=1; $TAIL" 2>&1) + # stray_procs_report reads the real ps; a stub stands in, and its line proves the call runs. + out=$(bash -c "LOG='$LOGDIR'; fail=1; stray_procs_report() { echo 'gauntlet: warning: stub'; }; $TAIL" 2>&1) + stub_at=$(printf '%s\n' "$out" | /usr/bin/grep -n -m1 -F 'gauntlet: warning: stub' | cut -d: -f1) + list_at=$(printf '%s\n' "$out" | /usr/bin/grep -n -m1 '^failing:' | cut -d: -f1) + if [ -n "$stub_at" ] && [ -n "$list_at" ] && [ "$stub_at" -lt "$list_at" ]; then + ok "a failed run reports stray processes, above the failing: list (GH #158)" + else + bad "no stray report above the failing: list (stub line '$stub_at', failing: line '$list_at'): <$out>" + fi last=$(printf '%s\n' "$out" | tail -n 1) if /usr/bin/grep -qF 'failing: tests/scripts/test-red.sh' <<<"$last"; then ok "the last line of a failed run repeats the failing: list" diff --git a/tests/scripts/test-gauntlet-stray-procs.sh b/tests/scripts/test-gauntlet-stray-procs.sh new file mode 100755 index 000000000..15105242e --- /dev/null +++ b/tests/scripts/test-gauntlet-stray-procs.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# test-gauntlet-stray-procs.sh — GH #158: scripts/_lib/stray-procs.sh names the operator's own +# long-running CPU-heavy process groups. Two real floods drive the rows: 4 `yes` at 100% CPU for +# ~4 h (2026-10-25) and 289 stress workers at load ~248, about 4% each on 12 cores (2026-10-05). +# Each case sources the library in a fresh `bash -c` and replaces its ps/load readers with fixture +# text, so no row depends on real CPU time. +set -uo pipefail +# An operator may set these in settings.json env; the rows below assume the defaults. +unset GAUNTLET_STRAY_MIN_SECS GAUNTLET_STRAY_MIN_CPU +HERE="$(cd -P "$(dirname "$0")" && pwd)" +LIB="$HERE/../../scripts/_lib/stray-procs.sh" +GAUNTLET="$HERE/../../scripts/run-gauntlet.sh" +pass=0; fail=0 +ok() { pass=$((pass + 1)); echo " PASS: $1"; } +bad() { fail=$((fail + 1)); echo " FAIL: $1" >&2; } + +echo "=== stray-procs: name leftover CPU-heavy process groups (GH #158) ===" + +# pid etime %cpu command, as `ps -o pid=,etime=,%cpu=,command=` prints them. +ROWS='82797 04:27:19 99.4 yes +82799 04:27:19 99.0 yes + 1234 1-00:00:30 85.0 /usr/bin/python3 stress.py --n 4 + 2222 30:00 250.0 node build + 3333 00:30 100.0 yes + 4444 05:00:00 2.0 /bin/zsh + 5555 29:59 99.0 sleeper + 6666 01:00:00 80.0 exact80 + 7777 01:00:00 79.9 under80' +for i in $(seq 1 30); do ROWS="$ROWS + $((9000 + i)) 45:00 4.0 /tmp/racer.sh"; done +for i in $(seq 1 43); do ROWS="$ROWS + $((8000 + i)) 05:00:00 0.0 /usr/libexec/SetStoreUpdateService"; done + +# run [env...]: report on stdout+stderr, then "rc=". +run() { + local body=$1; shift + env "$@" ROWS="$ROWS" bash -c ". '$LIB' +_sp_ps() { $body; } +_sp_load() { echo 6.25; } +stray_procs_report; echo \"rc=\$?\"" 2>&1 +} +line() { printf '%s\n' "$out" | /usr/bin/grep "^ $1:"; } + +out=$(run 'printf "%s\n" "$ROWS"') +l=$(line yes) +case "$l" in *"2 processes"*"pids 82797 82799"*) ok "two 4 h yes at 99% are one group with both pids" ;; *) bad "yes group wrong: <$l> in $out" ;; esac +case "$l" in *3333*) bad "a 30 s yes joined the old yes group: <$l>" ;; *) ok "a young copy of the same command is left out of the group" ;; esac +case "$(line python3)" in *"1 process"*"pids 1234"*"stress.py --n 4"*) ok "a dd-hh:mm:ss elapsed counts its days (1-00:00:30), and the command line is shown" ;; *) bad "1-day row missing: $out" ;; esac +case "$(line node)" in *"pids 2222"*) ok "exactly the minimum age and %CPU above 100 (multi-core) count" ;; *) bad "boundary age row missing: $out" ;; esac +case "$(line exact80)" in *"pids 6666"*) ok "exactly the minimum %CPU counts" ;; *) bad "80.0% row missing: $out" ;; esac +case "$out" in *under80*) bad "a 79.9% process was named: $out" ;; *) ok "just under the minimum %CPU is not named" ;; esac +case "$(line racer.sh)" in *"30 processes at 120"*) ok "30 workers at 4% each sum to one named group (the 2026-10-05 flood shape)" ;; *) bad "low-CPU flood not named: $out" ;; esac +case "$out" in *SetStoreUpdateService*) bad "43 idle copies were named: $out" ;; *) ok "many idle copies of one command are not named" ;; esac +case "$out" in *zsh*) bad "a 2% process was named: $out" ;; *) ok "a long but idle process is not named" ;; esac +case "$out" in *sleeper*) bad "a 29:59 process was named: $out" ;; *) ok "one second under the minimum age is not named" ;; esac +case "$out" in *"load 6.25"*) ok "the warning states the load average" ;; *) bad "load missing: $out" ;; esac +case "$out" in *"rc=0") ok "it returns 0 even when it finds strays (a warning, never a gate)" ;; *) bad "non-zero rc: $out" ;; esac + +out=$(env ROWS="$ROWS" bash -c ". '$LIB'; _sp_ps() { printf '%s\n' \"\$ROWS\"; }; stray_procs_report" 2>/dev/null) +[ -z "$out" ] && ok "the warning goes to stderr, never stdout (a caller may capture stdout)" || bad "stdout got: $out" + +out=$(run 'printf "%s\n" " 3333 00:30 100.0 yes"') +[ "$out" = "rc=0" ] && ok "no strays: silent, rc 0" || bad "no-stray run printed: $out" + +out=$(run 'return 1') +case "$out" in "gauntlet: stray-process check skipped (ps failed)"*"rc=0") ok "ps failing: one diagnostic line, rc 0" ;; *) bad "ps failure not reported: $out" ;; esac + +out=$(env ROWS="$ROWS" bash -c ". '$LIB'; _sp_ps() { printf '%s\n' \"\$ROWS\"; }; _sp_load() { return 1; }; stray_procs_report" 2>&1) +case "$out" in *"load unknown"*) ok "a failing load reader prints 'load unknown', not an empty value" ;; *) bad "empty load: $out" ;; esac + +out=$(run 'printf "%s\n" "$ROWS"' GAUNTLET_STRAY_MIN_SECS=20 GAUNTLET_STRAY_MIN_CPU=1) +case "$(line yes)" in *"3 processes"*) ok "GAUNTLET_STRAY_MIN_SECS lowers the age threshold" ;; *) bad "env age ignored: $out" ;; esac +case "$(line zsh)" in *"pids 4444"*) ok "GAUNTLET_STRAY_MIN_CPU lowers the CPU threshold" ;; *) bad "env cpu ignored: $out" ;; esac + +out=$(run 'printf "%s\n" "$ROWS"' GAUNTLET_STRAY_MIN_SECS=1h GAUNTLET_STRAY_MIN_CPU=x) +case "$out" in *zsh*|*"3 processes"*) bad "a non-numeric threshold changed the result: $out" ;; *"pids 82797"*) ok "a non-numeric threshold falls back to the default" ;; *) bad "fallback run lost the yes group: $out" ;; esac + +out=$(bash -ec ". '$LIB'; _sp_ps() { return 1; }; stray_procs_report; echo after" 2>/dev/null) +[ "$out" = "after" ] && ok "a set -e caller is not aborted" || bad "set -e caller aborted: $out" + +# Wiring: the gauntlet reports before it starts the layers (the failure-path call and its place +# above the failing: list are covered by test-gauntlet-failed-summary.sh). Text check only; +# running the gauntlet from a gauntlet test would recurse. +first=$(/usr/bin/grep -n -m1 '^stray_procs_report' "$GAUNTLET" | cut -d: -f1) +layers=$(/usr/bin/grep -n -m1 '^run_hook_tests >' "$GAUNTLET" | cut -d: -f1) +if [ -n "$first" ] && [ -n "$layers" ] && [ "$first" -lt "$layers" ]; then + ok "run-gauntlet.sh calls stray_procs_report before it starts the layers" +else + bad "no stray_procs_report call before the layers start (call line '$first', layers line '$layers')" +fi + +echo "stray-procs: $pass passed, $fail failed" +[ "$fail" -eq 0 ]