Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 35 additions & 11 deletions hooks/stop/cost-tracker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -79,21 +79,45 @@ sonnet_rate='{"i":2.0,"o":10.0,"cw":2.50,"cw1h":4.00,"cr":0.20}'
# on top of the two more full scans emit_rows and emit_codex_invocations did
# independently. Measured 9.5s/Stop-call against a real 158MB/42,838-line
# transcript before this fix (docs/research/cost-tracker-single-scan-perf-2026-09-22.md).
#
# Two jq calls in all, not 2-3 per file (GH #379: 6.5 s per Stop at 193 subagents). The first
# reads every meta at once and prints one line per file: its agentType when that is a non-empty
# one-line string, else "". A "" line (no meta, an unreadable one, any other shape) takes the
# per-file path below, so every type is the one that path would give. The second builds the map.
# ponytail: the second call takes 3 args per file (~400 bytes), so macOS ARG_MAX (1 MB) is near
# 2,000 subagents (the max seen is 193; raw_records' --argjson typemap has the same ceiling). Past
# it the map falls back to {} (agent_type null); feed the triples on stdin if that ever happens.
build_type_map() {
local parent_map="$1" vmap="$2" out='{}' f meta t tu id; shift 2
local parent_map="$1" vmap="$2" f meta t tu id i=0 rf=() fast types=() args=(); shift 2
for f in "$@"; do
id=$(basename "$f" .jsonl); id="${id#agent-}"
meta="${f%.jsonl}.meta.json"
t=$([[ -f "$meta" ]] && jq -r '.agentType // empty' "$meta" 2>/dev/null)
if [[ -z "$t" && -f "$meta" ]]; then
tu=$(jq -r '.toolUseId // empty' "$meta" 2>/dev/null)
[[ -n "$tu" ]] && t=$(printf '%s' "$parent_map" | jq -r --arg k "$tu" '.[$k] // empty' 2>/dev/null)
[[ -f "$meta" ]] && rf+=(--rawfile "m$i" "$meta")
i=$((i + 1))
done
fast=$(jq -nr ${rf[@]+"${rf[@]}"} --argjson n "$#" '
range(0; $n) as $i | ($ARGS.named["m\($i)"] // null)
| if . == null then "" else (try (fromjson | .agentType) catch null)
| if type == "string" and . != "" and (test("[\n\u0000]") | not) then . else "" end end' 2>/dev/null) || fast=""
while IFS= read -r t; do types+=("$t"); done <<< "$fast"
i=0
for f in "$@"; do
id=${f##*/}; id=${id%.jsonl}; id=${id#agent-}
t=${types[$i]:-}
if [[ -z "$t" ]]; then
meta="${f%.jsonl}.meta.json"
t=$([[ -f "$meta" ]] && jq -r '.agentType // empty' "$meta" 2>/dev/null)
if [[ -z "$t" && -f "$meta" ]]; then
tu=$(jq -r '.toolUseId // empty' "$meta" 2>/dev/null)
[[ -n "$tu" ]] && t=$(printf '%s' "$parent_map" | jq -r --arg k "$tu" '.[$k] // empty' 2>/dev/null)
fi
[[ -z "$t" ]] && t="unknown"
fi
[[ -z "$t" ]] && t="unknown"
out=$(printf '%s' "$out" | jq -c --arg f "$f" --arg t "$t" --arg id "$id" --argjson vmap "$vmap" \
'. + {($f): {t: $t, v: ($vmap[$id] // [])}}' 2>/dev/null) || out='{}'
args+=(--arg "f$i" "$f" --arg "i$i" "$id" --arg "t$i" "$t")
i=$((i + 1))
done
printf '%s' "$out"
jq -nc ${args[@]+"${args[@]}"} --argjson n "$#" --argjson vmap "$vmap" '$ARGS.named as $a
| reduce range(0; $n) as $i ({}; . + {($a["f\($i)"]): {t: $a["t\($i)"], v: ($vmap[$a["i\($i)"]] // [])}})' \
2>/dev/null || printf '{}'
}

# The third handoff cost (docs/research/delegation-criteria-field-survey-2026-09-04.md
Expand Down Expand Up @@ -425,7 +449,7 @@ emit_rows() {
scan_transcript() {
local _d="${1%.jsonl}/subagents" _f _ids=()
shopt -s nullglob
for _f in "$_d"/agent-*.jsonl; do _f=$(basename "$_f" .jsonl); _ids+=("${_f#agent-}"); done
for _f in "$_d"/agent-*.jsonl; do _f=${_f##*/}; _f=${_f%.jsonl}; _ids+=("${_f#agent-}"); done
shopt -u nullglob
# ponytail: [inputs|try fromjson] materializes the whole parsed transcript
# in jq's heap (the original code streamed `inputs` directly through each
Expand Down
54 changes: 52 additions & 2 deletions tests/hooks/test-cost-tracker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,57 @@ fi
trash "$fx" 2>/dev/null || true
echo "$pass4 passed, $fail4 failed (phase 4)"

pass=$((pass + pass2 + pass3 + pass4))
fail=$((fail + fail2 + fail3 + fail4))
# Phase 5 -- build_type_map (GH #379). Failure class: per-file data misattributed
# by a batched rewrite. Five subagents, one row each: two plain types, a type
# that starts with "--" (it must reach jq as an option value, never as a
# positional arg), a meta holding only toolUseId (resolved through the parent's
# Agent tool_use), and no meta at all ("unknown"). Two return windows (a1, a3)
# must land on their own file's row, keyed by the id from the file name.
echo "=== typemap: each subagent row carries its own type and its own return windows ==="
source "$ROOT/tests/_lib/harness.sh"
trap _cleanup_trash EXIT
pass5=0
fail5=0
tm_dir=$(fresh_tmpdir)
tm_home=$(fresh_tmpdir)
mkdir -p "$tm_dir/s/subagents"
python3 - "$tm_dir" <<'PY'
import json, sys
d = sys.argv[1]
def use(mid, inp, content=None):
m = {"model": "claude-sonnet-5", "id": mid, "usage": {"input_tokens": inp, "output_tokens": 0,
"cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}
if content:
m["content"] = content
return json.dumps({"type": "assistant", "message": m})
def ret(i):
return json.dumps({"type": "user", "message": {"role": "user",
"content": "<task-notification>\n<task-id>%s</task-id>\n</task-notification>" % i}})
spawn = [{"type": "tool_use", "name": "Agent", "id": "tu1", "input": {"subagent_type": "mh:reviewer"}}]
with open(d + "/s.jsonl", "w") as f:
f.write("\n".join([use("m0", 1, spawn), ret("a1"), use("m1", 100), ret("a3"), use("m2", 40)]) + "\n")
metas = {"a1": {"agentType": "Explore"}, "a2": {"agentType": "--help"}, "a3": {"toolUseId": "tu1"},
"a4": None, "a5": {"agentType": "general-purpose"}}
for n, (i, meta) in enumerate(metas.items()):
with open("%s/s/subagents/agent-%s.jsonl" % (d, i), "w") as f:
f.write(use("s" + i, 2 ** n) + "\n")
if meta is not None:
with open("%s/s/subagents/agent-%s.meta.json" % (d, i), "w") as f:
json.dump(meta, f)
PY
tm_payload=$(python3 -c 'import json,sys; print(json.dumps({"transcript_path": sys.argv[1], "session_id": "t379"}))' "$tm_dir/s.jsonl")
printf '%s' "$tm_payload" | HOME="$tm_home" MH_COST_TRACKER_SETTLE_S=0 bash "$SCRIPT" >/dev/null 2>/dev/null
got=$(jq -r 'select(.stream == "subagent") | [.agent_type, .input_tokens, .returns, (.verify_tokens | tostring)] | @tsv' \
"$tm_home/.local/share/kbg/metrics/costs.jsonl" 2>/dev/null | LC_ALL=C sort)
want=$'--help\t2\t0\tnull\nExplore\t1\t1\t100\ngeneral-purpose\t16\t0\tnull\nmh:reviewer\t4\t1\t40\nunknown\t8\t0\tnull'
if [[ "$got" == "$want" ]]; then
pass5=$((pass5 + 1)); echo "PASS: 5 subagents -> 5 rows, each with its own type (meta, --help, toolUseId, none) and its own return window"
else
fail5=$((fail5 + 1)); echo "FAIL: subagent rows (type, in, returns, verify) were:"$'\n'"$got"$'\n'"expected:"$'\n'"$want" >&2
fi
echo "$pass5 passed, $fail5 failed (phase 5)"

pass=$((pass + pass2 + pass3 + pass4 + pass5))
fail=$((fail + fail2 + fail3 + fail4 + fail5))
echo "TOTAL: $pass passed, $fail failed"
[[ "$fail" -eq 0 ]]
Loading