diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc88ccc5e1..e69911b2bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,6 +217,7 @@ jobs: .github/workflows/claude-security-review.yml .github/workflows/dependabot-miro-bundle.yml .github/workflows/do-not-merge.yml + .github/workflows/hook-utils-timing.yml .github/workflows/link-check.yml .github/workflows/pr-issue-linkage.yml .github/workflows/pr-title.yml @@ -1417,9 +1418,19 @@ jobs: --requirement .github/requirements-ci.txt echo "$GITHUB_WORKSPACE/node_modules/.bin" >> "$GITHUB_PATH" echo "$HOME/.local/bin" >> "$GITHUB_PATH" + # Three suites at a time, not four. The runner has 4 vCPUs and the suites + # are spawn-bound, so four looked like the shape that pays, but at four + # three separate suites failed across two runs by producing EMPTY output + # from an external command on a path with no clock in it: an awk read of + # an agent file, a jq read of a local-markdown fixture, and a conformance + # run. Serialising each one in turn only moves the symptom to the next + # suite, so the job count is the lever rather than the allowlist. The + # default stays 1 for Windows dev boxes (see the runner's header); the + # suites that assert wall-clock ceilings run one at a time regardless, + # from scripts/run-plugin-tests-serial.txt. - name: Run plugin contract tests if: needs.scope.outputs.run_full == 'true' - run: scripts/run-plugin-tests.sh + run: scripts/run-plugin-tests.sh --jobs 3 # Explicit step: run-plugin-tests.sh discovers only plugins/**/*.test.sh, # so the cheat-sheet generator suite under scripts/ never runs without it. - name: Run cheat-sheet generator tests diff --git a/.github/workflows/hook-utils-timing.yml b/.github/workflows/hook-utils-timing.yml new file mode 100644 index 0000000000..d288ca8599 --- /dev/null +++ b/.github/workflows/hook-utils-timing.yml @@ -0,0 +1,52 @@ +name: hook-utils-timing + +# Weekly wall-clock comparisons for lib/hook-utils.sh's bounded stdin read. +# lib/hook-utils.test.sh carries two interleaved-pair clock comparisons (the +# late-EOF window count and the stall overshoot) that each wait out real read +# bounds: about 80 s per run on a hosted runner and identical on both operating +# systems, while the suite's own comments call them advisory next to the +# load-independent probes that guard the same regressions (#2105). Every +# ordinary run of the suite therefore reports them as deferred, and this lane +# is where they actually run: HOOK_UTILS_TIMING=1 enables them, once a week on +# each platform, off the required merge gate. Advisory by construction: nothing +# aggregates this workflow into ci-status, and a flake here blocks no merge. +# +# Two literal jobs rather than a matrix so the runner policy reads pinned +# labels (never `*-latest`), the same rule ci.yml's lanes follow. +on: + schedule: + - cron: "17 6 * * 1" # Mondays 06:17 UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Run the shared lib suite with the clock comparisons enabled + env: + HOOK_UTILS_TIMING: "1" + run: bash lib/hook-utils.test.sh + + windows: + runs-on: windows-2025 + timeout-minutes: 25 + defaults: + run: + shell: bash + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Run the shared lib suite with the clock comparisons enabled on Windows + env: + HOOK_UTILS_TIMING: "1" + run: bash lib/hook-utils.test.sh diff --git a/lib/hook-utils.test.sh b/lib/hook-utils.test.sh index 3c758ea8c1..fc95862d37 100755 --- a/lib/hook-utils.test.sh +++ b/lib/hook-utils.test.sh @@ -1738,7 +1738,18 @@ bs_time_late_eof() { # $1 = shell prelude; prints elapsed ms (empty if untimed) # returns 1 regardless. Same spawn, opposite verdict. # shellcheck disable=SC2016 # $1 is the overriding function's own positional, not this shell's bs_reads_on='hook::json_complete() { printf "%s" "$1" | jq -e . >/dev/null 2>&1; return 1; }' -if bs_samples 6 bs_time_late_eof "" "$bs_reads_on"; then +# HOOK_UTILS_TIMING gates the two interleaved-pair CLOCK comparisons in this +# suite (this late-EOF one and the stall-overshoot one in Test 18g). Each is +# six pairs of two arms waiting out real bounds, about 80 s of wall time on a +# hosted runner, and both are advisory by their own terms: the load-independent +# probes beside them (the chunk-boundary engagement check and the stall +# read-count check) are the regression guards (#2105). Unset, which is every +# ordinary run, the comparison is reported as deferred and the probes carry the +# coverage; the weekly `hook-utils-timing` workflow sets HOOK_UTILS_TIMING=1 +# and runs the comparisons on both operating systems. +if [[ -z "${HOOK_UTILS_TIMING:-}" ]]; then + ok "buffer_stdin: late-EOF clock comparison deferred (HOOK_UTILS_TIMING unset; the weekly hook-utils-timing lane runs it, and the chunk-boundary engagement probe is the regression guard)" +elif bs_samples 6 bs_time_late_eof "" "$bs_reads_on"; then bs_paired_verdict "buffer_stdin: late-EOF stops at the payload, not the bound" \ 400 fast slow elif [[ -n "${EPOCHREALTIME:-}" ]]; then @@ -2390,7 +2401,10 @@ rm -f "$bs_stall_read_file" # Stall overshoot is load-sensitive when asserted as an absolute wall-clock gap # (#2105, #2080). The idle-slice probe above is the load-independent guard; this # relative check is advisory — fail only when every timed pair contradicts slicing. -if bs_samples 6 bs_time_stall "" "$bs_unsliced"; then +# Gated on HOOK_UTILS_TIMING like the late-EOF comparison (see Test 18b). +if [[ -z "${HOOK_UTILS_TIMING:-}" ]]; then + ok "buffer_stdin: stall overshoot clock comparison deferred (HOOK_UTILS_TIMING unset; the weekly hook-utils-timing lane runs it, and the read-count probe above is the regression guard, #2105)" +elif bs_samples 6 bs_time_stall "" "$bs_unsliced"; then bs_rel_ok=1 bs_rel_detail="deltas ${bs_deltas_a_first[*]} | ${bs_deltas_b_first[*]}" bs_rel_neg=0 diff --git a/plugins/context-guard/.claude-plugin/plugin.json b/plugins/context-guard/.claude-plugin/plugin.json index a7b7cf83cd..b0afafb0f0 100644 --- a/plugins/context-guard/.claude-plugin/plugin.json +++ b/plugins/context-guard/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "context-guard", - "version": "0.7.36", + "version": "0.7.37", "description": "Per-session context-window observability plus the first shipped consumer: a statusline wrapper tees each session's context_window fields to a per-session snapshot file, a zone resolver classifies usage into smart/acceptable/dumb bands (percentage bands plus window-class token bands, conservative-min combination, zones.json SSOT with shipped defaults), a reader contract fixes how consuming sessions interpret the snapshots, and zone-crossing hooks report once per transition into a worse zone across two channels \u2014 the continuation menu to the operator, who owns that choice, and to the model only the zone determination plus the counter-steer that a zone word is not a decay signal (advisory by default; an optional blocking mode gates new mutating work on a fresh dumb-zone snapshot with handoff-writing exempt), with a PostCompact hook persisting an evidence-degraded marker.", "author": { "name": "Melodic Software", diff --git a/plugins/context-guard/CHANGELOG.md b/plugins/context-guard/CHANGELOG.md index aa47342f04..ab11e699f8 100644 --- a/plugins/context-guard/CHANGELOG.md +++ b/plugins/context-guard/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to the `context-guard` plugin. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.7.37] + +### Changed + +- **The statusline-tee cancellation cases park their `mv` shim for two seconds + instead of ten.** Bash defers the TERM trap until the foreground `mv` + returns, so the shim's own sleep was the floor for both cancellation cases + and the suite spent most of its wall time waiting on a delay that proved + nothing. Two seconds exercises the same cancellation window behind the same + readiness marker. Test-side only; no hook, script or shipped behaviour + changes. + ## [0.7.36] ### Changed diff --git a/plugins/context-guard/scripts/statusline-tee.test.sh b/plugins/context-guard/scripts/statusline-tee.test.sh index e1eacd2bbb..ecd91ab178 100755 --- a/plugins/context-guard/scripts/statusline-tee.test.sh +++ b/plugins/context-guard/scripts/statusline-tee.test.sh @@ -376,16 +376,21 @@ if [[ -f "$CTXDIR17/sess-42.json" ]]; then ok "prune pass still wrote the live s # Claude Code "cancels the in-flight script" when a new update arrives while # this one is still running, so a kill between the write and the rename is # routine rather than exceptional. Driven by an `mv` shim that parks, so the -# signal lands inside the window deterministically. +# signal lands inside the window deterministically. The shim announces that it +# has parked by touching a marker, and the kill waits for that marker instead of +# a fixed sleep; the park itself is short because bash defers a trap until the +# foreground `mv` returns, so the shim's sleep is the floor on how long the +# reclaim takes to run. HOME18="$WORK/home18" mkdir -p "$HOME18" SHIM18="$WORK/shim18" mkdir -p "$SHIM18" -printf '#!/usr/bin/env bash\nsleep 10\n' >"$SHIM18/mv" +printf '#!/usr/bin/env bash\n: >"%s/parked"\nsleep 2\n' "$SHIM18" >"$SHIM18/mv" chmod +x "$SHIM18/mv" printf '%s' "$(build_input)" | HOME="$HOME18" PATH="$SHIM18:$PATH" bash "$TEE" >/dev/null 2>&1 & TEE_PID=$! -sleep 2 +tries=250 +while ((tries-- > 0)) && [[ ! -e "$SHIM18/parked" ]]; do sleep 0.02; done kill -TERM "$TEE_PID" 2>/dev/null wait "$TEE_PID" 2>/dev/null sleep 0.5 diff --git a/plugins/rate-limit-guard/.claude-plugin/plugin.json b/plugins/rate-limit-guard/.claude-plugin/plugin.json index 36729e421e..5d1147ed1b 100644 --- a/plugins/rate-limit-guard/.claude-plugin/plugin.json +++ b/plugins/rate-limit-guard/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "rate-limit-guard", - "version": "0.7.28", + "version": "0.7.29", "description": "Shared rate-limit guard for loop lanes: a statusline wrapper tees the subscription rate-limit windows to a fixed machine-scope file, a StopFailure hook records rate-limit stops reactively, and a reader contract fixes how consuming sessions pause and resume.", "author": { "name": "Melodic Software", diff --git a/plugins/rate-limit-guard/CHANGELOG.md b/plugins/rate-limit-guard/CHANGELOG.md index 40f979479e..1bc141de04 100644 --- a/plugins/rate-limit-guard/CHANGELOG.md +++ b/plugins/rate-limit-guard/CHANGELOG.md @@ -3,6 +3,18 @@ All notable changes to the `rate-limit-guard` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.7.29] + +### Changed + +- **The statusline-tee cancellation cases park their `mv` shim for two seconds + instead of ten.** Bash defers the TERM trap until the foreground `mv` + returns, so the shim's own sleep was the floor for both cancellation cases + and the suite spent most of its wall time waiting on a delay that proved + nothing. Two seconds exercises the same cancellation window behind the same + readiness marker. Test-side only; no hook, script or shipped behaviour + changes. + ## [0.7.28] ### Changed diff --git a/plugins/rate-limit-guard/scripts/statusline-tee.test.sh b/plugins/rate-limit-guard/scripts/statusline-tee.test.sh index 4df5d230db..4eaa0a18be 100755 --- a/plugins/rate-limit-guard/scripts/statusline-tee.test.sh +++ b/plugins/rate-limit-guard/scripts/statusline-tee.test.sh @@ -279,16 +279,21 @@ if [[ "$(jq -r '.session_id' <"$TEEFILE")" == "sess-later" ]]; then ok "snapshot # Claude Code "cancels the in-flight script" when a new update arrives while # this one is still running, so a kill between the write and the rename is # routine rather than exceptional. Driven by an `mv` shim that parks, so the -# signal lands inside the window deterministically. +# signal lands inside the window deterministically. The shim announces that it +# has parked by touching a marker, and the kill waits for that marker instead of +# a fixed sleep; the park itself is short because bash defers a trap until the +# foreground `mv` returns, so the shim's sleep is the floor on how long the +# reclaim takes to run. HOME14="$WORK/home14" mkdir -p "$HOME14" SHIM14="$WORK/shim14" mkdir -p "$SHIM14" -printf '#!/usr/bin/env bash\nsleep 10\n' >"$SHIM14/mv" +printf '#!/usr/bin/env bash\n: >"%s/parked"\nsleep 2\n' "$SHIM14" >"$SHIM14/mv" chmod +x "$SHIM14/mv" printf '%s' "$(build_input)" | HOME="$HOME14" PATH="$SHIM14:$PATH" bash "$TEE" >/dev/null 2>&1 & TEE_PID=$! -sleep 2 +tries=250 +while ((tries-- > 0)) && [[ ! -e "$SHIM14/parked" ]]; do sleep 0.02; done kill -TERM "$TEE_PID" 2>/dev/null wait "$TEE_PID" 2>/dev/null sleep 0.5 diff --git a/plugins/session-flow/.claude-plugin/plugin.json b/plugins/session-flow/.claude-plugin/plugin.json index fc7bc7e7f1..571062344f 100644 --- a/plugins/session-flow/.claude-plugin/plugin.json +++ b/plugins/session-flow/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "session-flow", - "version": "0.34.21", + "version": "0.34.22", "description": "Session-lifecycle toolkit of fourteen skills: workflow (navigate a staged dev workflow and suggest the next stage), handoff (write a save-point and resume prompt for /clear-and-resume), continue-in-background (delegate the task to a fresh background agent that continues it now \u2014 same save-point engine as handoff, delivered by launching a detached claude --bg session seeded with the resume prompt; launches only on explicit user request), keep-going (recover and continue after any interruption OR when live off-thread work looks stalled \u2014 inventory off-thread work, inspect its real output, act only on evidence, then continue; after a usage limit lifts it continues rather than summarizing-and-stalling), find-handoff (recover a lost handoff after /clear \u2014 when the resume prompt was written but never copied \u2014 via a read-only detection ladder: known-location glob of the handoffs dir, then a bounded, recency-ranked transcript scan for the handoff directive and dashed-rail markers, then a confirm-before-resume gate; surfaces only the resume prompt + metadata, never raw transcript content), clean-stop (get to a durable, linked stopping point before the machine may go away \u2014 sweep every repo/worktree for uncommitted, unpushed, or PR-less work, push it durable, put breadcrumbs in PR/issue bodies, then give a free-and-clear verdict), retro (structured end-of-session retrospective with transcript metrics and learning codification), running-retro (in-flight retrospective checkpoints that spawn a subagent to analyze the transcript so far and append classified findings to a cumulative running ledger \u2014 capture and route only, the live counterpart to retro; also owns a detached-observer substrate that can watch a session out-of-band and run the checkpoint autonomously after the session ends), orient (read-only session orientation \u2014 synthesize where we stand, what we are doing, and why, from durable + off-thread state the built-in /recap never sees: ledgers, handoffs, workflow checklists, running-retro ledgers, open PRs and work-items, and git), orchestrate (arm a session or worker with proactive-orchestration imperatives), reanchor (verify a session's working assumptions are still true against live reality \u2014 referenced PRs/issues/branches, base-branch drift, renamed/version-drifted surfaces, stale memory-tier files, and the goal a handoff records, compared across the chain so a re-derived goal reports as drift \u2014 before building on them), reconcile (retire finished off-thread work and reconcile this session's task ledger with reality \u2014 the prune-and-reconcile counterpart to keep-going's resume: inventory the work this session spawned, inspect its real state, retire the finished and close proven-done tasks, auto-settling the finished and gating any kill of still-running work; sibling sessions in the project are reported read-only), setup (check-centric verification of the observer's runtime prerequisites and configuration), and show-options (lay out which skills fit this moment as a ranked, nothing-hidden menu \u2014 a shortlist per bucket plus the complete remainder by name, resolved from the full installed catalog rather than the truncated in-context listing, so the human decides and no option is withheld for looking already-done).", "author": { "name": "Melodic Software", diff --git a/plugins/session-flow/CHANGELOG.md b/plugins/session-flow/CHANGELOG.md index dfaedf0fd3..66519800b2 100644 --- a/plugins/session-flow/CHANGELOG.md +++ b/plugins/session-flow/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog — session-flow plugin +## [0.34.22] + +### Changed + +- **The running-retro observer growth case sets its own idle confirmation.** The + case inherited the production 30-second idle confirmation from the test + harness, so it could only ever end at the lifetime deadline rather than when + the tail actually went idle. It now sets a one-second confirmation and a + ten-second deadline, which is what the case was always asserting. Test-side + only; the observer's own defaults are unchanged. + ## [0.34.21] ### Changed diff --git a/plugins/session-flow/skills/running-retro/scripts/test_observer.py b/plugins/session-flow/skills/running-retro/scripts/test_observer.py index be364da0e7..53857882a1 100755 --- a/plugins/session-flow/skills/running-retro/scripts/test_observer.py +++ b/plugins/session-flow/skills/running-retro/scripts/test_observer.py @@ -807,8 +807,13 @@ def test_tail_no_duplicate_events_on_growth(self): # grows in batches and assert exactly one observation per record. with tempfile.TemporaryDirectory() as d: tmp = Path(d) + # Growth ends about 1.8 s in; the 2 s idle window then opens and + # the 1 s idle confirmation closes the tail near 5 s. The idle + # confirmation is set explicitly because the harness default is + # the production 30 s, under which this case could only ever end + # at the lifetime deadline (a 30 s wait, measured in CI). ob = make_observer(tmp, analysis=False, idle_seconds=2.0, poll_seconds=0.15, - max_seconds=30.0) + idle_confirm_seconds=1.0, max_seconds=10.0) rec = '{"type":"user","message":{"content":"x"}}\n' total = 9 @@ -820,7 +825,7 @@ def test_tail_no_duplicate_events_on_growth(self): for _ in range(3): f.write(rec) time.sleep(0.5) - tailer.join(30) + tailer.join(10) lines = [ln for ln in ob.obs_path.read_text(encoding="utf-8").splitlines() if ln] self.assertEqual(len(lines), total, f"expected {total} distilled events, got {len(lines)} (dupes/underread?)") diff --git a/scripts/run-plugin-tests-serial.txt b/scripts/run-plugin-tests-serial.txt new file mode 100644 index 0000000000..d08c8a911d --- /dev/null +++ b/scripts/run-plugin-tests-serial.txt @@ -0,0 +1,73 @@ +# Suites scripts/run-plugin-tests.sh runs ONE AT A TIME, before the parallel +# group and never concurrently with any other suite. +# +# Membership criterion: the suite asserts a wall-clock ceiling (a `timeout` +# whose expiry is read as a failure, an elapsed-seconds bound, a latency +# budget) or drives a concurrency probe of its own (it spawns a burst of hooks +# and counts what got through). Both fail spuriously when other suites compete +# for the same cores, and a spurious red on a required check is worse than the +# seconds parallelism would have saved on these few. A suite that is merely +# slow does NOT belong here; parallelism is exactly what it wants. +# +# One repo-relative suite path per line; `#` comments and blank lines are +# ignored. An entry that matches no discovered suite fails the runner (exit 2): +# an allowlist must not outlive what it excuses. Keep the reason beside each +# entry, the way every other data file in scripts/ does. +# +# PROVISIONAL (github-iac ci-perf Phase 2.2): this first list is the union of +# the suites whose source reads a clock and compares it, plus the guardrails +# suites scripts/affected-tests.sh names. It is finalised against real +# pull-request runs on the 4-vCPU ubuntu-24.04 runner: a suite that passes +# under --jobs 4 across the verification window leaves the list, and a suite +# that flakes there joins it. + +# Alias-traversal cases assert a hard wall-clock ceiling through `timeout` +# (run_bounded reads 124 as the failure it is). +plugins/guardrails/hooks/block-dangerous-git.test.sh + +# The divergent-spelling walk runs under a hang guard sized against a loaded +# host; contention widens exactly the margin it depends on. +plugins/guardrails/hooks/block-noncanonical-commit.test.sh + +# Asserts a 1000-line hunk scans inside the hook's 30 s timeout. +plugins/guardrails/hooks/skill-reference-verify.test.sh + +# Case 13 fires 24 concurrent hooks against the grace budget and counts what +# landed inside it; other suites on the same cores skew that count. +plugins/context-guard/hooks/zone-gate.test.sh + +# A/B timing harness; its verdicts are elapsed-time comparisons. +plugins/performance/scripts/ab.test.sh + +# The statusline bench measures render latency. +plugins/rate-limit-guard/bench/bench.test.sh + +# Cancellation-window cases kill a parked rename shim on a short fuse. +plugins/rate-limit-guard/scripts/statusline-tee.test.sh +plugins/context-guard/scripts/statusline-tee.test.sh + +# Always-on hooks assert the marketplace latency budget (.claude/rules/hook-budget.md). +plugins/desktop-notification/hooks/desktop-notification.test.sh +plugins/instruction-placement/hooks/index-drift.test.sh +plugins/markdown-format/hooks/markdown-format.test.sh +plugins/source-control/hooks/pr-body-linkage-gate.test.sh +plugins/typos-format/hooks/typos-format.test.sh + +# The listing-budget reporter times its own whole-marketplace pass. +plugins/skill-quality/scripts/check-listing-budget.test.sh + +# Deadline-based cases: a suite that fails when a bounded operation overruns is +# measuring the box once other suites are competing for it. +# 2,000 ms spawnSync deadline on the ReDoS-shaped title case (measure.test.sh:471-478). +plugins/context-budget/skills/audit/scripts/measure.test.sh +# Two-second watchdog asserted to finish inside four seconds (audit-fleet.test.sh:1928-1936). +plugins/repo-fleet-hygiene/skills/audit/scripts/audit-fleet.test.sh + +# The two entries below are NOT wall-clock cases. Both fail under --jobs 4 on a +# hosted runner (run 33787030286), pass serially in the same job on main, and +# pass locally; no hypothesis so far explains either. They are listed to fail +# toward running the suite rather than toward a red lane nobody can diagnose, +# and the debt is tracked so the listing does not become permanent by default. +# Root cause tracked in #3694; both come off this list when it is found. +plugins/discovery/agents/tool-honesty.test.sh +plugins/work-items/tools/work-item-tracker/conformance/bindings/jira.test.sh diff --git a/scripts/run-plugin-tests.sh b/scripts/run-plugin-tests.sh index 742f00c734..75c89887bb 100755 --- a/scripts/run-plugin-tests.sh +++ b/scripts/run-plugin-tests.sh @@ -3,6 +3,8 @@ # hook tests (.claude/hooks/*.test.sh) and fail if any fails. The repo-local # hooks are tracked policy with the same test conventions as plugin hooks. # +# scripts/run-plugin-tests.sh [--strict-skips] [--jobs N] [--root DIR] +# # Each test is self-contained and cwd-independent; an individual test SKIPs # (exit 0) when an optional tool it needs (shellcheck, shfmt, ...) is absent, so # this runner gates on real failures without requiring every tool to be present. @@ -13,6 +15,27 @@ # --strict-skips turns any optional SKIP into a failure, for callers that have # provisioned the full toolchain and want a skip to mean the environment # regressed rather than that a tool is optional. +# +# PARALLELISM. --jobs N (default 1, or $PLUGIN_TEST_JOBS) runs up to N suites +# at once. The suites are spawn-bound: on a 4-vCPU Linux runner four at a time +# is the shape that pays, while on Windows Git Bash a parallel run measured +# sublinear (the box saturates on process creation; see the foot of +# scripts/affected-tests.sh), which is why the default stays 1 and CI passes the +# count explicitly. The suites named in scripts/run-plugin-tests-serial.txt +# assert wall-clock ceilings or drive concurrency probes of their own and fail +# spuriously under load, so they run one at a time, before the parallel group, +# and never overlap anything. An entry there that matches no discovered suite +# is an error rather than an ignored line: an allowlist must not outlive what +# it excuses. +# +# Output stays readable under parallelism. Every suite's output is captured to +# a file and replayed as one block under a lock (`=== path ===`, the suite's own +# lines, then `PASS:` or `FAIL:`), so blocks from concurrent suites never +# interleave and the per-suite markers that CI log tooling reads survive. The +# block is printed the moment its suite finishes, so progress stays visible. +# +# --root DIR discovers suites under DIR instead of the repository (test +# injection for this runner's own suite, scripts/run-plugin-tests.test.sh). set -uo pipefail # Fixture isolation (#2840). `-C` only changes directory, while an exported @@ -31,13 +54,81 @@ set -uo pipefail # GIT_DIR, and of the working directory. unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_COMMON_DIR GIT_PREFIX GIT_OBJECT_DIRECTORY GIT_CONFIG +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +runner="$script_dir/${BASH_SOURCE[0]##*/}" +SERIAL_LIST="${PLUGIN_TEST_SERIAL_LIST:-$script_dir/run-plugin-tests-serial.txt}" + +usage() { + echo "usage: run-plugin-tests.sh [--strict-skips] [--jobs N] [--root DIR]" >&2 + exit 2 +} + +# --worker : internal. Runs ONE suite with its output captured +# under $PLUGIN_TEST_LOG_DIR, records its exit status beside the capture, then +# prints the whole block under the print lock. Reached only from the dispatch +# below; it exits 0 regardless of the suite's result so xargs never stops +# early, and the summary reads the recorded status instead. +# +# The capture is keyed on the suite's INDEX, not on its path. A path-derived +# key (slashes rewritten to a separator) collides whenever a path segment +# already contains that separator: plugins/foo__bar/baz.test.sh and +# plugins/foo/bar__baz.test.sh flatten to the same name, and two suites in one +# parallel batch would then overwrite each other's capture and exit status. +# The index is unique by construction, so no path shape can collide. It is +# fixed-width and colon-terminated, which also keeps a path containing a colon +# from being split at the wrong separator. +if [[ "${1:-}" == "--worker" ]]; then + keyed="${2:?--worker needs an indexed suite path}" + key="${keyed%%:*}" + suite="${keyed#*:}" + log_dir="${PLUGIN_TEST_LOG_DIR:?--worker needs PLUGIN_TEST_LOG_DIR}" + log="$log_dir/$key" + rc=0 + bash "$suite" >"$log.out" 2>&1 || rc=$? + printf '%s\n' "$rc" >"$log.rc" + until mkdir "$log_dir/.print-lock" 2>/dev/null; do sleep 0.02; done + echo "=== $suite ===" + cat "$log.out" + if ((rc == 0)); then + echo "PASS: $suite" + else + echo "FAIL: $suite" >&2 + fi + rmdir "$log_dir/.print-lock" + exit 0 +fi + strict_skips=0 -if [[ "${1:-}" == "--strict-skips" ]]; then - strict_skips=1 +jobs="${PLUGIN_TEST_JOBS:-1}" +root="" +while (($# > 0)); do + case "$1" in + --strict-skips) strict_skips=1 ;; + --jobs) + [[ $# -ge 2 ]] || usage + jobs="$2" + shift + ;; + --jobs=*) jobs="${1#--jobs=}" ;; + --root) + [[ $# -ge 2 ]] || usage + root="$2" + shift + ;; + *) usage ;; + esac shift +done +if [[ ! "$jobs" =~ ^[1-9][0-9]*$ ]]; then + echo "error: --jobs must be a positive integer (got '$jobs')" >&2 + exit 2 fi -cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 1 +if [[ -n "$root" ]]; then + cd "$root" || exit 2 +else + cd "$script_dir/.." || exit 1 +fi mapfile -t tests < <(find plugins .claude/hooks -type f -name '*.test.sh' 2>/dev/null | sort) @@ -46,38 +137,96 @@ if [[ ${#tests[@]} -eq 0 ]]; then exit 2 fi -failed=0 -total_optional_skips=0 -total_discriminating_skips=0 -skipped_suites=() +# The serial allowlist: one repo-relative suite path per line, `#` comments and +# blank lines ignored. Read before anything runs so a stale entry fails the run +# up front rather than after minutes of suites. +serial_entries=() +if [[ -f "$SERIAL_LIST" ]]; then + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line%%#*}" + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -n "$line" ]] || continue + serial_entries+=("$line") + done <"$SERIAL_LIST" +fi +declare -A is_serial=() +for s in ${serial_entries[@]+"${serial_entries[@]}"}; do + found=0 + for t in "${tests[@]}"; do + if [[ "$t" == "$s" ]]; then + found=1 + break + fi + done + if ((found == 0)); then + echo "error: $SERIAL_LIST names '$s', which matches no discovered suite; remove the stale entry" >&2 + exit 2 + fi + is_serial["$s"]=1 +done +serial_suites=() +parallel_suites=() +suite_keys=() +for i in "${!tests[@]}"; do + t="${tests[$i]}" + # One key per suite, reused by the dispatch below and by the summary loop so + # both address the same capture. Assigned BY INDEX rather than appended: the + # key has to follow the position in `tests`, and an append would only agree + # with that while the array stays dense. + suite_keys[i]="$(printf '%06d' "$i")" + if [[ -n "${is_serial[$t]+x}" ]]; then + serial_suites+=("${suite_keys[$i]}:$t") + else + parallel_suites+=("${suite_keys[$i]}:$t") + fi +done + +log_dir="$(mktemp -d)" +trap 'rm -rf "$log_dir"' EXIT +export PLUGIN_TEST_LOG_DIR="$log_dir" if command -v git >/dev/null 2>&1; then echo "Runner git: $(git --version)" else echo "Runner git: unavailable" fi +echo "Suites: ${#tests[@]} (${#serial_suites[@]} serial, ${#parallel_suites[@]} across up to $jobs job(s))" -for t in "${tests[@]}"; do - echo "=== $t ===" - log="$(mktemp)" - if bash "$t" 2>&1 | tee "$log"; then - echo "PASS: $t" - else - echo "FAIL: $t" >&2 +for t in ${serial_suites[@]+"${serial_suites[@]}"}; do + bash "$runner" --worker "$t" +done +if [[ ${#parallel_suites[@]} -gt 0 ]]; then + # xargs appends one indexed suite path after `--worker` per invocation. + printf '%s\0' "${parallel_suites[@]}" | xargs -0 -n 1 -P "$jobs" bash "$runner" --worker +fi + +failed=0 +total_optional_skips=0 +total_discriminating_skips=0 +skipped_suites=() + +for i in "${!tests[@]}"; do + t="${tests[$i]}" + log="$log_dir/${suite_keys[$i]}" + if [[ ! -f "$log.rc" ]]; then + echo "FAIL: $t (no result recorded; the worker never reported)" >&2 failed=1 + continue fi - opt_skips="$(grep -c '^SKIP:' "$log" 2>/dev/null || true)" - disc_skips="$(grep -c '^DISCRIMINATING SKIP:' "$log" 2>/dev/null || true)" + rc="$(<"$log.rc")" + ((rc == 0)) || failed=1 + opt_skips="$(grep -c '^SKIP:' "$log.out" 2>/dev/null || true)" + disc_skips="$(grep -c '^DISCRIMINATING SKIP:' "$log.out" 2>/dev/null || true)" total_optional_skips=$((total_optional_skips + opt_skips)) total_discriminating_skips=$((total_discriminating_skips + disc_skips)) if ((opt_skips > 0)); then - first_reason="$(grep -m1 '^SKIP:' "$log")" + first_reason="$(grep -m1 '^SKIP:' "$log.out")" skipped_suites+=("$t — ${opt_skips} SKIP(s), first: ${first_reason#SKIP: }") fi if ((disc_skips > 0)); then echo "DISCRIMINATING SKIP: $t vacated $disc_skips discriminating case(s)" >&2 fi - rm -f "$log" done echo "" diff --git a/scripts/run-plugin-tests.test.sh b/scripts/run-plugin-tests.test.sh new file mode 100755 index 0000000000..f999a1185c --- /dev/null +++ b/scripts/run-plugin-tests.test.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# Self-test for scripts/run-plugin-tests.sh: the parallel runner, the serial +# allowlist, the print lock, the skip accounting, and the exit contract. +# +# Fixtures are throwaway suites under a mktemp root handed to the runner with +# --root, so the repository's own corpus is never discovered here. No git state +# is created (claude-code-plugins#2839). +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUNNER="$ROOT/scripts/run-plugin-tests.sh" +# shellcheck source=lib/test-harness.sh +. "$ROOT/scripts/lib/test-harness.sh" + +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT + +# make_root -> prints a fresh fixture root with the plugin/hook layout. +make_root() { + local r="$scratch/$1" + mkdir -p "$r/plugins" "$r/.claude/hooks" + printf '%s' "$r" +} + +# write_suite -> a suite script under the fixture root. +write_suite() { + local path="$1/$2" + mkdir -p "$(dirname "$path")" + printf '#!/usr/bin/env bash\n%s\n' "$3" >"$path" +} + +# run_runner