diff --git a/.changes/planning/feat-12478-plan-stress-test-7c41e2a9.md b/.changes/planning/feat-12478-plan-stress-test-7c41e2a9.md new file mode 100644 index 00000000000..18693378054 --- /dev/null +++ b/.changes/planning/feat-12478-plan-stress-test-7c41e2a9.md @@ -0,0 +1,8 @@ +--- +bump: minor +--- + +### Added + +- `plan`: `plan_stress_test` sets, per mode of the draft (`plan`, `proposal`, `incumbent`), which stress-tests `/planning:plan` runs: `auto` (the default, current behavior), `reviewer`, `devils-advocate` or `both`. It resolves through the convention home; `reference/config.md` defines the modes and values (#12478). +- `evals`: the `plan-stress-test` suite, 12 seeded drafts across the three modes with deterministic graders, and `fixtures/stress-arm/prepare-arm.sh`, which sets up a plugin copy for one arm or for the suite's negative control (#12478). diff --git a/plugins/planning/README.md b/plugins/planning/README.md index c29e86b81bc..0c95718eb04 100644 --- a/plugins/planning/README.md +++ b/plugins/planning/README.md @@ -20,7 +20,7 @@ where artifacts land in the consuming repo. | `/planning:design` | Design space | Explores types, contracts, module boundaries, and package topology through collaborative discussion rounds, producing capability-matrix / type-inventory / design-threads / topology artifacts; its `handoff` action delegates to `/planning:design-handoff`. Its `data`, `integration`, and `system` artifacts are typed: each carries a `scope` + `dialect` label and a fenced block in the dialect the `diagram_dialect` convention key resolves to (mermaid `erDiagram` or DBML for data; mermaid `sequenceDiagram` plus an OpenAPI 3.1 sketch for integration; a LikeC4 or C4-PlantUML container view for system, and no C4 view at all when that key is unset). `library` and `module` emit no typed artifact. | | `/planning:design-handoff` | Design→plan gate | Gates a finished design for `/planning:plan`. The gate is a binary check that every `design-threads.md` thread is RESOLVED, directional, or TAGGED-DEFERRED. Then it packages the plan-ready summary and resume prompt, or FAILs and routes back to `/planning:design`. On PASS and FAIL alike it also emits an advisory six-dimension coverage table (what, how, where, who, when, why) that never contributes to the verdict; uncovered dimensions ride along in the handoff summary and the resume prompt. | | `/planning:devils-advocate` | Adversarial review | Stress-tests plans via assumption extraction, evidence checks, failure scenarios, and operational-gotcha sweeps. Every finding evidence-backed, never generic warnings. An `incumbent` mode turns the same lens on the status quo: an Alternatives Sweep that stress-tests keeping an incumbent tool/approach against alternatives (native > official > vetted ladder, coupling priced, KEEP / MIGRATE / RESEARCH verdict), exploring the incumbent first-hand in a fresh sub-agent. | -| `/planning:plan` | Implementation plan | Produces a structured plan (goal, approach, test strategy, blast radius, parallelism analysis, tagged unilateral decisions) with a mandatory fresh-context `plan-reviewer` stress-test and a user approval gate, persisted to PLAN.md. | +| `/planning:plan` | Implementation plan | Produces a structured plan (goal, approach, test strategy, blast radius, parallelism analysis, tagged unilateral decisions) with a fresh-context `plan-reviewer` stress-test, on by default and set per mode by `plan_stress_test`, and a user approval gate, persisted to PLAN.md. | | `/planning:setup` | Configuration | `check` inspects the standards index read-only; `apply` interviews the consumer and bootstraps the standards index (idempotent; re-run to reconfigure). | The pipeline composes end-to-end. `wayfind` charts the fog upstream when an effort diff --git a/plugins/planning/evals/fixtures/stress-arm/apply-arm.sh b/plugins/planning/evals/fixtures/stress-arm/apply-arm.sh new file mode 100755 index 00000000000..41ce92ba61a --- /dev/null +++ b/plugins/planning/evals/fixtures/stress-arm/apply-arm.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Run by each st-* case's scaffold, in the eval workspace: writes the arm's plan_stress_test +# setting where /planning:plan resolves it and commits the seeded project. +set -euo pipefail + +mkdir -p docs/conventions +cp "$(dirname "${BASH_SOURCE[0]}")/planning.yaml" docs/conventions/planning.yaml +git add . +git commit -q -m "chore: import project" diff --git a/plugins/planning/evals/fixtures/stress-arm/devils-advocate-degraded-body.md.txt b/plugins/planning/evals/fixtures/stress-arm/devils-advocate-degraded-body.md.txt new file mode 100644 index 00000000000..bdbb36106a0 --- /dev/null +++ b/plugins/planning/evals/fixtures/stress-arm/devils-advocate-degraded-body.md.txt @@ -0,0 +1,9 @@ +**Arguments.** `[deep|shallow] [incumbent [target]|]`. + +## Variables + +Arguments: `$ARGUMENTS` + +## Purpose + +Read the plan or incumbent named in the arguments and report what you think of it. diff --git a/plugins/planning/evals/fixtures/stress-arm/planning.yaml b/plugins/planning/evals/fixtures/stress-arm/planning.yaml new file mode 100644 index 00000000000..2fb2fc995af --- /dev/null +++ b/plugins/planning/evals/fixtures/stress-arm/planning.yaml @@ -0,0 +1,4 @@ +plan_stress_test: + plan: auto + proposal: auto + incumbent: auto diff --git a/plugins/planning/evals/fixtures/stress-arm/prepare-arm.sh b/plugins/planning/evals/fixtures/stress-arm/prepare-arm.sh new file mode 100755 index 00000000000..17e89d149f9 --- /dev/null +++ b/plugins/planning/evals/fixtures/stress-arm/prepare-arm.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Copies the planning plugin to a new directory outside any repository, set up for one arm of the +# st-* suite, which `claude plugin eval` then runs with --ablation none. +# +# prepare-arm.sh [--degrade-devils-advocate] +# +# The first argument is the plan_stress_test value every mode gets in the copy. With +# --degrade-devils-advocate, the copy's devils-advocate skill keeps its frontmatter and takes its +# body from devils-advocate-degraded-body.md.txt: the suite's negative control. +# +# Exit: 0 written; 2 a usage error, or a dest that already exists. +set -euo pipefail + +usage() { + echo "usage: prepare-arm.sh [--degrade-devils-advocate]" >&2 + exit 2 +} + +[[ $# -ge 2 && $# -le 3 ]] || usage +value="$1" +dest="$2" +degrade="${3:-}" +case "$value" in +reviewer | devils-advocate | both | auto) ;; +*) usage ;; +esac +[[ -z "$degrade" || "$degrade" == "--degrade-devils-advocate" ]] || usage +if [[ -e "$dest" ]]; then + echo "prepare-arm.sh: $dest already exists" >&2 + exit 2 +fi + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$here/../../.." && pwd)" + +mkdir -p "$dest" +cp -R "$root/." "$dest/" +rm -rf "${dest:?}/evals/results" + +fixture="$dest/evals/fixtures/stress-arm" +printf 'plan_stress_test:\n plan: %s\n proposal: %s\n incumbent: %s\n' "$value" "$value" "$value" >"$fixture/planning.yaml" + +if [[ -n "$degrade" ]]; then + skill="$dest/skills/devils-advocate/SKILL.md" + frontmatter="$(awk 'NR == 1 || !done { print } NR > 1 && /^---$/ { done = 1 }' "$skill")" + printf '%s\n\n' "$frontmatter" >"$skill" + cat "$fixture/devils-advocate-degraded-body.md.txt" >>"$skill" +fi + +echo "$dest" diff --git a/plugins/planning/evals/st-incumbent-1/case.yaml b/plugins/planning/evals/st-incumbent-1/case.yaml new file mode 100644 index 00000000000..0094e544629 --- /dev/null +++ b/plugins/planning/evals/st-incumbent-1/case.yaml @@ -0,0 +1,6 @@ +schema_version: "1.1" +name: st-incumbent-1 +description: "A draft plan for a veterinary clinic reminder service decides whether to keep its in-process job scheduler." +expected_outcome: "A good review reports that the kept scheduler's literal per-clinic UTC offset cannot hold 09:00 local time across daylight saving changes, that the job store the draft relies on is absent from the pinned 2.8.1 release, and that one store file shared by the two sharded workers makes both fire every job. It does not attack the batch size of 200, which matches the gateway limit." +context: + scaffold_script: scaffold.sh diff --git a/plugins/planning/evals/st-incumbent-1/graders/defect-a.md b/plugins/planning/evals/st-incumbent-1/graders/defect-a.md new file mode 100644 index 00000000000..953400059ad --- /dev/null +++ b/plugins/planning/evals/st-incumbent-1/graders/defect-a.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^(?![^\n]*\bNo plan gaps found)(?![^\n]*\b(?:is|are|was|were|looks?|seems?|remains?)\s+(?!not\b|never\b)(?:(?:perfectly|entirely|totally|already|also|still|actually)\s+)?(?:fine|ok|okay|good|correct|accurate|sound|acceptable|safe|justified|reasonable)\b(?![^\n]*\b(?:but|however|yet)\b))(?![^\n]*\b(?:no|not\s+an?|n''t\s+an?)\s+(?:\w+\s+){0,2}(?:issue|problem|concern|risk|defect|gap|finding|blocker)s?\b)(?![^\n]*\b(?:nothing\s+wrong|no\s+need\s+to|need\s+not|can\s+be\s+ignored|no\s+action\s+needed|no\s+changes?\s+needed)\b)(?![^\n]*\b(?requirements.txt <<'TXT' +httpx==0.27.2 +pyyaml==6.0.2 +tickwheel==2.8.1 +TXT + +cat >reminders/scheduler.py <<'PY' +"""Registers one daily reminder job per clinic and runs the wheel.""" +import os +from zlib import crc32 + +import yaml +from tickwheel import Wheel + +from reminders.sender import send_due_reminders + + +def clinics_for_shard(clinics, shard_index, shard_count): + return [c for c in clinics if crc32(c["id"].encode()) % shard_count == shard_index] + + +def build_wheel(config_path="config/clinics.yaml"): + with open(config_path, encoding="utf-8") as handle: + clinics = yaml.safe_load(handle)["clinics"] + shard_index = int(os.environ["SHARD_INDEX"]) + shard_count = int(os.environ["SHARD_COUNT"]) + wheel = Wheel() + for clinic in clinics_for_shard(clinics, shard_index, shard_count): + wheel.daily( + send_due_reminders, + at=clinic["send_at"], + utc_offset=clinic["utc_offset"], + args=(clinic["id"],), + ) + return wheel + + +if __name__ == "__main__": + build_wheel().run_forever() +PY + +cat >reminders/sender.py <<'PY' +"""Sends one clinic's due reminders through the Postrider gateway.""" +import os +import sqlite3 +from collections import namedtuple + +import httpx + +BATCH_SIZE = 200 +GATEWAY_URL = "https://gateway.postrider.example/v2/messages" + +Reminder = namedtuple("Reminder", "id phone") + + +def _connect(): + return sqlite3.connect(os.environ["REMINDER_DB"]) + + +def due_reminders(clinic_id): + with _connect() as db: + rows = db.execute( + "SELECT id, phone FROM reminders" + " WHERE clinic_id = ? AND sent_at IS NULL AND due_on = date('now')", + (clinic_id,), + ).fetchall() + return [Reminder(*row) for row in rows] + + +def mark_sent(ids): + with _connect() as db: + db.executemany( + "UPDATE reminders SET sent_at = datetime('now') WHERE id = ?", + [(reminder_id,) for reminder_id in ids], + ) + + +def send_due_reminders(clinic_id): + reminders = due_reminders(clinic_id) + headers = {"Authorization": f"Bearer {os.environ['POSTRIDER_TOKEN']}"} + with httpx.Client(timeout=20.0, headers=headers) as client: + for start in range(0, len(reminders), BATCH_SIZE): + batch = reminders[start : start + BATCH_SIZE] + response = client.post( + GATEWAY_URL, + json={"recipients": [r.phone for r in batch], "template": "appointment"}, + ) + response.raise_for_status() + mark_sent([r.id for r in batch]) +PY + +cat >config/clinics.yaml <<'YAML' +clinics: + - id: maple-street + display_name: Maple Street Veterinary + timezone: America/Chicago + utc_offset: "-06:00" + send_at: "09:00" + - id: harbor-road + display_name: Harbor Road Animal Clinic + timezone: America/New_York + utc_offset: "-05:00" + send_at: "09:00" + - id: ridgeview + display_name: Ridgeview Pet Care + timezone: America/Denver + utc_offset: "-07:00" + send_at: "09:00" +YAML + +cat >deploy/compose.yaml <<'YAML' +services: + worker-0: + image: registry.example/reminders:latest + command: python -m reminders.scheduler + environment: + SHARD_INDEX: "0" + SHARD_COUNT: "2" + REMINDER_DB: /data/clinic.db + volumes: + - clinic-data:/data + worker-1: + image: registry.example/reminders:latest + command: python -m reminders.scheduler + environment: + SHARD_INDEX: "1" + SHARD_COUNT: "2" + REMINDER_DB: /data/clinic.db + volumes: + - clinic-data:/data +volumes: + clinic-data: +YAML + +cat >docs/vendor/tickwheel-changelog.md <<'MD' +# Tickwheel changelog + +## 3.0.0 + +- New `FileJobStore(path)`: pass as `Wheel(store=...)` to keep jobs across restarts. +- On start, the wheel loads every job in the store file and schedules it. +- A store file serves one Wheel at a time. The store takes no file lock. + +## 2.8.1 + +- Fixed a drift of up to one second in `daily` triggers under load. + +## 2.8.0 + +- `daily(fn, at, utc_offset)`: `utc_offset` is a literal value such as `-06:00`. +- The offset applies unchanged to every firing of the job. +- Tickwheel ships no zone database and does not read zone names. +MD + +cat >docs/vendor/cadenza-notes.md <<'MD' +# Cadenza scheduler notes + +Cadenza runs jobs from a schedule held in a message broker. + +- Requires a running Starling broker; there is no brokerless mode. +- Triggers accept zone names such as `Europe/Paris`. +- One firing per job across all consumers of a queue. +- Python 3.11 or later. +MD + +cat >docs/vendor/postrider-api.md <<'MD' +# Postrider gateway API, v2 + +`POST /v2/messages` sends one template to a list of recipients. + +- `recipients`: at most 200 entries; a longer list is refused with HTTP 413. +- Rate limit: 20 requests per second per token. +- A request is accepted or refused as a whole. +MD + +cat >.work/review-target/design/design-resolution.md <<'MD' +--- +outcome: early-exit +tier: C +reason: the Brief names the scheduling decision and its options; no type or module design is open +--- +MD + +cat >.work/review-target/PLAN.md <<'MD' +# Appointment reminder scheduling + +## Brief + +### Problem + +The reminder workers hold each day's schedule in memory. A redeploy between midnight and 09:00 +drops that morning's jobs, and clinics find out when owners miss appointments. We have also been +asked whether Tickwheel is still the right scheduler, or whether this is the moment to move. + +### Goal + +Decide whether to keep Tickwheel, and make scheduled reminders dependable for all three clinics. + +### Acceptance criteria + +- AC1: Each clinic's reminders leave at 09:00 clinic-local time on every day of the year. +- AC2: Restarting or redeploying the workers neither loses nor repeats a scheduled reminder. +- AC3: A peak day of 40,000 reminders is fully sent by 09:30 clinic-local time. + +### Out-of-scope + +- Reminder wording and templates. +- New channels beyond SMS. +- The clinic portal. + +### Captured assumptions + +- Quality bar: production +- The workers stay in Python. +- Reminder volume stays within the peak in AC3 for the next year. + +## Plan + +### Decision + +Keep Tickwheel. Turn on its persistent job store and leave the send path on Postrider as it is. + +### Why + +- Tickwheel's `daily` trigger takes a `utc_offset` per job, and `config/clinics.yaml` holds one for + each clinic, so AC1 is met by the schedule we have today. +- The Tickwheel release in our image includes `FileJobStore`, which gives AC2 without new + infrastructure: a job store on disk, read back when a worker starts. +- Cadenza is the main alternative. It needs a running Starling broker, and `deploy/compose.yaml` + runs none, so moving means operating a new stateful service for three clinics. + +### Phase 1: Persist the schedule [STATUS] + +**Satisfies:** AC2 + +Change `build_wheel` in `reminders/scheduler.py` to create +`Wheel(store=FileJobStore("/state/jobs.db"))`. Add a `reminders-state` volume in +`deploy/compose.yaml`, mounted at `/state` on the worker services. + +**Sanity Check:** in staging, restart the workers at 08:00 and confirm the 09:00 jobs still fire. + +### Phase 2: Stable job registration [STATUS] + +**Satisfies:** AC1, AC2 + +Give each `daily` job a stable id of `clinic:` and register it with `replace=True`, so a worker +start refreshes a clinic's job instead of adding another. Keep `send_at` and `utc_offset` in +`config/clinics.yaml` as the single place a clinic's schedule is set. + +**Sanity Check:** start a worker twice against one store and confirm the job list has one entry per +clinic. + +### Phase 3: Peak-day send rate [STATUS] + +**Satisfies:** AC3 + +Keep `BATCH_SIZE` at 200 in `reminders/sender.py`, since the Postrider request limit in +`docs/vendor/postrider-api.md` is 200 recipients. Add a pause between requests that holds a worker +to 15 requests per second, under the gateway's rate limit. At that rate one worker sends 40,000 +reminders in under 15 seconds. + +**Sanity Check:** run 40,000 reminders against the gateway sandbox and record the finish time. + +### Load-bearing assumptions + +- A worker can read the job store as soon as its volume is mounted. +- `mark_sent` runs after each accepted batch, so a restart mid-send resumes from the unsent rows. + +### Test strategy + +- Unit: `build_wheel` registers one job per clinic for the shard, with the configured time and + offset. +- Integration: a worker restart in staging keeps its jobs, checked with the store's job list. +- Load: the sandbox run in Phase 3 finishes inside the AC3 window. +MD + +# Each phase heading takes its status tag here, outside the heredoc. +plan=.work/review-target/PLAN.md +text="$(cat "$plan")" +printf '%s\n' "${text//'[STATUS]'/[TODO]}" >"$plan" diff --git a/plugins/planning/evals/st-incumbent-2/case.yaml b/plugins/planning/evals/st-incumbent-2/case.yaml new file mode 100644 index 00000000000..15a958705e8 --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/case.yaml @@ -0,0 +1,6 @@ +schema_version: "1.1" +name: st-incumbent-2 +description: "A draft plan for an event ticketing API decides whether to replace the locking library behind its seat holds." +expected_outcome: "A good review reports that the Emberstore row claim feature already in the deployed database was never weighed, that the one-module claim is wrong because src/checkout/complete.ts also calls Holdfast's extend, and that holds under the hf: prefix are invisible to Leaseline during the rollout, so a seat can get a second holder. It does not call the Node version a problem, since the project runs Node 22." +context: + scaffold_script: scaffold.sh diff --git a/plugins/planning/evals/st-incumbent-2/graders/defect-a.md b/plugins/planning/evals/st-incumbent-2/graders/defect-a.md new file mode 100644 index 00000000000..397945ec72e --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/graders/defect-a.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^(?![^\n]*\bNo plan gaps found)(?![^\n]*\b(?:is|are|was|were|looks?|seems?|remains?)\s+(?!not\b|never\b)(?:(?:perfectly|entirely|totally|already|also|still|actually)\s+)?(?:fine|ok|okay|good|correct|accurate|sound|acceptable|safe|justified|reasonable)\b(?![^\n]*\b(?:but|however|yet)\b))(?![^\n]*\b(?:no|not\s+an?|n''t\s+an?)\s+(?:\w+\s+){0,2}(?:issue|problem|concern|risk|defect|gap|finding|blocker)s?\b)(?![^\n]*\b(?:nothing\s+wrong|no\s+need\s+to|need\s+not|can\s+be\s+ignored|no\s+action\s+needed|no\s+changes?\s+needed)\b)(?![^\n]*\bnot\s+an?\s+(?:\w+\s+)?(?:alternative|option)\b)(?![^\n]*\b(?:doesn''t|does\s+not|cannot|can''t)\s+(?:meet|cover|satisfy|replace)\b)(?![^\n]*\bwas\s+(?:weighed|considered|compared)\b)(?=[^\n]*(?:\bEmberstore\b|\bHOLD_UNTIL\b|\bCLAIM\b|\brow\s+claims?\b))[^\n]*(?:\bnever\s+(?:\w+\s+)?(?:weigh|consider|compar|evaluat|mention|address)\w*|\bnot\s+(?:\w+\s+){0,2}(?:weighed|considered|compared|evaluated|mentioned|discussed|assessed)\b|\b(?:doesn''t|does\s+not|fails?\s+to)\s+(?:weigh|consider|compare|evaluate|mention)\b|\bunweighed\b|\bunconsidered\b|\bignor\w*|\boverlook\w*|\bomit\w*|\balready\s+(?:provides?|offers?|has|supports?|runs?|meets?|covers?)\b|\bbuilt[- ]?in\b|\bnative\w*|\bwithout\s+(?:a\s+|any\s+)?(?:new\s+)?(?:library|dependency|Redis)\b|\b(?:could|would|can)\s+(?:\w+\s+){0,3}(?:replace|meet|cover|satisfy|remove|retire)\b|\bmeets?\s+(?:AC\d|the\s+(?:requirement|need|criteria))\b|\binstead\s+of\b|\balternative\b|\bsimpler\b)' +flags: mi +target: last_message +--- diff --git a/plugins/planning/evals/st-incumbent-2/graders/defect-b.md b/plugins/planning/evals/st-incumbent-2/graders/defect-b.md new file mode 100644 index 00000000000..c0083d2aa83 --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/graders/defect-b.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^(?![^\n]*\bNo plan gaps found)(?![^\n]*\b(?:is|are|was|were|looks?|seems?|remains?)\s+(?!not\b|never\b)(?:(?:perfectly|entirely|totally|already|also|still|actually)\s+)?(?:fine|ok|okay|good|correct|accurate|sound|acceptable|safe|justified|reasonable)\b(?![^\n]*\b(?:but|however|yet)\b))(?![^\n]*\b(?:no|not\s+an?|n''t\s+an?)\s+(?:\w+\s+){0,2}(?:issue|problem|concern|risk|defect|gap|finding|blocker)s?\b)(?![^\n]*\b(?:nothing\s+wrong|no\s+need\s+to|need\s+not|can\s+be\s+ignored|no\s+action\s+needed|no\s+changes?\s+needed)\b)(?![^\n]*\b(?:does\s+not|doesn''t|never)\s+(?:\w+\s+)?(?:call|use|import)\b)(?=[^\n]*(?:\bcomplete\.ts\b|\bcompleteCheckout\b|\bextendLock\b|\bsrc/checkout\b))[^\n]*(?:\balso\s+(?:calls?|uses?|imports?)\b|\bsecond\s+(?:caller|call\s*site|use\w*|import\w*|module)\b|\banother\s+(?:caller|call\s*site|use\w*|import\w*|module)\b|\bnot\s+(?:the\s+)?(?:only|one)\b|\bnot\s+(?:\w+\s+){0,2}(?:listed|covered|migrated|mentioned|included|changed|updated)\b|\bmisse[sd]\b|\boverlook\w*|\bstill\s+(?:calls?|uses?|imports?)\b|\bomit\w*|\bimports?\s+(?:\w+\s+){0,2}holdfast\b|\bcalls?\s+(?:\w+\s+){0,2}holdfast\b|\bcontradict\w*|\bunmigrated\b|\bignor\w*|\bholdfast(?:''s)?\s+(?:\w+\s+)?(?:extend|caller|import)\w*)' +flags: mi +target: last_message +--- diff --git a/plugins/planning/evals/st-incumbent-2/graders/defect-c.md b/plugins/planning/evals/st-incumbent-2/graders/defect-c.md new file mode 100644 index 00000000000..738ef4400dc --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/graders/defect-c.md @@ -0,0 +1,6 @@ +--- +type: regex +pattern: '^(?![^\n]*\bNo plan gaps found)(?![^\n]*\b(?:is|are|was|were|looks?|seems?|remains?)\s+(?!not\b|never\b)(?:(?:perfectly|entirely|totally|already|also|still|actually)\s+)?(?:fine|ok|okay|good|correct|accurate|sound|acceptable|safe|justified|reasonable)\b(?![^\n]*\b(?:but|however|yet)\b))(?![^\n]*\b(?:no|not\s+an?|n''t\s+an?)\s+(?:\w+\s+){0,2}(?:issue|problem|concern|risk|defect|gap|finding|blocker)s?\b)(?![^\n]*\b(?:nothing\s+wrong|no\s+need\s+to|need\s+not|can\s+be\s+ignored|no\s+action\s+needed|no\s+changes?\s+needed)\b)(?![^\n]*\b(?:won''t|will\s+not|cannot|can''t|not|never|no)\s+(?:be\s+)?(?:\w+\s+)?(?:double[- ]?\w+|oversold|oversell\w*))(?=[^\n]*(?:\b(?:in[- ]?flight|live|active|existing|outstanding|open|unexpired|current|pre-cutover|old)\s+(?:\w+\s+){0,2}(?:holds?|leases?|locks?|keys?)\b|\bhf:|\bmixed\s+(?:deploy\w*|fleet|versions?)\b|\brolling\s+(?:deploy|update|restart|cutover|rollout)\w*|\bcut-?over\b|\bduring\s+(?:the\s+)?(?:deploy\w*|rollout|switch\w*|flip)\b|\bdifferent\s+(?:key\s+)?prefix\w*|\blease:))[^\n]*(?:\bdouble[- ]?(?:sold|sell\w*|book\w*|hold\w*|held)\b|\boversell\w*|\boversold\b|\bsame\s+seat\b|\bsecond\s+(?:hold\w*|buyer|lease)\b|\btwo\s+(?:holds|holders|buyers|customers|leases)\b|\binvisible\b|\b(?:can''t|cannot|can\s+not|doesn''t|does\s+not|won''t|will\s+not)\s+(?:see|read|honou?r|recogni[sz]e|detect)\b|\bnot\s+(?:visible|honou?red|recogni[sz]ed|seen)\b|\bunaware\b|\bbreaks?\s+AC1\b|\bviolates?\s+AC1\b)' +flags: mi +target: last_message +--- diff --git a/plugins/planning/evals/st-incumbent-2/graders/sound-part.md b/plugins/planning/evals/st-incumbent-2/graders/sound-part.md new file mode 100644 index 00000000000..9077ed33e06 --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/graders/sound-part.md @@ -0,0 +1,7 @@ +--- +type: regex +pattern: '^(?![^\n]*\bNo plan gaps found)(?![^\n]*\b(?:is|are|was|were|looks?|seems?|remains?)\s+(?!not\b|never\b)(?:(?:perfectly|entirely|totally|already|also|still|actually)\s+)?(?:fine|ok|okay|good|correct|accurate|sound|acceptable|safe|justified|reasonable)\b(?![^\n]*\b(?:but|however|yet)\b))(?![^\n]*\b(?:no|not\s+an?|n''t\s+an?)\s+(?:\w+\s+){0,2}(?:issue|problem|concern|risk|defect|gap|finding|blocker)s?\b)(?![^\n]*\b(?:nothing\s+wrong|no\s+need\s+to|need\s+not|can\s+be\s+ignored|no\s+action\s+needed|no\s+changes?\s+needed)\b)(?![^\n]*\b(?:no|without(?:\s+an?)?|needs\s+no)\s+(?:Node\s+)?(?:upgrade|bump)\b)(?![^\n]*(?:\bnot|n''t)\s+(?:\w+\s+)?(?:incompatib\w*|unsupported|too\s+old|a\s+conflict|a\s+mismatch))(?=[^\n]*(?:\bNode(?:\.js)?\b))[^\n]*(?:\bupgrades?\b|\btoo\s+old\b|\bolder\s+than\b|\bincompatib\w*|\bunsupported\b|\bnot\s+(?:met|satisfied)\b|\bbump\b|\bconflicts?\b|\bmismatch\w*|\bwrong\b)' +flags: mi +match: not_contains +target: last_message +--- diff --git a/plugins/planning/evals/st-incumbent-2/prompt.md b/plugins/planning/evals/st-incumbent-2/prompt.md new file mode 100644 index 00000000000..e6235ad5892 --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/prompt.md @@ -0,0 +1,9 @@ +--- +tags: [plan-stress-test] +runs: 3 +max_turns: 40 +timeout_seconds: 1500 +allowed_tools: [Read, Glob, Grep, Skill, Bash, Agent] +--- + +/planning:plan Continue the draft at `.work/review-target/PLAN.md` from Step 3. The Brief and the design gate are settled; `.work/review-target/design/design-resolution.md` records the design early exit. Run Steps 3, 3b and 4 as this skill and this repository's settings say. Then list every finding you verified as one bullet, naming the file, phase or claim it concerns and saying what is wrong. Stop there: no Step 4.5 onward, and edit no file. diff --git a/plugins/planning/evals/st-incumbent-2/samples/defect-a.json b/plugins/planning/evals/st-incumbent-2/samples/defect-a.json new file mode 100644 index 00000000000..be9dcecb8de --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/samples/defect-a.json @@ -0,0 +1,54 @@ +{ + "pass": [ + { + "answer": "- Emberstore 6.1 row claims (`HOLD_UNTIL`) were never weighed: the seats database already in compose could hold seats with no library.\n", + "why": "bullet naming the feature" + }, + { + "answer": "| 1 | IMPORTANT | Plan | The draft does not consider Emberstore's `CLAIM ... UNTIL`, which meets AC1 and AC2 | Compare it to Leaseline |\n", + "why": "table row" + }, + { + "answer": "- `HOLD_UNTIL` in docs/vendor/emberstore-release-notes.md is an alternative that would retire holds-redis.\n", + "why": "alternative" + }, + { + "answer": "| 2 | IMPORTANT | Plan | Emberstore 6.2 could replace Leaseline and Redis for holds; Why overlooks it | Add it to Why |\n", + "why": "overlooks" + }, + { + "answer": "- Holds could use the built-in row claim in Emberstore instead of a new library.\n", + "why": "built-in" + } + ], + "fail": [ + { + "answer": "", + "why": "empty" + }, + { + "answer": "No plan gaps found.\n", + "why": "clean pass" + }, + { + "answer": "- Emberstore's HOLD_UNTIL is not an alternative here, because holds must stay in Redis.\n", + "why": "dismissed" + }, + { + "answer": "- Emberstore row claims cannot replace Leaseline.\n", + "why": "negated" + }, + { + "answer": "- Emberstore 6.2 is the current image in compose.\n", + "why": "named, no nature" + }, + { + "answer": "- Phase 3 has no rollback step for the lockfile refresh.\n", + "why": "another finding" + }, + { + "answer": "- Consider a simpler rollout for Phase 2.\n", + "why": "generic, nature word without the feature" + } + ] +} diff --git a/plugins/planning/evals/st-incumbent-2/samples/defect-b.json b/plugins/planning/evals/st-incumbent-2/samples/defect-b.json new file mode 100644 index 00000000000..472a438f5f2 --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/samples/defect-b.json @@ -0,0 +1,54 @@ +{ + "pass": [ + { + "answer": "- Why says Holdfast is called from one module, but `src/checkout/complete.ts` also imports it for `extend`.\n", + "why": "bullet naming the file" + }, + { + "answer": "| 1 | CRITICAL | Plan | `completeCheckout` still calls Holdfast's `extend`; Phase 3's removal breaks the build | Add it to Phase 1 |\n", + "why": "table row with function" + }, + { + "answer": "- `extendLock` in complete.ts is a second call site the draft missed.\n", + "why": "second call site" + }, + { + "answer": "| 2 | IMPORTANT | Plan | src/checkout is not covered by Phase 1, and Holdfast's extend lives there | Migrate it |\n", + "why": "not covered" + }, + { + "answer": "- complete.ts contradicts the one-module claim.\n", + "why": "contradicts" + } + ], + "fail": [ + { + "answer": "", + "why": "empty" + }, + { + "answer": "No plan gaps found.\n", + "why": "clean pass" + }, + { + "answer": "- `complete.ts` does not call Holdfast directly, so the one-module claim holds.\n", + "why": "negated" + }, + { + "answer": "- completeCheckout is fine as it is.\n", + "why": "dismissed" + }, + { + "answer": "- `src/checkout/complete.ts` has a five-minute payment window.\n", + "why": "named, no nature" + }, + { + "answer": "- Phase 3 has no rollback step for the lockfile refresh.\n", + "why": "another finding" + }, + { + "answer": "- The holds module also uses `ioredis`.\n", + "why": "nature without the file" + } + ] +} diff --git a/plugins/planning/evals/st-incumbent-2/samples/defect-c.json b/plugins/planning/evals/st-incumbent-2/samples/defect-c.json new file mode 100644 index 00000000000..329081b521c --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/samples/defect-c.json @@ -0,0 +1,54 @@ +{ + "pass": [ + { + "answer": "- During the rollout, Leaseline cannot see `hf:` keys, so a seat held under Holdfast can get a second holder: AC1 breaks for up to ten minutes.\n", + "why": "bullet" + }, + { + "answer": "| 1 | CRITICAL | Plan | Live holds under `hf:` are invisible to `lease:` code; the same seat can be double-sold in the rollout | Check both prefixes |\n", + "why": "table row" + }, + { + "answer": "- Phase 2's rolling update leaves mixed versions for minutes, and two buyers can hold the same seat.\n", + "why": "rolling" + }, + { + "answer": "| 2 | IMPORTANT | Plan | Existing Holdfast locks are not honored at cutover, so seats get oversold | Drain first |\n", + "why": "cutover" + }, + { + "answer": "- Phase 2 says there is no data to migrate, but unexpired holds are unaware of leases, which violates AC1.\n", + "why": "quotes then contradicts" + } + ], + "fail": [ + { + "answer": "", + "why": "empty" + }, + { + "answer": "No plan gaps found.\n", + "why": "clean pass" + }, + { + "answer": "- Seats will not be double-sold during the rollout; holds expire in ten minutes.\n", + "why": "negated" + }, + { + "answer": "- The rolling update is fine since holds are short-lived.\n", + "why": "dismissed" + }, + { + "answer": "- Phase 2 needs a rollback if the hold error rate rises.\n", + "why": "generic rollout warning" + }, + { + "answer": "- Phase 3 has no rollback step for the lockfile refresh.\n", + "why": "another finding" + }, + { + "answer": "- Two buyers can lose out when a card is declined.\n", + "why": "nature without the cutover" + } + ] +} diff --git a/plugins/planning/evals/st-incumbent-2/samples/sound-part.json b/plugins/planning/evals/st-incumbent-2/samples/sound-part.json new file mode 100644 index 00000000000..4e87753ef2d --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/samples/sound-part.json @@ -0,0 +1,46 @@ +{ + "pass": [ + { + "answer": "", + "why": "empty" + }, + { + "answer": "No plan gaps found.\n", + "why": "clean pass" + }, + { + "answer": "- Node 22 meets Leaseline's floor of Node 20, so no upgrade is needed.\n", + "why": "praises the element" + }, + { + "answer": "- Leaseline's Node floor is not a conflict: `package.json` says >=22.\n", + "why": "negated attack" + }, + { + "answer": "- `src/checkout/complete.ts` also imports Holdfast.\n", + "why": "other finding" + } + ], + "fail": [ + { + "answer": "- Leaseline requires an upgrade of Node before Phase 1.\n", + "why": "bullet attack" + }, + { + "answer": "| 1 | IMPORTANT | Plan | Leaseline is incompatible with our Node version | Upgrade |\n", + "why": "table row" + }, + { + "answer": "- The API's Node.js version is too old for Leaseline.\n", + "why": "too old" + }, + { + "answer": "- There is a Node version mismatch between package.json and Leaseline.\n", + "why": "mismatch" + }, + { + "answer": "| 2 | SUGGESTION | Plan | Node floor not satisfied by the image | Bump the image |\n", + "why": "not satisfied" + } + ] +} diff --git a/plugins/planning/evals/st-incumbent-2/scaffold.sh b/plugins/planning/evals/st-incumbent-2/scaffold.sh new file mode 100755 index 00000000000..e4827a793f4 --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/scaffold.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Seeds the workspace, then applies this arm's plan_stress_test setting. +set -euo pipefail +here="$(dirname "${BASH_SOURCE[0]}")" +bash "$here/seed.sh" +bash "$here/../fixtures/stress-arm/apply-arm.sh" diff --git a/plugins/planning/evals/st-incumbent-2/seed.sh b/plugins/planning/evals/st-incumbent-2/seed.sh new file mode 100755 index 00000000000..02b5cc5cef6 --- /dev/null +++ b/plugins/planning/evals/st-incumbent-2/seed.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +set -euo pipefail + +git init -q -b main +git config user.name eval +git config user.email eval@example.invalid + +mkdir -p src/seats src/checkout src/db deploy docs/vendor .work/review-target/design + +cat >package.json <<'JSON' +{ + "name": "boxoffice-api", + "version": "4.3.0", + "private": true, + "type": "module", + "engines": { "node": ">=22" }, + "scripts": { + "build": "tsc -p .", + "test": "vitest run" + }, + "dependencies": { + "fastify": "4.28.1", + "holdfast": "2.9.4", + "ioredis": "5.4.1" + }, + "devDependencies": { + "typescript": "5.5.4", + "vitest": "2.0.5" + } +} +JSON + +cat >src/seats/holds.ts <<'TS' +import { acquire, release, type Lock } from "holdfast"; +import { redis } from "../db/redis.js"; + +const HOLD_MS = 10 * 60 * 1000; +const open = new Map(); + +export async function placeHold(seatId: string, buyerId: string): Promise { + const lock = await acquire(redis, `seat:${seatId}`, { ttlMs: HOLD_MS, owner: buyerId }); + if (lock === null) return false; + open.set(seatId, lock); + return true; +} + +export async function dropHold(seatId: string): Promise { + const lock = open.get(seatId); + if (lock === undefined) return; + await release(redis, lock); + open.delete(seatId); +} +TS + +cat >src/checkout/complete.ts <<'TS' +import { extend as extendLock } from "holdfast"; +import { redis } from "../db/redis.js"; +import { chargeBuyer } from "./payments.js"; +import { dropHold } from "../seats/holds.js"; + +const PAYMENT_WINDOW_MS = 5 * 60 * 1000; + +export async function completeCheckout(seatId: string, buyerId: string, cardToken: string) { + const extended = await extendLock(redis, `seat:${seatId}`, { + ttlMs: PAYMENT_WINDOW_MS, + owner: buyerId, + }); + if (!extended) throw new Error("hold expired"); + const receipt = await chargeBuyer(buyerId, cardToken); + await dropHold(seatId); + return receipt; +} +TS + +cat >src/db/schema.sql <<'SQL' +CREATE TABLE seats ( + seat_id TEXT PRIMARY KEY, + event_id TEXT NOT NULL, + section TEXT NOT NULL, + sold_to TEXT +); + +CREATE TABLE orders ( + order_id TEXT PRIMARY KEY, + seat_id TEXT NOT NULL REFERENCES seats (seat_id), + buyer_id TEXT NOT NULL, + paid_at TIMESTAMP NOT NULL +); +SQL + +cat >deploy/compose.yaml <<'YAML' +services: + api: + image: node:22-alpine + command: node dist/server.js + deploy: + replicas: 4 + update_config: + parallelism: 1 + delay: 60s + environment: + REDIS_URL: redis://holds-redis:6379 + EMBERSTORE_URL: emberstore://seats-db:7300/boxoffice + holds-redis: + image: redis:7 + seats-db: + image: emberstore:6.2.0 + volumes: + - seats:/var/lib/emberstore +volumes: + seats: +YAML + +cat >docs/vendor/holdfast-changelog.md <<'MD' +# Holdfast changelog + +## 2.9.4 + +- Final 2.x release. The maintainers have ended support for Holdfast. + +## 2.9.0 + +- `extend(client, name, { ttlMs, owner })` moves the expiry of a named lock for its owner. + +## 2.0.0 + +- `acquire` stores a lock under the Redis key `hf:`. +MD + +cat >docs/vendor/leaseline-notes.md <<'MD' +# Leaseline notes + +- Node 20 or later. +- `acquire(client, name, { ttlMs, holder })` returns a `Lease`, or `null` when `name` is taken. +- `lease.renew(ttlMs)` and `lease.release()` act on the `Lease` object. +- `renewByName(client, name, { ttlMs, holder })` is available from 1.4. +- Leases are stored under the Redis key `lease:`, as a hash. +- Leaseline reads and writes keys with that prefix and no others. +- Current release: 1.6.2, actively maintained. +MD + +cat >docs/vendor/emberstore-release-notes.md <<'MD' +# Emberstore release notes + +## 6.2 + +- Faster secondary index builds. + +## 6.1 + +- Row claims: `CLAIM FOR UNTIL