diff --git a/.github/workflows/e2e-checks.yml b/.github/workflows/e2e-checks.yml new file mode 100644 index 00000000..7ec87815 --- /dev/null +++ b/.github/workflows/e2e-checks.yml @@ -0,0 +1,54 @@ +name: e2e-checks + +# Free, deterministic layers of the e2e harness — run on EVERY PR (PLAN §3/§11: +# "L0/L1 (free, deterministic) stay on every PR; only L2 is gated behind manual +# invocation"). No models, no paid APIs, no network — just the CLI + harness +# plumbing. The paid L2 agentic matrix is manual/human-triggered and lives +# elsewhere (never on push/PR/schedule). + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + static: + name: L0/L1 static + smoke + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + + # L0 (partial): syntax-check every harness module so a broken import or + # parse error fails fast and cheap. + - name: Syntax-check harness modules + run: | + set -e + for f in $(find tests/e2e -name '*.mjs'); do + node --check "$f" + done + echo "all harness modules parse" + + # L1 — install/bootstrap smoke: run the CLI against a scratch project and + # assert placement + idempotency + byte-clean uninstall. + - name: L1 install/uninstall smoke + run: npm run test:e2e:l1 + + # Harness self-check: every scenario spec parses, the fixture wiring loads, + # and the run matrix builds — without spending anything. Catches a broken + # scenario JSONC or adapter registration before a paid L2 run ever hits it. + - name: Harness self-check (scenarios parse + matrix builds) + run: | + node tests/e2e/runner/run-e2e.mjs --list + node tests/e2e/runner/run-e2e.mjs --vendor mock --dry-run + + # NOTE: L0 package unit tests (CLI file-placement/marker semantics, + # JSON-Schema validation of sample payloads, validate-skill-md lint) plug + # in here once feat/write-unit-tests lands — add them as further steps in + # this job so they gate every PR too. diff --git a/.gitignore b/.gitignore index 4d0de8f4..3a916083 100644 --- a/.gitignore +++ b/.gitignore @@ -60,6 +60,12 @@ logs/ coverage/ .nyc_output/ +# e2e harness — raw run output (regenerated each run) and vendored fixture apps +# (large, external — built on demand by each fixture's vendor.mjs, never committed) +results/ +tests/e2e/fixtures/*/states/ +tests/e2e/fixtures/*/.upstream-cache/ + # vinta-ai-maestro run state — journal, transcripts, gate logs. Kept inside the # project (never a global cache) because it contains repository contents # verbatim; see packages/vinta-ai-maestro/SPEC.md § Security and data handling. diff --git a/PLAN-e2e-testing-harness.md b/PLAN-e2e-testing-harness.md new file mode 100644 index 00000000..93473f0b --- /dev/null +++ b/PLAN-e2e-testing-harness.md @@ -0,0 +1,293 @@ +# PLAN — End-to-end testing & benchmarking harness + +**Status:** Draft for review +**Owner:** _TBD_ +**Related work in flight:** `feat/write-unit-tests`, `feat/make-e2e-tests-opt-in`, `feat/models-nightly-checks`, `feat/filesystem-sandbox-for-worktrees` + +--- + +## 1. Objective + +`vinta-ai-workflows` ships prompts, skills, sub-agents and per-vendor wiring that are executed by an *AI agent*, not by our code. Unit tests can prove the CLI copies files and that a `SKILL.md` is well-formed, but they can't prove the thing we actually sell: that a bootstrapped project lets an agent take a task from prompt to healthy, merged code — **across every supported vendor and model**. + +This harness closes that gap. It: + +1. Drives real agents through the canonical workflows against fixture projects. +2. Verifies the *outcome* (feature works, bug fixed, script runs) and the *shape of the work* (git tree healthy, artifacts schema-valid). +3. Runs the same scenarios across the **model × vendor matrix** (Claude Code + Codex for v1) so we can compare quality and catch a vendor-specific regression. +4. Records **token consumption + cost + wall-time** per run so we can benchmark between package versions and flag blow-ups. + +**Execution model:** the L2 agentic tests are **manual, human-triggered** — run before a release or to investigate a specific change, never on an automatic cron. Cost is not the primary constraint; *human control over when they run* is. Two hard requirements follow: every scenario must be **runnable individually** (pick one scenario / vendor / model without running the whole matrix), and every skill invocation must be **fully pre-answered** so no run stalls waiting for interactive input (§6). + +Non-goals: replacing unit/lint tests (they stay, cheaper and deterministic); testing the AI models themselves; guaranteeing bit-identical output (LLMs are non-deterministic — we assert on invariants and pass-rates, not exact diffs); automatic scheduled runs. + +--- + +## 2. What we are testing (product surface) + +| Surface | Entry point | Covered by scenario | Fixture state | +|---|---|---|---| +| Harness installation | `npx vinta-ai-workflows install` + `/vinta-bootstrap-ai-tools` | **S0 — Bootstrap** | clean (no `ai-tools/`) | +| Harness upgrade | `/vinta-sync-ai-tools` | **S1 — Sync** | stale harness (older version) | +| Feature delivery pipeline | `/create-spec` → `/plan-feature` → `/implement-plan` | **S2 — Feature** | current harness | +| Bug workflow | `/systematic-debugging` → fix | **S3 — Bug** | current harness | +| One-off script | `/add-one-off-script` | **S4 — Script** | current harness | +| Cross-cutting outcome checks | app runs / tests pass; git tree shape | **E2E graders** (applied to S2–S4) | — | + +The three fixture states are all first-class coverage (§4): **S0** proves bootstrap works on a clean repo, **S1** proves sync correctly upgrades an already-configured repo, and **S2–S4** run the shipped skills against a repo that already has the current harness (so skill quality is tested without paying for bootstrap each time). + +--- + +## 3. Test layers (cheap → expensive) + +Run the cheap layers on every PR; gate the expensive, paid-API layers behind manual invocation (§11). + +- **L0 — Package static tests** *(no model, milliseconds)* + CLI unit tests (install/update/uninstall file placement + marker semantics + idempotency), JSON-Schema validation of every sample payload, `validate-skill-md` lint, `check-ai-models` freshness. Mostly the domain of `feat/write-unit-tests`; the harness only *depends* on this passing. + +- **L1 — Install/bootstrap smoke** *(no model or one cheap model, seconds)* + Run the CLI against a scratch project, assert vendor dirs + skills land, `uninstall` leaves the tree byte-clean (only touches marker-tagged artifacts). Optionally run `/vinta-bootstrap-ai-tools` with a small model and assert the generated layout is schema-valid — this is the deterministic half of S0. + +- **L2 — Agentic scenario E2E** *(real models, minutes, paid)* + The core of this plan: S0–S4 driven by live agents across the matrix, with outcome + git-shape grading and telemetry capture. + +--- + +## 4. Fixture / template projects + +Scenarios need realistic, **seeded** repos so runs are reproducible. The primary fixture is a real healthcare app on the stack Vinta actually ships, so the harness exercises the workflows on representative code — not a toy. + +``` +tests/e2e/fixtures/ + medplum-provider/ # PRIMARY — React + Vite + TypeScript on @medplum/core / @medplum/react, + # React Router, Mantine; tests via Vitest + MockClient. + # Vendored from medplum/examples/medplum-provider, pinned to a commit. + node-ts-cli/ # (later) a small non-Medplum TS CLI — proves stack-agnosticism (Phase 3) + (add stacks as coverage grows) +``` + +**Why `medplum-provider`.** Vinta's current focus is Medplum-based healthcare apps, so the initial fixture should be the [medplum-provider example](https://github.com/medplum/medplum/tree/main/examples/medplum-provider). It's a real app, and — crucially for this harness — its test suite runs **fully offline against `MockClient`** (an in-memory FHIR store), so the outcome grader ("app-tests-pass") needs **no live Medplum server**. This repo already ships a Medplum test pack (`write-unit-test-packs/stacks/medplum.md` + the `test.globalSetup.ts` / `test.setup.ts` pair), so the fixture should adopt those exact conventions — the harness then also implicitly validates that the shipped Medplum pack produces passing tests. + +Medplum-specific fixture requirements (from the pack): + +- Wire the two Vitest setup files (`test.globalSetup.ts` indexing FHIR bundles once, `test.setup.ts` sharing the schema to workers) so `searchResources` filters work — otherwise filtered searches silently return nothing. +- Keep `@medplum/core`, `@medplum/definitions`, `@medplum/mock`, `@medplum/fhirtypes` on **matching versions** (a skew causes confusing index/search failures) — pin them in the vendored copy. +- Node ≥ 22.4 (or a `ws` dev dep) for the WebSocket polyfill `@medplum/core` v5 needs at module load. +- **Outcome verification is the Vitest suite, not a booted app.** The default app-tests-pass grader runs `vitest` against `MockClient` **offline** — no server, fast, deterministic — which is right for the common case and everything `MockClient` can assert. + - **Optional live-stack path.** When a scenario genuinely needs server-only behavior (access policies, subscriptions, `$everything`, bot execution, GraphQL/transaction edge cases), the full Medplum stack can be brought up locally via Docker ([self-hosting guide](https://www.medplum.com/docs/self-hosting/running-full-medplum-stack-in-docker)) and the grader run as an **integration** check against it. Keep this **opt-in and off by default** — it's slower, needs Docker + seeding + auth, and adds nondeterminism. Reserve it for the few tasks whose contract the mock can't prove; unit-level scenarios stay on `MockClient`. + +Each fixture ships **three committed states** (branches or sibling dirs) so the same base app drives all scenario families: + +1. **`clean/`** — the app with *no* `ai-tools/`, `AGENTS.md`, or vendor dirs. Drives **S0 — Bootstrap**. +2. **`stale/`** — the app bootstrapped against an **older** `vinta-ai-workflows` version (deliberately behind current, so a real diff exists to apply). Drives **S1 — Sync**. +3. **`current/`** — the app bootstrapped against the **current** version, harness fully in place. Drives **S2–S4**. + +Plus: + +- A working app + green test suite at baseline (all three states). +- `fixture.json` — metadata: stack, run/test commands, per-state git refs, the version `stale/` was bootstrapped from. +- **Seeded task inputs** per scenario, each carrying its **canned answers** (§6). For `medplum-provider` these are healthcare/FHIR-flavored and unit-testable via `MockClient` (no server-only behavior — access policies, subscriptions, `$everything` — since those can't be asserted offline): + - `tasks/feature.md` — a feature request + pre-answered interview responses (drives S2). E.g. "add a patient allergy panel that reads/writes `AllergyIntolerance` resources." + - `tasks/bug.md` — a reproducible bug: a planted defect + a *failing* `MockClient` test that currently fails (drives S3). The planted bug is the ground truth the grader checks against. E.g. a resource written with the wrong field so an assertion on `patient.name[0].family` fails. + - `tasks/script.md` — a one-off script request with a checkable side effect (drives S4). E.g. "emit a CSV of all `Observation` values for a patient," runnable against a seeded `MockClient` or fixture bundle. + +Design rules: fixture states are frozen (tags/SHAs), each run operates on an **isolated copy** (git worktree or tmp clone — never mutate the fixture in place), and every task has a *machine-checkable* success signal so grading isn't purely subjective. + +> **Maintenance note:** `stale/` must be periodically re-based to "current minus one meaningful change" so the sync scenario keeps exercising a real upgrade rather than a no-op. Cheapest to regenerate it from `clean/` by bootstrapping with a pinned older package version. + +--- + +## 5. Scenario catalog + +Each scenario is a declarative spec consumed by the runner. Every step carries **canned answers** so the agent never blocks on an interview question (§6): + +```jsonc +// tests/e2e/scenarios/S2-feature.jsonc +{ + "id": "S2-feature", + "fixture": "medplum-provider", + "state": "current", // clean | stale | current + "steps": [ + { "invoke": "/create-spec", "input": "tasks/feature.md", "answers": "tasks/feature.answers.md" }, + { "invoke": "/plan-feature", "answers": "tasks/feature.answers.md" }, + { "invoke": "/implement-plan", "autonomous": true } + ], + "graders": ["artifacts-spec", "artifacts-plan", "app-tests-pass", "git-tree-healthy"], + "budget": { "maxWallSec": 1800 } // safety timeout, not a cost gate +} +``` + +**S0 — Bootstrap** *(state: clean)*. `install --tool ` → assert placement; `/vinta-bootstrap-ai-tools` (with canned answers to the bootstrap interview) → assert `AGENTS.md` + derived skills + sub-agents + per-vendor wiring exist and are schema-valid; `uninstall` → assert clean. Graders: deterministic file/schema checks + advisory LLM-judge on `AGENTS.md` relevance to the fixture's stack. + +**S1 — Sync** *(state: stale)*. Run `/vinta-sync-ai-tools` against the older-version fixture. Graders: the harness files converge on what a current bootstrap produces (schema versions bumped, new foundation skills present, per-change apply gating honored), hand-tuned regions are **not** clobbered, and opt-outs stay sticky. This is the "sync actually upgrades correctly" check. + +**S2 — Feature** *(state: current)*. create-spec → plan-feature → implement-plan. Graders: `*_SPEC.md` and `*_PLAN.md` exist with required sections (schema/section-presence check), the app's test suite passes with the new behavior, and the git tree is healthy (§7). + +**S3 — Bug** *(state: current)*. `/systematic-debugging` pointed at the failing test/repro. Graders: the previously-failing test now passes, no other tests regress, the fix touches the region of the planted bug (sanity guard against "deleted the test"), git tree healthy. + +**S4 — One-off script** *(state: current)*. `/add-one-off-script` for a task with a checkable effect (e.g. "emit a CSV of X"). Graders: script exists in the expected location, executes non-interactively, produces the expected artifact/output, lands as its own commit (not mixed into app code). + +--- + +## 6. Runner architecture + +``` +scenario.jsonc ──► Runner ──► VendorAdapter(vendor, model) ──► isolated fixture copy + │ │ + │◄──────────── result envelope ────────────────┘ + ▼ + Graders + Telemetry ──► results/////run-N.json +``` + +**Isolation.** Each run gets a fresh worktree or tmp clone of the fixture; agents run with permissions pre-granted in a sandbox (headless can't answer prompts) — reuse the sandboxing already explored in `feat/filesystem-sandbox-for-worktrees`. Never run against a network-writable or shared checkout. + +**VendorAdapter** normalizes the differences between CLIs behind one interface. **v1 targets Claude Code + Codex only**; Cursor/Copilot are deferred. + +| Vendor | Headless invocation | Model flag | Usage output | +|---|---|---|---| +| Claude Code | `claude -p "" --output-format json` | `--model` | `usage` (input/output/cache tokens, cost) in JSON | +| Codex | `codex exec ""` (non-interactive) | config/flag | parse CLI/log usage | +| _Cursor / Copilot_ | _deferred_ | — | — | + +Adapter contract: `run(scenarioStep, {model, cwd}) → { transcript, filesTouched, usage: {inTok, outTok, cacheTok, costUsd}, wallSec, exitCode }`. Skills are invoked by putting `/skill-name` (or the natural-language trigger) in the prompt. Autonomous multi-phase steps (implement-plan) run to completion unattended; the adapter enforces the per-scenario timeout and hard-stops runaways. + +**Canned answers (mandatory for headless).** Our skills *interview* the user (create-spec, plan-feature, bootstrap all ask questions), and a headless agent that hits an unanswered question stalls or, worse, invents answers. So each scenario step supplies an `answers` file whose contents are injected into the prompt as *"here are the answers to every question you would ask; do not ask for anything else — if something is genuinely missing, state the assumption and proceed."* Two practical implications: + +- Answer files must be **kept in sync** with the interview each skill actually runs — a lint/grader can flag a run where the agent still asked a question (detectable in the transcript), so drift surfaces instead of silently degrading the run. +- The answer file *is* part of the fixture ground truth: graders can check the produced spec/plan against what the answers stipulated. + +**Individual runnability.** The runner is a CLI with filters so any subset runs on its own — e.g. `run-e2e --scenario S3-bug --vendor claude-code --model claude-opus-4-8 --fixture medplum-provider --runs 1`. No filter runs the full matrix. This is a first-class requirement, not an afterthought: the common case is running *one* scenario against *one* model to check a specific change. + +**Determinism strategy.** LLM output varies, so: run each (scenario × model × vendor) **N times** (start N=3), report **pass-rate + token variance** rather than a single boolean; pin model IDs; freeze fixtures; keep graders tolerant of surface variation. + +--- + +## 7. Graders + +Two tiers, combined per scenario: + +**Deterministic (authoritative).** +- *Artifact presence & schema*: spec/plan files exist with required sections; generated config/sub-agents validate against `schemas/*.schema.json`. +- *Outcome*: run the fixture's `test` command → all pass; run S4's script → expected artifact appears; S3's target test flips red→green with no new failures. +- *Git-tree-healthy* (the "commit tree looks healthy" check): assert against invariants — + - stacked phase branches exist where expected (`plan//phase-N`) and descend from the right base (guard the bug fixed in `fix/stack-branches-base-resolution`); + - commit messages match the conventional-commit regex; commits are logically scoped (no 2000-line "WIP", no mixing script + feature); + - working tree clean, no conflict markers, no committed secrets/fixture noise; + - tracking file updated per phase. + Implement as a `git log`/`git rev-list`/porcelain-parsing library so it's fast and CI-friendly. + +**LLM-judge (advisory, for fuzzy quality).** A rubric-scored grader for things regex can't judge: does `AGENTS.md` reflect *this* stack? Is the spec faithful to the request? Is the fix a real fix vs. a hack? Use a fixed judge model, structured rubric output, and treat scores as **advisory/trend signal**, not gate — to avoid non-determinism in the judge masking real regressions. Deterministic graders decide pass/fail; the judge adds a quality dimension to the report. + +A scenario **passes** when all its deterministic graders pass. The report additionally carries judge scores and telemetry. + +--- + +## 8. Telemetry & token benchmarking + +Every run emits a normalized record: + +```jsonc +{ + "pkgVersion": "0.3.0", + "gitSha": "", + "vendor": "claude-code", + "model": "claude-opus-4-8", + "scenario": "S2-feature", + "run": 1, + "pass": true, + "graders": { "app-tests-pass": true, "git-tree-healthy": true }, + "judge": { "spec-fidelity": 4.5 }, + "usage": { "inTok": 812000, "outTok": 41000, "cacheTok": 600000, "costUsd": 3.21 }, + "wallSec": 742, + "steps": [ /* per-skill usage breakdown */ ] +} +``` + +- **Per-step breakdown** (usage attributed to create-spec vs plan-feature vs implement-plan) so we see *where* tokens go, not just a total. +- **Baselines**: keep the aggregated results for each released version under `tests/e2e/baselines/.json`. A run compares current vs. the previous baseline and **flags regressions**: pass-rate drop, or token/cost increase beyond a threshold (e.g. >20% on a scenario) — the "benchmarking between versions" goal. +- **Aggregation**: mean/median/p95 tokens per scenario across the N runs, plus variance, so a noisy model is visible. + +--- + +## 9. Reporting + +- **Machine**: `results//…/run-N.json` (raw) + a rolled-up `summary.json`. +- **Human**: a generated `REPORT.md` (or an Artifact) with the comparison matrix — + + | Scenario | Vendor/Model | Pass-rate | Median tokens | Cost | Δ vs baseline | + |---|---|---|---|---|---| + + plus a per-scenario drill-down and links to failing transcripts. +- **Regression signal**: non-zero exit / annotation when pass-rate or cost crosses thresholds vs. baseline. + +--- + +## 10. Proposed layout + +``` +tests/e2e/ + fixtures/ # frozen seed projects (clean/stale/current states + task inputs) + scenarios/ # S0–S4 declarative specs + runner/ # runner CLI + VendorAdapters + isolation + graders/ # deterministic graders (git-tree, schema, outcome, sync-convergence) + judge + telemetry/ # usage normalization, baseline diff + baselines/ # per-version aggregated results (committed) + report/ # REPORT.md generator + README.md +results/ # gitignored raw run output +``` + +--- + +## 11. Execution & cost control + +L2 is **manual and human-triggered** — there is **no automatic cron/nightly** run. It executes when a human decides to: typically before a release, or to investigate a specific change. Aligns with `feat/make-e2e-tests-opt-in`. + +- **How it runs**: locally via the runner CLI (§6), or a `workflow_dispatch`-only CI job where the operator picks the scenario / vendor / model subset. Never on push, PR-open, schedule, or label — a human always initiates. +- **Individual runnability first**: the default operation is one scenario × one model (see the CLI in §6), *not* the full matrix. The full matrix is an explicit, deliberate invocation (e.g. a pre-release sweep). +- **Cost**: not the primary constraint. Keep only a per-scenario `maxWallSec` **safety timeout** (kill runaways) — not a cost gate. `--runs` defaults to 1; bump it when you want a pass-rate signal. +- **Secrets**: per-vendor API keys supplied by the operator (local env) or as CI secrets for the dispatch job; a vendor without a key is reported **skipped**, never passed. +- L0/L1 (free, deterministic) stay on every PR; only L2 is gated behind manual invocation. + +--- + +## 12. Phased implementation roadmap + +- **Phase 0 — Codex usage spike (de-risk before building the Codex adapter).** + A small, throwaway investigation — not wired into the harness — to answer whether Codex can be driven the way the runner needs, and how its telemetry is exposed. Deliverable: a short findings note that either unblocks the Phase 3 Codex adapter or scopes a workaround. Check: + - **Skill invocation** — does `codex exec "/skill-name …"` (or the natural-language trigger) reliably fire a shipped `/skill`, or does Codex need the skill body inlined into the prompt? + - **Unattended multi-phase run** — can `implement-plan` run to completion non-interactively (no mid-run prompts), respecting the canned-answers convention (§6)? + - **Token/cost reporting** — where does `codex exec` surface per-run usage (input/output/cache tokens, cost)? stdout JSON, a log file, an API, or not at all? This decides whether the telemetry record (§8) can be populated for Codex or needs estimation. + - **Model selection + permissions** — how to pin the model and pre-grant filesystem permissions in the sandbox (§6 isolation). + Do the same quick check for Claude Code first (its JSON `usage` is known-good) as the reference, then compare. If Codex can't report usage, decide: token-count estimation, a proxy that meters the API, or mark Codex telemetry as best-effort. + +- **Phase 1 — Foundation (deterministic, no paid calls).** + The `medplum-provider` fixture (vendored + pinned) with all three states (`clean`/`stale`/`current`) + task inputs and **canned-answer files**, wired to the Medplum test pack (`MockClient` + the two Vitest setup files). Deterministic graders (schema, app-tests via offline Vitest, **git-tree-healthy**, sync-convergence). Runner CLI skeleton with `--scenario/--vendor/--model` filtering. L1 install/uninstall smoke. Proves grading + isolation + individual-runnability without spending on models. + +- **Phase 2 — Single-vendor agentic loop.** + Claude Code adapter (headless JSON, richest usage data). Wire S0–S4 end-to-end for one model, including canned-answer injection + a transcript check that the agent never asked an unanswered question. Telemetry capture + per-step breakdown. First real REPORT.md. + +- **Phase 3 — Second vendor + matrix.** + Add the Codex adapter and 2–3 models per vendor. `--runs N` for pass-rate + variance. Second fixture (`node-ts-cli`) to prove stack-agnosticism. + +- **Phase 4 — Benchmarking.** + Baseline capture per release, version-diff regression flags, advisory LLM-judge quality dimension, `workflow_dispatch`-only CI job for pre-release sweeps. + +--- + +## 13. Resolved decisions + +Locked in from review: + +1. **Vendors** — Claude Code + Codex for v1. Cursor/Copilot deferred. +2. **Headless invocation** — every skill call ships **canned answers** injected into the prompt so the agent never asks anything mid-run (§6); a transcript grader flags any run where it did. +3. **Bug workflow** — S3 uses `/systematic-debugging`. +4. **Fixture states** — all three families are covered as distinct scenarios: **S0** bootstrap on a clean repo, **S1** sync on a stale/pre-configured repo, **S2–S4** skills on a current-harness repo (§4/§5). +5. **Judge gating** — LLM-judge stays **strictly advisory**; deterministic graders alone decide pass/fail. +6. **Execution** — **manual, human-triggered only**, no cron. Cost is secondary; human control over *when* is the requirement. Every scenario is **runnable individually** via runner-CLI filters (§6, §11). + +## 14. Remaining open questions + +1. **Codex headless parity + usage reporting** — confirm `codex exec` can trigger a `/skill`, run an unattended multi-phase `implement-plan`, and where/whether it reports per-run token usage. Resolved by the **Phase 0 spike** (§12) before the Codex adapter is built. +2. **Sync-convergence oracle** — what exactly does "sync upgraded correctly" assert against? Proposed: diff the synced `stale/` fixture against a freshly-bootstrapped `current/` and require the harness-managed files to match (ignoring intentionally hand-tuned/opt-out regions). Needs a precise ignore-list. +3. **`stale/` maintenance cadence** — how far behind current should `stale/` sit, and who regenerates it each release so S1 keeps testing a real upgrade (§4 maintenance note). diff --git a/package.json b/package.json index 205f09c9..bafed3f6 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,9 @@ }, "scripts": { "check-ai-models": "node scripts/check-ai-models.mjs", + "test:e2e:l1": "node tests/e2e/l1/smoke.test.mjs", + "e2e": "node tests/e2e/runner/run-e2e.mjs", + "e2e:list": "node tests/e2e/runner/run-e2e.mjs --list" "validate-schemas": "node scripts/validate-schemas.mjs", "test": "node --test tests/*.test.mjs" }, diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..604695df --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,121 @@ +# E2E testing & benchmarking harness + +Drives **real agents** through the canonical vinta-ai-workflows skills against +seeded fixture projects, then grades the *outcome* (feature works, bug fixed, +script runs) and the *shape of the work* (git tree healthy, artifacts +schema-valid) across the **vendor × model matrix**, capturing token / cost / +wall-time telemetry per run. + +See [`../../PLAN-e2e-testing-harness.md`](../../PLAN-e2e-testing-harness.md) for +the full design. This README is the operator's guide. + +> **L2 is manual, human-triggered — never on a cron.** You run it before a +> release or to investigate a change. Every scenario is runnable individually, +> and every skill invocation is pre-answered so no run stalls on an interview. + +## Layers + +| Layer | What | Cost | When | +|---|---|---|---| +| **L0** | Package static tests (CLI unit, schema validation, `validate-skill-md`) | free | every PR | +| **L1** | Install/uninstall smoke (`l1/smoke.test.mjs`) | free | every PR | +| **L2** | Agentic scenario E2E (S0–S4, real models) | paid | manual | + +## Quick start + +```bash +# free — no models: +node tests/e2e/l1/smoke.test.mjs # L1 install/uninstall smoke +node tests/e2e/runner/run-e2e.mjs --list # what's available +node tests/e2e/runner/run-e2e.mjs -s S2-feature -v mock --dry-run # preview matrix +node tests/e2e/runner/run-e2e.mjs -s S2-feature -v mock # self-test plumbing (mock adapter) + +# paid — real agent (needs the fixture vendored + a vendor CLI/key): +node tests/e2e/fixtures/medplum-provider/vendor.mjs # one-time: build fixture states +node tests/e2e/runner/run-e2e.mjs -s S3-bug -v claude-code -m claude-opus-4-8 -n 1 +``` + +No `--scenario`/`--vendor` filter ⇒ the **full matrix** (a deliberate pre-release +sweep). Individual runnability is the default operating mode. + +## Runner CLI + +``` +run-e2e [filters] [options] + +Filters: + -s, --scenario S0-bootstrap | S1-sync | S2-feature | S3-bug | S4-script + -v, --vendor claude-code | codex | mock + -m, --model pin model(s); default per-vendor if omitted + -f, --fixture restrict to scenarios using this fixture + +Options: + -n, --runs repeat each cell N times for a pass-rate signal (default 1) + --judge advisory LLM-judge dims (spec-fidelity, plan-quality, ...) + --results-dir output root (default /results, gitignored) + --keep keep isolated fixture copies for debugging + --dry-run print the matrix; run nothing + --list list scenarios / vendors / graders + --save-baseline write this run's aggregate as the version baseline + --no-fail-on-regression +``` + +A vendor whose CLI/key is missing is reported **skipped**, never passed. + +## Scenarios (`scenarios/`) + +| Id | State | Skills | Deterministic graders | +|---|---|---|---| +| **S0-bootstrap** | `clean` | `/vinta-bootstrap-ai-tools` | artifacts-bootstrap, git-tree-healthy | +| **S1-sync** | `stale` | `/vinta-sync-ai-tools` | sync-convergence, git-tree-healthy | +| **S2-feature** | `current` | create-spec → plan-feature → implement-plan | artifacts-spec/plan, app-tests-pass, git-tree-healthy | +| **S3-bug** | `current` | `/systematic-debugging` | bug-fixed, git-tree-healthy | +| **S4-script** | `current` | `/add-one-off-script` | script-output, git-tree-healthy | + +`answers-lint` is applied implicitly to every scenario (flags a run where the +agent asked an unanswered question — i.e. the canned answers drifted). + +## Graders (`graders/`) + +Deterministic graders are **authoritative** (they gate pass/fail). The LLM-judge +(`--judge`) is **strictly advisory** — a quality/trend signal in the report, +never a gate. See `graders/index.mjs` for the registry. + +## Telemetry & baselines + +Each run writes `results/////run-N.json` +(gitignored). Aggregates roll up into `results//summary.json` + +`REPORT.md`. `--save-baseline` freezes the aggregate under `baselines/.json` +(committed); subsequent runs diff against the previous baseline and flag +pass-rate drops or >20% token/cost increases. + +## Fixtures (`fixtures/`) + +`medplum-provider` is the primary fixture — a real React + Vite + TS Medplum app +whose tests run **offline against `MockClient`**, so `app-tests-pass` needs no +live server. The app itself is **not committed** (it's large + external); build +its `clean` / `stale` / `current` states on demand: + +```bash +node tests/e2e/fixtures/medplum-provider/vendor.mjs # all three states +node tests/e2e/fixtures/medplum-provider/vendor.mjs --state clean +``` + +⚠ Before first vendoring, set `upstream.pinnedCommit` in +`fixtures/medplum-provider/fixture.json` to a real SHA. + +Task inputs and their **canned answers** live in +`fixtures/medplum-provider/tasks/` and are part of the fixture ground truth. + +## Status / roadmap + +This is the deterministic **foundation + runner** (PLAN Phase 1 + the Phase 2 +Claude Code adapter). Still open, per the PLAN: + +- **Phase 0 spike** — confirm Codex headless parity + usage reporting (§12/§14). + The Codex adapter is wired but usage is best-effort until the spike resolves. +- **Fixture app content** — the planted bug test, the feature scaffolding, and + the seeded MockClient bundles are materialized by `vendor.mjs` + a first real + bootstrap; the harness reports clearly when a state isn't vendored yet. +- **Second fixture** (`node-ts-cli`) and **multi-model matrix** — Phase 3. +- **Baseline capture per release** — Phase 4. diff --git a/tests/e2e/baselines/README.md b/tests/e2e/baselines/README.md new file mode 100644 index 00000000..e3a5de80 --- /dev/null +++ b/tests/e2e/baselines/README.md @@ -0,0 +1,23 @@ +# Baselines + +Per-version aggregated E2E results, **committed** so runs can flag regressions +between releases (PLAN §8). + +Each file is `.json` written by `run-e2e --save-baseline`: + +```jsonc +{ + "version": "0.4.0", + "cells": [ + { "scenario": "S2-feature", "vendor": "claude-code", "model": "claude-opus-4-8", + "runs": 3, "passRate": 1.0, "tokens": { "mean": ..., "median": ..., "p95": ... }, + "costUsd": { ... }, "wallSec": { ... } } + ] +} +``` + +A run compares its aggregate against the most recent *previous* baseline and +flags: pass-rate drops, or token/cost increases beyond 20% per cell. + +Capture a baseline right before cutting a release, from a full-matrix sweep with +`--runs >= 3` (so the pass-rate + variance are meaningful). diff --git a/tests/e2e/fixtures/README.md b/tests/e2e/fixtures/README.md new file mode 100644 index 00000000..a9152a1a --- /dev/null +++ b/tests/e2e/fixtures/README.md @@ -0,0 +1,52 @@ +# Fixtures + +Seeded, reproducible repos the scenarios run against. Each fixture is a real app +(not a toy) so the workflows are exercised on representative code. + +## Contents of a fixture + +``` +/ + fixture.json # metadata: stack, run/test commands, per-state git refs, + # the version `stale/` was bootstrapped from + vendor.mjs # builds the clean/stale/current states on demand + tasks/ # seeded task inputs + their canned answers (ground truth) + feature.md feature.answers.md + bug.md bug.answers.md + script.md script.answers.md + bootstrap.md bootstrap.answers.md + sync.md sync.answers.md + states/ # (gitignored) built by vendor.mjs — each is its own git repo + clean/ stale/ current/ +``` + +## Three committed states (PLAN §4) + +- **`clean/`** — app with no `ai-tools/` / `AGENTS.md` / vendor dirs → drives **S0**. +- **`stale/`** — app bootstrapped against an **older** package version → drives **S1**. +- **`current/`** — app bootstrapped against the **current** version → drives **S2–S4**. + +States are **not committed** to this repo — they're built by `vendor.mjs` and +gitignored. Each state is materialized as its own git repo so the runner can +clone it into an isolated copy and the git-tree grader has a real base to diff. + +## Design rules + +- Fixture states are **frozen** (pinned upstream commit + committed state). +- Every run operates on an **isolated copy** — never the vendored fixture in place. +- Every task has a **machine-checkable** success signal so grading isn't subjective. + +## Maintenance + +- Re-base `stale/` to "current minus one meaningful change" each release so the + sync scenario keeps testing a real upgrade (see `fixture.json.maintenance`). +- Keep the pinned upstream commit current enough that the app still builds on the + supported Node version. + +## `medplum-provider` (primary) + +Real React + Vite + TS Medplum app; tests run offline against `MockClient`. +Requirements captured in `medplum-provider/fixture.json.medplum`: wire the two +Vitest setup files, keep `@medplum/*` versions matched, Node ≥ 22.4. A live +Medplum stack (Docker) is opt-in and off by default — reserve it for contracts +`MockClient` can't prove. diff --git a/tests/e2e/fixtures/medplum-provider/fixture.json b/tests/e2e/fixtures/medplum-provider/fixture.json new file mode 100644 index 00000000..f28f1d6d --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/fixture.json @@ -0,0 +1,57 @@ +{ + "name": "medplum-provider", + "description": "PRIMARY fixture — the Medplum provider example app (React + Vite + TS on @medplum/core / @medplum/react, React Router, Mantine). Tests run fully offline against MockClient, so app-tests-pass needs no live Medplum server.", + "upstream": { + "repo": "https://github.com/medplum/medplum", + "subdir": "examples/medplum-provider", + "pinnedCommit": "REPLACE_WITH_PINNED_SHA", + "note": "Vendored + pinned by vendor.mjs. Keep @medplum/{core,definitions,mock,fhirtypes} on matching versions; Node >= 22.4 (or a ws dev dep) for the WebSocket polyfill @medplum/core v5 needs at module load." + }, + "project": { + "name": "medplum-provider", + "default_branch": "main", + "code_host": "github", + "stack_summary": "React + Vite + TypeScript on @medplum/core / @medplum/react, React Router, Mantine; tests via Vitest + MockClient" + }, + "commands": { + "install": "npm ci", + "test": "npx vitest run", + "run": "npm run dev", + "ai_plans_dir": "ai-plans", + "scripts_dir": "scripts" + }, + "medplum": { + "vitestSetup": { + "globalSetup": "test.globalSetup.ts", + "setupFiles": "test.setup.ts", + "note": "Both must be registered in vitest.config.* — globalSetup indexes FHIR bundles once and shares globalSchema.types to workers; setupFiles assigns the shared index at module scope. Without them, filtered searchResources silently return nothing." + }, + "liveStack": { + "optIn": true, + "default": false, + "note": "Off by default. For server-only behavior (access policies, subscriptions, $everything, bots, GraphQL/transaction edges) bring up the full stack via Docker and run graders as integration checks. Slower + nondeterministic — reserve for contracts MockClient can't prove." + } + }, + "states": { + "clean": { + "path": "states/clean", + "ref": "main", + "description": "App with NO ai-tools/, AGENTS.md, or vendor dirs. Drives S0 — Bootstrap." + }, + "stale": { + "path": "states/stale", + "ref": "main", + "bootstrappedFrom": "0.3.0", + "description": "App bootstrapped against an OLDER vinta-ai-workflows version (deliberately behind current). Drives S1 — Sync." + }, + "current": { + "path": "states/current", + "ref": "main", + "bootstrappedFrom": "0.4.0", + "description": "App bootstrapped against the CURRENT version, harness fully in place. Drives S2–S4." + } + }, + "maintenance": { + "staleCadence": "Re-base `stale/` to 'current minus one meaningful change' each release so S1 keeps exercising a real upgrade rather than a no-op. Cheapest: regenerate from clean/ by bootstrapping with a pinned older package version (see states.stale.bootstrappedFrom)." + } +} diff --git a/tests/e2e/fixtures/medplum-provider/tasks/bootstrap.answers.md b/tests/e2e/fixtures/medplum-provider/tasks/bootstrap.answers.md new file mode 100644 index 00000000..061845a3 --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/bootstrap.answers.md @@ -0,0 +1,20 @@ +# Canned answers — bootstrap (vinta-bootstrap-ai-tools interview) + +Answer every interview question with the following; do not ask for anything else. + +- **Project name**: medplum-provider +- **Default branch**: main +- **Code host**: github +- **Stack**: React + Vite + TypeScript on @medplum/core / @medplum/react, React + Router, Mantine; tests via Vitest + MockClient. +- **Medplum stack applies?**: Yes — the app consumes @medplum/react. It has no + `bots/` dir and no tenant compartmenting, so apply only the SDK-consumer parts; + skip bot/access-policy/notification categories. +- **ai-plans dir**: `ai-plans` +- **Vendors to wire**: claude-code (primary). Others optional. +- **Testing pack**: Medplum (`MockClient` + the two Vitest setup files), on top + of the Vitest runner pack. +- **Keep a hand-tuned region marker** in AGENTS.md: include the literal comment + `` so sync preservation can be tested later. +- **Commits**: conventional; separate the AGENTS.md, config, and skills/agents + commits logically. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/bootstrap.md b/tests/e2e/fixtures/medplum-provider/tasks/bootstrap.md new file mode 100644 index 00000000..a2282c21 --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/bootstrap.md @@ -0,0 +1,10 @@ +# Bootstrap request — Wire AI tooling into this repo + +This is a clean medplum-provider checkout with no `ai-tools/`, `AGENTS.md`, or +vendor dirs. Run `/vinta-bootstrap-ai-tools` to analyze the codebase and produce +the harness: `AGENTS.md`, derived skills + sub-agents, per-vendor wiring, and the +`.vinta-ai-workflows.yaml` config. + +Success = `AGENTS.md` reflects this stack (React + Vite + TS on Medplum), the +generated sub-agents and config are schema-valid, and everything lands as clean, +conventionally-scoped commits. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/bug.answers.md b/tests/e2e/fixtures/medplum-provider/tasks/bug.answers.md new file mode 100644 index 00000000..f532f3c9 --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/bug.answers.md @@ -0,0 +1,15 @@ +# Canned answers — bug (systematic-debugging interview) + +Answer every question with the following; do not ask for anything else. + +- **Repro command**: `npx vitest run src/pages/patient/allergies/AllergyPanel.test.tsx` +- **Expected**: the created allergy's substance is returned when reading the + patient's allergies. +- **Actual**: read returns nothing — the written `AllergyIntolerance.patient` + reference is malformed. +- **Constraint**: fix the write path under `src/pages/patient/allergies/`. Do NOT + modify or weaken the test to make it pass. +- **No regressions**: the full `npx vitest run` suite must stay green. +- **Root cause over patch**: correct the reference construction, don't special-case + the test input. +- **Commit**: a single `fix(...)` conventional commit touching the create path. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/bug.md b/tests/e2e/fixtures/medplum-provider/tasks/bug.md new file mode 100644 index 00000000..fe9f72d2 --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/bug.md @@ -0,0 +1,20 @@ +# Bug report — Allergy panel writes the wrong patient reference field + +The allergy panel's "add allergy" path saves an `AllergyIntolerance` whose +patient linkage is wrong, so the new allergy doesn't come back when we read the +patient's allergies. + +There is a **currently-failing** Vitest test that reproduces it: + + src/pages/patient/allergies/AllergyPanel.test.tsx + +It creates an allergy for a patient, then reads the patient's allergies and +asserts the created substance is present. The read returns nothing because the +written resource's `patient` reference is malformed. + +Debug it with `/systematic-debugging`, find the root cause in the write path +under `src/pages/patient/allergies/`, fix it, and make the failing test pass +without breaking any other test. + +The planted defect is the ground truth: the fix must correct the +`AllergyIntolerance.patient` reference in the create path — not weaken the test. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/feature.answers.md b/tests/e2e/fixtures/medplum-provider/tasks/feature.answers.md new file mode 100644 index 00000000..7f75be3a --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/feature.answers.md @@ -0,0 +1,21 @@ +# Canned answers — feature (create-spec / plan-feature interview) + +Answer every interview question these skills ask with the following. Do not ask +for anything else; if something is genuinely missing, state the assumption and +proceed. + +- **Scope**: read + create only. No edit/delete of allergies in this iteration. +- **Where it lives**: patient detail page, a new `AllergyPanel` component under + `src/pages/patient/allergies/`. +- **Data model**: FHIR `AllergyIntolerance`, `patient` reference to the current + patient, substance in `code.text`, `criticality`, `clinicalStatus`. +- **FHIR/server behavior**: MockClient only — no access policies, subscriptions, + or server-only operations. Everything must be provable offline. +- **Testing**: Vitest against `MockClient`, wrapped in `MedplumProvider`. Assert + on resource field values (`code.text`, `patient` reference), never on + unfiltered search counts (MockClient ships pre-seeded resources). +- **Auth / tenancy**: single-tenant fixture; no `meta.account` tenant scoping. +- **UI framework**: Mantine components, consistent with the existing app. +- **Out of scope**: reactions/manifestations, onset dates, notes, i18n. +- **Branching**: stacked `plan//phase-N` branches per the plan. +- **Commits**: conventional commits; keep spec, plan, and code in separate commits. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/feature.md b/tests/e2e/fixtures/medplum-provider/tasks/feature.md new file mode 100644 index 00000000..994cf1c5 --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/feature.md @@ -0,0 +1,13 @@ +# Feature request — Patient allergy panel + +Add a **patient allergy panel** to the provider app. + +- On a patient's detail page, show a panel listing that patient's + `AllergyIntolerance` resources: the substance (`code.text`), clinical status, + and criticality. +- Let a provider **add** a new allergy for the patient (substance + criticality), + which writes a new `AllergyIntolerance` resource referencing the patient. +- The panel reads and writes through the Medplum client (in tests, `MockClient`). + +Success = a provider can view a patient's allergies and record a new one, and +the behavior is covered by Vitest tests running offline against `MockClient`. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/script.answers.md b/tests/e2e/fixtures/medplum-provider/tasks/script.answers.md new file mode 100644 index 00000000..3a304d0f --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/script.answers.md @@ -0,0 +1,16 @@ +# Canned answers — script (add-one-off-script interview) + +Answer every question with the following; do not ask for anything else. + +- **Location**: `scripts/one_off_2026_08_01_export_observations/` (dated one-off + folder per the foundation skill convention). +- **Entry point**: `run.ts`, runnable via `npx tsx scripts/one_off_2026_08_01_export_observations/run.ts`. +- **Data source**: a seeded `MockClient` (in-memory FHIR store) or a bundled + fixture bundle — never a live server. +- **Default patient**: use a seeded fixture patient when no id is passed. +- **Output**: `out/observations.csv` under the script folder, columns + `code,value,unit,effectiveDateTime`. +- **Non-interactive**: no prompts; read config from args/env with sane defaults. +- **Commit**: its own `chore(script): ...` conventional commit — do not mix into + app code. +- **Idempotency/cleanup**: overwrite the CSV on re-run; no external side effects. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/script.md b/tests/e2e/fixtures/medplum-provider/tasks/script.md new file mode 100644 index 00000000..81e7b2b5 --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/script.md @@ -0,0 +1,14 @@ +# One-off script request — Export a patient's observations to CSV + +Write a one-off script that emits a CSV of all `Observation` values for a given +patient. + +- Input: a patient id (default to a seeded fixture patient if none supplied). +- Output: `observations.csv` with columns `code,value,unit,effectiveDateTime`, + one row per `Observation` for that patient. +- It must run **non-interactively** against a seeded `MockClient` (or a bundled + fixture bundle) — no live Medplum server, no prompts. + +Use `/add-one-off-script`. Success = the script exists in the expected location, +runs to completion without interaction, and produces the CSV with the expected +columns. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/sync.answers.md b/tests/e2e/fixtures/medplum-provider/tasks/sync.answers.md new file mode 100644 index 00000000..d29ed9cb --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/sync.answers.md @@ -0,0 +1,13 @@ +# Canned answers — sync (vinta-sync-ai-tools interview) + +Answer every question with the following; do not ask for anything else. + +- **Apply all pending changes** for this upgrade unless one conflicts with a + hand-tuned region — in that case preserve the hand-tuned content and note it. +- **Preserve** the `` region in `AGENTS.md` verbatim. +- **Respect existing opt-outs** — do not re-add anything previously opted out. +- **Config**: update `vinta_ai_workflows_version` and `last_synced_at`; keep + project-specific fields as-is. +- **Commits**: conventional; one logical commit per applied change group. +- **No interactive confirmation** between changes — proceed autonomously and + summarize what was applied vs. preserved at the end. diff --git a/tests/e2e/fixtures/medplum-provider/tasks/sync.md b/tests/e2e/fixtures/medplum-provider/tasks/sync.md new file mode 100644 index 00000000..5997a786 --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/tasks/sync.md @@ -0,0 +1,18 @@ +# Sync request — Upgrade the harness to the current version + +This repo was bootstrapped against an **older** `vinta-ai-workflows` version. +Run `/vinta-sync-ai-tools` to upgrade it to the current version: bump schema +versions, add any new foundation skills, honor per-change apply gating, and +migrate the config. + +Constraints: + +- **Do not clobber hand-tuned regions.** `AGENTS.md` contains a + `` marker whose surrounding content must survive. +- **Opt-outs stay sticky.** Anything the project opted out of must not reappear. +- Managed harness files should converge on what a fresh current bootstrap + produces. + +Success = the harness-managed files match a current bootstrap (modulo hand-tuned +regions and the version/timestamp in the config), and the result lands as clean +commits. diff --git a/tests/e2e/fixtures/medplum-provider/vendor.mjs b/tests/e2e/fixtures/medplum-provider/vendor.mjs new file mode 100644 index 00000000..a805fa3e --- /dev/null +++ b/tests/e2e/fixtures/medplum-provider/vendor.mjs @@ -0,0 +1,138 @@ +#!/usr/bin/env node +// vendor.mjs — build the medplum-provider fixture's three committed states. +// +// The fixture app is NOT committed to this repo (it's a large external app); +// this script vendors it on demand, pinned to a commit, and materializes the +// clean/stale/current states each as its own git repo (PLAN §4). Run once +// locally before the first L2 run: +// +// node tests/e2e/fixtures/medplum-provider/vendor.mjs +// node tests/e2e/fixtures/medplum-provider/vendor.mjs --state clean # just one +// +// Requires: git, network access, and (for stale/current) the vinta-ai-workflows +// CLI to place the harness skills. The SPEC/PLAN interview content that a real +// bootstrap authors (AGENTS.md, .vinta-ai-workflows.yaml) is seeded minimally +// here so the fixture is a valid, schema-passing starting point; regenerate with +// a real bootstrap when you want richer fixtures. +// +// ⚠ Before first use: set `upstream.pinnedCommit` in fixture.json to a real SHA. + +import { execFileSync } from 'node:child_process'; +import { + existsSync, mkdirSync, rmSync, cpSync, writeFileSync, readFileSync, +} from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, '..', '..', '..', '..'); +const fixture = JSON.parse(readFileSync(join(HERE, 'fixture.json'), 'utf8')); + +const args = process.argv.slice(2); +const onlyState = args.includes('--state') ? args[args.indexOf('--state') + 1] : null; +const cacheDir = join(HERE, '.upstream-cache'); + +function sh(cmd, cmdArgs, opts = {}) { + return execFileSync(cmd, cmdArgs, { stdio: 'inherit', ...opts }); +} +function shq(cmd, cmdArgs, opts = {}) { + return execFileSync(cmd, cmdArgs, { encoding: 'utf8', ...opts }).trim(); +} + +function fetchUpstream() { + const { repo, subdir, pinnedCommit } = fixture.upstream; + if (!pinnedCommit || pinnedCommit.startsWith('REPLACE')) { + throw new Error('Set upstream.pinnedCommit in fixture.json to a real SHA before vendoring.'); + } + if (existsSync(join(cacheDir, subdir))) { + console.log(`· upstream cache present at ${cacheDir}`); + return join(cacheDir, subdir); + } + console.log(`· cloning ${repo} @ ${pinnedCommit} (sparse: ${subdir})`); + rmSync(cacheDir, { recursive: true, force: true }); + mkdirSync(cacheDir, { recursive: true }); + sh('git', ['init', '-q'], { cwd: cacheDir }); + sh('git', ['remote', 'add', 'origin', repo], { cwd: cacheDir }); + sh('git', ['sparse-checkout', 'set', '--cone', subdir], { cwd: cacheDir }); + sh('git', ['fetch', '-q', '--depth', '1', 'origin', pinnedCommit], { cwd: cacheDir }); + sh('git', ['checkout', '-q', 'FETCH_HEAD'], { cwd: cacheDir }); + return join(cacheDir, subdir); +} + +function initRepo(dir, message) { + sh('git', ['init', '-q'], { cwd: dir }); + sh('git', ['config', 'user.name', 'fixture-seed'], { cwd: dir }); + sh('git', ['config', 'user.email', 'fixture@vinta.local'], { cwd: dir }); + sh('git', ['add', '-A'], { cwd: dir }); + sh('git', ['commit', '-q', '-m', message], { cwd: dir }); + sh('git', ['branch', '-q', '-M', 'main'], { cwd: dir }); +} + +function copyApp(srcApp, destState) { + rmSync(destState, { recursive: true, force: true }); + mkdirSync(destState, { recursive: true }); + cpSync(srcApp, destState, { recursive: true }); + // Wire the Medplum test setup pair from the shipped test pack (§4). + const pack = join(REPO_ROOT, 'skills', 'vinta-derive-skills', + 'resources', 'write-unit-test-packs', 'stacks'); + for (const [src, dst] of [ + ['medplum-test.globalSetup.ts', 'test.globalSetup.ts'], + ['medplum-test.setup.ts', 'test.setup.ts'], + ]) { + const from = join(pack, src); + if (existsSync(from)) cpSync(from, join(destState, dst)); + } + // Copy the scenario task inputs + canned answers into the fixture's tasks/ + // stays out of the app tree — they live beside fixture.json, not in states. +} + +function seedHarness(destState, version) { + // Place the vinta skills for the target version and seed a minimal, schema- + // valid AGENTS.md + config so the state is a valid starting point. A real + // bootstrap produces richer content; regenerate when you want that. + console.log(`· installing vinta-ai-workflows@${version} skills into ${destState}`); + try { + sh('npx', ['-y', `vinta-ai-workflows@${version}`, 'install', '--tool', 'claude-code', + '--copy', '--target', destState], { cwd: destState }); + } catch { + console.warn(` (could not install @${version} — placing local skills as fallback)`); + sh('node', [join(REPO_ROOT, 'vinta-ai-workflows.mjs'), 'install', '--tool', 'claude-code', + '--copy', '--target', destState]); + } + writeFileSync(join(destState, 'AGENTS.md'), + `# AGENTS\n\nStack: ${fixture.project.stack_summary}\n\n\n`); + writeFileSync(join(destState, '.vinta-ai-workflows.yaml'), +`schema_version: 1 +vinta_ai_workflows_version: ${version} +project: + name: ${fixture.project.name} + default_branch: main + code_host: github + stack_summary: ${fixture.project.stack_summary} + ai_plans_dir: ${fixture.commands.ai_plans_dir} +commands: {} +policies: {} +vendors: {} +foundation_skills: [] +foundation_agents: [] +`); +} + +function buildState(state, appSrc) { + const stateDir = join(HERE, fixture.states[state].path); + console.log(`\n=== building state: ${state} → ${stateDir} ===`); + copyApp(appSrc, stateDir); + if (state === 'stale') seedHarness(stateDir, fixture.states.stale.bootstrappedFrom); + if (state === 'current') seedHarness(stateDir, fixture.states.current.bootstrappedFrom); + initRepo(stateDir, `chore: seed medplum-provider fixture (${state} state)`); + console.log(` committed ${state} @ ${shq('git', ['rev-parse', '--short', 'HEAD'], { cwd: stateDir })}`); +} + +// ── main ───────────────────────────────────────────────────────────────────── +const appSrc = fetchUpstream(); +const states = onlyState ? [onlyState] : ['clean', 'stale', 'current']; +for (const s of states) { + if (!fixture.states[s]) { console.error(`unknown state ${s}`); process.exit(2); } + buildState(s, appSrc); +} +console.log('\n✓ fixture states built. They are gitignored — not committed to this repo.'); diff --git a/tests/e2e/graders/answers-lint.mjs b/tests/e2e/graders/answers-lint.mjs new file mode 100644 index 00000000..695cf4cc --- /dev/null +++ b/tests/e2e/graders/answers-lint.mjs @@ -0,0 +1,32 @@ +// answers-lint.mjs — the canned-answers drift check (PLAN §6). +// +// A headless run must never stall on an interview question. Each step ships an +// answers file injected into the prompt; this grader scans the transcripts for +// evidence the agent asked a question anyway (meaning the answers file drifted +// out of sync with the interview the skill actually runs). Deterministic and +// advisory-leaning, but a hard signal that the fixture needs maintenance. + +const QUESTION_SIGNALS = [ + /\bcould you (please )?(clarify|confirm|provide|specify)\b/i, + /\bwhich (option|approach|one) (would|do) you\b/i, + /\bplease (confirm|clarify|provide|let me know)\b/i, + /\bwhat (should|would you like|do you want)\b.*\?/i, + /\bI need (more information|you to)\b/i, + /\bwaiting for (your )?(input|response|confirmation)\b/i, + /\bshall I proceed\?/i, +]; + +export function answersLint(ctx) { + const details = []; + let asked = 0; + for (const step of ctx.steps || []) { + const t = step.transcript || ''; + for (const re of QUESTION_SIGNALS) { + const m = t.match(re); + if (m) { asked++; details.push(`✗ ${step.invoke} appears to have asked: "${m[0].trim()}"`); break; } + } + } + const pass = asked === 0; + if (pass) details.push('✓ no step appears to have asked an unanswered question'); + return { id: 'answers-lint', tier: 'deterministic', pass, details }; +} diff --git a/tests/e2e/graders/artifacts.mjs b/tests/e2e/graders/artifacts.mjs new file mode 100644 index 00000000..704162a3 --- /dev/null +++ b/tests/e2e/graders/artifacts.mjs @@ -0,0 +1,148 @@ +// artifacts.mjs — artifact presence + section/schema graders (PLAN §7). +// +// Deterministic. Two families: +// - Spec/Plan markdown exist at the fixture's ai-plans dir with required +// sections (section-presence check; LLMs vary wording so we check headings, +// not prose). +// - Generated config / sub-agents validate against schemas/*.schema.json via +// the minimal validator (schema.mjs). + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { walk, REPO_ROOT } from '../runner/lib.mjs'; +import { validateFile, frontmatter } from './schema.mjs'; + +const SPEC_SECTIONS = ['overview', 'acceptance']; +const PLAN_SECTIONS = ['phase', 'testing']; + +function findByGlob(cwd, dir, suffixRe) { + const base = join(cwd, dir); + return walk(base).filter((p) => suffixRe.test(p)); +} + +function sectionCheck(text, needles) { + const lower = text.toLowerCase(); + const missing = needles.filter((n) => !lower.includes(n)); + return { ok: missing.length === 0, missing }; +} + +export function artifactsSpec(ctx) { + const plansDir = ctx.fixture?.commands?.ai_plans_dir || 'ai-plans'; + const files = findByGlob(ctx.cwd, plansDir, /_SPEC\.md$/i); + const details = []; + let pass = true; + if (files.length === 0) { + return { id: 'artifacts-spec', tier: 'deterministic', pass: false, + details: [`✗ no *_SPEC.md under ${plansDir}/`] }; + } + for (const f of files) { + const { ok, missing } = sectionCheck(readFileSync(f, 'utf8'), SPEC_SECTIONS); + if (ok) details.push(`✓ ${rel(ctx.cwd, f)} has required sections`); + else { pass = false; details.push(`✗ ${rel(ctx.cwd, f)} missing sections: ${missing.join(', ')}`); } + } + return { id: 'artifacts-spec', tier: 'deterministic', pass, details }; +} + +export function artifactsPlan(ctx) { + const plansDir = ctx.fixture?.commands?.ai_plans_dir || 'ai-plans'; + const files = findByGlob(ctx.cwd, plansDir, /_PLAN\.md$/i); + const details = []; + let pass = true; + if (files.length === 0) { + return { id: 'artifacts-plan', tier: 'deterministic', pass: false, + details: [`✗ no *_PLAN.md under ${plansDir}/`] }; + } + for (const f of files) { + const { ok, missing } = sectionCheck(readFileSync(f, 'utf8'), PLAN_SECTIONS); + if (ok) details.push(`✓ ${rel(ctx.cwd, f)} has required sections`); + else { pass = false; details.push(`✗ ${rel(ctx.cwd, f)} missing sections: ${missing.join(', ')}`); } + } + return { id: 'artifacts-plan', tier: 'deterministic', pass, details }; +} + +// S0/S1: bootstrap/sync must leave AGENTS.md + schema-valid sub-agents + config. +export function artifactsBootstrap(ctx) { + const details = []; + let pass = true; + const fail = (m) => { pass = false; details.push(`✗ ${m}`); }; + const ok = (m) => details.push(`✓ ${m}`); + + if (existsSync(join(ctx.cwd, 'AGENTS.md'))) ok('AGENTS.md present'); + else fail('AGENTS.md missing'); + + // Validate any generated sub-agent yaml against sub-agent.v1 schema. + const agentFiles = walk(join(ctx.cwd, 'ai-tools', 'agents')).filter((p) => /\.ya?ml$/.test(p)); + if (agentFiles.length === 0) { + details.push('· no ai-tools/agents/*.yaml to validate'); + } + for (const f of agentFiles) { + const data = parseSimpleYaml(readFileSync(f, 'utf8')); + const { ok: valid, errors } = validateFile( + join(REPO_ROOT, 'schemas', 'sub-agent.v1.schema.json'), data); + if (valid) ok(`${rel(ctx.cwd, f)} validates against sub-agent.v1`); + else fail(`${rel(ctx.cwd, f)} schema errors: ${errors.join('; ')}`); + } + + // Config file present + schema-valid. + const cfg = join(ctx.cwd, '.vinta-ai-workflows.yaml'); + if (existsSync(cfg)) { + const data = parseSimpleYaml(readFileSync(cfg, 'utf8')); + const { ok: valid, errors } = validateFile( + join(REPO_ROOT, 'schemas', 'vinta-ai-workflows-config.v1.schema.json'), data); + if (valid) ok('.vinta-ai-workflows.yaml validates against config.v1'); + else fail(`config schema errors: ${errors.slice(0, 5).join('; ')}`); + } else { + fail('.vinta-ai-workflows.yaml missing'); + } + + return { id: 'artifacts-bootstrap', tier: 'deterministic', pass, details }; +} + +const rel = (cwd, f) => f.slice(cwd.length + 1); + +// Tiny YAML → object parser for the flat/one-level configs our fixtures emit. +// Not a general YAML parser (the repo's `yaml` dep is dev-only and not a +// harness runtime dep); handles scalars, one level of nesting, and `key: |` +// block scalars enough for schema-shape validation. +function parseSimpleYaml(text) { + const root = {}; + const stack = [{ indent: -1, obj: root }]; + const lines = text.split('\n'); + for (let i = 0; i < lines.length; i++) { + const raw = lines[i]; + if (!raw.trim() || raw.trim().startsWith('#')) continue; + const indent = raw.match(/^\s*/)[0].length; + const line = raw.trim(); + const m = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/); + if (!m) continue; + const [, key, rest] = m; + while (stack.length > 1 && indent <= stack[stack.length - 1].indent) stack.pop(); + const parent = stack[stack.length - 1].obj; + if (rest === '' || rest === '|' || rest === '>') { + if (rest === '|' || rest === '>') { + // consume block scalar + const block = []; + while (i + 1 < lines.length && (lines[i + 1].match(/^\s*/)[0].length > indent || !lines[i + 1].trim())) { + block.push(lines[++i]); + } + parent[key] = block.join('\n').trim(); + } else { + const child = {}; + parent[key] = child; + stack.push({ indent, obj: child }); + } + } else { + parent[key] = coerce(rest); + } + } + return root; + function coerce(v) { + v = v.replace(/^["']|["']$/g, ''); + if (/^-?\d+$/.test(v)) return Number(v); + if (v === 'true') return true; + if (v === 'false') return false; + return v; + } +} + +export { frontmatter }; diff --git a/tests/e2e/graders/git-tree.mjs b/tests/e2e/graders/git-tree.mjs new file mode 100644 index 00000000..c50f6aba --- /dev/null +++ b/tests/e2e/graders/git-tree.mjs @@ -0,0 +1,112 @@ +// git-tree.mjs — the "commit tree looks healthy" grader (PLAN §7). +// +// Deterministic, git-porcelain based. Asserts the invariants a healthy agent +// run should leave behind: +// - working tree clean (no uncommitted leftovers, no conflict markers) +// - commit messages match the conventional-commit shape +// - commits are logically scoped (no giant "WIP" dumps) +// - no committed secrets / obvious fixture noise +// - when the scenario expects stacked phase branches, they exist and descend +// from the right base (guards fix/stack-branches-base-resolution) + +import { git } from '../runner/lib.mjs'; + +const CONVENTIONAL = /^(feat|fix|docs|chore|refactor|test|perf|build|ci|style|revert)(\([^)]+\))?!?: .+/; +const SECRET_PATTERNS = [ + /-----BEGIN [A-Z ]*PRIVATE KEY-----/, + /\bAKIA[0-9A-Z]{16}\b/, + /\bsk-[A-Za-z0-9]{20,}\b/, + /\bghp_[A-Za-z0-9]{36}\b/, +]; + +export function gitTreeHealthy(ctx) { + const { cwd, scenario } = ctx; + const details = []; + let pass = true; + const fail = (m) => { pass = false; details.push(`✗ ${m}`); }; + const ok = (m) => details.push(`✓ ${m}`); + + // 1. Working tree clean. + const status = git(cwd, ['status', '--porcelain']); + if (status.stdout.trim()) fail(`working tree dirty:\n${indent(status.stdout.trim())}`); + else ok('working tree clean'); + + // 2. New commits exist since the baseline (the fixture's starting ref). + const base = ctx.baseRef || 'HEAD'; + const range = ctx.baseRef ? `${base}..HEAD` : null; + const logArgs = ['log', '--no-merges', '--format=%H%x1f%s%x1f%an']; + if (range) logArgs.push(range); + else logArgs.push('-20'); + const logRes = git(cwd, logArgs); + const commits = logRes.stdout.split('\n').filter(Boolean).map((l) => { + const [hash, subject, author] = l.split('\x1f'); + return { hash, subject, author }; + }); + + if (range && commits.length === 0) { + fail('no new commits since fixture baseline — agent produced nothing committed'); + } else { + ok(`${commits.length} commit(s) evaluated`); + } + + // 3. Conventional-commit subjects. + for (const c of commits) { + if (!CONVENTIONAL.test(c.subject)) { + fail(`non-conventional commit subject: "${c.subject}"`); + } + if (/\bWIP\b/i.test(c.subject)) fail(`WIP commit left in history: "${c.subject}"`); + } + if (commits.every((c) => CONVENTIONAL.test(c.subject)) && commits.length) { + ok('all commit subjects conventional'); + } + + // 4. Logical scoping — no single monster commit. + for (const c of commits) { + const st = git(cwd, ['show', '--stat', '--format=', c.hash]); + const changed = st.stdout.match(/(\d+) insertions?\(\+\)/); + const inserted = changed ? Number(changed[1]) : 0; + if (inserted > 2000) { + fail(`commit ${short(c.hash)} adds ${inserted} lines — likely unscoped dump`); + } + } + + // 5. No conflict markers anywhere tracked. + const grep = git(cwd, ['grep', '-lE', '^(<<<<<<<|=======|>>>>>>>)', 'HEAD']); + if (grep.code === 0 && grep.stdout.trim()) { + fail(`conflict markers present in: ${grep.stdout.trim().split('\n').join(', ')}`); + } else { + ok('no conflict markers'); + } + + // 6. No obvious secrets in the new diff. + if (range) { + const diff = git(cwd, ['diff', base, 'HEAD']); + for (const re of SECRET_PATTERNS) { + if (re.test(diff.stdout)) fail(`possible committed secret matching ${re}`); + } + } + + // 7. Stacked phase branches, when the scenario declares them. + const expectStack = scenario?.expect?.stackedBranches; + if (expectStack) { + const branches = git(cwd, ['branch', '--list', 'plan/*']).stdout + .split('\n').map((b) => b.replace(/^[*+]?\s*/, '').trim()).filter(Boolean); + if (branches.length === 0) { + fail('scenario expects stacked plan//phase-N branches; none found'); + } else { + ok(`stacked branches present: ${branches.join(', ')}`); + // Each phase-N must descend from phase-(N-1) or the base — never orphaned. + for (const b of branches) { + const mergeBase = git(cwd, ['merge-base', b, base]); + if (mergeBase.code !== 0 || !mergeBase.stdout.trim()) { + fail(`branch ${b} does not descend from base ${base} (base-resolution bug)`); + } + } + } + } + + return { id: 'git-tree-healthy', tier: 'deterministic', pass, details }; +} + +const short = (h) => (h || '').slice(0, 8); +const indent = (s) => s.split('\n').map((l) => ` ${l}`).join('\n'); diff --git a/tests/e2e/graders/index.mjs b/tests/e2e/graders/index.mjs new file mode 100644 index 00000000..e5989a6d --- /dev/null +++ b/tests/e2e/graders/index.mjs @@ -0,0 +1,51 @@ +// index.mjs — grader registry + orchestration (PLAN §7). +// +// A scenario passes when ALL its deterministic graders pass. Judge scores are +// attached to the report as advisory signal only. + +import { gitTreeHealthy } from './git-tree.mjs'; +import { artifactsSpec, artifactsPlan, artifactsBootstrap } from './artifacts.mjs'; +import { appTestsPass, scriptOutput, bugFixed } from './outcome.mjs'; +import { syncConvergence } from './sync-convergence.mjs'; +import { answersLint } from './answers-lint.mjs'; + +// id → grader fn. All deterministic graders are synchronous (git/fs/spawn). +export const GRADERS = { + 'git-tree-healthy': gitTreeHealthy, + 'artifacts-spec': artifactsSpec, + 'artifacts-plan': artifactsPlan, + 'artifacts-bootstrap': artifactsBootstrap, + 'app-tests-pass': appTestsPass, + 'script-output': scriptOutput, + 'bug-fixed': bugFixed, + 'sync-convergence': syncConvergence, + 'answers-lint': answersLint, +}; + +export function runGraders(graderIds, ctx) { + const results = []; + for (const id of graderIds) { + const fn = GRADERS[id]; + if (!fn) { + results.push({ id, tier: 'deterministic', pass: false, details: [`✗ unknown grader "${id}"`] }); + continue; + } + try { + results.push(fn(ctx)); + } catch (e) { + results.push({ id, tier: 'deterministic', pass: false, details: [`✗ grader threw: ${e.message}`] }); + } + } + return results; +} + +export function scenarioPassed(graderResults) { + const deterministic = graderResults.filter((g) => g.tier === 'deterministic'); + // A scenario with no deterministic graders can't pass — that's a spec error. + if (deterministic.length === 0) return false; + return deterministic.every((g) => g.pass); +} + +export function knownGraders() { + return Object.keys(GRADERS); +} diff --git a/tests/e2e/graders/judge.mjs b/tests/e2e/graders/judge.mjs new file mode 100644 index 00000000..6842af33 --- /dev/null +++ b/tests/e2e/graders/judge.mjs @@ -0,0 +1,61 @@ +// judge.mjs — LLM-judge grader (PLAN §7). STRICTLY ADVISORY. +// +// Scores fuzzy quality dimensions regex can't judge (does AGENTS.md reflect +// *this* stack? is the spec faithful to the request? is the fix real vs a hack?) +// using a fixed judge model with a structured rubric. Its scores are a +// trend/quality signal in the report — they NEVER decide pass/fail (§13.5). +// Deterministic graders alone gate. +// +// Enabled only with `--judge` (it costs tokens). Uses an adapter (default the +// Claude Code adapter) so the judge model is pinned and swappable. + +import { makeAdapter } from '../runner/adapters/index.mjs'; + +const RUBRICS = { + 'spec-fidelity': 'How faithfully does the produced *_SPEC.md capture the task request and its canned answers? 1=ignores it, 5=captures every stated requirement.', + 'plan-quality': 'Is the *_PLAN.md a real, buildable plan (phased, testable, risks noted) vs a stub? 1=stub, 5=excellent.', + 'fix-is-real': 'Is the bug fix a genuine root-cause fix vs a hack that games the test? 1=hack, 5=real fix.', + 'agents-md-relevance': 'Does AGENTS.md reflect *this* fixture stack specifically vs generic boilerplate? 1=generic, 5=stack-specific.', +}; + +export async function runJudge(ctx, dimensions, { model, adapter = 'claude-code' } = {}) { + const results = {}; + const ad = makeAdapter(adapter); + const avail = ad.available(); + if (!avail.available) { + return { skipped: true, reason: avail.reason, scores: {} }; + } + + for (const dim of dimensions) { + const rubric = RUBRICS[dim]; + if (!rubric) continue; + const prompt = [ + 'You are an impartial code-quality judge. Score the following on a 1.0–5.0 scale.', + `Rubric — ${dim}: ${rubric}`, + '', + 'Context to judge (files + transcript excerpts):', + summarizeContext(ctx), + '', + 'Respond with ONLY a JSON object: {"score": , "reason": ""}.', + ].join('\n'); + + try { + const res = await ad.run({ prompt }, { model, cwd: ctx.cwd, timeoutMs: 120000, tasksDir: ctx.tasksDir }); + const m = res.transcript.match(/\{[\s\S]*"score"[\s\S]*\}/); + results[dim] = m ? JSON.parse(m[0]) : { score: null, reason: 'unparseable judge output' }; + } catch (e) { + results[dim] = { score: null, reason: `judge error: ${e.message}` }; + } + } + return { skipped: false, scores: results }; +} + +function summarizeContext(ctx) { + const bits = []; + for (const step of ctx.steps || []) { + bits.push(`### ${step.invoke}\n${(step.transcript || '').slice(0, 2000)}`); + } + return bits.join('\n\n').slice(0, 8000); +} + +export { RUBRICS }; diff --git a/tests/e2e/graders/outcome.mjs b/tests/e2e/graders/outcome.mjs new file mode 100644 index 00000000..f5ecfb4d --- /dev/null +++ b/tests/e2e/graders/outcome.mjs @@ -0,0 +1,108 @@ +// outcome.mjs — outcome graders (PLAN §7): the app actually works. +// +// app-tests-pass — run the fixture's test command; all pass. For medplum this +// is `vitest` against MockClient, fully offline (§4). +// script-output — S4: the produced one-off script exists, runs +// non-interactively, and emits its expected artifact. +// bug-fixed — S3: the previously-failing target test flips red→green with +// no new regressions, and the fix touches the planted region. + +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { run, git, walk } from '../runner/lib.mjs'; + +// Split a "cmd arg arg" string into argv (fixtures declare commands as strings). +function argv(cmd) { + return cmd.match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g).map((s) => s.replace(/^["']|["']$/g, '')); +} + +export function appTestsPass(ctx) { + const cmd = ctx.fixture?.commands?.test; + if (!cmd) { + return { id: 'app-tests-pass', tier: 'deterministic', pass: false, + details: ['✗ fixture.json declares no `commands.test`'] }; + } + const [bin, ...args] = argv(cmd); + const res = run(bin, args, { cwd: ctx.cwd, timeoutMs: (ctx.timeoutMs || 1800000) }); + const pass = res.code === 0; + return { + id: 'app-tests-pass', + tier: 'deterministic', + pass, + details: [ + pass ? `✓ \`${cmd}\` passed (${res.wallSec}s)` : `✗ \`${cmd}\` exited ${res.code}`, + ...(pass ? [] : [tail(res.stdout + res.stderr, 40)]), + ], + }; +} + +export function scriptOutput(ctx) { + const spec = ctx.scenario?.expect?.script; + if (!spec) { + return { id: 'script-output', tier: 'deterministic', pass: false, + details: ['✗ scenario declares no expect.script { run, artifact }'] }; + } + const details = []; + let pass = true; + const fail = (m) => { pass = false; details.push(`✗ ${m}`); }; + + // The script should have landed *somewhere* under the scripts dir. + const scriptsDir = ctx.fixture?.commands?.scripts_dir || 'scripts'; + const found = walk(join(ctx.cwd, scriptsDir)); + if (found.length === 0) fail(`no script created under ${scriptsDir}/`); + else details.push(`✓ script(s) present under ${scriptsDir}/`); + + // Run it non-interactively and check the expected artifact appears. + if (spec.run) { + const [bin, ...args] = argv(spec.run); + const res = run(bin, args, { cwd: ctx.cwd, timeoutMs: 300000, input: '' }); + if (res.code !== 0) fail(`\`${spec.run}\` exited ${res.code}: ${tail(res.stderr, 10)}`); + else details.push(`✓ \`${spec.run}\` ran clean`); + } + if (spec.artifact) { + if (existsSync(join(ctx.cwd, spec.artifact))) details.push(`✓ produced ${spec.artifact}`); + else fail(`expected artifact ${spec.artifact} not produced`); + } + return { id: 'script-output', tier: 'deterministic', pass, details }; +} + +export function bugFixed(ctx) { + const spec = ctx.scenario?.expect?.bug; + if (!spec) { + return { id: 'bug-fixed', tier: 'deterministic', pass: false, + details: ['✗ scenario declares no expect.bug { testCmd, region }'] }; + } + const details = []; + let pass = true; + const fail = (m) => { pass = false; details.push(`✗ ${m}`); }; + const ok = (m) => details.push(`✓ ${m}`); + + // 1. The full suite passes now (target test green + no regressions). + const cmd = spec.testCmd || ctx.fixture?.commands?.test; + const [bin, ...args] = argv(cmd); + const res = run(bin, args, { cwd: ctx.cwd, timeoutMs: (ctx.timeoutMs || 1800000) }); + if (res.code === 0) ok(`test suite green after fix (\`${cmd}\`)`); + else fail(`test suite still failing:\n${tail(res.stdout + res.stderr, 30)}`); + + // 2. The fix touched the planted-bug region (guard against "deleted the test"). + if (spec.region) { + const base = ctx.baseRef || 'HEAD~1'; + const diff = git(ctx.cwd, ['diff', '--name-only', base, 'HEAD']); + const touched = diff.stdout.split('\n').filter(Boolean); + if (touched.some((f) => f.includes(spec.region))) ok(`fix touched planted region ${spec.region}`); + else fail(`fix did not touch planted region ${spec.region} (touched: ${touched.join(', ') || 'nothing'})`); + + // Sanity: the target test file must still exist and not be gutted. + if (spec.testFile) { + const tf = join(ctx.cwd, spec.testFile); + if (!existsSync(tf)) fail(`target test file ${spec.testFile} was deleted`); + else if ((readFileSync(tf, 'utf8').match(/\b(it|test)\s*\(/g) || []).length === 0) { + fail(`target test file ${spec.testFile} has no tests left — gutted?`); + } else ok('target test file intact'); + } + } + + return { id: 'bug-fixed', tier: 'deterministic', pass, details }; +} + +const tail = (s, n) => (s || '').split('\n').slice(-n).join('\n'); diff --git a/tests/e2e/graders/schema.mjs b/tests/e2e/graders/schema.mjs new file mode 100644 index 00000000..4d579dcf --- /dev/null +++ b/tests/e2e/graders/schema.mjs @@ -0,0 +1,117 @@ +// schema.mjs — a *minimal* JSON-Schema-subset validator + schema graders. +// +// The harness must be dependency-free (the repo ships with only `yaml` as a +// dev dep), so rather than pull in ajv we implement just the keywords our +// sample payloads and generated artifacts actually use: type, required, +// properties, additionalProperties (bool), enum, const, pattern, minLength, +// items, minItems. This is deliberately small — it validates the shapes this +// repo produces, not arbitrary Draft 2020-12. + +import { loadJson, readText } from '../runner/lib.mjs'; + +const TYPE_CHECKS = { + object: (v) => v !== null && typeof v === 'object' && !Array.isArray(v), + array: (v) => Array.isArray(v), + string: (v) => typeof v === 'string', + integer: (v) => Number.isInteger(v), + number: (v) => typeof v === 'number', + boolean: (v) => typeof v === 'boolean', + null: (v) => v === null, +}; + +export function validate(schema, data, path = '') { + const errors = []; + const at = path || '(root)'; + + if (schema.type) { + const types = Array.isArray(schema.type) ? schema.type : [schema.type]; + if (!types.some((t) => TYPE_CHECKS[t]?.(data))) { + errors.push(`${at}: expected type ${types.join('|')}, got ${describe(data)}`); + return errors; // no point checking further against the wrong type + } + } + + if ('const' in schema && !deepEqual(schema.const, data)) { + errors.push(`${at}: expected const ${JSON.stringify(schema.const)}, got ${JSON.stringify(data)}`); + } + + if (schema.enum && !schema.enum.some((e) => deepEqual(e, data))) { + errors.push(`${at}: ${JSON.stringify(data)} not in enum ${JSON.stringify(schema.enum)}`); + } + + if (typeof data === 'string') { + if (schema.minLength != null && data.length < schema.minLength) { + errors.push(`${at}: string shorter than minLength ${schema.minLength}`); + } + if (schema.pattern && !new RegExp(schema.pattern).test(data)) { + errors.push(`${at}: string does not match pattern ${schema.pattern}`); + } + } + + if (Array.isArray(data)) { + if (schema.minItems != null && data.length < schema.minItems) { + errors.push(`${at}: array shorter than minItems ${schema.minItems}`); + } + if (schema.items) { + data.forEach((item, i) => errors.push(...validate(schema.items, item, `${at}[${i}]`))); + } + } + + if (TYPE_CHECKS.object(data) && (schema.properties || schema.required || 'additionalProperties' in schema)) { + for (const key of schema.required || []) { + if (!(key in data)) errors.push(`${at}: missing required property "${key}"`); + } + const props = schema.properties || {}; + for (const [key, val] of Object.entries(data)) { + if (props[key]) { + errors.push(...validate(props[key], val, path ? `${path}.${key}` : key)); + } else if (schema.additionalProperties === false) { + errors.push(`${at}: additional property "${key}" not allowed`); + } + } + } + + return errors; +} + +function describe(v) { + if (v === null) return 'null'; + if (Array.isArray(v)) return 'array'; + return typeof v; +} + +function deepEqual(a, b) { + if (a === b) return true; + if (typeof a !== typeof b) return false; + if (a && b && typeof a === 'object') { + const ka = Object.keys(a); const kb = Object.keys(b); + if (Array.isArray(a) !== Array.isArray(b)) return false; + if (ka.length !== kb.length) return false; + return ka.every((k) => deepEqual(a[k], b[k])); + } + return false; +} + +// Validate a data object against a schema file on disk. Returns {ok, errors}. +export function validateFile(schemaPath, data) { + const schema = loadJson(schemaPath); + const errors = validate(schema, data); + return { ok: errors.length === 0, errors }; +} + +// Parse YAML frontmatter (--- ... ---) from a markdown file body. Returns the +// raw frontmatter block string, or null. We only need presence + a couple of +// scalar keys, so a tiny line parser beats pulling in a YAML dep here. +export function frontmatter(text) { + if (!text) return null; + const m = text.match(/^---\n([\s\S]*?)\n---/); + if (!m) return null; + const obj = {}; + for (const line of m[1].split('\n')) { + const mm = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/); + if (mm) obj[mm[1]] = mm[2].replace(/^["']|["']$/g, '').trim(); + } + return { raw: m[1], fields: obj }; +} + +export { readText }; diff --git a/tests/e2e/graders/sync-convergence.mjs b/tests/e2e/graders/sync-convergence.mjs new file mode 100644 index 00000000..d5584eb5 Binary files /dev/null and b/tests/e2e/graders/sync-convergence.mjs differ diff --git a/tests/e2e/l1/smoke.test.mjs b/tests/e2e/l1/smoke.test.mjs new file mode 100644 index 00000000..6494fc3c --- /dev/null +++ b/tests/e2e/l1/smoke.test.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +// smoke.test.mjs — L1 install/bootstrap smoke (PLAN §3 L1, §12 Phase 1). +// +// No model, no paid API — deterministic. Runs the CLI against a scratch project +// and asserts: +// 1. install --tool lands the vinta- skills under the vendor dir +// 2. install is idempotent (re-running doesn't duplicate/corrupt) +// 3. uninstall leaves the tree byte-clean — it only touches marker-tagged +// artifacts, so a pre-existing user file with the same name is preserved +// +// Usage: node tests/e2e/l1/smoke.test.mjs +// Exit 0 = all assertions pass; 1 = a failure. + +import { mkdtempSync, rmSync, existsSync, writeFileSync, readFileSync, readdirSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(HERE, '..', '..', '..'); +const CLI = join(REPO_ROOT, 'vinta-ai-workflows.mjs'); + +let failures = 0; +const pass = (m) => console.log(` \x1b[32m✓\x1b[0m ${m}`); +const fail = (m) => { failures++; console.error(` \x1b[31m✗\x1b[0m ${m}`); }; +function assert(cond, m) { cond ? pass(m) : fail(m); } + +function cli(args, cwd) { + return execFileSync('node', [CLI, ...args], { cwd, encoding: 'utf8' }); +} + +// Snapshot every file path + content hash under a dir (excluding a skip list). +function snapshot(dir) { + const out = {}; + const walk = (d, base) => { + for (const name of readdirSync(d, { withFileTypes: true })) { + const p = join(d, name.name); + const rel = join(base, name.name); + if (name.isDirectory()) walk(p, rel); + else out[rel] = readFileSync(p, 'utf8'); + } + }; + if (existsSync(dir)) walk(dir, ''); + return out; +} + +function scenario(tool, vendorDir) { + console.log(`\n[L1] tool=${tool} → ${vendorDir}`); + const proj = mkdtempSync(join(tmpdir(), 'e2e-l1-')); + try { + // Seed a pre-existing user file to prove uninstall doesn't nuke it. + mkdirSync(join(proj, vendorDir), { recursive: true }); + const userFile = join(proj, vendorDir, 'my-own-skill'); + mkdirSync(userFile, { recursive: true }); + writeFileSync(join(userFile, 'SKILL.md'), '# my own skill\n'); + const before = snapshot(proj); + + // 1. install (copy mode so it's self-contained + carries the marker). + cli(['install', '--tool', tool, '--copy', '--target', proj], proj); + const installed = readdirSync(join(proj, vendorDir)); + assert(installed.includes('vinta-bootstrap-ai-tools'), + 'install placed vinta-bootstrap-ai-tools'); + assert(existsSync(join(proj, vendorDir, 'vinta-bootstrap-ai-tools', 'SKILL.md')), + 'installed skill has SKILL.md'); + assert(existsSync(join(proj, vendorDir, 'vinta-bootstrap-ai-tools', '.installed-by-vinta-ai-workflows')), + 'installed skill carries the marker file'); + + // 2. idempotency: re-install, expect no error + same set. + const set1 = readdirSync(join(proj, vendorDir)).sort().join(','); + cli(['install', '--tool', tool, '--copy', '--target', proj], proj); + const set2 = readdirSync(join(proj, vendorDir)).sort().join(','); + assert(set1 === set2, 'install is idempotent (skill set unchanged on re-run)'); + + // 3. uninstall → tree byte-clean vs. the pre-install snapshot. + cli(['uninstall', '--tool', tool, '--target', proj], proj); + const after = snapshot(proj); + const beforeKeys = Object.keys(before).sort().join('\n'); + const afterKeys = Object.keys(after).sort().join('\n'); + assert(beforeKeys === afterKeys, + 'uninstall leaves the tree byte-clean (only marker-tagged artifacts removed)'); + assert(existsSync(join(userFile, 'SKILL.md')), + 'uninstall preserved the pre-existing user skill'); + } finally { + rmSync(proj, { recursive: true, force: true }); + } +} + +console.log('=== L1 install/uninstall smoke ==='); +scenario('claude-code', '.claude/skills'); +scenario('agents', '.agents/skills'); + +if (failures) { + console.error(`\n\x1b[31mL1 smoke: ${failures} failure(s)\x1b[0m`); + process.exit(1); +} +console.log('\n\x1b[32mL1 smoke: all assertions passed\x1b[0m'); diff --git a/tests/e2e/report/generate.mjs b/tests/e2e/report/generate.mjs new file mode 100644 index 00000000..4d69feb9 --- /dev/null +++ b/tests/e2e/report/generate.mjs @@ -0,0 +1,69 @@ +// generate.mjs — human + machine reporting (PLAN §9). +// +// Emits summary.json (rolled-up cells + baseline diff) and REPORT.md (the +// comparison matrix + per-scenario drill-down + links to failing transcripts). + +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { writeJson, pct, round } from '../runner/lib.mjs'; + +export function writeSummary(resultsDir, { version, gitSha, cells, baseline }) { + const summary = { version, gitSha, generatedFrom: 'tests/e2e/runner', cells, baseline }; + writeJson(join(resultsDir, 'summary.json'), summary); + return summary; +} + +export function writeReport(resultsDir, summary) { + const { version, gitSha, cells, baseline } = summary; + const lines = []; + lines.push(`# E2E Report — vinta-ai-workflows \`${version}\``); + lines.push(''); + lines.push(`- Harness commit: \`${gitSha || 'unknown'}\``); + lines.push(`- Cells: ${cells.length}`); + if (baseline?.baselineVersion) lines.push(`- Compared against baseline: \`${baseline.baselineVersion}\``); + lines.push(''); + + if (baseline?.flags?.length) { + lines.push('## ⚠️ Regression flags'); + lines.push(''); + for (const f of baseline.flags) lines.push(`- **${f}**`); + lines.push(''); + } + + lines.push('## Comparison matrix'); + lines.push(''); + lines.push('| Scenario | Vendor / Model | Pass-rate | Median tokens | Median cost | Δ tokens vs baseline |'); + lines.push('|---|---|---|---|---|---|'); + for (const c of cells) { + const cmp = baseline?.comparisons?.find((x) => x.key === `${c.scenario}|${c.vendor}|${c.model}`); + const delta = cmp && cmp.tokDelta != null ? `${cmp.tokDelta > 0 ? '+' : ''}${round(cmp.tokDelta * 100, 1)}%` : '—'; + lines.push( + `| ${c.scenario} | ${c.vendor} / ${c.model} | ${pct(c.passRate)} | ` + + `${fmt(c.tokens.median)} | $${round(c.costUsd.median, 4)} | ${delta} |`, + ); + } + lines.push(''); + + lines.push('## Per-scenario drill-down'); + lines.push(''); + for (const c of cells) { + lines.push(`### ${c.scenario} — ${c.vendor} / ${c.model}`); + lines.push(''); + lines.push(`- Runs: ${c.runs}, pass-rate ${pct(c.passRate)}`); + lines.push(`- Tokens: mean ${fmt(c.tokens.mean)}, median ${fmt(c.tokens.median)}, p95 ${fmt(c.tokens.p95)}, variance ${fmt(c.tokens.variance)}`); + lines.push(`- Cost: mean $${round(c.costUsd.mean, 4)}, wall mean ${round(c.wallSec.mean, 1)}s`); + if (c.failures?.length) { + lines.push('- Failing runs:'); + for (const f of c.failures) { + lines.push(` - run ${f.run}: ${f.failedGraders.join(', ')} — \`${f.transcriptPath}\``); + } + } + lines.push(''); + } + + const md = lines.join('\n'); + writeFileSync(join(resultsDir, 'REPORT.md'), `${md}\n`); + return md; +} + +const fmt = (n) => (n >= 1000 ? `${round(n / 1000, 1)}k` : `${round(n, 0)}`); diff --git a/tests/e2e/runner/adapters/base.mjs b/tests/e2e/runner/adapters/base.mjs new file mode 100644 index 00000000..d9b62ec7 --- /dev/null +++ b/tests/e2e/runner/adapters/base.mjs @@ -0,0 +1,83 @@ +// base.mjs — VendorAdapter contract + shared helpers. +// +// Adapter contract (from PLAN §6): +// run(step, { model, cwd, timeoutMs }) → { +// transcript, filesTouched, usage: {inTok, outTok, cacheTok, costUsd}, +// wallSec, exitCode +// } +// +// A step is one entry from a scenario's `steps[]`: +// { invoke: "/create-spec", input?: "tasks/feature.md", +// answers?: "tasks/feature.answers.md", autonomous?: true, prompt?: "..." } +// +// The adapter's job is only to *drive the agent* for one step and report what +// happened. Grading and telemetry aggregation live elsewhere. + +import { readText, git } from '../lib.mjs'; +import { join } from 'node:path'; + +// Build the prompt for a step. This is the heart of the "canned answers" +// contract (§6): the answers file is injected so a headless agent never stalls +// on an interview question, and is told not to invent anything beyond it. +export function buildPrompt(step, { tasksDir }) { + if (step.prompt) return step.prompt; // fully explicit override + + const parts = []; + // Invoking a skill is just putting its trigger in the prompt. + parts.push(step.invoke); + + const input = step.input ? readText(join(tasksDir, step.input)) : null; + if (input) { + parts.push('\n## Task\n'); + parts.push(input.trim()); + } + + const answers = step.answers ? readText(join(tasksDir, step.answers)) : null; + if (answers) { + parts.push('\n## Canned answers to every question you might ask\n'); + parts.push(answers.trim()); + parts.push( + '\nThese are the answers to every interview question this skill would ask. ' + + 'Do not ask for anything else. If something is genuinely missing, state the ' + + 'assumption explicitly and proceed — never block waiting for input.', + ); + } + + if (step.autonomous) { + parts.push( + '\nRun this to completion autonomously across all phases. Do not stop for ' + + 'confirmation between phases; make reasonable decisions and keep going until done.', + ); + } + + return parts.join('\n'); +} + +// Snapshot which files a step touched, using git porcelain in the isolated +// fixture copy. Adapters call this before/after when the CLI itself doesn't +// report file writes. +export function gitDirtyFiles(cwd) { + const res = git(cwd, ['status', '--porcelain']); + if (res.code !== 0) return []; + return res.stdout.split('\n').map((l) => l.slice(3).trim()).filter(Boolean); +} + +export const EMPTY_USAGE = { inTok: 0, outTok: 0, cacheTok: 0, costUsd: 0 }; + +// Base class documents the contract; concrete adapters override run(). +export class VendorAdapter { + constructor({ vendor, models } = {}) { + this.vendor = vendor; + this.models = models || []; + } + + // Return { available: boolean, reason?: string }. When unavailable the runner + // reports the cell as *skipped*, never passed (§11: a vendor without a key is + // skipped). + available() { return { available: true }; } + + // eslint-disable-next-line no-unused-vars + async run(step, ctx) { + throw new Error(`${this.vendor}: run() not implemented`); + } +} diff --git a/tests/e2e/runner/adapters/claude-code.mjs b/tests/e2e/runner/adapters/claude-code.mjs new file mode 100644 index 00000000..f6574250 --- /dev/null +++ b/tests/e2e/runner/adapters/claude-code.mjs @@ -0,0 +1,61 @@ +// claude-code.mjs — Claude Code adapter (headless, richest usage data). +// +// Invocation: `claude -p "" --output-format json [--model ]`. +// Headless runs can't answer permission prompts, so we run inside the isolated +// fixture copy with permissions bypassed (§6 isolation — always a throwaway +// worktree/clone, never a shared checkout). The JSON envelope carries `usage` +// (input/output/cache tokens) and `total_cost_usd`, which map straight onto our +// telemetry record (§8). + +import { VendorAdapter, buildPrompt, gitDirtyFiles, EMPTY_USAGE } from './base.mjs'; +import { run, hasCommand } from '../lib.mjs'; + +export class ClaudeCodeAdapter extends VendorAdapter { + constructor(opts = {}) { + super({ vendor: 'claude-code', ...opts }); + } + + available() { + if (!hasCommand('claude')) { + return { available: false, reason: 'claude CLI not on PATH' }; + } + // Auth is via subscription or ANTHROPIC_API_KEY; we can't fully verify here, + // so presence of the CLI is the gate. A run that fails auth surfaces as a + // non-zero exit, reported (not silently passed). + return { available: true }; + } + + async run(step, { model, cwd, timeoutMs, tasksDir }) { + const prompt = buildPrompt(step, { tasksDir }); + const args = ['-p', prompt, '--output-format', 'json', '--dangerously-skip-permissions']; + if (model) args.push('--model', model); + + const res = run('claude', args, { cwd, timeoutMs }); + + let usage = { ...EMPTY_USAGE }; + let transcript = res.stdout; + try { + const json = JSON.parse(res.stdout); + transcript = json.result ?? res.stdout; + const u = json.usage || {}; + usage = { + inTok: u.input_tokens ?? 0, + outTok: u.output_tokens ?? 0, + cacheTok: (u.cache_read_input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0), + costUsd: json.total_cost_usd ?? 0, + }; + } catch { + // Non-JSON output (e.g. an early crash) — keep raw transcript, zero usage. + } + + return { + transcript, + filesTouched: gitDirtyFiles(cwd), + usage, + wallSec: res.wallSec, + exitCode: res.timedOut ? 124 : res.code, + timedOut: res.timedOut, + stderr: res.stderr, + }; + } +} diff --git a/tests/e2e/runner/adapters/codex.mjs b/tests/e2e/runner/adapters/codex.mjs new file mode 100644 index 00000000..3e31e949 --- /dev/null +++ b/tests/e2e/runner/adapters/codex.mjs @@ -0,0 +1,61 @@ +// codex.mjs — OpenAI Codex adapter (best-effort; see Phase 0 spike, §12/§14). +// +// Invocation: `codex exec ""` runs non-interactively. Open questions +// the Phase 0 spike must confirm before this is considered production-ready: +// 1. Does `codex exec "/skill-name …"` reliably fire a shipped skill, or must +// the skill body be inlined? (buildPrompt puts the trigger first either way.) +// 2. Can a multi-phase `implement-plan` run unattended to completion? +// 3. Where/whether does `codex exec` surface per-run token usage? +// +// Until the spike resolves (3), usage is parsed best-effort from stdout and +// otherwise reported as zero with `usageEstimated: true` so telemetry never +// silently claims Codex numbers it doesn't have. + +import { VendorAdapter, buildPrompt, gitDirtyFiles, EMPTY_USAGE } from './base.mjs'; +import { run, hasCommand } from '../lib.mjs'; + +export class CodexAdapter extends VendorAdapter { + constructor(opts = {}) { + super({ vendor: 'codex', ...opts }); + } + + available() { + if (!hasCommand('codex')) { + return { available: false, reason: 'codex CLI not on PATH' }; + } + return { available: true }; + } + + async run(step, { model, cwd, timeoutMs, tasksDir }) { + const prompt = buildPrompt(step, { tasksDir }); + // Non-interactive; pre-grant workspace writes for the sandboxed copy. + const args = ['exec', '--full-auto', prompt]; + if (model) args.push('--model', model); + + const res = run('codex', args, { cwd, timeoutMs }); + + const usage = parseCodexUsage(res.stdout) || { ...EMPTY_USAGE }; + + return { + transcript: res.stdout, + filesTouched: gitDirtyFiles(cwd), + usage, + usageEstimated: !parseCodexUsage(res.stdout), + wallSec: res.wallSec, + exitCode: res.timedOut ? 124 : res.code, + timedOut: res.timedOut, + stderr: res.stderr, + }; + } +} + +// Best-effort: look for a "tokens used" summary line Codex prints. Returns null +// when nothing parseable is found so the caller can flag the estimate. +function parseCodexUsage(stdout) { + if (!stdout) return null; + const m = stdout.match(/tokens?\s+used[:\s]+([\d,]+)/i); + if (!m) return null; + const total = Number(m[1].replace(/,/g, '')); + if (!Number.isFinite(total)) return null; + return { inTok: total, outTok: 0, cacheTok: 0, costUsd: 0 }; +} diff --git a/tests/e2e/runner/adapters/index.mjs b/tests/e2e/runner/adapters/index.mjs new file mode 100644 index 00000000..add8931d --- /dev/null +++ b/tests/e2e/runner/adapters/index.mjs @@ -0,0 +1,37 @@ +// index.mjs — adapter registry. v1 targets Claude Code + Codex (§6); `mock` +// is the no-spend self-test adapter. Cursor / Copilot are deferred. + +import { ClaudeCodeAdapter } from './claude-code.mjs'; +import { CodexAdapter } from './codex.mjs'; +import { MockAdapter } from './mock.mjs'; + +const REGISTRY = { + 'claude-code': ClaudeCodeAdapter, + codex: CodexAdapter, + mock: MockAdapter, +}; + +const ALIASES = { + claude: 'claude-code', + 'claude-code': 'claude-code', + codex: 'codex', + 'openai-codex': 'codex', + mock: 'mock', +}; + +// The real vendors for a full-matrix run (mock is opt-in only). +export const MATRIX_VENDORS = ['claude-code', 'codex']; + +export function resolveVendor(name) { + return ALIASES[name?.toLowerCase()] || null; +} + +export function makeAdapter(vendor, opts = {}) { + const Cls = REGISTRY[vendor]; + if (!Cls) throw new Error(`unknown vendor "${vendor}". Known: ${Object.keys(REGISTRY).join(', ')}`); + return new Cls(opts); +} + +export function knownVendors() { + return Object.keys(REGISTRY); +} diff --git a/tests/e2e/runner/adapters/mock.mjs b/tests/e2e/runner/adapters/mock.mjs new file mode 100644 index 00000000..f1cbb245 --- /dev/null +++ b/tests/e2e/runner/adapters/mock.mjs @@ -0,0 +1,149 @@ +// mock.mjs — a no-spend adapter that fabricates plausible, grader-passing work. +// +// Purpose: exercise the *whole* harness pipeline (isolation → steps → graders → +// telemetry → report) deterministically, with zero API cost. It is NOT a vendor +// — `--vendor mock` is how you self-test the runner and graders after changing +// them, and how CI can smoke the plumbing without keys. +// +// It reproduces the *shape* a good agent run would leave behind: spec/plan files +// with the required sections, a fixed defect, a one-off script, conventional +// commits on properly-based stacked branches. It does NOT run real skills. + +import { VendorAdapter, buildPrompt, gitDirtyFiles } from './base.mjs'; +import { git } from '../lib.mjs'; +import { writeFileSync, mkdirSync, existsSync, readFileSync } from 'node:fs'; +import { join, dirname } from 'node:path'; + +export class MockAdapter extends VendorAdapter { + constructor(opts = {}) { + super({ vendor: 'mock', ...opts }); + } + + available() { return { available: true }; } + + async run(step, ctx) { + const { cwd, model, tasksDir, fixture, scenario } = ctx; + // Realize the prompt so buildPrompt (canned-answer injection) is exercised + // and the transcript reflects it — the answers-lint grader reads transcripts. + const prompt = buildPrompt(step, { tasksDir }); + const started = Date.now(); + + const plansDir = fixture?.commands?.ai_plans_dir || 'ai-plans'; + const feature = 'patient-allergy-panel'; + const lines = [`[mock] resolved prompt (${prompt.length} chars)`, `[mock] model=${model}`]; + + if (step.invoke.includes('create-spec')) { + write(cwd, `${plansDir}/2026-08-01-${feature}_SPEC.md`, mockSpec()); + commit(cwd, `docs(spec): add ${feature} spec`); + lines.push('[mock] wrote spec'); + } else if (step.invoke.includes('plan-feature')) { + branchStack(cwd, feature, 1); + write(cwd, `${plansDir}/2026-08-01-${feature}_PLAN.md`, mockPlan()); + commit(cwd, `docs(plan): add ${feature} implementation plan`); + lines.push('[mock] wrote plan on stacked branch'); + } else if (step.invoke.includes('implement-plan')) { + write(cwd, `src/mock-${feature}.ts`, `export const allergyPanel = () => 'ok';\n`); + commit(cwd, `feat(${feature}): implement allergy panel`); + lines.push('[mock] implemented feature'); + } else if (step.invoke.includes('systematic-debugging')) { + // "Fix" the planted bug by touching the region named in the bug task. + write(cwd, 'src/mock-bugfix.ts', `export const fixed = true;\n`); + commit(cwd, 'fix(allergy): correct AllergyIntolerance patient reference field'); + lines.push('[mock] applied bug fix'); + } else if (step.invoke.includes('add-one-off-script')) { + const scriptsDir = fixture?.commands?.scripts_dir || 'scripts'; + write(cwd, `${scriptsDir}/one_off_2026_08_01_export_observations.ts`, + `// mock one-off script\nconsole.log('rows,written');\n`); + commit(cwd, 'chore(script): add observations CSV export one-off'); + lines.push('[mock] wrote one-off script'); + } else if (step.invoke.includes('bootstrap') || step.invoke.includes('sync')) { + write(cwd, 'AGENTS.md', `# AGENTS\n\nStack: ${fixture?.project?.stack_summary || 'unknown'}\n`); + mkdirSync(join(cwd, 'ai-tools', 'skills'), { recursive: true }); + write(cwd, 'ai-tools/agents/deploy-author.yaml', mockAgentYaml()); + write(cwd, '.vinta-ai-workflows.yaml', mockConfigYaml(fixture)); + commit(cwd, `chore(ai-tools): ${step.invoke.includes('sync') ? 'sync' : 'bootstrap'} harness`); + lines.push('[mock] wrote ai-tools scaffold'); + } + + void scenario; + return { + transcript: `${lines.join('\n')}\n\n(resolved prompt)\n${prompt}`, + filesTouched: gitDirtyFiles(cwd), + usage: { inTok: 1000, outTok: 200, cacheTok: 500, costUsd: 0 }, + wallSec: Math.max(0.01, (Date.now() - started) / 1000), + exitCode: 0, + }; + } +} + +function write(cwd, rel, content) { + const p = join(cwd, rel); + mkdirSync(dirname(p), { recursive: true }); + writeFileSync(p, content); +} + +function commit(cwd, message) { + git(cwd, ['add', '-A']); + git(cwd, ['commit', '-q', '-m', message, + '--author', 'mock-agent '], + { env: { GIT_COMMITTER_NAME: 'mock-agent', GIT_COMMITTER_EMAIL: 'mock@example.com' } }); +} + +// Create/checkout a stacked phase branch `plan//phase-N` based on the +// current branch — mirrors the real stacked-branch convention the git-tree +// grader asserts (guarding fix/stack-branches-base-resolution). +function branchStack(cwd, feature, phase) { + git(cwd, ['checkout', '-q', '-b', `plan/${feature}/phase-${phase}`]); +} + +const mockSpec = () => `# ${'Patient Allergy Panel'} — SPEC + +## Overview +Add a patient allergy panel reading/writing AllergyIntolerance resources. + +## Goals +- Read AllergyIntolerance for the active patient. + +## Non-goals +- Server-side access policies. + +## Acceptance criteria +- Tests pass against MockClient. +`; + +const mockPlan = () => `# Patient Allergy Panel — PLAN + +## Context +Implements the SPEC. + +## Phases +### Phase 1 +- Add the panel component. + +## Testing +- Vitest against MockClient. + +## Risks +- Version skew across @medplum packages. +`; + +const mockAgentYaml = () => `schema_version: 1 +name: deploy-author +description: Specialist for deploy-affecting Medplum changes. +access: read-write +body: | + You handle deploy-affecting changes. +`; + +const mockConfigYaml = (fixture) => `schema_version: 1 +vinta_ai_workflows_version: 0.4.0 +project: + name: ${fixture?.project?.name || 'fixture'} + default_branch: main + code_host: github + stack_summary: ${fixture?.project?.stack_summary || 'unknown'} + ai_plans_dir: ${fixture?.commands?.ai_plans_dir || 'ai-plans'} +`; + +// Keep import used even if a future path reads the file back. +export { existsSync, readFileSync }; diff --git a/tests/e2e/runner/isolation.mjs b/tests/e2e/runner/isolation.mjs new file mode 100644 index 00000000..56ec9a9e --- /dev/null +++ b/tests/e2e/runner/isolation.mjs @@ -0,0 +1,64 @@ +// isolation.mjs — per-run isolated fixture copies (PLAN §6 isolation). +// +// Every run operates on a throwaway copy of the fixture state — never the +// vendored fixture in place. We clone the state's git repo into a tmp dir and +// check out the frozen ref, so each run starts from an identical, committed +// baseline and the git-tree grader has a real base to diff against. +// +// (The sandboxing of *agent permissions* — pre-granting filesystem writes so a +// headless agent doesn't stall — is handled per-adapter, e.g. Claude Code's +// --dangerously-skip-permissions. This module handles filesystem isolation. +// See feat/filesystem-sandbox-for-worktrees for stronger OS-level sandboxing.) + +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { git, run } from './lib.mjs'; + +// Resolve a fixture state to { repoDir, ref } from fixture.json. +export function resolveState(fixture, fixtureDir, state) { + const s = fixture.states?.[state]; + if (!s) throw new Error(`fixture "${fixture.name}" has no state "${state}" in fixture.json`); + const repoDir = resolve(fixtureDir, s.path); + return { repoDir, ref: s.ref || 'HEAD', vendored: existsSync(repoDir) }; +} + +// Create an isolated copy of a fixture state. Returns { dir, baseRef, cleanup }. +export function isolate(fixture, fixtureDir, state, { keep = false } = {}) { + const { repoDir, ref, vendored } = resolveState(fixture, fixtureDir, state); + if (!vendored) { + throw new Error( + `fixture state not vendored: ${repoDir}\n` + + `Run the fixture vendoring script first (e.g. \`node tests/e2e/fixtures/${fixture.name}/vendor.mjs\`).`, + ); + } + if (!existsSync(join(repoDir, '.git'))) { + throw new Error(`fixture state at ${repoDir} is not a git repo — vendoring must commit each state`); + } + + const dir = mkdtempSync(join(tmpdir(), `e2e-${fixture.name}-${state}-`)); + // Local clone is fast (hardlinks) and gives us a real repo to diff against. + const clone = run('git', ['clone', '--quiet', '--no-hardlinks', repoDir, dir]); + if (clone.code !== 0) { + rmSync(dir, { recursive: true, force: true }); + throw new Error(`git clone of fixture failed: ${clone.stderr}`); + } + const co = git(dir, ['checkout', '--quiet', ref]); + if (co.code !== 0) { + rmSync(dir, { recursive: true, force: true }); + throw new Error(`git checkout ${ref} failed: ${co.stderr}`); + } + // Detach & record base so graders diff new work against exactly this ref. + const baseSha = git(dir, ['rev-parse', 'HEAD']).stdout.trim(); + // Ensure a committer identity exists for agent commits inside the copy. + git(dir, ['config', 'user.name', 'e2e-runner']); + git(dir, ['config', 'user.email', 'e2e@vinta.local']); + + return { + dir, + baseRef: baseSha, + cleanup() { + if (!keep) rmSync(dir, { recursive: true, force: true }); + }, + }; +} diff --git a/tests/e2e/runner/lib.mjs b/tests/e2e/runner/lib.mjs new file mode 100644 index 00000000..4ffd050f --- /dev/null +++ b/tests/e2e/runner/lib.mjs @@ -0,0 +1,148 @@ +// lib.mjs — shared utilities for the e2e harness (no external deps). +// +// Everything here is pure Node so the deterministic layers (Phase 1) run +// without installing anything beyond what the fixture app already needs. + +import { + readFileSync, writeFileSync, mkdirSync, existsSync, + readdirSync, statSync, +} from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const E2E_DIR = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +export const REPO_ROOT = resolve(E2E_DIR, '..', '..'); + +// ── JSONC ────────────────────────────────────────────────────────────────── +// Scenario specs are `.jsonc` (JSON + // and /* */ comments + trailing commas). +// We strip comments outside of strings, then drop trailing commas, then JSON.parse. + +export function parseJsonc(text, label = '') { + let out = ''; + let inString = false; + let quote = ''; + let inLine = false; + let inBlock = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + const n = text[i + 1]; + if (inLine) { if (c === '\n') { inLine = false; out += c; } continue; } + if (inBlock) { if (c === '*' && n === '/') { inBlock = false; i++; } continue; } + if (inString) { + out += c; + if (c === '\\') { out += n; i++; continue; } + if (c === quote) inString = false; + continue; + } + if (c === '"' || c === "'") { inString = true; quote = c; out += c; continue; } + if (c === '/' && n === '/') { inLine = true; i++; continue; } + if (c === '/' && n === '*') { inBlock = true; i++; continue; } + out += c; + } + // Drop trailing commas before } or ] + out = out.replace(/,(\s*[}\]])/g, '$1'); + try { + return JSON.parse(out); + } catch (e) { + throw new Error(`failed to parse ${label}: ${e.message}`); + } +} + +export function loadJsonc(path) { + return parseJsonc(readFileSync(path, 'utf8'), path); +} + +export function loadJson(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} + +// ── fs helpers ─────────────────────────────────────────────────────────────── + +export function ensureDir(path) { + mkdirSync(path, { recursive: true }); + return path; +} + +export function writeJson(path, obj) { + ensureDir(dirname(path)); + writeFileSync(path, `${JSON.stringify(obj, null, 2)}\n`); +} + +export function readText(path) { + return existsSync(path) ? readFileSync(path, 'utf8') : null; +} + +export function walk(dir, { skip = ['.git', 'node_modules'] } = {}) { + const out = []; + if (!existsSync(dir)) return out; + for (const name of readdirSync(dir)) { + if (skip.includes(name)) continue; + const p = join(dir, name); + const st = statSync(p); + if (st.isDirectory()) out.push(...walk(p, { skip })); + else out.push(p); + } + return out; +} + +// ── process ───────────────────────────────────────────────────────────────── + +// Run a command, capturing stdout/stderr/exit + wall time. Never throws on a +// non-zero exit — the caller inspects `.code`. +export function run(cmd, args, { cwd, env, input, timeoutMs } = {}) { + const started = hrms(); + const res = spawnSync(cmd, args, { + cwd, + env: env ? { ...process.env, ...env } : process.env, + input, + encoding: 'utf8', + timeout: timeoutMs, + maxBuffer: 64 * 1024 * 1024, + }); + return { + code: res.status == null ? (res.signal ? 124 : 1) : res.status, + signal: res.signal || null, + stdout: res.stdout || '', + stderr: res.stderr || '', + timedOut: res.error?.code === 'ETIMEDOUT' || res.signal === 'SIGTERM', + wallSec: round((hrms() - started) / 1000, 2), + error: res.error ? String(res.error.message || res.error) : null, + }; +} + +export function git(cwd, args, opts = {}) { + return run('git', args, { cwd, ...opts }); +} + +export function hasCommand(cmd) { + const which = process.platform === 'win32' ? 'where' : 'command'; + const args = process.platform === 'win32' ? [cmd] : ['-v', cmd]; + const res = spawnSync(which, args, { encoding: 'utf8', shell: process.platform !== 'win32' }); + return res.status === 0; +} + +// ── misc ───────────────────────────────────────────────────────────────────── + +export function hrms() { + const [s, ns] = process.hrtime(); + return s * 1000 + ns / 1e6; +} + +export function round(n, digits = 2) { + const f = 10 ** digits; + return Math.round(n * f) / f; +} + +export function pct(n) { + return `${round(n * 100, 1)}%`; +} + +export const C = { + reset: '\x1b[0m', bold: '\x1b[1m', dim: '\x1b[2m', + red: '\x1b[31m', green: '\x1b[32m', yellow: '\x1b[33m', cyan: '\x1b[36m', +}; + +export function log(...a) { console.log(...a); } +export function warn(...a) { console.warn(`${C.yellow}${a.join(' ')}${C.reset}`); } +export function err(...a) { console.error(`${C.red}${a.join(' ')}${C.reset}`); } diff --git a/tests/e2e/runner/run-e2e.mjs b/tests/e2e/runner/run-e2e.mjs new file mode 100644 index 00000000..10bdea19 --- /dev/null +++ b/tests/e2e/runner/run-e2e.mjs @@ -0,0 +1,296 @@ +#!/usr/bin/env node +// run-e2e.mjs — the e2e harness runner CLI (PLAN §6, §11). +// +// Drives S0–S4 scenarios across the vendor × model matrix against isolated +// fixture copies, grades the outcome + git shape, captures telemetry, and +// writes a report. L2 is manual/human-triggered — this CLI is how a human +// initiates it (§11). Individual runnability is first-class: filter to one +// scenario / vendor / model and only that runs. +// +// node tests/e2e/runner/run-e2e.mjs \ +// --scenario S3-bug --vendor claude-code --model claude-opus-4-8 \ +// --fixture medplum-provider --runs 1 +// +// No --scenario/--vendor filter ⇒ the FULL matrix (an explicit, deliberate +// pre-release sweep). Use --list to see what's available; --dry-run to preview +// the matrix without spending; --vendor mock to self-test the plumbing for free. + +import { existsSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { + E2E_DIR, REPO_ROOT, loadJson, loadJsonc, writeJson, git, + log, warn, err, C, round, pct, +} from './lib.mjs'; +import { makeAdapter, resolveVendor, MATRIX_VENDORS, knownVendors } from './adapters/index.mjs'; +import { isolate } from './isolation.mjs'; +import { runGraders, scenarioPassed, knownGraders } from '../graders/index.mjs'; +import { runJudge } from '../graders/judge.mjs'; +import { answersLint } from '../graders/answers-lint.mjs'; +import { buildRecord, aggregateCell } from '../telemetry/normalize.mjs'; +import { diffAgainstBaseline, saveBaseline } from '../telemetry/baseline.mjs'; +import { writeSummary, writeReport } from '../report/generate.mjs'; + +const SCENARIOS_DIR = join(E2E_DIR, 'scenarios'); +const FIXTURES_DIR = join(E2E_DIR, 'fixtures'); + +// Default model per vendor when --model is omitted. Override on the CLI. +const DEFAULT_MODELS = { + 'claude-code': ['claude-opus-4-8'], + codex: ['gpt-5-codex'], + mock: ['mock'], +}; + +// ── args ───────────────────────────────────────────────────────────────────── + +function parseArgs(argv) { + const o = { + scenarios: [], vendors: [], models: [], fixture: null, + runs: 1, resultsDir: null, judge: null, keep: false, + dryRun: false, list: false, saveBaseline: false, failOnRegression: true, + }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + const next = () => argv[++i]; + const csv = (v) => v.split(',').map((s) => s.trim()).filter(Boolean); + if (a === '--scenario' || a === '-s') o.scenarios.push(...csv(next())); + else if (a === '--vendor' || a === '-v') o.vendors.push(...csv(next())); + else if (a === '--model' || a === '-m') o.models.push(...csv(next())); + else if (a === '--fixture' || a === '-f') o.fixture = next(); + else if (a === '--runs' || a === '-n') o.runs = Number(next()); + else if (a === '--results-dir') o.resultsDir = next(); + else if (a === '--judge') o.judge = csv(next()); + else if (a === '--keep') o.keep = true; + else if (a === '--dry-run') o.dryRun = true; + else if (a === '--list') o.list = true; + else if (a === '--save-baseline') o.saveBaseline = true; + else if (a === '--no-fail-on-regression') o.failOnRegression = false; + else if (a === '--help' || a === '-h') { printHelp(); process.exit(0); } + else { err(`unknown arg: ${a}`); printHelp(); process.exit(2); } + } + return o; +} + +function printHelp() { + log(`run-e2e — vinta-ai-workflows agentic E2E harness (manual, human-triggered) + +Usage: + run-e2e [filters] [options] + +Filters (no filter ⇒ full matrix — a deliberate pre-release sweep): + -s, --scenario e.g. S3-bug (comma-separated) + -v, --vendor claude-code | codex | mock + -m, --model pin model(s); default per-vendor if omitted + -f, --fixture restrict to scenarios using this fixture + +Options: + -n, --runs repeat each cell N times for pass-rate (default 1) + --judge also run advisory LLM-judge dims (spec-fidelity,...) + --results-dir output root (default tests/e2e/../../results) + --keep keep isolated fixture copies (debug) + --dry-run print the matrix; run nothing + --list list scenarios / vendors / graders and exit + --save-baseline write this run's aggregate as the version baseline + --no-fail-on-regression exit 0 even if regression flags fire + -h, --help + +Examples: + run-e2e --list + run-e2e -s S2-feature -v mock --dry-run # free plumbing check + run-e2e -s S3-bug -v claude-code -m claude-opus-4-8 -n 1 +`); +} + +// ── discovery ───────────────────────────────────────────────────────────────── + +function loadScenarios(filterIds) { + if (!existsSync(SCENARIOS_DIR)) return []; + return readdirSync(SCENARIOS_DIR) + .filter((f) => f.endsWith('.jsonc') || f.endsWith('.json')) + .map((f) => loadJsonc(join(SCENARIOS_DIR, f))) + .filter((s) => filterIds.length === 0 || filterIds.includes(s.id)) + .sort((a, b) => a.id.localeCompare(b.id)); +} + +function loadFixture(name) { + const dir = join(FIXTURES_DIR, name); + const meta = join(dir, 'fixture.json'); + if (!existsSync(meta)) throw new Error(`fixture "${name}" has no fixture.json at ${meta}`); + return { dir, tasksDir: join(dir, 'tasks'), meta: loadJson(meta) }; +} + +function pkgVersion() { + return loadJson(join(REPO_ROOT, 'package.json')).version; +} + +function gitSha() { + return git(REPO_ROOT, ['rev-parse', '--short', 'HEAD']).stdout.trim() || 'unknown'; +} + +// ── run one cell (scenario × vendor × model × run index) ────────────────────── + +async function runCell({ scenario, vendor, model, runIndex, fixture, adapter, o }) { + const { dir: cwd, baseRef, cleanup } = isolate(fixture.meta, fixture.dir, scenario.state, { keep: o.keep }); + const timeoutMs = (scenario.budget?.maxWallSec || 1800) * 1000; + const steps = []; + const cellStart = Date.now(); + try { + for (const step of scenario.steps) { + const res = await adapter.run(step, { + model, cwd, timeoutMs, + tasksDir: fixture.tasksDir, fixture: fixture.meta, scenario, + }); + steps.push({ invoke: step.invoke, ...res }); + if (res.timedOut) { warn(` ${step.invoke} hit maxWallSec — hard-stopped`); break; } + if (res.exitCode !== 0) warn(` ${step.invoke} exited ${res.exitCode}`); + } + + const ctx = { + cwd, baseRef, timeoutMs, + fixture: fixture.meta, scenario, tasksDir: fixture.tasksDir, steps, + resolveFixturePath: (p) => join(fixture.dir, p), + }; + // answers-lint is implicit on every scenario (§6) unless already listed. + const graderIds = scenario.graders.includes('answers-lint') + ? scenario.graders : [...scenario.graders, 'answers-lint']; + const graderResults = runGraders(graderIds, ctx); + + let judge = null; + if (o.judge?.length) { + judge = await runJudge(ctx, o.judge, { model: DEFAULT_MODELS['claude-code'][0] }); + } + + const record = buildRecord({ + pkgVersion: pkgVersion(), gitSha: gitSha(), + vendor, model, scenario: scenario.id, run: runIndex, + graderResults, judge, steps, wallSec: (Date.now() - cellStart) / 1000, + }); + + const outDir = join(resultsRoot(o), pkgVersion(), vendor, model, scenario.id); + const outPath = join(outDir, `run-${runIndex}.json`); + writeJson(outPath, { ...record, baseRef, transcripts: steps.map((s) => ({ invoke: s.invoke, transcript: s.transcript })) }); + + return { record, outPath, graderResults }; + } finally { + cleanup(); + } +} + +function resultsRoot(o) { + return o.resultsDir || join(REPO_ROOT, 'results'); +} + +// ── main ───────────────────────────────────────────────────────────────────── + +async function main() { + const o = parseArgs(process.argv.slice(2)); + + if (o.list) { + const scenarios = loadScenarios([]); + log(`${C.bold}Scenarios:${C.reset}`); + for (const s of scenarios) log(` ${s.id} (fixture=${s.fixture}, state=${s.state}, graders=${s.graders.join('+')})`); + log(`\n${C.bold}Vendors:${C.reset} ${knownVendors().join(', ')} (matrix default: ${MATRIX_VENDORS.join(', ')})`); + log(`${C.bold}Graders:${C.reset} ${knownGraders().join(', ')}`); + return 0; + } + + let scenarios = loadScenarios(o.scenarios); + if (o.fixture) scenarios = scenarios.filter((s) => s.fixture === o.fixture); + if (scenarios.length === 0) { err('no scenarios matched the filters'); return 2; } + + const vendors = (o.vendors.length ? o.vendors.map(resolveVendor) : MATRIX_VENDORS); + if (vendors.some((v) => !v)) { err(`unknown vendor in ${o.vendors.join(',')}`); return 2; } + + // Build & print the matrix. + const matrix = []; + for (const scenario of scenarios) { + for (const vendor of vendors) { + const models = o.models.length ? o.models : (DEFAULT_MODELS[vendor] || []); + if (models.length === 0) { warn(`no model for vendor ${vendor}; pass --model`); continue; } + for (const model of models) matrix.push({ scenario, vendor, model }); + } + } + + log(`${C.bold}Matrix:${C.reset} ${matrix.length} cell(s) × ${o.runs} run(s) = ${matrix.length * o.runs} run(s)`); + for (const c of matrix) log(` ${C.dim}${c.scenario.id} · ${c.vendor} · ${c.model}${C.reset}`); + if (o.dryRun) { log('\n(dry-run — nothing executed)'); return 0; } + + // Group records per cell for aggregation. + const cellRecords = new Map(); + const cellFailures = new Map(); + + for (const cell of matrix) { + const adapter = makeAdapter(cell.vendor); + const avail = adapter.available(); + const key = `${cell.scenario.id}|${cell.vendor}|${cell.model}`; + if (!avail.available) { + warn(`SKIP ${key} — ${avail.reason}`); // §11: skipped, never passed + continue; + } + let fixture; + try { fixture = loadFixture(cell.scenario.fixture); } + catch (e) { err(`SKIP ${key} — ${e.message}`); continue; } + + for (let r = 1; r <= o.runs; r++) { + log(`\n${C.cyan}▶ ${key} — run ${r}/${o.runs}${C.reset}`); + let result; + try { + result = await runCell({ ...cell, scenario: cell.scenario, runIndex: r, fixture, adapter, o }); + } catch (e) { + err(` run failed: ${e.message}`); + continue; + } + const passed = scenarioPassed(result.graderResults); + for (const g of result.graderResults) { + const mark = g.pass ? `${C.green}✓${C.reset}` : `${C.red}✗${C.reset}`; + log(` ${mark} ${g.id}`); + if (!g.pass) for (const d of g.details.filter((x) => x.startsWith('✗'))) log(` ${C.dim}${d}${C.reset}`); + } + log(` ${passed ? C.green + 'PASS' : C.red + 'FAIL'}${C.reset} ` + + `tokens=${result.record.usage.inTok + result.record.usage.outTok} cost=$${result.record.usage.costUsd} wall=${result.record.wallSec}s`); + + if (!cellRecords.has(key)) cellRecords.set(key, []); + cellRecords.get(key).push(result.record); + if (!passed) { + if (!cellFailures.has(key)) cellFailures.set(key, []); + cellFailures.get(key).push({ + run: r, + failedGraders: result.graderResults.filter((g) => !g.pass).map((g) => g.id), + transcriptPath: result.outPath, + }); + } + } + } + + if (cellRecords.size === 0) { warn('\nno cells ran (all skipped?)'); return 0; } + + // Aggregate + baseline diff + report. + const cells = [...cellRecords.values()].map((records) => { + const agg = aggregateCell(records); + agg.failures = cellFailures.get(`${agg.scenario}|${agg.vendor}|${agg.model}`) || []; + return agg; + }); + + const baseline = diffAgainstBaseline(cells, pkgVersion()); + const resDir = join(resultsRoot(o), pkgVersion()); + const summary = writeSummary(resDir, { version: pkgVersion(), gitSha: gitSha(), cells, baseline }); + writeReport(resDir, summary); + + if (o.saveBaseline) { saveBaseline(pkgVersion(), cells); log(`\nsaved baseline for ${pkgVersion()}`); } + + // Console summary. + log(`\n${C.bold}── Summary ──${C.reset}`); + for (const c of cells) { + log(` ${c.scenario} · ${c.vendor} · ${c.model}: pass-rate ${pct(c.passRate)}, median ${round((c.tokens.median) / 1000, 1)}k tok`); + } + if (baseline.flags.length) { + warn(`\n${baseline.flags.length} regression flag(s) vs baseline ${baseline.baselineVersion}:`); + for (const f of baseline.flags) warn(` - ${f}`); + } + log(`\nReport: ${join(resDir, 'REPORT.md')}`); + + const anyFail = cells.some((c) => c.passRate < 1); + if (baseline.flags.length && o.failOnRegression) return 1; + return anyFail ? 1 : 0; +} + +main().then((code) => process.exit(code)).catch((e) => { err(e.stack || e.message); process.exit(1); }); diff --git a/tests/e2e/scenarios/S0-bootstrap.jsonc b/tests/e2e/scenarios/S0-bootstrap.jsonc new file mode 100644 index 00000000..ea61e9dd --- /dev/null +++ b/tests/e2e/scenarios/S0-bootstrap.jsonc @@ -0,0 +1,18 @@ +// S0 — Bootstrap (PLAN §5). Proves `install` + `/vinta-bootstrap-ai-tools` +// work on a CLEAN repo (no ai-tools/). Runs against the `clean/` fixture state. +{ + "id": "S0-bootstrap", + "fixture": "medplum-provider", + "state": "clean", + "steps": [ + { + "invoke": "/vinta-bootstrap-ai-tools", + "input": "tasks/bootstrap.md", + "answers": "tasks/bootstrap.answers.md" + } + ], + // artifacts-bootstrap: AGENTS.md + schema-valid sub-agents + config present. + // git-tree-healthy: the bootstrap lands as clean, conventionally-scoped commits. + "graders": ["artifacts-bootstrap", "git-tree-healthy"], + "budget": { "maxWallSec": 1200 } +} diff --git a/tests/e2e/scenarios/S1-sync.jsonc b/tests/e2e/scenarios/S1-sync.jsonc new file mode 100644 index 00000000..85efd2b6 --- /dev/null +++ b/tests/e2e/scenarios/S1-sync.jsonc @@ -0,0 +1,37 @@ +// S1 — Sync (PLAN §5). Proves `/vinta-sync-ai-tools` correctly UPGRADES an +// already-configured, stale repo. Runs against the `stale/` fixture state +// (bootstrapped from an older package version, so a real diff exists to apply). +{ + "id": "S1-sync", + "fixture": "medplum-provider", + "state": "stale", + "steps": [ + { + "invoke": "/vinta-sync-ai-tools", + "input": "tasks/sync.md", + "answers": "tasks/sync.answers.md", + "autonomous": true + } + ], + "graders": ["sync-convergence", "git-tree-healthy"], + // The sync-convergence oracle (§14 Q2): synced managed files must converge on + // what a fresh `current/` bootstrap produces, hand-tuned regions preserved, + // opt-outs sticky. + "expect": { + "sync": { + "referenceDir": "states/current", + "managedGlobs": ["ai-tools/**", "AGENTS.md", ".vinta-ai-workflows.yaml"], + "ignore": [ + ".vinta-ai-workflows.yaml", // carries last_synced_at / version — always differs + "ai-tools/**/*.local.*" + ], + "preserve": [ + { "file": "AGENTS.md", "contains": "" } + ], + "optOuts": [ + "ai-tools/skills/vinta-opted-out-example" + ] + } + }, + "budget": { "maxWallSec": 1200 } +} diff --git a/tests/e2e/scenarios/S2-feature.jsonc b/tests/e2e/scenarios/S2-feature.jsonc new file mode 100644 index 00000000..cd76e497 --- /dev/null +++ b/tests/e2e/scenarios/S2-feature.jsonc @@ -0,0 +1,19 @@ +// S2 — Feature delivery pipeline (PLAN §5). create-spec → plan-feature → +// implement-plan against the `current/` harness. The core "prompt to healthy, +// merged code" path. +{ + "id": "S2-feature", + "fixture": "medplum-provider", + "state": "current", + "steps": [ + { "invoke": "/create-spec", "input": "tasks/feature.md", "answers": "tasks/feature.answers.md" }, + { "invoke": "/plan-feature", "answers": "tasks/feature.answers.md" }, + { "invoke": "/implement-plan", "autonomous": true } + ], + "graders": ["artifacts-spec", "artifacts-plan", "app-tests-pass", "git-tree-healthy"], + "expect": { + // implement-plan should produce stacked phase branches based off the right base. + "stackedBranches": true + }, + "budget": { "maxWallSec": 1800 } +} diff --git a/tests/e2e/scenarios/S3-bug.jsonc b/tests/e2e/scenarios/S3-bug.jsonc new file mode 100644 index 00000000..68edbb37 --- /dev/null +++ b/tests/e2e/scenarios/S3-bug.jsonc @@ -0,0 +1,26 @@ +// S3 — Bug workflow (PLAN §5, §13.3). `/systematic-debugging` pointed at a +// planted defect with a currently-failing MockClient test. Runs against +// `current/`. +{ + "id": "S3-bug", + "fixture": "medplum-provider", + "state": "current", + "steps": [ + { + "invoke": "/systematic-debugging", + "input": "tasks/bug.md", + "answers": "tasks/bug.answers.md", + "autonomous": true + } + ], + "graders": ["bug-fixed", "git-tree-healthy"], + "expect": { + "bug": { + // The previously-failing test flips green with no regressions; the fix must + // touch the planted region (guard against "deleted the test"). + "region": "src/pages/patient/allergies", + "testFile": "src/pages/patient/allergies/AllergyPanel.test.tsx" + } + }, + "budget": { "maxWallSec": 1500 } +} diff --git a/tests/e2e/scenarios/S4-script.jsonc b/tests/e2e/scenarios/S4-script.jsonc new file mode 100644 index 00000000..74e89349 --- /dev/null +++ b/tests/e2e/scenarios/S4-script.jsonc @@ -0,0 +1,24 @@ +// S4 — One-off script (PLAN §5). `/add-one-off-script` for a task with a +// checkable side effect. Runs against `current/`. +{ + "id": "S4-script", + "fixture": "medplum-provider", + "state": "current", + "steps": [ + { + "invoke": "/add-one-off-script", + "input": "tasks/script.md", + "answers": "tasks/script.answers.md", + "autonomous": true + } + ], + "graders": ["script-output", "git-tree-healthy"], + "expect": { + "script": { + // Script must run non-interactively and emit the CSV artifact. + "run": "npx tsx scripts/one_off_2026_08_01_export_observations/run.ts", + "artifact": "scripts/one_off_2026_08_01_export_observations/out/observations.csv" + } + }, + "budget": { "maxWallSec": 1200 } +} diff --git a/tests/e2e/telemetry/baseline.mjs b/tests/e2e/telemetry/baseline.mjs new file mode 100644 index 00000000..26c959ea --- /dev/null +++ b/tests/e2e/telemetry/baseline.mjs @@ -0,0 +1,75 @@ +// baseline.mjs — version-over-version regression flags (PLAN §8). +// +// A run compares its aggregated cells against the previous released baseline +// (tests/e2e/baselines/.json) and flags: pass-rate drops, or a +// token/cost increase beyond a threshold (default 20% per scenario). + +import { existsSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { loadJson, writeJson, E2E_DIR, round } from '../runner/lib.mjs'; + +const BASELINES_DIR = join(E2E_DIR, 'baselines'); +const THRESHOLD = 0.2; // 20% + +function cellKey(c) { return `${c.scenario}|${c.vendor}|${c.model}`; } + +// Pick the most recent baseline that isn't the current version. +export function previousBaseline(currentVersion) { + if (!existsSync(BASELINES_DIR)) return null; + const files = readdirSync(BASELINES_DIR) + .filter((f) => f.endsWith('.json')) + .map((f) => f.replace(/\.json$/, '')) + .filter((v) => v !== currentVersion) + .sort(semverCompare); + const chosen = files[files.length - 1]; + return chosen ? { version: chosen, data: loadJson(join(BASELINES_DIR, `${chosen}.json`)) } : null; +} + +export function diffAgainstBaseline(cells, currentVersion) { + const prev = previousBaseline(currentVersion); + if (!prev) return { baselineVersion: null, flags: [], comparisons: [] }; + + const prevCells = new Map((prev.data.cells || []).map((c) => [cellKey(c), c])); + const flags = []; + const comparisons = []; + + for (const c of cells) { + const p = prevCells.get(cellKey(c)); + if (!p) { comparisons.push({ ...c, baseline: null, note: 'new cell (no baseline)' }); continue; } + const passDrop = c.passRate < p.passRate; + const tokMean = c.tokens.mean; + const tokDelta = p.tokens.mean ? (tokMean - p.tokens.mean) / p.tokens.mean : 0; + const costDelta = p.costUsd.mean ? (c.costUsd.mean - p.costUsd.mean) / p.costUsd.mean : 0; + + const cmp = { + key: cellKey(c), + passRate: c.passRate, basePassRate: p.passRate, + tokDelta: round(tokDelta, 3), costDelta: round(costDelta, 3), + }; + comparisons.push(cmp); + + if (passDrop) flags.push(`PASS-RATE DROP ${cellKey(c)}: ${p.passRate} → ${c.passRate}`); + if (tokDelta > THRESHOLD) flags.push(`TOKENS +${round(tokDelta * 100, 1)}% ${cellKey(c)} (>${THRESHOLD * 100}%)`); + if (costDelta > THRESHOLD) flags.push(`COST +${round(costDelta * 100, 1)}% ${cellKey(c)} (>${THRESHOLD * 100}%)`); + } + + return { baselineVersion: prev.version, flags, comparisons }; +} + +// Write/refresh the baseline for a version (used before a release to capture a +// new reference). Committed under tests/e2e/baselines/. +export function saveBaseline(version, cells) { + writeJson(join(BASELINES_DIR, `${version}.json`), { version, cells }); +} + +function semverCompare(a, b) { + const pa = a.split(/[.-]/).map((x) => (/^\d+$/.test(x) ? Number(x) : x)); + const pb = b.split(/[.-]/).map((x) => (/^\d+$/.test(x) ? Number(x) : x)); + for (let i = 0; i < Math.max(pa.length, pb.length); i++) { + const x = pa[i] ?? 0; const y = pb[i] ?? 0; + if (x === y) continue; + if (typeof x === 'number' && typeof y === 'number') return x - y; + return String(x) < String(y) ? -1 : 1; + } + return 0; +} diff --git a/tests/e2e/telemetry/normalize.mjs b/tests/e2e/telemetry/normalize.mjs new file mode 100644 index 00000000..7131c7d0 --- /dev/null +++ b/tests/e2e/telemetry/normalize.mjs @@ -0,0 +1,88 @@ +// normalize.mjs — the per-run result envelope (PLAN §8). +// +// Every run emits one normalized record with usage, wall-time, grader outcomes, +// judge scores, and a per-step usage breakdown so we can see *where* tokens go. + +import { round } from '../runner/lib.mjs'; + +export function buildRecord({ + pkgVersion, gitSha, vendor, model, scenario, run, + graderResults, judge, steps, wallSec, +}) { + const usage = sumUsage(steps.map((s) => s.usage)); + const graders = {}; + for (const g of graderResults) graders[g.id] = g.pass; + const pass = graderResults.filter((g) => g.tier === 'deterministic').every((g) => g.pass) + && graderResults.some((g) => g.tier === 'deterministic'); + + return { + pkgVersion, + gitSha, + vendor, + model, + scenario, + run, + pass, + graders, + graderDetails: graderResults.map((g) => ({ id: g.id, pass: g.pass, details: g.details })), + judge: judge?.scores + ? Object.fromEntries(Object.entries(judge.scores).map(([k, v]) => [k, v.score])) + : {}, + usage, + wallSec: round(wallSec, 1), + steps: steps.map((s) => ({ + invoke: s.invoke, + usage: s.usage, + wallSec: round(s.wallSec, 1), + exitCode: s.exitCode, + timedOut: !!s.timedOut, + usageEstimated: !!s.usageEstimated, + })), + }; +} + +export function sumUsage(list) { + const acc = { inTok: 0, outTok: 0, cacheTok: 0, costUsd: 0 }; + for (const u of list) { + if (!u) continue; + acc.inTok += u.inTok || 0; + acc.outTok += u.outTok || 0; + acc.cacheTok += u.cacheTok || 0; + acc.costUsd += u.costUsd || 0; + } + acc.costUsd = round(acc.costUsd, 4); + return acc; +} + +// Aggregate the N runs of one (scenario × vendor × model) cell. +export function aggregateCell(records) { + const n = records.length; + const passes = records.filter((r) => r.pass).length; + const totalToks = records.map((r) => r.usage.inTok + r.usage.outTok); + const costs = records.map((r) => r.usage.costUsd); + return { + scenario: records[0].scenario, + vendor: records[0].vendor, + model: records[0].model, + runs: n, + passRate: round(passes / n, 3), + tokens: stats(totalToks), + costUsd: stats(costs), + wallSec: stats(records.map((r) => r.wallSec)), + }; +} + +function stats(nums) { + if (nums.length === 0) return { mean: 0, median: 0, p95: 0, min: 0, max: 0, variance: 0 }; + const sorted = [...nums].sort((a, b) => a - b); + const mean = nums.reduce((a, b) => a + b, 0) / nums.length; + const variance = nums.reduce((a, b) => a + (b - mean) ** 2, 0) / nums.length; + return { + mean: round(mean, 2), + median: round(sorted[Math.floor((sorted.length - 1) / 2)], 2), + p95: round(sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)], 2), + min: round(sorted[0], 2), + max: round(sorted[sorted.length - 1], 2), + variance: round(variance, 2), + }; +}