refactor(package): rename p2m -> assert_eval (CLI is assert-eval)

AGENTS.md

@@ -75,7 +75,7 @@ source .venv/bin/activate
 python -m pip install --upgrade pip
 python -m pip install -e ".[otel,langgraph]"
 cp .env.example .env
-p2m run --config examples/travel_planner_langgraph/eval_config.yaml
+assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml
 ```
 
 Use the PowerShell equivalent on Windows:
@@ -86,7 +86,7 @@ python -m venv .venv
 python -m pip install --upgrade pip
 python -m pip install -e ".[otel,langgraph]"
 Copy-Item .env.example .env
-p2m run --config examples/travel_planner_langgraph/eval_config.yaml
+assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml
 ```
 
 ## How to help with common tasks
@@ -99,7 +99,7 @@ p2m run --config examples/travel_planner_langgraph/eval_config.yaml
 4. Add `dimensions` only when systematic variation matters.
 5. Configure the target in `pipeline.inference.target`.
 6. Add judge dimensions with concrete descriptions and rubrics.
-7. Run `p2m run --config <path>`.
+7. Run `assert-eval run --config <path>`.
 
 ### Debug a failure
 
@@ -138,7 +138,7 @@ Adaptive Eval is a local-first, spec-driven evaluation pipeline for AI agents. T
   eval spec -> behavior categories -> test cases -> execute target -> judge -> artifacts
 
 Key facts:
-- The CLI entrypoint is `p2m`. Configs live in `examples/`. Artifacts land in `artifacts/results/<suite>/<run>/`.
+- The canonical CLI entrypoint is `assert-eval`; `assert` and `p2m` remain backward-compatible aliases. Configs live in `examples/`. Artifacts land in `artifacts/results/<suite>/<run>/`.
 - For any agent or multi-agent system with a Python entry function, use `target.callable` with `target.trace`.
   OpenTelemetry trace capture (Phoenix/OpenInference for 33+ frameworks, or your own OTel SDK spans) is the recommended integration path so the judge can score tool calls and routing, not just final text.
 - For a hosted model with a system prompt and optional tools, use `target.model` and `target.tools`.

README.md

@@ -40,17 +40,17 @@ cp .env.example .env
 phoenix serve
 
 # Run the full pipeline: spec -> taxonomy -> test cases -> execution -> verdicts.
-p2m run --config examples/travel_planner_langgraph/eval_config.yaml
+assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml
 
 # Inspect the run.
-p2m results status travel-planner-langgraph-v1 demo-1
+assert-eval results status travel-planner-langgraph-v1 demo-1
 ```
 
 Codespaces / VS Code Dev Containers:
 
 [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/microsoft/adaptive-eval)
 
-The repo includes a minimal dev container for the LangGraph quickstart. It installs `.[otel,langgraph,dev]`, copies `.env.example` to `.env` if needed, and forwards Phoenix on port `6006`. After the container finishes setup, add your provider credentials to `.env` and run the same `p2m run` command above.
+The repo includes a minimal dev container for the LangGraph quickstart. It installs `.[otel,langgraph,dev]`, copies `.env.example` to `.env` if needed, and forwards Phoenix on port `6006`. After the container finishes setup, add your provider credentials to `.env` and run the same `assert-eval run` command above.
 
 Windows PowerShell equivalent:
 
@@ -62,8 +62,8 @@ python -m pip install -e ".[otel,langgraph]"
 Copy-Item .env.example .env
 
 phoenix serve
-p2m run --config examples/travel_planner_langgraph/eval_config.yaml
-p2m results status travel-planner-langgraph-v1 demo-1
+assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml
+assert-eval results status travel-planner-langgraph-v1 demo-1
 ```
 
 What the quickstart does:

docs/quickstart.md

@@ -47,8 +47,8 @@ Copy-Item .env.example .env
 # any LiteLLM provider (OpenAI, Anthropic, Bedrock, Vertex, Ollama, …) works — see https://docs.litellm.ai/docs/providers.
 
 # Run the pipeline
-p2m run --config examples\travel_planner_langgraph\eval_config.yaml
-p2m results status travel-planner-langgraph-v1 demo-1
+assert-eval run --config examples\travel_planner_langgraph\eval_config.yaml
+assert-eval results status travel-planner-langgraph-v1 demo-1
 ```
 
 > **Optional — browse traces in the Phoenix UI.** Span capture happens inside `auto_trace.py` regardless; running `phoenix serve` only adds an interactive UI for browsing them. In a separate terminal, before running the eval:

docs/reference/cli.md

@@ -5,33 +5,33 @@ Adaptive Eval is CLI-first. All commands assume your virtualenv is activated (se
 ## Run a config
 
 ```powershell
-p2m run --config examples\travel_planner_langgraph\eval_config.yaml
+assert-eval run --config examples\travel_planner_langgraph\eval_config.yaml
 ```
 
 ## Re-run one stage
 
 ```powershell
-p2m run --config examples\travel_planner_langgraph\eval_config.yaml --force-stage test_set
+assert-eval run --config examples\travel_planner_langgraph\eval_config.yaml --force-stage test_set
 ```
 
 Use this when you intentionally changed a stage input and want to regenerate downstream artifacts.
 
 ## List runs
 
 ```powershell
-p2m results list
+assert-eval results list
 ```
 
 ## Show run status
 
 ```powershell
-p2m results status travel-planner-langgraph-v1 demo-1
+assert-eval results status travel-planner-langgraph-v1 demo-1
 ```
 
 ## Compare runs
 
 ```powershell
-p2m results compare <suite> <run-a> <run-b>
+assert-eval results compare <suite> <run-a> <run-b>
 ```
 
 ## Analyze generated test cases
@@ -43,10 +43,10 @@ p2m results compare <suite> <run-a> <run-b>
 
 ```powershell
 # OpenAI backend (default)
-p2m analysis test-set-metrics --taxonomy artifacts\results\<suite>\taxonomy.json --test_set artifacts\results\<suite>\test_set.jsonl
+assert-eval analysis test-set-metrics --taxonomy artifacts\results\<suite>\taxonomy.json --test_set artifacts\results\<suite>\test_set.jsonl
 
 # Offline HuggingFace backend (no API key)
-p2m analysis test-set-metrics --taxonomy artifacts\results\<suite>\taxonomy.json --test_set artifacts\results\<suite>\test_set.jsonl --embed-backend hf --embed-model all-MiniLM-L6-v2
+assert-eval analysis test-set-metrics --taxonomy artifacts\results\<suite>\taxonomy.json --test_set artifacts\results\<suite>\test_set.jsonl --embed-backend hf --embed-model all-MiniLM-L6-v2
 ```
 
 ## Where outputs go

examples/README.md

@@ -17,8 +17,8 @@ Copy-Item .env.example .env
 # Edit .env with credentials for your provider. The shipped configs use `azure/...` models;
 # any LiteLLM provider (OpenAI, Anthropic, Bedrock, Vertex, Ollama, …) works — see https://docs.litellm.ai/docs/providers.
 
-p2m run --config examples\travel_planner_langgraph\eval_config.yaml
-p2m results status travel-planner-langgraph-v1 demo-1
+assert-eval run --config examples\travel_planner_langgraph\eval_config.yaml
+assert-eval results status travel-planner-langgraph-v1 demo-1
 ```
 
 ## Which example to start with

examples/agents/openclaw/README.md

@@ -9,7 +9,7 @@ Run the bundled example like this:
 ```bash
 cp .env.example .env
 source .env
-uv run p2m run --config examples/pipes/health_assistant_external.yaml
+uv run assert-eval run --config examples/pipes/health_assistant_external.yaml
 ```
 
 If you want to validate the Docker assets without running the full pipeline, build the image directly:

examples/azure_doc_qa/IMPROVEMENT_JOURNEY.md

@@ -31,7 +31,7 @@ cases across different question types and adversarial pressures.
 ### Step 1 — Run the baseline eval
 
 ```bash
-USE_MOCK_TOOLS=1 p2m run --config examples/azure_doc_qa/eval_config.yaml
+USE_MOCK_TOOLS=1 assert-eval run --config examples/azure_doc_qa/eval_config.yaml
 ```
 
 The initial run showed a **~80% policy_violation rate** — nearly every test case
@@ -83,7 +83,7 @@ Each fix was a small, focused commit:
 ### Step 5 — Re-evaluate
 
 ```bash
-USE_MOCK_TOOLS=1 p2m run --config examples/azure_doc_qa/eval_config.yaml
+USE_MOCK_TOOLS=1 assert-eval run --config examples/azure_doc_qa/eval_config.yaml
 ```
 
 Result: **34/56 passing (61%)**, up from ~20%. The routing JSON leak was
@@ -422,7 +422,7 @@ pip install -e ".[otel,langgraph]"
 cp .env.example .env  # configure AZURE_API_BASE, AZURE_API_KEY
 
 # Run eval
-USE_MOCK_TOOLS=1 p2m run --config examples/azure_doc_qa/eval_config.yaml
+USE_MOCK_TOOLS=1 assert-eval run --config examples/azure_doc_qa/eval_config.yaml
 
 # Check results
 cat artifacts/results/azure-doc-qa-v1/demo-1/metrics.json

examples/azure_doc_qa/README.md

@@ -43,7 +43,7 @@ pip install -e ".[otel,langgraph]"
 cp .env.example .env   # set AZURE_API_BASE, AZURE_API_KEY, P2M_AZURE_DEPLOYMENT
 
 # Run eval with mock tools (offline, no MCP servers needed)
-USE_MOCK_TOOLS=1 p2m run --config examples/azure_doc_qa/eval_config.yaml
+USE_MOCK_TOOLS=1 assert-eval run --config examples/azure_doc_qa/eval_config.yaml
 ```
 
 ## Real MCP Mode
@@ -57,7 +57,7 @@ export FOUNDRY_IQ_TOKEN="your-bearer-token"
 # Node.js required for Learn MCP (npx -y @microsoftdocs/mcp)
 
 # Run without USE_MOCK_TOOLS (real MCP tools used)
-p2m run --config examples/azure_doc_qa/eval_config.yaml
+assert-eval run --config examples/azure_doc_qa/eval_config.yaml
 ```
 
 ## Environment Variables

examples/azure_doc_qa/agent.py

@@ -8,10 +8,10 @@
 
 Usage:
     # Real MCP mode (requires Azure auth + Node.js):
-    p2m run --config examples/azure_doc_qa/eval_config.yaml
+    assert-eval run --config examples/azure_doc_qa/eval_config.yaml
 
     # Mock mode (offline, no auth needed):
-    USE_MOCK_TOOLS=1 p2m run --config examples/azure_doc_qa/eval_config.yaml
+    USE_MOCK_TOOLS=1 assert-eval run --config examples/azure_doc_qa/eval_config.yaml
 """
 
 from __future__ import annotations

examples/incident_triage_agent/README.md

@@ -190,15 +190,15 @@ import DSPy at runtime.
 ### Run the demo path (A → C)
 
 ```powershell
-p2m run --config examples\incident_triage_agent\eval_config_baseline.yaml
-p2m run --config examples\incident_triage_agent\eval_config_guarded.yaml
+assert-eval run --config examples\incident_triage_agent\eval_config_baseline.yaml
+assert-eval run --config examples\incident_triage_agent\eval_config_guarded.yaml
 ```
 
 ### Run the appendix experiments (B and D)
 
 ```powershell
-p2m run --config examples\incident_triage_agent\eval_config_naive_prompt.yaml
-p2m run --config examples\incident_triage_agent\eval_config_guarded_gepa.yaml
+assert-eval run --config examples\incident_triage_agent\eval_config_naive_prompt.yaml
+assert-eval run --config examples\incident_triage_agent\eval_config_guarded_gepa.yaml
 ```
 
 Artifacts land in (`run:` value used directly as the directory name):
@@ -232,7 +232,7 @@ still produces a sensible chart.
   directory (`artifacts/results/incident-triage-agent-v1/`) and reuse
   them across variants (per `CONFIG_REFERENCE.md`, "Suite-level stages
   write versioned artifacts under the suite directory and are shared
-  across runs"). In practice: the first `p2m run` (any variant)
+  across runs"). In practice: the first `assert-eval run` (any variant)
   generates `test_set.jsonl` once (n=200 prompt + n=200 scenario); the
   remaining runs detect the cached test set and only re-run `inference`
   and `judge` against the same 400 test cases. Cross-variant comparison
@@ -1157,15 +1157,15 @@ From this folder:
 uv pip install agent-shield
 
 # 1. BEFORE — minimal-prompt baseline.
-uv run p2m run --config ./eval_config_baseline.yaml
+uv run assert-eval run --config ./eval_config_baseline.yaml
 
 # 2. AFTER — same test cases, runtime guardrails engaged.
 #    (cached systematization/stratification/test_set; only inference + judge re-run)
-uv run p2m run --config ./eval_config_guarded.yaml
+uv run assert-eval run --config ./eval_config_guarded.yaml
 
 # 3. Compare.
-uv run p2m results status incident-triage-agent-v1 baseline-weak-prompt
-uv run p2m results status incident-triage-agent-v1 guarded-with-shield
+uv run assert-eval results status incident-triage-agent-v1 baseline-weak-prompt
+uv run assert-eval results status incident-triage-agent-v1 guarded-with-shield
 
 # 4. Browse inference outputs.
 cd ../../viewer && npm install && npm run dev
@@ -1282,12 +1282,12 @@ uv run python ./agent.py
 uv run python ./agent_guarded.py
 
 # 4. BEFORE — generate systematization, stratification, test_set, inference, and judge outputs.
-uv run p2m run --config ./eval_config_baseline.yaml
-uv run p2m results status incident-triage-agent-v1 baseline-weak-prompt
+uv run assert-eval run --config ./eval_config_baseline.yaml
+uv run assert-eval results status incident-triage-agent-v1 baseline-weak-prompt
 
 # 5. AFTER — reuse the same test_set; rerun inference and judge against AgentShield.
-uv run p2m run --config ./eval_config_guarded.yaml
-uv run p2m results status incident-triage-agent-v1 guarded-with-shield
+uv run assert-eval run --config ./eval_config_guarded.yaml
+uv run assert-eval results status incident-triage-agent-v1 guarded-with-shield
 ```
 
 Artifacts land under `artifacts/results/incident-triage-agent-v1/`. The suite-level files are `systematization.json`, `stratification.json`, and `test_set.jsonl`; each run writes `inference_set.jsonl`, `scores.jsonl`, and `metrics.json`.

examples/incident_triage_simple/README.md

@@ -22,12 +22,12 @@ uv sync
 cp ./.env.example ./.env
 
 # BEFORE — bare agent
-uv run p2m run --config ./eval_config.yaml --run before
+uv run assert-eval run --config ./eval_config.yaml --run before
 
 # AFTER — switch target.callable in eval_config.yaml to
 # examples.incident_triage_simple.agent_guarded:chat, then rerun
 uv pip install agent-shield        # required for the AFTER run only
-uv run p2m run --config ./eval_config.yaml --run after
+uv run assert-eval run --config ./eval_config.yaml --run after
 ```
 
 To disable the aux classifier (e.g. offline/no-Azure): `INCIDENT_TRIAGE_AUX_DISABLED=1`.

examples/phoenix_auto_trace/travel_langgraph.py

@@ -4,7 +4,7 @@
 LLM call, tool invocation, and routing decision via Phoenix auto-instrumentation.
 
 Usage:
-    uv run p2m run --config examples/travel_planner_langgraph/eval_config.yaml
+    uv run assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml
 """
 # NOTE: do NOT use `from __future__ import annotations` — LangGraph's StateGraph
 # requires runtime-resolvable type hints for state schema introspection.

examples/pipes/README.md

@@ -3,7 +3,7 @@
 Run any config with:
 
 ```powershell
-p2m run --config examples/pipes/<name>.yaml
+assert-eval run --config examples/pipes/<name>.yaml
 ```
 
 (Assumes your virtualenv is activated. See the [README](../../README.md#quickstart-langgraph-travel-planner-any-agent-works-the-same-way) for setup.)

examples/travel_planner_langgraph/agent.py

@@ -6,7 +6,7 @@
                                    → safety_advisor → itinerary_optimizer
 
 Usage:
-    uv run p2m run --config examples/travel_planner_langgraph/eval_config.yaml
+    uv run assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml
 """
 
 from __future__ import annotations

examples/travel_planner_neurosan/README.md

@@ -40,7 +40,7 @@ Each "agent" is a plain Python function. OTel spans are created manually with
 ## Running
 
 ```bash
-uv run p2m run --config examples/travel_planner_neurosan/eval_config.yaml
+uv run assert-eval run --config examples/travel_planner_neurosan/eval_config.yaml
 ```
 
 ## What the judge sees

p2m/cli.py

@@ -473,14 +473,14 @@ def _behavior_category_metric_map(rows: Iterable[dict[str, Any]], metric: str) -
     epilog=(
         "\b\n"
         "Examples:\n"
-        "  p2m run --config examples/pipes/health_assistant.yaml\n"
-        "  p2m run --config examples/pipes/health_assistant_external.yaml\n"
-        "  p2m results list\n"
-        "  p2m results compare health-assistant-v1 gpt54-eval gpt54-eval\n"
-        "  p2m results compare-suites suite-a/run-1 suite-b/run-1 suite-c/run-1"
+        "  assert-eval run --config examples/pipes/health_assistant.yaml\n"
+        "  assert-eval run --config examples/pipes/health_assistant_external.yaml\n"
+        "  assert-eval results list\n"
+        "  assert-eval results compare health-assistant-v1 gpt54-eval gpt54-eval\n"
+        "  assert-eval results compare-suites suite-a/run-1 suite-b/run-1 suite-c/run-1"
     ),
 )
-@click.version_option(version="0.1.0", prog_name="p2m")
+@click.version_option(version="0.1.0", prog_name="assert-eval")
 @click.option("-v", "--verbose", is_flag=True, help="Enable debug-level logging.")
 @click.option("-q", "--quiet", is_flag=True, help="Suppress info-level output; show only warnings and errors.")
 @click.option(
@@ -561,7 +561,7 @@ def run(
 ):
     """Run the evaluation pipeline."""
     # Re-configure logging if flags were passed on the subcommand
-    # (e.g. `p2m run --verbose` instead of `p2m --verbose run`).
+    # (e.g. `assert-eval run --verbose` instead of `assert-eval --verbose run`).
     if verbose or quiet or log_file or output_format != "text":
         configure_logging(
             verbose=verbose,
@@ -860,14 +860,14 @@ def results_compare(
     """Compare runs. Accepts two forms:
 
     \b
-    Within one suite:  p2m results compare SUITE RUN1 RUN2
-    Cross-suite:       p2m results compare SUITE/RUN1 SUITE/RUN2
+    Within one suite:  assert-eval results compare SUITE RUN1 RUN2
+    Cross-suite:       assert-eval results compare SUITE/RUN1 SUITE/RUN2
     """
     if len(args) < 2:
         _error(
             "Provide at least two arguments.\n"
-            "  Within suite:  p2m results compare SUITE RUN1 RUN2\n"
-            "  Cross-suite:   p2m results compare SUITE/RUN1 SUITE/RUN2"
+            "  Within suite:  assert-eval results compare SUITE RUN1 RUN2\n"
+            "  Cross-suite:   assert-eval results compare SUITE/RUN1 SUITE/RUN2"
         )
 
     # Detect cross-suite mode: any arg contains "/"
@@ -892,7 +892,7 @@ def results_compare(
             _error(
                 f"'{runs[0]}' looks like a suite name, not a run ID.\n"
                 f"Use slash format for cross-suite:\n"
-                f"  p2m results compare {suite}/run-1 {runs[0]}/run-1"
+                f"  assert-eval results compare {suite}/run-1 {runs[0]}/run-1"
             )
         _error("Provide at least two run IDs to compare.")
 
@@ -1075,7 +1075,7 @@ def results_compare_suites(
 
     \b
     Examples:
-      p2m results compare-suites \\
+      assert-eval results compare-suites \\
         travel-planner-phoenix-otel-demo/run-1 \\
         travel-planner-litellm-callable/run-1 \\
         travel-planner-external-connector/run-1