diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 615884c9c..e6e857f1c 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -59,8 +59,7 @@ jobs: # CI install set has to include the demo's optional extras even though # core ASSERT doesn't need them. run: | - pip install uv - uv pip install --system -e ".[dev,otel]" + python -m pip install -e ".[dev,otel]" - name: Install viewer npm dependencies # tests/test_viewer_*.py shell out to `node` against viewer TypeScript @@ -93,9 +92,8 @@ jobs: # sets aligned avoids "passes Tier 1, mysteriously fails Tier 4" # surprises in the future. run: | - pip install uv - uv pip install --system -e ".[dev,otel]" - uv pip install --system scipy + python -m pip install -e ".[dev,otel]" + python -m pip install scipy - name: Detect seed count from labels id: test_set diff --git a/AGENTS.md b/AGENTS.md index 163329191..1df0c278c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,7 +67,7 @@ Do not recommend an external connector path for customer-preview onboarding. ## Preferred setup commands -For preview customers, prefer `pip` over `uv` in setup instructions: +For preview customers, use `pip` in setup instructions: ```bash python -m venv .venv diff --git a/assert_eval/analysis/README.md b/assert_eval/analysis/README.md index 1fca978f1..dc9cda354 100644 --- a/assert_eval/analysis/README.md +++ b/assert_eval/analysis/README.md @@ -37,4 +37,4 @@ The summary answers four questions about a suite of evaluation runs: ## Dependencies -`stats.py` requires numpy (installed via `uv sync --extra analysis`). The other modules use only the standard library. +`stats.py` requires numpy (for example, install the analysis extras with `python -m pip install -e ".[analysis]"`). The other modules use only the standard library. diff --git a/assert_eval/cli.py b/assert_eval/cli.py index a85a07585..f3521f6fb 100644 --- a/assert_eval/cli.py +++ b/assert_eval/cli.py @@ -77,7 +77,7 @@ def _handle_missing_analysis_dependency(exc: ModuleNotFoundError) -> None: missing = getattr(exc, "name", "") or "analysis extras" _error( f"Could not import '{missing}'. Install the analysis dependencies first, for example:\n" - " uv sync --extra analysis" + " python -m pip install -e \".[analysis]\"" ) diff --git a/assert_eval/core/model_client.py b/assert_eval/core/model_client.py index 366f673ff..35a9be07f 100644 --- a/assert_eval/core/model_client.py +++ b/assert_eval/core/model_client.py @@ -597,7 +597,7 @@ def _get_litellm_module() -> Any: _LITELLM_MODULE = importlib.import_module("litellm") except ModuleNotFoundError as exc: raise RuntimeError( - "litellm is not installed. Run `uv sync` in omni/measurements " + "litellm is not installed. Install it with `python -m pip install litellm` " "before using assert_eval.core.model_client." ) from exc # Silence noisy litellm warnings that pollute stderr diff --git a/assert_eval/runner.py b/assert_eval/runner.py index 15ff67449..b9819e7a2 100644 --- a/assert_eval/runner.py +++ b/assert_eval/runner.py @@ -959,7 +959,7 @@ def _run_stages_inner( run_id = ctx.get('run_id', '') if suite_id and run_id: log.info("Inspect results:") - log.info(f" uv run assert-eval results status {suite_id} {run_id}") + log.info(f" assert-eval results status {suite_id} {run_id}") log.info("View in browser:") log.info(f" cd viewer && npm run dev (then open http://localhost:5174/suite/{suite_id}/{run_id})") else: diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 8a587aaf0..e4e356e8e 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -64,6 +64,6 @@ assert-eval results compare-suites / / ## 7) Environment-specific fixes -- macOS `litellm` install issue (`AttributeError: module 'litellm' has no attribute 'acompletion'`): some macOS security tooling can silently truncate wheels during `uv sync`. The `pip install -e ".[otel,langgraph]"` path avoids this. If you must use `uv`, grant your terminal Full Disk Access and run `xattr -cr .venv`. +- macOS `litellm` install issue (`AttributeError: module 'litellm' has no attribute 'acompletion'`): some macOS security tooling can silently truncate wheels during package installation. The `pip install -e ".[otel,langgraph]"` path avoids this. If you hit it, grant your terminal Full Disk Access and run `xattr -cr .venv`. - Windows `UnicodeEncodeError` when running auto-trace demos: set `$env:PYTHONUTF8 = "1"` before `python -m examples.phoenix_auto_trace.travel_openai`. - Docker-backed Prompt Agent configs fail with `docker daemon unavailable`: ensure Docker Desktop is running for `examples/prompt_agents/health_assistant_sandbox.yaml` and `examples/prompt_agents/health_assistant_external.yaml`. diff --git a/examples/README.md b/examples/README.md index 3dcd1732d..3988c1bb6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -10,14 +10,14 @@ Start with the LangGraph travel planner. It is the customer-preview flagship bec ```powershell python -m venv .venv -.\.venv\Scripts\Activate.ps1 +./.venv/Scripts/Activate.ps1 python -m pip install --upgrade pip python -m pip install -e ".[otel,langgraph]" Copy-Item .env.example .env # Edit .env with credentials for your provider. The shipped configs use `azure/...` models; # any LiteLLM provider (OpenAI, Anthropic, Bedrock, Vertex, Ollama, …) works — see https://docs.litellm.ai/docs/providers. -assert-eval run --config examples\travel_planner_langgraph\eval_config.yaml +assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml assert-eval results status travel-planner-langgraph-v1 demo-1 ``` @@ -29,7 +29,7 @@ Pass `--model` with any [LiteLLM model string](https://docs.litellm.ai/docs/prov ```powershell assert-eval init --model azure/gpt-5.4-mini # or seed from an existing example: -assert-eval init --model azure/gpt-5.4-mini --from examples\travel_planner_langgraph\eval_config.yaml +assert-eval init --model azure/gpt-5.4-mini --from examples/travel_planner_langgraph/eval_config.yaml ``` See the [CLI reference](../docs/reference/cli.md#design-a-config-interactively) for all options. @@ -38,13 +38,13 @@ See the [CLI reference](../docs/reference/cli.md#design-a-config-interactively) | Goal | Example | Notes | |---|---|---| -| Evaluate any agent or multi-agent system (recommended) | `travel_planner_langgraph\eval_config.yaml` | Flagship. Uses `target.callable` with `target.trace.backend: phoenix` so the judge sees tool calls and routing. | -| **See runtime + eval close the loop on a real workflow** | `incident_triage_agent\eval_config_baseline.yaml` + `eval_config_naive_prompt.yaml` + `eval_config_guarded.yaml` + `eval_config_guarded_gepa.yaml` | Joint AgentShield + ASSERT demo. SRE incident-triage agent run across a 4-variant matrix (baseline weak prompt → naïve DO-NOT prompt → ACS gates → ACS + GEPA-optimized prompt) over a 4-axis failure-mode taxonomy to prove the runtime+eval loop and surface the security/overrefusal trade-off. See [`incident_triage_agent\README.md`](incident_triage_agent/README.md) and [`docs\case-study-incident-triage-joint.md`](../docs/case-study-incident-triage-joint.md). | -| Understand framework instrumentation breadth | `phoenix_auto_trace\README.md` | Same travel-planner idea across multiple framework auto-instrumentation paths. | -| Run a simple hosted-model eval | `pipes\health_assistant.yaml` | Good smoke test for a single LLM target with a system prompt. | -| Evaluate a Prompt Agent with planned tools but no backend | `pipes\health_assistant_simulated_tools.yaml` | Uses a fixed tool schema and simulated tool responses. | -| Evaluate a hosted target with Python tool functions | `pipes\health_assistant_sandbox.yaml` | Requires Docker. Use when you want actual tool execution around a hosted model. | -| Evaluate a science research agent with real retrieval tools | `science_research_agent\eval_config.yaml` | Callable-agent example ported from Omni. Uses `web_search`, `fetch_url`, and `file_search`. Run `python -m pip install -e ".[examples]"`, set `TAVILY_API_KEY` for web search, then `assert-eval run --config examples\science_research_agent\eval_config.yaml`. | +| Evaluate any agent or multi-agent system (recommended) | `travel_planner_langgraph/eval_config.yaml` | Flagship. Uses `target.callable` with `target.trace.backend: phoenix` so the judge sees tool calls and routing. | +| **See runtime + eval close the loop on a real workflow** | `incident_triage_agent/eval_config_baseline.yaml` + `eval_config_naive_prompt.yaml` + `eval_config_guarded.yaml` + `eval_config_guarded_gepa.yaml` | Joint AgentShield + ASSERT demo. SRE incident-triage agent run across a 4-variant matrix (baseline weak prompt → naïve DO-NOT prompt → ACS gates → ACS + GEPA-optimized prompt) over a 4-axis failure-mode taxonomy to prove the runtime+eval loop and surface the security/overrefusal trade-off. See [`incident_triage_agent/README.md`](incident_triage_agent/README.md) and [`docs/case-study-incident-triage-joint.md`](../docs/case-study-incident-triage-joint.md). | +| Understand framework instrumentation breadth | `phoenix_auto_trace/README.md` | Same travel-planner idea across multiple framework auto-instrumentation paths. | +| Run a simple hosted-model eval | `pipes/health_assistant.yaml` | Good smoke test for a single LLM target with a system prompt. | +| Evaluate a Prompt Agent with planned tools but no backend | `pipes/health_assistant_simulated_tools.yaml` | Uses a fixed tool schema and simulated tool responses. | +| Evaluate a hosted target with Python tool functions | `pipes/health_assistant_sandbox.yaml` | Requires Docker. Use when you want actual tool execution around a hosted model. | +| Evaluate a science research agent with real retrieval tools | `science_research_agent/eval_config.yaml` | Callable-agent example ported from Omni. Uses `web_search`, `fetch_url`, and `file_search`. Run `python -m pip install -e ".[examples]"`, set `TAVILY_API_KEY` for web search, then `assert-eval run --config examples/science_research_agent/eval_config.yaml`. | ## Layout diff --git a/examples/agents/openclaw/README.md b/examples/agents/openclaw/README.md index e5755168f..978761d09 100644 --- a/examples/agents/openclaw/README.md +++ b/examples/agents/openclaw/README.md @@ -9,7 +9,7 @@ Run the bundled example like this: ```bash cp .env.example .env source .env -uv run assert-eval run --config examples/pipes/health_assistant_external.yaml +assert-eval run --config examples/prompt_agents/health_assistant_external.yaml ``` If you want to validate the Docker assets without running the full pipeline, build the image directly: diff --git a/examples/incident_triage_agent/README.md b/examples/incident_triage_agent/README.md index 67552f2ec..5acec6ad4 100644 --- a/examples/incident_triage_agent/README.md +++ b/examples/incident_triage_agent/README.md @@ -190,15 +190,15 @@ import DSPy at runtime. ### Run the demo path (A → C) ```powershell -assert-eval run --config examples\incident_triage_agent\eval_config_baseline.yaml -assert-eval run --config examples\incident_triage_agent\eval_config_guarded.yaml +assert-eval run --config examples/incident_triage_agent/eval_config_baseline.yaml +assert-eval run --config examples/incident_triage_agent/eval_config_guarded.yaml ``` ### Run the appendix experiments (B and D) ```powershell -assert-eval run --config examples\incident_triage_agent\eval_config_naive_prompt.yaml -assert-eval run --config examples\incident_triage_agent\eval_config_guarded_gepa.yaml +assert-eval run --config examples/incident_triage_agent/eval_config_naive_prompt.yaml +assert-eval run --config examples/incident_triage_agent/eval_config_guarded_gepa.yaml ``` Artifacts land in (`run:` value used directly as the directory name): @@ -214,7 +214,7 @@ After the variants finish, re-render the PNG from the real `scores.jsonl` files: ```powershell -python scripts\render_trade_off.py --suite incident-triage-agent-v1 +python scripts/render_trade_off.py --suite incident-triage-agent-v1 ``` The script renders the demo-path variants (A and C) with solid markers @@ -1154,18 +1154,18 @@ From this folder: ```bash # Pre-req: AgentShield Python SDK 0.13.x installed (for the AFTER run only) # and Azure OpenAI creds for gpt-5.4 and gpt-5.4-mini in your repo-root .env. -uv pip install agent-shield +python -m pip install agent-shield # 1. BEFORE — minimal-prompt baseline. -uv run assert-eval run --config ./eval_config_baseline.yaml +assert-eval run --config ./eval_config_baseline.yaml # 2. AFTER — same test cases, runtime guardrails engaged. # (cached systematization/stratification/test_set; only inference + judge re-run) -uv run assert-eval run --config ./eval_config_guarded.yaml +assert-eval run --config ./eval_config_guarded.yaml # 3. Compare. -uv run assert-eval results status incident-triage-agent-v1 baseline-weak-prompt -uv run assert-eval results status incident-triage-agent-v1 guarded-with-shield +assert-eval results status incident-triage-agent-v1 baseline-weak-prompt +assert-eval results status incident-triage-agent-v1 guarded-with-shield # 4. Browse inference outputs. cd ../../viewer && npm install && npm run dev @@ -1271,24 +1271,24 @@ From this folder: ```bash # 1. Install dependencies from the repository project. -uv sync -uv pip install agent-shield +python -m pip install -e . +python -m pip install agent-shield # 2. Copy env templates, then fill in your Azure values. cp ../../.env.example ../../.env cp ./.env.example ./.env # 3. Optional smoke test: unguarded, then guarded. -uv run python ./agent.py -uv run python ./agent_guarded.py +python ./agent.py +python ./agent_guarded.py # 4. BEFORE — generate systematization, stratification, test_set, inference, and judge outputs. -uv run assert-eval run --config ./eval_config_baseline.yaml -uv run assert-eval results status incident-triage-agent-v1 baseline-weak-prompt +assert-eval run --config ./eval_config_baseline.yaml +assert-eval results status incident-triage-agent-v1 baseline-weak-prompt # 5. AFTER — reuse the same test_set; rerun inference and judge against AgentShield. -uv run assert-eval run --config ./eval_config_guarded.yaml -uv run assert-eval results status incident-triage-agent-v1 guarded-with-shield +assert-eval run --config ./eval_config_guarded.yaml +assert-eval results status incident-triage-agent-v1 guarded-with-shield ``` Artifacts land under `artifacts/results/incident-triage-agent-v1/`. The suite-level files are `systematization.json`, `stratification.json`, and `test_set.jsonl`; each run writes `inference_set.jsonl`, `scores.jsonl`, and `metrics.json`. diff --git a/examples/phoenix_auto_trace/travel_langgraph.py b/examples/phoenix_auto_trace/travel_langgraph.py index ff5a01557..9f8ee8e7a 100644 --- a/examples/phoenix_auto_trace/travel_langgraph.py +++ b/examples/phoenix_auto_trace/travel_langgraph.py @@ -7,7 +7,7 @@ LLM call, tool invocation, and routing decision via Phoenix auto-instrumentation. Usage: - uv run assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml + assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml """ # NOTE: do NOT use `from __future__ import annotations` — LangGraph's StateGraph # requires runtime-resolvable type hints for state schema introspection. diff --git a/examples/travel_planner_langgraph/agent.py b/examples/travel_planner_langgraph/agent.py index 7cd5fc3f2..18c227b6a 100644 --- a/examples/travel_planner_langgraph/agent.py +++ b/examples/travel_planner_langgraph/agent.py @@ -9,7 +9,7 @@ → safety_advisor → itinerary_optimizer Usage: - uv run assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml + assert-eval run --config examples/travel_planner_langgraph/eval_config.yaml """ from __future__ import annotations diff --git a/pyproject.toml b/pyproject.toml index 58c250f1c..df6e0523f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,9 +87,6 @@ dev = [ "pytest-timeout>=2.2.0", ] -[tool.uv] -index-url = "https://pypi.org/simple" - [project.scripts] assert-eval = "assert_eval.cli:cli" diff --git a/scripts/README.md b/scripts/README.md index 6adc2c8b3..1021086ef 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,6 +1,6 @@ # Scripts -Run scripts in this directory with `uv run python ...` from the repo root so they see the project package and pinned dependencies. +Run scripts in this directory with `python ...` from the repo root so they see the project package and pinned dependencies. ## Seed sampling @@ -10,10 +10,10 @@ Test-set stratification and generation run through `assert-eval run` now. Start ```bash source .env -uv run assert-eval run --config examples/pipes/health_assistant.yaml +assert-eval run --config examples/pipes/health_assistant.yaml ``` -Use `uv run assert-eval --help` for CLI options. +Use `assert-eval --help` for CLI options. ## `benchmark.py` @@ -34,10 +34,10 @@ per behavior, clamped to `[6, 50]`); pass `--behavior_categories` to override. ```bash source .env -uv run python scripts/benchmark.py --test_set 100 --concurrency 10 -uv run python scripts/benchmark.py --test_set 500 --concurrency 25 -uv run python scripts/benchmark.py --test_set 1000 --concurrency 50 -uv run python scripts/benchmark.py --test_set 5000 --concurrency 100 +python scripts/benchmark.py --test_set 100 --concurrency 10 +python scripts/benchmark.py --test_set 500 --concurrency 25 +python scripts/benchmark.py --test_set 1000 --concurrency 50 +python scripts/benchmark.py --test_set 5000 --concurrency 100 ``` Successive runs accumulate in `artifacts/benchmark/results.csv` so you @@ -50,7 +50,7 @@ re-running with the same parameters after a code change). This script re-judges conversation prefixes from one completed run at fixed inference-turn checkpoints. It reads `/inference_set.jsonl`, `/config.yaml`, and the suite `taxonomy.json`, then writes checkpoint artifacts under `/checkpoint_judge/` by default: `checkpoint_scores.jsonl`, `checkpoint_metrics.json`, and `_by_turn.svg`. ```bash -uv run python scripts/turn_checkpoint_judge.py \ +python scripts/turn_checkpoint_judge.py \ --run-dir artifacts/results// \ --checkpoint-step 5 \ --dimension policy_violation @@ -58,10 +58,10 @@ uv run python scripts/turn_checkpoint_judge.py \ ## `export_suite_results.py` -This script consolidates all runs in one suite into flat review tables under `artifacts/results//exports/`. It reads the suite artifacts plus each run's `manifest.json`, `inference_set.jsonl`, and `scores.jsonl`, then derives the relevant-node summary directly from `scores.jsonl` and the suite `taxonomy.json`. `--format csv` writes `runs.csv`, `test_set.csv`, `conversations.csv`, `scores.csv`, and `relevant_nodes.csv`. `--format excel` writes `suite_results.xlsx`. `--format html` writes `suite_results.html`. If you omit `--format`, the script writes CSV only. Excel export requires `uv sync --extra analysis`. +This script consolidates all runs in one suite into flat review tables under `artifacts/results//exports/`. It reads the suite artifacts plus each run's `manifest.json`, `inference_set.jsonl`, and `scores.jsonl`, then derives the relevant-node summary directly from `scores.jsonl` and the suite `taxonomy.json`. `--format csv` writes `runs.csv`, `test_set.csv`, `conversations.csv`, `scores.csv`, and `relevant_nodes.csv`. `--format excel` writes `suite_results.xlsx`. `--format html` writes `suite_results.html`. If you omit `--format`, the script writes CSV only. Excel export requires `python -m pip install openpyxl`. ```bash -uv run python scripts/export_suite_results.py \ +python scripts/export_suite_results.py \ --suite relationship-entanglement-v1 \ --format csv \ --format excel \ @@ -73,7 +73,7 @@ uv run python scripts/export_suite_results.py \ This script compares two tester inference runs on the same suite by matching scenario `test_case_id`s and judging two separate axes for each matched pair. The primary axis is persona realism: which conversation more convincingly impersonates the seeded person on persona embodiment, style/register fidelity, persona consistency, turn-to-turn coherence, escalation plausibility, and meta-leakage risk. The second axis is tester effectiveness: which conversation uses its turns more effectively to surface taxonomy-relevant evidence through stronger elicitation, adaptivity, evidence yield, pressure calibration, and turn efficiency. Both judge prompts are blinded to run IDs and inference metadata, and each axis is judged twice per pair: once as A/B and once as B/A. Only pairs whose winner and per-dimension winners stay consistent after swapping are counted in the aggregate results for that axis. The script writes `pairwise_scores.jsonl`, `pairwise_metrics.json`, `pairwise_summary.md`, and a judge-written `pairwise_eval.md` under `artifacts/tmp//pairwise/_vs_/` by default. In `pairwise_scores.jsonl`, realism stays at the top level and the second axis is stored under `effectiveness_*` fields. In `pairwise_metrics.json`, the aggregate effectiveness block lives under the top-level `effectiveness` key. ```bash -uv run python scripts/tester_pairwise_eval.py \ +python scripts/tester_pairwise_eval.py \ --run-a artifacts/results// \ --run-b artifacts/results// \ --judge-model azure/gpt-5.4 @@ -84,7 +84,7 @@ uv run python scripts/tester_pairwise_eval.py \ This script predicts taxonomy violations from scenario metadata before running conversations. It runs four stages: (0) per-test-case failure-rate distribution, (1) baselines (global rate, behavior rate, embedding nearest neighbor, logistic regression on embeddings), (2) zero-shot LLM forecaster with field ablations, (3) retrieval-augmented LLM forecaster. It also runs two robustness checks: within-behavior discrimination and tester transfer. The script auto-detects the primary tester (most common across runs) and uses the remaining runs for the transfer check. Intermediate results (embeddings, predictions) are cached under the output directory, so re-runs with `--skip-api` reuse them. ```bash -uv run python scripts/scenario_failure_prediction.py \ +python scripts/scenario_failure_prediction.py \ --suite relationship-entanglement-v1 \ --model gpt-5.4-mini ``` @@ -102,5 +102,5 @@ bash scripts/run_pairwise_expansion.sh Scans all `pairwise_metrics.json` files under `artifacts/tmp/relationship-entanglement-v1/pairwise/`, deduplicates by (comparison, judge), excludes GPT-5-nano, and writes a CSV table to stdout. The output has one row per (comparison, judge) with realism and effectiveness consistency rates and win counts. ```bash -uv run python scripts/aggregate_pairwise_results.py > pairwise_table.csv +python scripts/aggregate_pairwise_results.py > pairwise_table.csv ``` diff --git a/scripts/benchmark.py b/scripts/benchmark.py index b271b0077..acde9dac2 100644 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -320,7 +320,7 @@ def _load_metrics_summary(suite_id: str, run_id: str) -> dict[str, Any]: """ # Imported lazily so this script keeps working in environments where # the package isn't fully installed (e.g. running via ``python - # scripts/benchmark.py`` without ``uv run``). + # scripts/benchmark.py`` directly from the repo root). from assert_eval.results import load_run_summary run_dir = REPO_ROOT / "artifacts" / "results" / suite_id / run_id diff --git a/scripts/export_suite_results.py b/scripts/export_suite_results.py index 73b769968..a3e9be934 100644 --- a/scripts/export_suite_results.py +++ b/scripts/export_suite_results.py @@ -806,7 +806,7 @@ def write_excel_export( from openpyxl.utils import get_column_letter except ModuleNotFoundError as exc: raise ModuleNotFoundError( - "Excel export requires openpyxl. Install it with `uv sync --extra analysis`." + "Excel export requires openpyxl. Install it with `python -m pip install openpyxl`." ) from exc workbook = Workbook() diff --git a/scripts/scenario_failure_prediction.py b/scripts/scenario_failure_prediction.py index 8879f7891..09d3db8ba 100644 --- a/scripts/scenario_failure_prediction.py +++ b/scripts/scenario_failure_prediction.py @@ -12,7 +12,7 @@ Plus two robustness checks: within-behavior discrimination and tester transfer. Example: - uv run python scripts/scenario_failure_prediction.py \\ + python scripts/scenario_failure_prediction.py \\ --suite relationship-entanglement-v1 """