diff --git a/.gitleaks.toml b/.gitleaks.toml index 81df4055..c5d18c5b 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -30,3 +30,10 @@ condition = "AND" targetRules = ["generic-api-key"] regexes = ['''^cw-key-123456$'''] paths = ['''^tests/test_harbor_environment_contract\.py$'''] + +[[allowlists]] +description = "Bedrock model ID that follows a credential name in SkillSpector bridge tests" +condition = "AND" +targetRules = ["generic-api-key"] +regexes = ['''^us\.anthropic\.claude-opus-5$'''] +paths = ['''^tests/validators/test_security\.py$'''] diff --git a/CHANGELOG.md b/CHANGELOG.md index 562d9992..d3f30d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,65 @@ All notable changes to SkillEvaluator are documented in this file. ### Changed +- SkillSpector now has its own default model per provider instead of + following the chat default, so chat-default upgrades no longer reach + SkillSpector before it supports them. An explicit `SKILL_EVAL_LLM_MODEL` + still reaches it, and `SKILLSPECTOR_MODEL` still overrides both. The bridge + forwards your `SKILLSPECTOR_MODEL_`, `SKILLSPECTOR_MODEL_REGISTRY` and + `SKILLSPECTOR_REASONING_EFFORT` settings (never `SKILLSPECTOR_TEMPERATURE`), + defaults the reasoning effort to `medium` for GPT-6 models, and records the + requested `skillspector_model` and `skillspector_reasoning_effort` next to + `skillspector_version` in the security result. +- Install instructions and the missing-scanner hint now pin SkillSpector to + `v2.12.0` instead of its default branch. +- GPT-6 models get the GPT-5 request shape everywhere SkillEvaluator calls a + model: no `temperature`, and `max_completion_tokens` on the native OpenAI + endpoint. Gateway catalog IDs such as `aws/anthropic/bedrock-claude-opus-5` + are recognized as the Claude model they name. +- Output-token limits are sized per model family. OpenAI reasoning models get + at least 32,768 tokens; other models get 16,384 for `--llm-verify`, rubric + evaluation, deduplication verdicts and judges, 8,192 for report suggestions + and 16,000 for dataset generation. On the Anthropic API, Claude Opus 4 and + 4.1 stay at the 8,192 tokens the Anthropic SDK allows without streaming. +- Reasoning effort is sent only to GPT-6 models and to Claude 5.5 or later on + the Anthropic API: `high` for `--llm-verify`, `medium` for rubric evaluation, + deduplication verdicts and judges. +- Rubric evaluation and deduplication verdicts request structured output. The + verifier and deduplication prompts ask for `rationale` instead of + `reasoning`; replies that still say `reasoning` are accepted. +- The `goal_accuracy` RAGAS scorer is skipped for GPT-5 and GPT-6 judges, which + reject its sampling settings. +- `run_config.json` records the judge reasoning effort, and each standard + judge's details record `judge_model_used`, `judge_reasoning_effort` and + `judge_fallback_used`. Skill lift leaves out a case whose two conditions were + judged by different models and counts it in `judge_mixed_cases`. +- Removed the unused `CONTENT_DEDUP_LLM_*` constants and `LLM_VERIFY_MODEL`; an + LLM client built from explicit credentials uses the provider default model. +- Tier 3 pins the agent CLIs Harbor installs: Codex 0.161.0 and Claude Code + 2.1.292, instead of the latest release. Claude Code runs Claude 5.5 and later + models with `reasoning_effort=medium`; Codex keeps `high`. Local mode keeps + your installed CLIs and fails a trial early when they are too old for the + model: Claude 5.5 needs Claude Code 2.1.280 or newer, and GPT-6 needs Codex + 0.159.1 or newer. +- OpenAI-compatible gateway agent defaults: Codex moves to + `openai/openai/gpt-6.1-sol` and Claude Code to + `aws/anthropic/bedrock-claude-opus-5-5`. OpenCode stays on + `nvidia/nvidia/nemotron-3-super-120b-long-ctx`, no longer tied to the gateway + chat default. Claude Code on the shared gateway key runs with + `CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1`, so gateways that forward to + another provider do not reject its pre-release request fields. +- The OpenAI default model moves from `gpt-5.6-sol` to `gpt-6.1-sol` for chat, + judging, Codex and SkillSpector, which runs it at `medium` reasoning effort, + so the effort values above apply to the OpenAI defaults. OpenCode keeps + `openai/gpt-5.6-sol` unless `SKILL_EVAL_LLM_MODEL` is set, because + GPT-6.1 Sol supports tool calls only on the Responses API. +- The Anthropic and Bedrock default models move from Claude Opus 5 to Claude + Opus 5.5 (`claude-opus-5-5` and `us.anthropic.claude-opus-5-5`) for chat, + judging and Claude Code. On the Claude API they get the effort values above + and server-side refusal fallback, and Claude Code runs them at + `reasoning_effort=medium`. + SkillSpector stays on `claude-opus-5` and `us.anthropic.claude-opus-5`, which + SkillSpector v2.12.0 supports. - Docker Claude Code and Codex containers no longer get the distribution `nodejs` and `npm` packages from agent setup, except on Alpine images. On Debian-based images, including the default eval image, those packages also @@ -26,6 +85,18 @@ All notable changes to SkillEvaluator are documented in this file. ### Fixed +- Safety-classifier refusals, content filters and output-token truncation are + detected instead of being read as empty or malformed output. `--llm-verify` + and rubric evaluation report the cause, and `--llm-verify` keeps the + complete verdicts of a truncated reply. Claude 5.5 or later requests to the + native Claude API use server-side fallback to Claude Opus 4.8. In the Harbor + verifier, Bedrock retries a filtered Claude 5.5 reply on Claude Opus 4.8, and + a refusal or empty judge reply moves on to the next + `LLM_JUDGE_FALLBACK_MODELS` entry. +- The Harbor verifier's last-resort judge model follows the selected provider. + With no judge or chat model configured, it used to request an OpenAI model + from Anthropic, Bedrock, NVIDIA Build and gateways. The unused host-side + `llm_judge.DEFAULT_JUDGE_MODEL` constant is removed. - Fixed Docker Codex and Claude Code setup wasting its timeout installing distribution Node/npm before Harbor's own installers. Native, gateway Codex, and NVIDIA Build agents now provision Node/npm only on the installer branches diff --git a/README.md b/README.md index 5d3e4aa3..7d653883 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ export OPENAI_API_KEY='sk-...' NVIDIA Build defaults to `nvidia/nemotron-3-super-120b-a12b` for evaluator chat, Tier 3 agent execution, and judging. Its embedding default is -`nvidia/nemotron-3-embed-1b`; OpenAI defaults to `gpt-5.6-sol` and +`nvidia/nemotron-3-embed-1b`; OpenAI defaults to `gpt-6.1-sol` and `text-embedding-3-small`. These are configured defaults, not automatically updated selections of the latest models. To select a different evaluator model: diff --git a/docs/agents-and-sandboxes.mdx b/docs/agents-and-sandboxes.mdx index 6e8aef3f..cff01733 100644 --- a/docs/agents-and-sandboxes.mdx +++ b/docs/agents-and-sandboxes.mdx @@ -22,10 +22,14 @@ SkillEvaluator selects the provider-native default: `opencode` for NVIDIA Build, | Agent | Notes | | --- | --- | -| `claude-code` | Anthropic's Claude Code CLI. `claude` is an accepted alias. | -| `codex` | OpenAI's Codex CLI. The OpenAI-provider default. Runs with `reasoning_effort=high` so scores and cost stay comparable across Harbor releases. | +| `claude-code` | Anthropic's Claude Code CLI. `claude` is an accepted alias. Harbor installs Claude Code 2.1.292. Claude 5.5 and later models run with `reasoning_effort=medium`; older models keep Claude Code's default effort. | +| `codex` | OpenAI's Codex CLI. The OpenAI-provider default. Harbor installs Codex 0.161.0. Runs with `reasoning_effort=high` so scores and cost stay comparable across Harbor releases. | | `opencode` | The OpenCode CLI. | +The pinned CLI releases apply wherever Harbor installs the agent: Docker and +the remote environments. [Local mode](#local-mode) runs the CLI already on your +machine instead. + The `claude` alias is canonicalized to `claude-code` everywhere an agent name can appear — `-a/--agents`, `--agent-model`, and `harbor.agents` in `evals/config.yml` — so both spellings always name the same agent. @@ -350,6 +354,11 @@ hard-unsupported and fails closed before agent-runtime discovery — directs you to WSL2 or `--env-mode docker`. Only `claude-code`, `codex`, and `opencode` can run in local mode. +Local mode does not install the pinned CLI releases. Each trial checks the +installed version against the selected model instead and fails before the agent +starts when it is too old: Claude 5.5 and later models need Claude Code 2.1.280 +or newer, and GPT-6 models need Codex 0.159.1 or newer. + The sandbox policy is controlled by environment variables: | Variable | Default | Effect | diff --git a/docs/ci-integration.mdx b/docs/ci-integration.mdx index 513dac00..640fe6d5 100644 --- a/docs/ci-integration.mdx +++ b/docs/ci-integration.mdx @@ -66,7 +66,7 @@ jobs: - name: Install external Python scanners run: | uv tool install semgrep - uv tool install git+https://github.com/NVIDIA/SkillSpector.git + uv tool install git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0 - name: Install Gitleaks run: | diff --git a/docs/cli-reference.mdx b/docs/cli-reference.mdx index 10de1f8e..bc224cb6 100644 --- a/docs/cli-reference.mdx +++ b/docs/cli-reference.mdx @@ -396,7 +396,7 @@ Use `skillevaluator tier3 PATH` for the complete workflow with automatic missing | `--n-concurrent INTEGER` | unset | Concurrent eval cases per agent. | | `--max-agents INTEGER` | unset | Maximum agents to run in parallel. | | `--model TEXT` | unset | Global agent model override. | -| `--agent-model TEXT` | unset | Per-agent model override, `AGENT=MODEL` (repeatable), e.g. `--agent-model codex=gpt-5.6-sol`. Accepts the `claude` alias for `claude-code`. | +| `--agent-model TEXT` | unset | Per-agent model override, `AGENT=MODEL` (repeatable), e.g. `--agent-model codex=gpt-6.1-sol`. Accepts the `claude` alias for `claude-code`. | | `--custom-dockerfile-mode [preserve\|rebase]` | unset | How a custom `evals/environment/Dockerfile` is applied: `rebase` (built-in default) swaps its `FROM` line onto the eval base image; `preserve` keeps your `FROM` and appends the eval dependencies. | | `--skill-workspace-mode [isolated\|group]` | unset | `isolated` (built-in default) stages only the target skill in the agent workspace; `group` also stages the skills from `--include-skills` / `skill_workspace.include`. | | `--include-skills PATH` | none | Additional skill(s) to mount into the eval environment (repeatable; paths must exist). Requires group workspace mode (`--skill-workspace-mode group` or `skill_workspace.mode: group` in `evals/config.yml`). | @@ -415,7 +415,8 @@ Use `skillevaluator tier3 PATH` for the complete workflow with automatic missing | `--progress [auto\|rich\|plain\|off]` | `auto` | Progress display for the run: `rich` live view, `plain` line output, or `off`. | Without a `--model`/`--agent-model` override, agent execution uses -`gpt-5.6-sol` for OpenAI, `claude-opus-5` for Anthropic, and +`gpt-6.1-sol` for OpenAI (OpenCode: `openai/gpt-5.6-sol` unless +`SKILL_EVAL_LLM_MODEL` is set), `claude-opus-5-5` for Anthropic, and `nvidia/nemotron-3-super-120b-a12b` for NVIDIA Build. OpenCode displays the last value as `nvidia/nvidia/nemotron-3-super-120b-a12b` because its adapter namespace precedes the NVIDIA catalog ID. diff --git a/docs/configuration.mdx b/docs/configuration.mdx index 631aaa7b..8b3c7db5 100644 --- a/docs/configuration.mdx +++ b/docs/configuration.mdx @@ -89,7 +89,7 @@ Set `SKILL_EVAL_LLM_PROVIDER` to one of `openai`, `anthropic`, `nv_build`, `bedr export OPENAI_API_KEY='sk-...' ``` - Default model: `gpt-5.6-sol` (OpenAI GPT-5.6 Sol frontier). For a lower-cost override set `SKILL_EVAL_LLM_MODEL=gpt-5.4-mini`. Endpoint `https://api.openai.com/v1`. Override the endpoint with `SKILL_EVAL_LLM_BASE_URL` (takes precedence) or `OPENAI_BASE_URL` — but a base URL that is a non-canonical alias of the native OpenAI endpoint is rejected; use `https://api.openai.com/v1` verbatim. On the canonical endpoint, `gpt-5*` models automatically send `max_completion_tokens` instead of `max_tokens`. + Default model: `gpt-6.1-sol` (OpenAI GPT-6.1 Sol frontier). For a lower-cost override set `SKILL_EVAL_LLM_MODEL=gpt-5.4-mini`. Endpoint `https://api.openai.com/v1`. Override the endpoint with `SKILL_EVAL_LLM_BASE_URL` (takes precedence) or `OPENAI_BASE_URL` — but a base URL that is a non-canonical alias of the native OpenAI endpoint is rejected; use `https://api.openai.com/v1` verbatim. On the canonical endpoint, GPT-5 and GPT-6 models automatically send `max_completion_tokens` instead of `max_tokens`. Tier 3's OpenCode agent uses `openai/gpt-5.6-sol` unless `SKILL_EVAL_LLM_MODEL` is set, because OpenCode calls tools through Chat Completions and GPT-6.1 Sol supports tool calls only on the Responses API. ```bash title="Anthropic setup" @@ -97,7 +97,7 @@ Set `SKILL_EVAL_LLM_PROVIDER` to one of `openai`, `anthropic`, `nv_build`, `bedr export ANTHROPIC_API_KEY='sk-ant-...' ``` - Default model: `claude-opus-5`, using the Anthropic SDK's default endpoint. Override it with `SKILL_EVAL_LLM_BASE_URL` (highest precedence) or `ANTHROPIC_BASE_URL`; when neither is set, SkillEvaluator uses the Anthropic default. + Default model: `claude-opus-5-5`, using the Anthropic SDK's default endpoint. Override it with `SKILL_EVAL_LLM_BASE_URL` (highest precedence) or `ANTHROPIC_BASE_URL`; when neither is set, SkillEvaluator uses the Anthropic default. For Anthropic, either base-URL variable must be an absolute HTTP(S) API root. Supply an unversioned root such as `https://gateway.example.com` or a legacy URL whose final segment is `/v1`, such as `https://gateway.example.com/team/v1`. SkillEvaluator normalizes the latter to its API root so the Anthropic SDK, the Tier 3 verifier, and a matching Claude Code agent each form request paths with exactly one `/v1/messages` suffix. An independently credentialed Claude Code route applies the same normalization and validation to `ANTHROPIC_BASE_URL` when the evaluator uses another provider. Routing through path prefixes, ports, IP addresses, internationalized domain names, and ordinary safe percent escapes is preserved. @@ -125,7 +125,7 @@ Set `SKILL_EVAL_LLM_PROVIDER` to one of `openai`, `anthropic`, `nv_build`, `bedr # (environment variables, shared profile, or an assumed role). ``` - Default model: `us.anthropic.claude-opus-5` on the AWS Bedrock Runtime (US geo inference profile for Claude Opus 5). + Default model: `us.anthropic.claude-opus-5-5` on the AWS Bedrock Runtime (US geo inference profile for Claude Opus 5.5). ```bash title="OpenAI-compatible setup" @@ -140,15 +140,15 @@ Set `SKILL_EVAL_LLM_PROVIDER` to one of `openai`, `anthropic`, `nv_build`, `bedr | --- | --- | --- | | Evaluator and standard judge | `nvidia/nvidia/nemotron-3-super-120b-long-ctx` | `SKILL_EVAL_LLM_MODEL`; judge-only: `LLM_JUDGE_MODEL` then `SKILL_EVAL_JUDGE_MODEL` | | Tier 2 embeddings | `nvidia/nvidia/nemotron-3-embed-1b` | `SKILL_EVAL_EMBEDDING_MODEL` | - | Codex agent | `openai/openai/gpt-5.6-sol` | `--agent-model codex=MODEL` | - | Claude Code agent | `aws/anthropic/bedrock-claude-opus-5` | `--agent-model claude-code=MODEL` | + | Codex agent | `openai/openai/gpt-6.1-sol` | `--agent-model codex=MODEL` | + | Claude Code agent | `aws/anthropic/bedrock-claude-opus-5-5` | `--agent-model claude-code=MODEL` | | OpenCode agent | `nvidia/nvidia/nemotron-3-super-120b-long-ctx` | `--agent-model opencode=openai/MODEL` | Gateways can use different catalog IDs. Supply those exact IDs through the overrides; SkillEvaluator never substitutes a different model after a request fails. OpenCode's `openai/` prefix selects its adapter and is removed before sending the catalog ID. `SKILL_EVAL_LLM_MODEL` controls evaluation and judging; it does not replace these separate gateway agent defaults. Agent selection uses `--agent-model` / `--model`, then `harbor.agents.AGENT.model`, then the harness default, and records the model source in the run report. For Tier 3 in Docker, OpenCode requires Chat Completions, Codex requires Responses, and Claude Code requires Anthropic Messages, each with tool-call support. Selecting a shared URL does not add those APIs to a server. See the [Claude gateway requirements](https://code.claude.com/docs/en/llm-gateway). - Claude Code inherits the shared key and URL, stripping a terminal `/v1` from its API root while retaining any path prefix. An explicit `ANTHROPIC_BASE_URL` overrides that root. An explicit `ANTHROPIC_API_KEY` selects an independent Claude route: without `ANTHROPIC_BASE_URL`, it keeps the native Anthropic endpoint and defaults to `claude-opus-5`. CLI and skill-config model overrides still take precedence. Existing native-provider routes are unchanged. + Claude Code inherits the shared key and URL, stripping a terminal `/v1` from its API root while retaining any path prefix. An explicit `ANTHROPIC_BASE_URL` overrides that root. With the shared key, Claude Code runs with `CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1`, so it does not send pre-release request fields that a gateway forwarding to another provider rejects. An explicit `ANTHROPIC_API_KEY` selects an independent Claude route: without `ANTHROPIC_BASE_URL`, it keeps the native Anthropic endpoint and defaults to `claude-opus-5-5`. CLI and skill-config model overrides still take precedence. Existing native-provider routes are unchanged. @@ -258,7 +258,7 @@ schema_version: 1 harbor: agents: codex: - model: gpt-5.6-sol + model: gpt-6.1-sol ``` You can also supply that model at invocation time with `--agent-model codex=MODEL`. See [Tier 3: Live Evaluation](tier3-live-evaluation.mdx) for the full run workflow. @@ -313,7 +313,9 @@ variables. The `--llm` security analysis runs [SkillSpector](https://github.com/NVIDIA/SkillSpector), which has its own provider environment (`SKILLSPECTOR_PROVIDER` plus per-provider credential variables). SkillEvaluator bridges your configured provider automatically for that invocation, so the one key above is all you need. For `nv_build`, the bridge uses SkillSpector's OpenAI-compatible provider path pointed at the public Build endpoint, so the same `NVIDIA_API_KEY` covers this too; it does not create a second NVIDIA credential name. -The subprocess environment is isolated. Only the selected provider settings and basic process environment variables (`PATH`, `HOME`, TLS bundles, temp dirs) pass through — other ambient credentials in your shell are never forwarded. +The subprocess environment is isolated. Only the selected provider settings and basic process environment variables (`PATH`, `HOME`, TLS bundles, temp dirs) pass through — other ambient credentials in your shell are never forwarded. Your SkillSpector model settings (`SKILLSPECTOR_MODEL_`, `SKILLSPECTOR_MODEL_REGISTRY`, `SKILLSPECTOR_REASONING_EFFORT`) pass through too; `SKILLSPECTOR_TEMPERATURE` does not. + +SkillSpector sends its own structured-output requests, so it has its own default model per provider instead of following the chat default. On Anthropic and Bedrock it stays on Claude Opus 5 while the chat default is Claude Opus 5.5. An explicit `SKILL_EVAL_LLM_MODEL` still reaches SkillSpector, and `SKILLSPECTOR_MODEL` overrides both. SkillSpector v2.12.0 does not return structured output for Claude 5.5 models: if `SKILL_EVAL_LLM_MODEL` names one, also set `SKILLSPECTOR_MODEL` to a supported model such as `claude-opus-5` (Bedrock: `us.anthropic.claude-opus-5`), or the `--llm` security stage is reported incomplete. See [SkillSpector bridge](environment-variables.mdx#skillspector-bridge) for the defaults. Setting `SKILLSPECTOR_PROVIDER` yourself overrides the bridge, but it must name a supported public provider (`openai`, `anthropic`, or `bedrock` — or `nv_build` to force the Build mapping); anything else fails closed and the subprocess receives no credential. diff --git a/docs/environment-variables.mdx b/docs/environment-variables.mdx index 1bf49116..7cdcc5b2 100644 --- a/docs/environment-variables.mdx +++ b/docs/environment-variables.mdx @@ -26,7 +26,7 @@ These variables select and configure the provider used for LLM-backed checks and | Variable | Default | Effect | | --- | --- | --- | | `SKILL_EVAL_LLM_PROVIDER` | auto-selected (see below) | Provider for chat and judging: `openai`, `anthropic`, `nv_build`, `bedrock`, or `openai-compatible`. | -| `SKILL_EVAL_LLM_MODEL` | OpenAI: `gpt-5.6-sol`; Anthropic: `claude-opus-5`; NVIDIA Build: `nvidia/nemotron-3-super-120b-a12b`; OpenAI-compatible: `nvidia/nvidia/nemotron-3-super-120b-long-ctx`; Bedrock: `us.anthropic.claude-opus-5` | Chat model override. For gateways, use the exact catalog ID when it differs from the default. Setting it to an empty string is a configuration error. For a lower-cost OpenAI option, use `gpt-5.4-mini`. | +| `SKILL_EVAL_LLM_MODEL` | OpenAI: `gpt-6.1-sol`; Anthropic: `claude-opus-5-5`; NVIDIA Build: `nvidia/nemotron-3-super-120b-a12b`; OpenAI-compatible: `nvidia/nvidia/nemotron-3-super-120b-long-ctx`; Bedrock: `us.anthropic.claude-opus-5-5` | Chat model override. For gateways, use the exact catalog ID when it differs from the default. Setting it to an empty string is a configuration error. For a lower-cost OpenAI option, use `gpt-5.4-mini`. | | `SKILL_EVAL_LLM_BASE_URL` | provider default endpoint | Endpoint override; takes precedence over `OPENAI_BASE_URL` and `ANTHROPIC_BASE_URL`. Required for `openai-compatible`. Ignored by `nv_build`, whose endpoint is fixed to `https://integrate.api.nvidia.com/v1` — point a custom endpoint at the `openai-compatible` provider instead. | | `SKILL_EVAL_LLM_API_KEY` | — | API key for the `openai-compatible` provider (local servers still require it to be set). | | `SKILL_EVAL_MODEL_CATALOG_ALLOW_HTTP_HOSTS` | unset | Comma-separated hosts whose model catalog may be read over plain HTTP. Catalog reads otherwise require HTTPS unless the host is loopback, because the request carries a bearer token. Use this only when the transport is already encrypted and authenticated below HTTP — a WireGuard or comparable tunnel peer, for example — where the tool cannot see that the link is protected. Each entry matches one whole host as written, with no resolution, no suffix matching, and no wildcards. A plain-HTTP request to an accepted host also bypasses any inherited HTTP proxy, so the bearer token is never offered to an intermediary. The transport rechecks authorization before dispatch and rejects hosts that are no longer allowed. HTTPS routing is unchanged. | @@ -98,12 +98,15 @@ If none is set, LLM-backed commands fail with a configuration error. If more tha ### SkillSpector bridge -The `--llm` security analysis runs [SkillSpector](https://github.com/NVIDIA/SkillSpector), which has its own provider environment. SkillEvaluator bridges your configured provider into that invocation with an isolated subprocess environment: only the selected provider's settings and a small set of basic process variables are forwarded, so ambient credentials for other providers never reach SkillSpector. The `nv_build` provider is bridged through SkillSpector's OpenAI-compatible path, pointing `OPENAI_BASE_URL` at the public NVIDIA Build endpoint. +The `--llm` security analysis runs [SkillSpector](https://github.com/NVIDIA/SkillSpector), which has its own provider environment. SkillEvaluator bridges your configured provider into that invocation with an isolated subprocess environment: only the selected provider's settings, the SkillSpector model settings below, and a small set of basic process variables are forwarded, so ambient credentials for other providers never reach SkillSpector. `SKILLSPECTOR_TEMPERATURE` is not forwarded, because current reasoning models reject a custom temperature. The `nv_build` provider is bridged through SkillSpector's OpenAI-compatible path, pointing `OPENAI_BASE_URL` at the public NVIDIA Build endpoint. | Variable | Default | Effect | | --- | --- | --- | | `SKILLSPECTOR_PROVIDER` | bridged from your provider | Overrides the provider SkillSpector uses. Must name a supported public provider (`openai`, `anthropic`, `bedrock`, or `nv_build`) — only that provider's credential variables are forwarded. Any other value fails closed: SkillSpector runs without a credential. | -| `SKILLSPECTOR_MODEL` | bridged from your configured model | Model SkillSpector uses for its LLM analysis. | +| `SKILLSPECTOR_MODEL` | `SKILL_EVAL_LLM_MODEL` when set, else the SkillSpector default for your provider | Model SkillSpector uses for its LLM analysis. SkillSpector defaults: OpenAI `gpt-6.1-sol`, Anthropic `claude-opus-5`, Bedrock `us.anthropic.claude-opus-5`, NVIDIA Build `nvidia/nemotron-3-super-120b-a12b`, OpenAI-compatible `nvidia/nvidia/nemotron-3-super-120b-long-ctx`. They track the models the pinned SkillSpector release supports, not the chat defaults. | +| `SKILLSPECTOR_MODEL_` | unset | Per-analyzer model override, such as `SKILLSPECTOR_MODEL_META_ANALYZER`. Forwarded as set. | +| `SKILLSPECTOR_MODEL_REGISTRY` | unset | Path to a complete custom SkillSpector model registry, for model IDs the bundled registry does not list. Forwarded as set. | +| `SKILLSPECTOR_REASONING_EFFORT` | `medium` for GPT-6 models, else unset | Reasoning effort for every SkillSpector analyzer. Your value wins. SkillEvaluator sets it only when every SkillSpector model, including `SKILLSPECTOR_MODEL_` overrides, is a GPT-6 model, so never for Claude. | ## Embeddings @@ -151,8 +154,8 @@ execution environments are covered in | `SKILLEVALUATOR_LOCAL_INHERIT_AGENT_KEYS` | `false` | When true, local-mode trials inherit the host's agent credential variables into their ambient environment. Off by default so a hostile skill command cannot read credentials — agents and verifiers receive credentials per-exec instead. | | `SKILLEVALUATOR_RUNTIME_DIR` | `~/.local/share/skillevaluator/runtimes` | Where local mode looks for managed agent CLIs. Must be a dedicated subdirectory — not the home directory or one of its parents. | | `SKILL_EVAL_HARBOR_PREBUILD_TASK_ENVS` | unset | Set to exactly `1` to opt in to pre-building Harbor task environment images before trials start (other truthy spellings are not honored). Opt-in because concurrent trial cleanup can remove a shared prebuilt image that sibling trials still need. | -| `SKILL_EVAL_JUDGE_MODEL` | falls back to `SKILL_EVAL_LLM_MODEL` | Operator-controlled model override for the Tier 3 dimension judge in standard grading. `LLM_JUDGE_MODEL` is honored as a higher-precedence spelling; when neither is set, the judge uses `SKILL_EVAL_LLM_MODEL`, then the built-in judge default. Both spellings change only the model used with the configured `SKILL_EVAL_LLM_PROVIDER`; they do not select a separate judge provider, so the model must be supported by that provider. Standard grading makes both spellings resolve to the selected override in Harbor's parent process, then forwards them through its verifier-only job layer, which takes precedence over task and step verifier declarations. Neither name is injected into the evaluated agent's initial environment; attempts to set or reference one from skill-authored `runtime_env` or native `[environment.env]` tables are rejected. Native custom-only graders keep ownership of their verifier configuration. Shared agent/verifier containers are a process boundary, not a confidentiality boundary; use a separate verifier environment when the model name itself must be hidden from untrusted agent processes. `run_config.json` records the configured judge provider, model, source, `override_applied`, and catalog-verification status separately from agent models; an allowed provider fallback may use a different model for an individual judge call. When `override_applied` is false for a native Harbor task, `effective_model_source` is `native_harbor_runtime` and catalog verification is explicitly inconclusive because an authored task or step verifier declaration may supersede the recorded fallback. | -| `LLM_JUDGE_FALLBACK_MODELS` | unset | Comma-separated fallback models for the Tier 3 standard judges in the Harbor verifier. When the provider rejects the judge model as not found, invalid, or not accessible to the key, the verifier tries each fallback in order with the same provider. Fallback applies to the `openai`, `nv_build`, and `openai-compatible` providers. The value is forwarded only to standard verifier jobs, never to the agent environment or `custom_only` runs. | +| `SKILL_EVAL_JUDGE_MODEL` | falls back to `SKILL_EVAL_LLM_MODEL` | Operator-controlled model override for the Tier 3 dimension judge in standard grading. `LLM_JUDGE_MODEL` is honored as a higher-precedence spelling; when neither is set, the judge uses `SKILL_EVAL_LLM_MODEL`, then the built-in judge default. Both spellings change only the model used with the configured `SKILL_EVAL_LLM_PROVIDER`; they do not select a separate judge provider, so the model must be supported by that provider. Standard grading makes both spellings resolve to the selected override in Harbor's parent process, then forwards them through its verifier-only job layer, which takes precedence over task and step verifier declarations. Neither name is injected into the evaluated agent's initial environment; attempts to set or reference one from skill-authored `runtime_env` or native `[environment.env]` tables are rejected. Native custom-only graders keep ownership of their verifier configuration. Shared agent/verifier containers are a process boundary, not a confidentiality boundary; use a separate verifier environment when the model name itself must be hidden from untrusted agent processes. `run_config.json` records the configured judge provider, model, source, `override_applied`, reasoning effort, and catalog-verification status separately from agent models; an allowed provider fallback may use a different model for an individual judge call. When `override_applied` is false for a native Harbor task, `effective_model_source` is `native_harbor_runtime` and catalog verification is explicitly inconclusive because an authored task or step verifier declaration may supersede the recorded fallback. | +| `LLM_JUDGE_FALLBACK_MODELS` | unset | Comma-separated fallback models for the Tier 3 standard judges in the Harbor verifier. When the provider rejects the judge model as not found, invalid, or not accessible to the key, the verifier tries each fallback in order with the same provider. Fallback applies to the `openai`, `nv_build`, and `openai-compatible` providers. A safety refusal or an empty reply from the judge model also moves to the next fallback, on every provider. The value is forwarded only to standard verifier jobs, never to the agent environment or `custom_only` runs. | | `SKILLEVALUATOR_HARBOR_DIAGNOSTIC_ARTIFACT_MAX_BYTES` | `5242880` (5 MiB) | Size cap per diagnostic artifact copied from a Harbor trial into the results directory. Values are bounded to 0–67108864 bytes (64 MiB); `0` skips non-empty artifacts. Oversized artifacts are skipped and recorded in the artifact manifest. | ### Harbor process environment diff --git a/docs/installation.mdx b/docs/installation.mdx index 9dfaf572..14729f6a 100644 --- a/docs/installation.mdx +++ b/docs/installation.mdx @@ -154,7 +154,7 @@ Install the required security scanners separately so their dependency stacks do ```bash title="External security scanners" brew install semgrep gitleaks # macOS uv tool install semgrep # isolated Python tool environment -uv tool install git+https://github.com/NVIDIA/SkillSpector.git +uv tool install git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0 ``` On other systems, install Gitleaks with `go install github.com/gitleaks/gitleaks/v8@latest` or a binary from the [Gitleaks releases](https://github.com/gitleaks/gitleaks/releases). Semgrep and SkillSpector must be on `PATH`, or configured with `SKILLEVALUATOR_SEMGREP_PATH` and `SKILLEVALUATOR_SKILLSPECTOR_PATH`. RHEL 8 and compatible enterprise Linux systems can supply compatible executables independently of the SkillEvaluator Python environment. diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 1cb4b575..8233972c 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -48,7 +48,7 @@ Default models are already configured for NVIDIA Build and OpenAI. ```bash title="Tier 1 scanner setup on macOS" brew install semgrep gitleaks - uv tool install git+https://github.com/NVIDIA/SkillSpector.git + uv tool install git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0 ``` See [Installation](installation.mdx#system-tools) for other platforms. diff --git a/docs/tier1-validation.mdx b/docs/tier1-validation.mdx index bfccc50b..a4f10c54 100644 --- a/docs/tier1-validation.mdx +++ b/docs/tier1-validation.mdx @@ -59,7 +59,7 @@ for skills. Its Tier 1 LLM checks still run only when requested. ```bash title="Complete offline Tier 1" brew install gitleaks semgrep - uv tool install git+https://github.com/NVIDIA/SkillSpector.git + uv tool install git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0 skillevaluator validate ./my-skill --tiers 1 ``` @@ -329,7 +329,7 @@ Tier 1 checks always gate the exit code inside `validate`. Tier 2 gates when it Install it with `brew install gitleaks` or download a binary from the official [Gitleaks releases](https://github.com/gitleaks/gitleaks/releases). - Install Bandit and pip-audit with `skillevaluator[security]`. Install Semgrep separately with `brew install semgrep` or `uv tool install semgrep`, and SkillSpector with `uv tool install git+https://github.com/NVIDIA/SkillSpector.git`. Required scanner failures remain `INCOMPLETE` until the scanner is installed and returns valid evidence. + Install Bandit and pip-audit with `skillevaluator[security]`. Install Semgrep separately with `brew install semgrep` or `uv tool install semgrep`, and SkillSpector with `uv tool install git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0`. Required scanner failures remain `INCOMPLETE` until the scanner is installed and returns valid evidence. Configure a supported provider or omit `--llm`/`--llm-verify`. Use `--no-tier2` (or `--no-dedup`) to disable dedup when a fully key-free Tier 1 run is required. diff --git a/docs/tier3-live-evaluation.mdx b/docs/tier3-live-evaluation.mdx index 0d6390b9..816903bb 100644 --- a/docs/tier3-live-evaluation.mdx +++ b/docs/tier3-live-evaluation.mdx @@ -78,6 +78,10 @@ using Harbor 0.24: `tier3 evaluate` fail the prerequisite check until you reinstall the extra. - Codex runs with `reasoning_effort=high`, the effort Harbor 0.22 used by default, so results stay comparable across Harbor upgrades. +- Harbor installs Codex 0.161.0 and Claude Code 2.1.292 rather than the latest + release, so a new CLI release does not change results. Claude Code runs + Claude 5.5 and later models with `reasoning_effort=medium`. Local mode uses + your installed CLIs and checks that they are new enough for the model. - Installed Codex adapters merge existing user and MCP configuration before the selected runtime provider and endpoint take precedence over conflicting route keys. @@ -606,7 +610,8 @@ in [Judge evidence budgets](environment-variables.mdx#judge-evidence-budgets). a strict JSON Schema `response_format`; Anthropic `/v1/messages` receives `output_config`. Bedrock judges use prompt-only JSON. - **Automatic downgrade.** If an endpoint rejects the schema option itself - with HTTP 400 or 422 (for example, `response_format` is unsupported), the + with HTTP 400 or 422 (for example, `response_format` is unsupported, or a + gateway turned the schema into a `tool_choice` the model rejects), the judge repeats that request without the schema and logs a warning. It then skips the schema for that provider, endpoint, and model for the remaining judge calls in that verifier run. Other 400 and 422 errors are not @@ -617,8 +622,24 @@ in [Judge evidence budgets](environment-variables.mdx#judge-evidence-budgets). format reminder. `behavior_check` can also salvage complete per-behavior results from a truncated reply and counts the missing behaviors as not passed. +- **Reasoning models.** GPT-5 and GPT-6 judges get no `temperature` and at + least 32,768 output tokens, because reasoning tokens count against the + limit; other judges get 16,384. GPT-6 judges and Claude 5.5 or later judges + on the Anthropic API are sent reasoning effort `medium`; other models keep + their default effort. +- **Refusals.** A safety-classifier refusal is never scored as an answer. On + the native Claude API, Claude 5.5 or later judges use server-side fallback to + `claude-opus-4-8`, and on Bedrock a filtered Claude 5.5 reply is retried + once on Claude Opus 4.8 in the same geography. A refusal or an empty reply + then moves on to the next `LLM_JUDGE_FALLBACK_MODELS` entry, if any. + Each judge's details in the trial's `skill_evaluator_reward.json` record + `judge_model_used`, `judge_reasoning_effort`, and `judge_fallback_used`. + When the two conditions of a case were judged by different models, the case + is left out of that judge's skill lift and counted in `judge_mixed_cases` + in `lift.json`. - **RAGAS goal scorer.** With the `openai` provider on the canonical - `https://api.openai.com/v1` endpoint, `goal_accuracy` first tries the RAGAS + `https://api.openai.com/v1` endpoint and a judge model that is not a GPT-5 or + GPT-6 reasoning model, `goal_accuracy` first tries the RAGAS `AgentGoalAccuracyWithReference` scorer through its own client, without the schema or retry handling above but within the same time budget. If RAGAS is unavailable or fails, the prompt judge runs instead. diff --git a/src/skillevaluator/constants.py b/src/skillevaluator/constants.py index ade04e71..f468ce28 100644 --- a/src/skillevaluator/constants.py +++ b/src/skillevaluator/constants.py @@ -6,7 +6,7 @@ Based on SkillEvaluator HOW_TO_CONTRIBUTE_SKILLS.md, HOW_TO_CONTRIBUTE_WORKFLOW_RULES.md specifications. """ -from skillevaluator.provider_config import CHAT_DEFAULT_OPENAI, EMBEDDING_DEFAULT_NVIDIA +from skillevaluator.provider_config import CHAT_DEFAULT_OPENAI, EMBEDDING_DEFAULT_NVIDIA, MAX_COMPLETION_TOKENS # ============================================================================= # SKILLS CONSTANTS @@ -276,8 +276,9 @@ SIMILARITY_MAX_DISCOVERED_PATHS = 4 * SIMILARITY_MAX_ENTRIES SIMILARITY_DEFAULT_MAX_SCALAR_COMPARISONS = 128_000_000 -CONTENT_DEDUP_LLM_DEFAULT_MODEL = "azure/anthropic/claude-opus-4-8" -CONTENT_DEDUP_LLM_TEMPERATURE = 0.1 +# Reasoning effort for the dedup verdict model (GPT-6 and Claude 5.5 or later only): +# a DUPLICATE verdict creates a blocking finding, so precision outweighs latency. +CONTENT_DEDUP_LLM_REASONING_EFFORT = "medium" CONTENT_DEDUP_SCANNABLE_EXTENSIONS = frozenset({".md", ".mdc", ".py", ".sh"}) CONTENT_DEDUP_BINARY_EXTENSIONS = frozenset( @@ -313,9 +314,11 @@ # LLM FINDING VERIFICATION CONSTANTS # ============================================================================= -LLM_VERIFY_MODEL = "azure/anthropic/claude-opus-4-8" -LLM_VERIFY_MAX_TOKENS = 512 +LLM_VERIFY_MAX_TOKENS = MAX_COMPLETION_TOKENS LLM_VERIFY_TEMPERATURE = 0.0 +# A wrong false_positive verdict hides a real finding in adversarial input, and +# verification is opt-in and low volume, so it gets the most reasoning. +LLM_VERIFY_REASONING_EFFORT = "high" # ============================================================================= @@ -389,8 +392,9 @@ # LLM RUBRIC EVALUATION CONSTANTS (ported from SkillEvaluator Tier 2) # ============================================================================= -RUBRIC_MAX_TOKENS = 4096 +RUBRIC_MAX_TOKENS = MAX_COMPLETION_TOKENS RUBRIC_MIN_SCORE = 60 +RUBRIC_REASONING_EFFORT = "medium" RUBRIC_CRITERIA = [ { @@ -581,16 +585,18 @@ # ============================================================================= DIMENSION_JUDGE_MODEL = CHAT_DEFAULT_OPENAI -DIMENSION_JUDGE_MAX_TOKENS = 4096 +DIMENSION_JUDGE_MAX_TOKENS = MAX_COMPLETION_TOKENS DIMENSION_JUDGE_TEMPERATURE: float | None = 0.0 +DIMENSION_JUDGE_REASONING_EFFORT = "medium" DIMENSION_VERDICT_PASS_THRESHOLD = 0.5 DIMENSION_VERDICT_NEUTRAL_THRESHOLD = 0.4 # LLM-as-Judge for the Insights tab (additional Conclusions and Recommendations # on top of the deterministic ones produced by tier3_normalizer). INSIGHTS_JUDGE_MODEL = DIMENSION_JUDGE_MODEL -INSIGHTS_JUDGE_MAX_TOKENS = 4096 +INSIGHTS_JUDGE_MAX_TOKENS = MAX_COMPLETION_TOKENS INSIGHTS_JUDGE_TEMPERATURE: float | None = None +INSIGHTS_JUDGE_REASONING_EFFORT = DIMENSION_JUDGE_REASONING_EFFORT INSIGHTS_JUDGE_MAX_CONCLUSIONS = 5 INSIGHTS_JUDGE_MAX_RECOMMENDATIONS = 5 diff --git a/src/skillevaluator/deduplication/intra_skill/intra_skill_validator.py b/src/skillevaluator/deduplication/intra_skill/intra_skill_validator.py index 62c21be5..de671ae7 100644 --- a/src/skillevaluator/deduplication/intra_skill/intra_skill_validator.py +++ b/src/skillevaluator/deduplication/intra_skill/intra_skill_validator.py @@ -16,6 +16,7 @@ from skillevaluator.constants import ( CONTENT_DEDUP_EMBEDDING_BATCH_SIZE, + CONTENT_DEDUP_LLM_REASONING_EFFORT, CONTENT_DEDUP_MAX_CHUNKS, CONTENT_DEDUP_MAX_LLM_CLUSTERS, CONTENT_DEDUP_SIMILARITY_THRESHOLD, @@ -265,7 +266,7 @@ def validate(self, skill_path: Path) -> ValidationResult: # Step 5: LLM analysis for each cluster (concurrent) logger.info("Running LLM analysis on %d cluster(s) concurrently...", len(clusters)) try: - llm = LLMClient(model=self._llm_model) + llm = LLMClient(model=self._llm_model, reasoning_effort=CONTENT_DEDUP_LLM_REASONING_EFFORT) config = llm._resolved_config() provider, model = safe_llm_labels(config.provider, config.model) except LLMClientError: diff --git a/src/skillevaluator/deduplication/intra_skill/llm_analyzer.py b/src/skillevaluator/deduplication/intra_skill/llm_analyzer.py index 3cf5ba7d..70825d4f 100644 --- a/src/skillevaluator/deduplication/intra_skill/llm_analyzer.py +++ b/src/skillevaluator/deduplication/intra_skill/llm_analyzer.py @@ -8,7 +8,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from skillevaluator.inference import LLMClient, LLMClientError, LLMVerdict from skillevaluator.models.result import Severity @@ -18,6 +18,18 @@ VALID_VERDICTS = {"DUPLICATE", "INTENTIONAL_DETAIL", "RELATED_BUT_DISTINCT"} +VERDICT_JSON_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "verdict": {"type": "string", "enum": sorted(VALID_VERDICTS)}, + "confidence": {"type": "number"}, + "rationale": {"type": "string"}, + "suggestion": {"type": "string"}, + }, + "required": ["verdict", "confidence", "rationale", "suggestion"], + "additionalProperties": False, +} + SYSTEM_PROMPT = """You are a technical content analyst for AI agent skill packages. You analyze groups of text chunks from different files within the same skill directory. @@ -37,7 +49,7 @@ { "verdict": "DUPLICATE" | "INTENTIONAL_DETAIL" | "RELATED_BUT_DISTINCT", "confidence": , - "reasoning": "<2-3 sentence explanation>", + "rationale": "<2-3 sentence explanation>", "suggestion": "" }""" @@ -62,7 +74,12 @@ def build_user_prompt(cluster: ContentCluster) -> str: def analyze_cluster(client: LLMClient, cluster: ContentCluster) -> LLMVerdict: """Run LLM analysis on a single content cluster.""" user_prompt = build_user_prompt(cluster) - data = client.extract_json_from_response(SYSTEM_PROMPT, user_prompt) + data = client.extract_json_from_response( + SYSTEM_PROMPT, + user_prompt, + response_schema=VERDICT_JSON_SCHEMA, + schema_name="dedup_verdict", + ) verdict = data.get("verdict", "") if verdict not in VALID_VERDICTS: @@ -71,7 +88,7 @@ def analyze_cluster(client: LLMClient, cluster: ContentCluster) -> LLMVerdict: return LLMVerdict( verdict=verdict, confidence=float(data.get("confidence", 0.0)), - reasoning=data.get("reasoning", ""), + reasoning=data.get("rationale", data.get("reasoning", "")), suggestion=data.get("suggestion", ""), ) diff --git a/src/skillevaluator/evaluation/dimension_judge.py b/src/skillevaluator/evaluation/dimension_judge.py index 3101bb29..40824537 100644 --- a/src/skillevaluator/evaluation/dimension_judge.py +++ b/src/skillevaluator/evaluation/dimension_judge.py @@ -21,6 +21,7 @@ from skillevaluator.constants import ( DIMENSION_JUDGE_MAX_TOKENS, DIMENSION_JUDGE_MODEL, + DIMENSION_JUDGE_REASONING_EFFORT, DIMENSION_JUDGE_TEMPERATURE, DIMENSION_MAPPING, DIMENSION_VERDICT_NEUTRAL_THRESHOLD, @@ -141,6 +142,7 @@ class DimensionJudge(LLMClient): default_model: str = DIMENSION_JUDGE_MODEL default_max_tokens: int | None = DIMENSION_JUDGE_MAX_TOKENS default_temperature: float | None = DIMENSION_JUDGE_TEMPERATURE + default_reasoning_effort: str | None = DIMENSION_JUDGE_REASONING_EFFORT def get_system_prompt(self) -> str: return _SYSTEM_PROMPT diff --git a/src/skillevaluator/evaluation/insights_judge.py b/src/skillevaluator/evaluation/insights_judge.py index 63e96218..66fe3573 100644 --- a/src/skillevaluator/evaluation/insights_judge.py +++ b/src/skillevaluator/evaluation/insights_judge.py @@ -27,6 +27,7 @@ INSIGHTS_JUDGE_MAX_RECOMMENDATIONS, INSIGHTS_JUDGE_MAX_TOKENS, INSIGHTS_JUDGE_MODEL, + INSIGHTS_JUDGE_REASONING_EFFORT, INSIGHTS_JUDGE_TEMPERATURE, ) from skillevaluator.inference.client import LLMClient @@ -480,6 +481,7 @@ class InsightsJudge(LLMClient): default_model: str = INSIGHTS_JUDGE_MODEL default_max_tokens: int | None = INSIGHTS_JUDGE_MAX_TOKENS default_temperature: float | None = INSIGHTS_JUDGE_TEMPERATURE + default_reasoning_effort: str | None = INSIGHTS_JUDGE_REASONING_EFFORT def get_system_prompt(self) -> str: return _SYSTEM_PROMPT diff --git a/src/skillevaluator/inference/__init__.py b/src/skillevaluator/inference/__init__.py index ceb8b93a..6f234e6d 100644 --- a/src/skillevaluator/inference/__init__.py +++ b/src/skillevaluator/inference/__init__.py @@ -9,13 +9,22 @@ from skillevaluator.inference.client import LLMClient from skillevaluator.inference.finding_verifier import FindingVerifier -from skillevaluator.inference.types import EmptyLLMResponseError, LLMClientError, LLMConfigError, LLMVerdict +from skillevaluator.inference.types import ( + EmptyLLMResponseError, + LLMClientError, + LLMClientRefusedError, + LLMClientTruncatedError, + LLMConfigError, + LLMVerdict, +) __all__ = [ "EmptyLLMResponseError", "FindingVerifier", "LLMClient", "LLMClientError", + "LLMClientRefusedError", + "LLMClientTruncatedError", "LLMConfigError", "LLMVerdict", ] diff --git a/src/skillevaluator/inference/client.py b/src/skillevaluator/inference/client.py index e90ffd8b..cde0dfe7 100644 --- a/src/skillevaluator/inference/client.py +++ b/src/skillevaluator/inference/client.py @@ -23,20 +23,34 @@ class supports two usage patterns: import os import re import urllib.error +from collections.abc import Callable from dataclasses import replace from typing import Any, NamedTuple from urllib.parse import urlsplit -from skillevaluator.constants import LLM_VERIFY_MODEL, LLM_VERIFY_TEMPERATURE +from skillevaluator.constants import LLM_VERIFY_TEMPERATURE +from skillevaluator.inference.diagnostics import llm_failure_diagnostic from skillevaluator.inference.retry import RetryConfig, resolve_retry_config, retry_call_with_backoff -from skillevaluator.inference.types import EmptyLLMResponseError, LLMClientError +from skillevaluator.inference.types import ( + EmptyLLMResponseError, + LLMClientError, + LLMClientRefusedError, + LLMClientTruncatedError, +) from skillevaluator.logging_config import get_logger from skillevaluator.provider_config import ( + ANTHROPIC_REFUSAL_FALLBACK_MODEL, + ANTHROPIC_SERVER_SIDE_FALLBACK_BETA, + CHAT_DEFAULT_MODELS, + MAX_COMPLETION_TOKENS, OPENAI_BASE_URL, ProviderConfig, ProviderConfigurationError, - _model_leaf, _supports_custom_temperature, + completion_token_limit, + effective_reasoning_effort, + is_claude_5_5_or_later, + is_openai_reasoning_model, resolve_llm_provider, ) @@ -94,14 +108,35 @@ def _sdk_targets_native_openai(client: Any) -> bool: return request_url.endswith(suffix) and _is_canonical_openai_base_url(request_url[: -len(suffix)]) +def _is_native_anthropic_endpoint(config: ProviderConfig) -> bool: + return config.provider == "anthropic" and ( + not config.base_url or urlsplit(config.base_url).hostname == "api.anthropic.com" + ) + + def _token_limit_kwargs(config: ProviderConfig, max_tokens: int | None) -> dict[str, int]: if max_tokens is None: return {} - if _is_native_openai_endpoint(config) and _model_leaf(config.model).startswith("gpt-5"): + max_tokens = completion_token_limit(config.model, max_tokens) + if _is_native_openai_endpoint(config) and is_openai_reasoning_model(config.model): return {"max_completion_tokens": max_tokens} return {"max_tokens": max_tokens} +def _anthropic_max_tokens(model: str, max_tokens: int) -> int: + """Clamp ``max_tokens`` to the Anthropic SDK's per-model non-streaming limit. + + The SDK refuses a non-streaming request above that limit (8,192 for + Claude Opus 4 and 4.1) before sending it. + """ + try: + from anthropic._constants import MODEL_NONSTREAMING_TOKENS + except ImportError: + return max_tokens + limit = MODEL_NONSTREAMING_TOKENS.get(model) + return min(max_tokens, limit) if isinstance(limit, int) else max_tokens + + def _temperature_kwargs(model: str, temperature: float | None) -> dict[str, float]: if temperature is None or not _supports_custom_temperature(model): return {} @@ -160,7 +195,10 @@ def _build_anthropic_output_config(schema: dict[str, Any]) -> dict[str, Any]: "disallowed", ) -_SCHEMA_OPTION_PATTERN = r"(?:response_format|response format|output_config|json_schema|structured[_ ]outputs?)" +# Gateways can turn a schema into a forced tool call, so a tool_choice rejection is a schema rejection too. +_SCHEMA_OPTION_PATTERN = ( + r"(?:response_format|response format|output_config|json_schema|structured[_ ]outputs?|tool_choice)" +) _SCHEMA_REJECTION_REASON = ( r"(?:unsupported|not supported|not permitted|not allowed|disallowed|" r"unknown (?:parameter|field|argument)|unrecognized (?:request argument|parameter)|" @@ -238,15 +276,19 @@ def _is_schema_unsupported_error(exc: Exception) -> bool: def _call_with_schema_fallback( call_fn: Any, - call_kwargs: dict[str, Any], + build_kwargs: Callable[[bool], dict[str, Any]], *, - schema_key: str, target_key: SchemaTargetKey, use_schema: bool, ) -> Any: - """Invoke call_fn and downgrade to prompt-only on confirmed HTTP 400/422 schema errors.""" + """Invoke call_fn and downgrade to prompt-only on confirmed HTTP 400/422 schema errors. + + ``build_kwargs(include_schema)`` returns the provider request with or + without its structured-output option, so each provider keeps its other + request fields on the downgrade. + """ try: - return call_fn(**call_kwargs) + return call_fn(**build_kwargs(use_schema)) except Exception as exc: if use_schema and _is_schema_unsupported_error(exc): logger.warning( @@ -255,22 +297,38 @@ def _call_with_schema_fallback( target_key.provider, target_key.model, ) - fallback_kwargs = dict(call_kwargs) - fallback_kwargs.pop(schema_key, None) - result = call_fn(**fallback_kwargs) + result = call_fn(**build_kwargs(False)) _SCHEMA_UNSUPPORTED_TARGETS.add(target_key) return result raise +# OpenAI-compatible finish_reason and Anthropic stop_reason values. +_REFUSAL_STOP_REASONS = frozenset({"content_filter", "refusal"}) +_TRUNCATION_STOP_REASONS = frozenset({"length", "max_tokens"}) + + +def _checked_content(content: str, stop_reason: object, refusal: object = None) -> str: + """Return model text, raising typed errors for refusals, truncation and empty replies.""" + if stop_reason in _REFUSAL_STOP_REASONS or (isinstance(refusal, str) and refusal): + raise LLMClientRefusedError(f"LLM provider declined the request (stop reason: {stop_reason})") + if stop_reason in _TRUNCATION_STOP_REASONS: + raise LLMClientTruncatedError("LLM response stopped at the output-token limit", content) + if not content: + raise EmptyLLMResponseError("LLM returned empty response content") + return content + + def _extract_choice_content(response: Any) -> str: choices = getattr(response, "choices", None) or [] first_choice = choices[0] if choices else None message = getattr(first_choice, "message", None) if first_choice is not None else None content = getattr(message, "content", None) if message is not None else "" - if not content: - return "" - return content.strip() + return _checked_content( + content.strip() if content else "", + getattr(first_choice, "finish_reason", None), + getattr(message, "refusal", None), + ) class LLMClient: @@ -286,12 +344,19 @@ class LLMClient: call :meth:`process`. Subclasses may override the ``default_*`` class attributes to change - model, token limit, or temperature without touching ``__init__``. + model, token limit, temperature, or reasoning effort without touching + ``__init__``, and set ``response_schema`` to request structured output + from :meth:`process`. A subclass whose :meth:`parse_response` keeps the + complete entries of a cut-off reply sets ``salvage_truncated_response``. """ - default_model: str = LLM_VERIFY_MODEL + default_model: str | None = None default_max_tokens: int | None = None default_temperature: float | None = LLM_VERIFY_TEMPERATURE + default_reasoning_effort: str | None = None + response_schema: dict[str, Any] | None = None + schema_name: str = "judge_response" + salvage_truncated_response: bool = False def __init__( self, @@ -301,6 +366,7 @@ def __init__( *, max_tokens: int | None = None, temperature: float | None = None, + reasoning_effort: str | None = None, max_retries: int | None = None, retry_base_delay: float | None = None, retry_max_delay: float | None = None, @@ -311,6 +377,7 @@ def __init__( self._api_key = api_key self._max_tokens = max_tokens if max_tokens is not None else self.default_max_tokens self._temperature = temperature if temperature is not None else self.default_temperature + self._reasoning_effort = reasoning_effort if reasoning_effort is not None else self.default_reasoning_effort retry_cfg = resolve_retry_config() self._retry_config = RetryConfig( max_retries=max_retries if max_retries is not None else retry_cfg.max_retries, @@ -320,6 +387,7 @@ def __init__( self._http_client = http_client self._client: Any = None self._provider_config: ProviderConfig | None = None + self.last_failure: str | None = None # -- public read-only properties for introspection -------------------- @@ -339,6 +407,12 @@ def api_key(self) -> str | None: def temperature(self) -> float | None: return self._temperature + @property + def reasoning_effort(self) -> str | None: + """Return the reasoning effort sent to the resolved model, or ``None`` for its default.""" + config = self._resolved_config() + return effective_reasoning_effort(config.provider, config.model, self._reasoning_effort) + @property def max_retries(self) -> int: """Return the maximum number of retry attempts for transient errors.""" @@ -362,10 +436,11 @@ def _resolved_config(self) -> ProviderConfig: return self._provider_config if self._api_key or self._base_url: - model = self._model or self.default_model base_url = _effective_openai_base_url(self._base_url) + provider = "openai" if _is_canonical_openai_base_url(base_url) else "openai-compatible" + model = self._model or self.default_model or CHAT_DEFAULT_MODELS[provider] self._provider_config = ProviderConfig( - provider="openai" if _is_canonical_openai_base_url(base_url) else "openai-compatible", + provider=provider, model=model, api_key=self._api_key, base_url=base_url, @@ -455,32 +530,44 @@ def completions( config = self._resolved_config() client = self._get_client() target_key = SchemaTargetKey(config.provider, config.base_url or "", config.model) + reasoning_effort = self.reasoning_effort def _invoke_provider() -> str: use_schema = response_schema is not None and target_key not in _SCHEMA_UNSUPPORTED_TARGETS if config.provider == "anthropic": - call_kwargs: dict[str, Any] = { - "model": config.model, - "max_tokens": self._max_tokens or 4096, - "system": system_prompt, - "messages": [{"role": "user", "content": user_prompt}], - **_temperature_kwargs(config.model, self._temperature), - } - if use_schema and response_schema is not None: - call_kwargs["output_config"] = _build_anthropic_output_config(response_schema) + + def _anthropic_kwargs(include_schema: bool) -> dict[str, Any]: + call_kwargs: dict[str, Any] = { + "model": config.model, + "max_tokens": _anthropic_max_tokens(config.model, self._max_tokens or MAX_COMPLETION_TOKENS), + "system": system_prompt, + "messages": [{"role": "user", "content": user_prompt}], + **_temperature_kwargs(config.model, self._temperature), + } + output_config: dict[str, Any] = {} + if include_schema and response_schema is not None: + output_config.update(_build_anthropic_output_config(response_schema)) + if reasoning_effort is not None: + output_config["effort"] = reasoning_effort + if output_config: + call_kwargs["output_config"] = output_config + if is_claude_5_5_or_later(config.model) and _is_native_anthropic_endpoint(config): + # Server-side fallback (beta): a safety-classifier refusal is retried on + # the fallback model within the same call. Older SDKs lack typed fields. + call_kwargs["extra_headers"] = {"anthropic-beta": ANTHROPIC_SERVER_SIDE_FALLBACK_BETA} + call_kwargs["extra_body"] = {"fallbacks": [{"model": ANTHROPIC_REFUSAL_FALLBACK_MODEL}]} + return call_kwargs + response = _call_with_schema_fallback( client.messages.create, - call_kwargs, - schema_key="output_config", + _anthropic_kwargs, target_key=target_key, use_schema=use_schema, ) content = "".join( str(block.text) for block in response.content if getattr(block, "type", None) == "text" ) - if not content: - raise EmptyLLMResponseError("LLM returned empty response content") - return content.strip() + return _checked_content(content.strip(), getattr(response, "stop_reason", None)) if config.provider == "bedrock": try: from litellm import completion @@ -498,40 +585,44 @@ def _invoke_provider() -> str: **_temperature_kwargs(config.model, self._temperature), **({"max_tokens": self._max_tokens} if self._max_tokens is not None else {}), ) - content = _extract_choice_content(response) - if not content: - raise EmptyLLMResponseError("LLM returned empty response content") - return content - call_kwargs: dict[str, Any] = { - "model": config.model, - "stream": False, - "messages": [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": user_prompt}, - ], - **_temperature_kwargs(config.model, self._temperature), - **_token_limit_kwargs(config, self._max_tokens), - } - if use_schema and response_schema is not None: - call_kwargs["response_format"] = _build_openai_response_format(response_schema, schema_name) + return _extract_choice_content(response) + + def _openai_kwargs(include_schema: bool) -> dict[str, Any]: + call_kwargs: dict[str, Any] = { + "model": config.model, + "stream": False, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + **_temperature_kwargs(config.model, self._temperature), + **_token_limit_kwargs(config, self._max_tokens), + **({"reasoning_effort": reasoning_effort} if reasoning_effort is not None else {}), + } + if include_schema and response_schema is not None: + call_kwargs["response_format"] = _build_openai_response_format(response_schema, schema_name) + return call_kwargs response = _call_with_schema_fallback( client.chat.completions.create, - call_kwargs, - schema_key="response_format", + _openai_kwargs, target_key=target_key, use_schema=use_schema, ) - content = _extract_choice_content(response) - if not content: - raise EmptyLLMResponseError("LLM returned empty response content") - return content + return _extract_choice_content(response) return retry_call_with_backoff(_invoke_provider, config=self._retry_config) - def extract_json_from_response(self, system_prompt: str, user_prompt: str) -> dict: + def extract_json_from_response( + self, + system_prompt: str, + user_prompt: str, + *, + response_schema: dict[str, Any] | None = None, + schema_name: str = "judge_response", + ) -> dict: """Send a completion and parse JSON from the response.""" - raw = self.completions(system_prompt, user_prompt) + raw = self.completions(system_prompt, user_prompt, response_schema=response_schema, schema_name=schema_name) cleaned = raw.strip() if cleaned.startswith("```"): @@ -567,20 +658,33 @@ def process(self, **kwargs: Any) -> Any: """Orchestrate a full LLM interaction: prompt -> call -> parse. On failure the fallback response is returned so callers always - receive a usable result. + receive a usable result, and ``last_failure`` records a bounded + diagnostic of the cause. When ``salvage_truncated_response`` is set, + a reply cut off at the token limit is parsed for its complete entries + instead, and ``last_failure`` still records the truncation. """ + self.last_failure = None try: system_prompt = self.get_system_prompt() user_prompt = self.create_user_prompt(**kwargs) - raw = self.completions(system_prompt, user_prompt) + raw = self.completions( + system_prompt, + user_prompt, + response_schema=self.response_schema, + schema_name=self.schema_name, + ) return self.parse_response(raw, **kwargs) - except LLMClientError: - logger.warning("LLM not configured - using fallback response") - return self.get_fallback_response(**kwargs) except NotImplementedError: raise except Exception as exc: - logger.warning(f"LLM call failed ({exc}) - using fallback response") + self.last_failure = llm_failure_diagnostic(exc) + if isinstance(exc, LLMClientTruncatedError) and exc.content and self.salvage_truncated_response: + logger.warning("LLM reply was truncated (%s) - keeping its complete entries", self.last_failure) + try: + return self.parse_response(exc.content, **kwargs) + except Exception: + pass + logger.warning("LLM call failed (%s) - using fallback response", self.last_failure) return self.get_fallback_response(**kwargs) diff --git a/src/skillevaluator/inference/diagnostics.py b/src/skillevaluator/inference/diagnostics.py index 68825f5b..fa630b64 100644 --- a/src/skillevaluator/inference/diagnostics.py +++ b/src/skillevaluator/inference/diagnostics.py @@ -7,7 +7,7 @@ import re -from skillevaluator.inference.types import LLMClientError +from skillevaluator.inference.types import LLMClientError, LLMClientRefusedError, LLMClientTruncatedError # Provider error bodies are untrusted. Only these known protocol values are # useful to callers; arbitrary messages, URLs, IDs, and headers stay private. @@ -80,6 +80,11 @@ def llm_failure_diagnostic(exc: Exception) -> str: remedy = "The LLM request failed. Check the selected provider and model configuration." return f"HTTP {status}{context}: {remedy}" + if isinstance(exc, LLMClientRefusedError): + return "The model declined the request (safety refusal). Retry with a different model or review the input." + if isinstance(exc, LLMClientTruncatedError): + return "LLM response stopped at the output-token limit before it was complete. Retry or use another model." + # These names are shared by the supported SDKs; inspecting the hierarchy # also recognizes SDK subclasses without loading optional dependencies. kinds = {kind.__name__ for kind in type(exc).__mro__} diff --git a/src/skillevaluator/inference/finding_verifier.py b/src/skillevaluator/inference/finding_verifier.py index 0eb151e4..c24e3bda 100644 --- a/src/skillevaluator/inference/finding_verifier.py +++ b/src/skillevaluator/inference/finding_verifier.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -from skillevaluator.constants import LLM_VERIFY_MAX_TOKENS +from skillevaluator.constants import LLM_VERIFY_MAX_TOKENS, LLM_VERIFY_REASONING_EFFORT from skillevaluator.inference.client import LLMClient from skillevaluator.logging_config import get_logger @@ -31,6 +31,9 @@ class FindingVerifier(LLMClient): """Classifies security / PII findings as true or false positives.""" default_max_tokens: int | None = LLM_VERIFY_MAX_TOKENS + default_reasoning_effort: str | None = LLM_VERIFY_REASONING_EFFORT + # Verdicts are one JSON object per line, so a cut-off reply still holds complete ones. + salvage_truncated_response = True # -- Prompt definitions ----------------------------------------------- @@ -40,7 +43,7 @@ class FindingVerifier(LLMClient): "true positives or false positives, given the surrounding code context.\n\n" "For EACH finding, respond with a JSON object on its own line:\n" '{"index": , "verdict": "true_positive"|"false_positive"|"uncertain", ' - '"confidence": "high"|"medium"|"low", "reasoning": ""}\n\n' + '"confidence": "high"|"medium"|"low", "rationale": ""}\n\n' "Guidelines:\n" "- Test card numbers (4111...), placeholder data, documentation examples, " "and CLI command samples are false positives.\n" @@ -127,7 +130,7 @@ def _build_prompt(self, findings: list[Finding], skill_path: Path) -> str: ) parts.append( - "\nFor each finding above, output one JSON object per line with index, verdict, confidence, and reasoning." + "\nFor each finding above, output one JSON object per line with index, verdict, confidence, and rationale." ) return "\n".join(parts) diff --git a/src/skillevaluator/inference/types.py b/src/skillevaluator/inference/types.py index 4a22315b..640d54ff 100644 --- a/src/skillevaluator/inference/types.py +++ b/src/skillevaluator/inference/types.py @@ -16,6 +16,21 @@ class EmptyLLMResponseError(LLMClientError): """Raised when a successful provider response contains no model text.""" +class LLMClientRefusedError(LLMClientError): + """Raised when the provider declines a request, such as a safety-classifier refusal.""" + + +class LLMClientTruncatedError(LLMClientError): + """Raised when a response stops at the output-token limit. + + ``content`` keeps the partial text for callers that can salvage it. + """ + + def __init__(self, message: str, content: str = "") -> None: + super().__init__(message) + self.content = content + + LLMConfigError = LLMClientError diff --git a/src/skillevaluator/provider_config.py b/src/skillevaluator/provider_config.py index bc7412b2..230958d1 100644 --- a/src/skillevaluator/provider_config.py +++ b/src/skillevaluator/provider_config.py @@ -21,11 +21,11 @@ # Pinned frontier chat defaults (not floating aliases like ``gpt-5`` / ``claude-opus-latest``). # Harbor ``templates/eval.py`` cannot import this module — keep its local -# ``DEFAULT_JUDGE_MODEL`` in sync via the drift test in +# ``DEFAULT_JUDGE_MODELS`` in sync via the drift test in # ``tests/tier3/test_judge_parse_robustness.py``. -CHAT_DEFAULT_OPENAI = "gpt-5.6-sol" -CHAT_DEFAULT_ANTHROPIC = "claude-opus-5" -CHAT_DEFAULT_BEDROCK = "us.anthropic.claude-opus-5" +CHAT_DEFAULT_OPENAI = "gpt-6.1-sol" +CHAT_DEFAULT_ANTHROPIC = "claude-opus-5-5" +CHAT_DEFAULT_BEDROCK = "us.anthropic.claude-opus-5-5" CHAT_DEFAULT_NVIDIA = "nvidia/nemotron-3-super-120b-a12b" # Gateway catalog IDs are independent of native-provider model names. Operators # can override these defaults without changing the configured endpoint or key. @@ -40,12 +40,28 @@ "bedrock": CHAT_DEFAULT_BEDROCK, "openai-compatible": CHAT_DEFAULT_GATEWAY, } +# SkillSpector sends its own structured-output requests, so its defaults move +# only to models the pinned SkillSpector release supports, not with the chat +# defaults. Every entry is a literal so a chat-default bump cannot move it. +# An explicit ``SKILL_EVAL_LLM_MODEL`` still reaches SkillSpector. +SKILLSPECTOR_DEFAULT_MODELS = { + "openai": "gpt-6.1-sol", + "anthropic": "claude-opus-5", + "bedrock": "us.anthropic.claude-opus-5", + "nv_build": "nvidia/nemotron-3-super-120b-a12b", + "openai-compatible": "nvidia/nvidia/nemotron-3-super-120b-long-ctx", +} +SKILLSPECTOR_DEFAULT_REASONING_EFFORT = "medium" # Agent harnesses have separate model defaults with their required capabilities. +# OpenCode keeps its own literal so a gateway chat-default change does not move it. GATEWAY_AGENT_DEFAULT_MODELS = { - "codex": "openai/openai/gpt-5.6-sol", - "claude-code": "aws/anthropic/bedrock-claude-opus-5", - "opencode": CHAT_DEFAULT_GATEWAY, + "codex": "openai/openai/gpt-6.1-sol", + "claude-code": "aws/anthropic/bedrock-claude-opus-5-5", + "opencode": "nvidia/nvidia/nemotron-3-super-120b-long-ctx", } +# OpenCode drives OpenAI through a Chat Completions tool loop, and GPT-6.1 Sol +# supports tool calls only on the Responses API. +OPENCODE_OPENAI_DEFAULT_MODEL = "gpt-5.6-sol" EMBEDDING_DEFAULT_NVIDIA = "nvidia/nemotron-3-embed-1b" EMBEDDING_DEFAULT_GATEWAY = "nvidia/nvidia/nemotron-3-embed-1b" _EMBEDDING_DEFAULT_MODELS = { @@ -63,31 +79,94 @@ _UNRESERVED_BYTES = frozenset(b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~") _NO_CUSTOM_TEMPERATURE_MODEL_IDS = frozenset({"claude-mythos-preview"}) _ANTHROPIC_BEDROCK_PREFIX_RE = re.compile(r"^(?:(?:[a-z]{2}|global)\.)?anthropic\.") +# Gateway catalogs can put vendor words before the model name, as in +# ``aws/anthropic/bedrock-claude-opus-5-5``. +_GATEWAY_LEAF_PREFIX_RE = re.compile(r"^(?:[a-z]+-)+?(?=(?:claude|gpt)-)") _VERSIONED_CLAUDE_MODEL_RE = re.compile( r"^claude-[a-z][a-z-]*-(?P\d+)" r"(?:-(?P\d{1,2}))?" r"(?:-(?:\d{8}|latest))?" r"(?:-v\d+)?(?::\d+)?$" ) +# Output-token ceilings. OpenAI recommends reserving at least 25K tokens for +# reasoning plus output on its reasoning models. Other models keep the role's +# own limit, at most MAX_COMPLETION_TOKENS. The Anthropic SDK also caps +# non-streaming requests per model (8,192 for Claude Opus 4 and 4.1), and the +# shared client clamps to that cap. +MAX_COMPLETION_TOKENS = 16384 +REASONING_MAX_COMPLETION_TOKENS = 32768 +# Claude 5.5 safety classifiers can decline benign security content. The +# native Claude API then retries the request on this model in the same call. +ANTHROPIC_REFUSAL_FALLBACK_MODEL = "claude-opus-4-8" +ANTHROPIC_SERVER_SIDE_FALLBACK_BETA = "server-side-fallback-2026-07-01" def _model_leaf(model: str) -> str: """Return a normalized model ID for capability checks only.""" leaf = str(model or "").strip().casefold().rsplit("/", 1)[-1] + leaf = _GATEWAY_LEAF_PREFIX_RE.sub("", leaf, count=1) return _ANTHROPIC_BEDROCK_PREFIX_RE.sub("", leaf, count=1) +def _claude_version(model: str) -> tuple[int, int] | None: + """Return the ``(major, minor)`` version of a Claude model ID, or ``None``.""" + match = _VERSIONED_CLAUDE_MODEL_RE.fullmatch(_model_leaf(model)) + if match is None: + return None + return int(match.group("major")), int(match.group("minor") or 0) + + +def is_openai_reasoning_model(model: str) -> bool: + """Return whether ``model`` is a GPT-5 or GPT-6 reasoning model.""" + return _model_leaf(model).startswith(("gpt-5", "gpt-6")) + + +def is_gpt6_model(model: str) -> bool: + """Return whether ``model`` is a GPT-6 model.""" + return _model_leaf(model).startswith("gpt-6") + + +def is_claude_5_5_or_later(model: str) -> bool: + """Return whether ``model`` is a Claude 5.5 or later model.""" + version = _claude_version(model) + return version is not None and version >= (5, 5) + + def _supports_custom_temperature(model: str) -> bool: """Return whether ``model`` accepts a non-default temperature value.""" - leaf = _model_leaf(model) - if leaf.startswith("gpt-5") or leaf in _NO_CUSTOM_TEMPERATURE_MODEL_IDS: + if is_openai_reasoning_model(model) or _model_leaf(model) in _NO_CUSTOM_TEMPERATURE_MODEL_IDS: return False + version = _claude_version(model) + return version is None or version < (4, 7) - match = _VERSIONED_CLAUDE_MODEL_RE.fullmatch(leaf) - if match is None: - return True - version = (int(match.group("major")), int(match.group("minor") or 0)) - return version < (4, 7) + +def completion_token_limit(model: str, max_tokens: int) -> int: + """Return ``max_tokens`` raised to the reasoning ceiling for OpenAI reasoning models.""" + return max(max_tokens, REASONING_MAX_COMPLETION_TOKENS) if is_openai_reasoning_model(model) else max_tokens + + +def effective_reasoning_effort(provider: str, model: str, effort: str | None) -> str | None: + """Return the reasoning effort SkillEvaluator sends for ``model`` on ``provider``. + + GPT-6 models get one on every provider except Bedrock, and Claude 5.5 or + later gets one on every ``anthropic`` endpoint, including a custom + ``ANTHROPIC_BASE_URL``. Bedrock and OpenAI-compatible Claude routes can + reject the field, so they and all other models keep their default effort. + """ + if provider == "anthropic": + supported = is_claude_5_5_or_later(model) + else: + supported = provider != "bedrock" and is_gpt6_model(model) + return effort if supported else None + + +def skillspector_reasoning_effort(model: str) -> str | None: + """Return the SkillSpector reasoning effort SkillEvaluator sets for ``model``. + + Only GPT-6 models get one. Claude models behind OpenAI-compatible gateways + can reject the parameter, so their effort stays at the model default. + """ + return SKILLSPECTOR_DEFAULT_REASONING_EFFORT if is_gpt6_model(model) else None class ProviderConfigurationError(ValueError): diff --git a/src/skillevaluator/tier3/eval_core/llm_judge.py b/src/skillevaluator/tier3/eval_core/llm_judge.py index f3a4c332..3c5b3f9b 100644 --- a/src/skillevaluator/tier3/eval_core/llm_judge.py +++ b/src/skillevaluator/tier3/eval_core/llm_judge.py @@ -15,10 +15,13 @@ import urllib.error import urllib.request from typing import Any -from urllib.parse import urlparse -from skillevaluator.inference.types import EmptyLLMResponseError -from skillevaluator.provider_config import CHAT_DEFAULT_OPENAI, _model_leaf, _supports_custom_temperature +from skillevaluator.inference.types import EmptyLLMResponseError, LLMClientTruncatedError +from skillevaluator.provider_config import ( + MAX_COMPLETION_TOKENS, + ProviderConfig, + effective_reasoning_effort, +) from skillevaluator.tier3.eval_core.atif_helpers import ( _SECTION_COMPACT_TOOL_HISTORY, _SECTION_FINAL_RESPONSE, @@ -32,7 +35,8 @@ OPENAI_CHAT_URL = "https://api.openai.com/v1/chat/completions" NVIDIA_BUILD_CHAT_URL = "https://integrate.api.nvidia.com/v1/chat/completions" -DEFAULT_JUDGE_MODEL = CHAT_DEFAULT_OPENAI +# Sent only where effective_reasoning_effort allows it (GPT-6, Claude 5.5+ on the anthropic provider). +JUDGE_REASONING_EFFORT = "medium" _ERROR_REDACTION_MARKER = "[REDACTED]" _JUDGE_ERROR_REASON_LIMIT = 512 @@ -94,37 +98,6 @@ def _resolve_url(provider: str) -> str: return base_url.rstrip("/") + "/chat/completions" if base_url else OPENAI_CHAT_URL -def _is_native_openai_chat_url(provider: str, request_url: str) -> bool: - if str(provider or "").strip().casefold() != "openai": - return False - - raw_url = str(request_url or "") - if raw_url != raw_url.strip() or any(ord(character) < 0x20 or ord(character) == 0x7F for character in raw_url): - return False - try: - parsed = urlparse(raw_url) - port = parsed.port - except ValueError: - return False - - return ( - parsed.scheme.casefold() == "https" - and parsed.hostname is not None - and parsed.hostname.casefold() == "api.openai.com" - and parsed.netloc.casefold() in {"api.openai.com", "api.openai.com:443"} - and port in {None, 443} - and parsed.path in {"/v1/chat/completions", "/v1/chat/completions/"} - and parsed.username is None - and parsed.password is None - and not parsed.params - and not parsed.query - and not parsed.fragment - and ";" not in raw_url - and "?" not in raw_url - and "#" not in raw_url - ) - - def _configured_secret_values(extra_secret_values: tuple[str | None, ...] = ()) -> list[str]: values = { value @@ -212,25 +185,36 @@ def _chat_completion_payload( response_schema: dict[str, Any] | None = None, schema_name: str = "judge_response", ) -> dict[str, Any]: + """Return the OpenAI-compatible judge request body ``LLMClient`` would send. + + Built from the shared client's request helpers, so the drift test holds the + Harbor template's in-container copy to what the client sends. + """ + from skillevaluator.inference.client import ( + _build_openai_response_format, + _temperature_kwargs, + _token_limit_kwargs, + ) + resolved_provider = _provider() if provider is None else provider resolved_request_url = _resolve_url(resolved_provider) if request_url is None else request_url - token_key = ( - "max_completion_tokens" - if _model_leaf(model).startswith("gpt-5") - and _is_native_openai_chat_url(resolved_provider, resolved_request_url) - else "max_tokens" + config = ProviderConfig( + provider=resolved_provider, + model=model, + api_key=None, + base_url=resolved_request_url.rstrip("/").removesuffix("/chat/completions"), + litellm_model=f"openai/{model}", ) payload: dict[str, Any] = { "model": model, - token_key: max_tokens, + **_token_limit_kwargs(config, max_tokens), "messages": [{"role": "user", "content": prompt}], "stream": False, + **_temperature_kwargs(model, temperature), } - if temperature is not None and _supports_custom_temperature(model): - payload["temperature"] = temperature + if effort := effective_reasoning_effort(resolved_provider, model, JUDGE_REASONING_EFFORT): + payload["reasoning_effort"] = effort if response_schema is not None: - from skillevaluator.inference.client import _build_openai_response_format - payload["response_format"] = _build_openai_response_format(response_schema, schema_name) return payload @@ -240,7 +224,7 @@ def call_public_llm( *, model: str | None = None, api_key: str | None = None, - max_tokens: int = 1024, + max_tokens: int = 4096, temperature: float = 0.0, timeout: int = 60, allow_model_fallback: bool = True, @@ -262,6 +246,7 @@ def call_public_llm( api_key=api_key, max_tokens=max_tokens, temperature=temperature, + reasoning_effort=JUDGE_REASONING_EFFORT, ) return ( client.completions( @@ -274,6 +259,9 @@ def call_public_llm( ) except EmptyLLMResponseError: return "", None + except LLMClientTruncatedError as exc: + # Hand back the partial reply so judges can salvage its complete entries. + return exc.content, None except Exception as exc: detail = f"Public provider call failed: {exc}" return None, _redact_configured_credentials(detail, (api_key,)) @@ -609,7 +597,7 @@ def _skip_whitespace(index: int) -> int: return [] -STRUCTURED_JUDGE_MAX_TOKENS = 4096 +STRUCTURED_JUDGE_MAX_TOKENS = MAX_COMPLETION_TOKENS _JUDGE_RETRY_REMINDER = ( "\n\nIMPORTANT: Your previous reply could not be parsed or validated. Respond with ONLY the " diff --git a/src/skillevaluator/tier3/generate_dataset.py b/src/skillevaluator/tier3/generate_dataset.py index 2ae3b41d..294153aa 100644 --- a/src/skillevaluator/tier3/generate_dataset.py +++ b/src/skillevaluator/tier3/generate_dataset.py @@ -58,6 +58,10 @@ from skillevaluator.tier3.eval_core.codex_tool_call_normalizer import atif_content_text from skillevaluator.validators.frontmatter_parser import FRONTMATTER_PATTERN +# Output budget for generation and refinement; OpenAI reasoning models get the +# larger reasoning ceiling from the shared client. +_GENERATION_MAX_TOKENS = 16000 + _INTERACTIVE_RE = re.compile( r"interactive|opens?\s+a?\s*browser|waits?\s+for\s+(the\s+)?user|device.code\s+flow", re.IGNORECASE, @@ -485,7 +489,7 @@ async def _generate_with_llm( if announce_provider: print(f" Using public provider: {provider.provider} / {provider.model}") text = await asyncio.to_thread( - LLMClient(max_tokens=2000, temperature=0.3).completions, + LLMClient(max_tokens=_GENERATION_MAX_TOKENS, temperature=0.3).completions, "You generate high-quality JSON evaluation datasets for AI agent skills.", prompt, ) @@ -885,7 +889,7 @@ async def _refine_with_llm( print(f" Refining with public provider: {provider.provider} / {provider.model}") text = await asyncio.to_thread( - LLMClient(max_tokens=4000, temperature=0.2).completions, + LLMClient(max_tokens=_GENERATION_MAX_TOKENS, temperature=0.2).completions, "You refine JSON evaluation datasets using execution evidence.", prompt, ) diff --git a/src/skillevaluator/tier3/harbor/collector.py b/src/skillevaluator/tier3/harbor/collector.py index 149c3cc7..7ef15d0f 100644 --- a/src/skillevaluator/tier3/harbor/collector.py +++ b/src/skillevaluator/tier3/harbor/collector.py @@ -4507,6 +4507,14 @@ def _logical_attempt_rewards(rewards: list[dict[str, Any]]) -> list[dict[str, An logical_reward["_logical_overall"] = overall if any(row.get("_has_trajectory") for row in rows): logical_reward["_has_trajectory"] = True + # Keep every step's judge models so lift can spot a judge fallback in any step. + judge_models = { + metric: sorted(models) + for metric in _LLM_JUDGE_METRICS + if (models := set().union(*(_reward_judge_models(row, metric) for row in rows))) + } + if judge_models: + logical_reward["_judge_models"] = judge_models logical.append(logical_reward) return logical @@ -4915,16 +4923,77 @@ def _pass_summary( } +_LLM_JUDGE_METRICS = ("accuracy", "goal_accuracy", "behavior_check") + + +def _reward_judge_models(reward: dict[str, Any], metric: str) -> set[str]: + """Return the judge models that scored ``metric`` in one reward row. + + A multi-step logical reward carries its steps' models in ``_judge_models``. + """ + carried = reward.get("_judge_models") + if isinstance(carried, dict): + models = carried.get(metric) + return {model for model in models if isinstance(model, str) and model} if isinstance(models, list) else set() + details = reward.get("details") + detail = details.get(metric) if isinstance(details, dict) else None + model = detail.get("judge_model_used") if isinstance(detail, dict) else None + return {model} if isinstance(model, str) and model else set() + + +def _judge_models_by_case( + rewards: list[dict[str, Any]], metric: str, expected_case_ids: set[str] | None +) -> dict[str, set[str]]: + """Map each case to the judge models that scored ``metric`` in these rewards.""" + models: dict[str, set[str]] = {} + for reward in rewards: + if reward_models := _reward_judge_models(reward, metric): + models.setdefault(_entry_id(reward, expected_case_ids), set()).update(reward_models) + return models + + +def _metric_average_excluding( + rewards: list[dict[str, Any]], metric: str, excluded_cases: set[str], expected_case_ids: set[str] | None +) -> float | None: + kept = [reward for reward in rewards if _entry_id(reward, expected_case_ids) not in excluded_cases] + return average_metrics(kept)[0].get(metric) + + def _compute_lift( with_scores: dict[str, float], without_scores: dict[str, float], + with_rewards: list[dict[str, Any]] | None = None, + without_rewards: list[dict[str, Any]] | None = None, + expected_case_ids: list[str] | None = None, ) -> dict[str, Any]: - """Compute skill lift (with-skill minus without-skill) per metric.""" + """Compute skill lift (with-skill minus without-skill) per metric. + + With the per-trial rewards, a case whose arms were judged by different + models (a judge fallback in one arm) is left out of that judge metric's + lift and counted in ``judge_mixed_cases``. + """ lift: dict[str, Any] = {} - metrics = tuple(m for m in DISPLAY_METRICS if m in with_scores and m in without_scores) - for metric in metrics: + paired_scores: dict[str, tuple[float, float]] = {} + mixed_counts: dict[str, int] = {} + expected_set = set(expected_case_ids) if expected_case_ids else None + for metric in (m for m in DISPLAY_METRICS if m in with_scores and m in without_scores): w = with_scores[metric] wo = without_scores[metric] + if metric in _LLM_JUDGE_METRICS and with_rewards and without_rewards: + with_models = _judge_models_by_case(with_rewards, metric, expected_set) + without_models = _judge_models_by_case(without_rewards, metric, expected_set) + mixed = { + case_id + for case_id in with_models.keys() & without_models.keys() + if len(with_models[case_id] | without_models[case_id]) > 1 + } + if mixed: + mixed_counts[metric] = len(mixed) + w = _metric_average_excluding(with_rewards, metric, mixed, expected_set) + wo = _metric_average_excluding(without_rewards, metric, mixed, expected_set) + if w is None or wo is None: + continue + paired_scores[metric] = (w, wo) delta = round(w - wo, 4) lift[metric] = { "with_skill": w, @@ -4932,14 +5001,17 @@ def _compute_lift( "delta": delta, "direction": "up" if delta > 0 else ("down" if delta < 0 else "flat"), } + metrics = tuple(paired_scores) if metrics in {DISPLAY_METRICS, LEGACY_METRICS}: - overall_with = sum(with_scores[m] for m in metrics) / len(metrics) - overall_without = sum(without_scores[m] for m in metrics) / len(metrics) + overall_with = sum(paired_scores[m][0] for m in metrics) / len(metrics) + overall_without = sum(paired_scores[m][1] for m in metrics) / len(metrics) lift["overall"] = { "with_skill": round(overall_with, 4), "without_skill": round(overall_without, 4), "delta": round(overall_with - overall_without, 4), } + if mixed_counts: + lift["judge_mixed_cases"] = mixed_counts return lift @@ -6495,7 +6567,13 @@ def collect_harbor_results( and without_execution.get("execution_status") == "succeeded" ) if paired_execution_succeeded and with_scores and without_scores: - lift = _compute_lift(with_scores, without_scores) + lift = _compute_lift( + with_scores, + without_scores, + with_logical_rewards, + without_logical_rewards, + expected_case_ids, + ) _write_generated_root_json(agent_dir / "lift.json", output_dir, lift) custom_lift: dict[str, Any] = {} diff --git a/src/skillevaluator/tier3/harbor/local_agents.py b/src/skillevaluator/tier3/harbor/local_agents.py index d38e9244..43943a3a 100644 --- a/src/skillevaluator/tier3/harbor/local_agents.py +++ b/src/skillevaluator/tier3/harbor/local_agents.py @@ -25,6 +25,7 @@ from harbor.agents.installed.opencode import OpenCode from harbor.models.trial.paths import EnvironmentPaths +from skillevaluator.provider_config import is_claude_5_5_or_later, is_gpt6_model from skillevaluator.tier3.harbor.installed_agents import SkillEvaluatorClaudeCode, SkillEvaluatorCodex from skillevaluator.tier3.harbor.nvidia_build_bridge import ( MAX_REQUESTS_PER_BRIDGE, @@ -48,6 +49,11 @@ _NVIDIA_BUILD_BRIDGE_CLIENT_TOKEN_ENV = "SKILLEVALUATOR_NVIDIA_BUILD_BRIDGE_CLIENT_TOKEN" _NVIDIA_BUILD_FILE_BACKED_SENTINEL_KEY = "skillevaluator-file-backed-nvidia-key" _NVIDIA_BUILD_HOST_KEY_FILE_ENV = "SKILLEVALUATOR_NVIDIA_API_KEY_FILE" +# Local mode runs the CLI installed on the host, not the release Harbor installs +# in other environments. These are the oldest releases that run each model family. +_CLAUDE_CODE_MIN_VERSION_FOR_CLAUDE_5_5 = (2, 1, 280) +_CODEX_MIN_VERSION_FOR_GPT_6 = (0, 159, 1) +_CLI_VERSION_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)") async def _await_task_uninterruptibly( @@ -105,6 +111,17 @@ def _merge_codex_openai_compatible_config(config: dict[str, Any], base_url: str) return config +def _require_cli_version(cli: str, version: str, minimum: tuple[int, int, int], model: str) -> None: + """Fail before the agent starts when the installed ``cli`` is too old for ``model``.""" + match = _CLI_VERSION_RE.search(version) + if match is None or tuple(int(part) for part in match.groups()) < minimum: + required = ".".join(str(part) for part in minimum) + raise RuntimeError( + f"{model} needs {cli} {required} or newer, but the installed {cli} reports version {version!r}. " + f"Update the {cli} CLI or use --env-mode docker." + ) + + def _rewrite_launcher_segment(command: str, rewrite: Callable[[str], str]) -> str: """Apply launcher rewrites before the CLI ``--`` prompt separator only.""" launcher, separator, prompt = command.partition(" -- ") @@ -119,7 +136,11 @@ class SkillEvaluatorLocalClaudeCode(ClaudeCode): _REMOTE_CLAUDE_TMP = PurePosixPath(EnvironmentPaths.agent_dir / "claude-tmp") async def install(self, environment: BaseEnvironment) -> None: - await self.exec_as_agent(environment, command="claude --version") + result = await self.exec_as_agent(environment, command="claude --version") + self._version = self.parse_version(result.stdout or "") + model = self.model_name or "" + if is_claude_5_5_or_later(model): + _require_cli_version("claude", self._version, _CLAUDE_CODE_MIN_VERSION_FOR_CLAUDE_5_5, model) def get_version_command(self) -> str | None: return "claude --version" @@ -197,7 +218,11 @@ class SkillEvaluatorLocalCodex(SkillEvaluatorGatewayCodex): _REMOTE_CODEX_SECRETS_DIR = PurePosixPath(EnvironmentPaths.agent_dir / "codex-secrets") async def install(self, environment: BaseEnvironment) -> None: - await self.exec_as_agent(environment, command="codex --version") + result = await self.exec_as_agent(environment, command="codex --version") + self._version = self.parse_version(result.stdout or "") + model = self.model_name or "" + if is_gpt6_model(model): + _require_cli_version("codex", self._version, _CODEX_MIN_VERSION_FOR_GPT_6, model) def get_version_command(self) -> str | None: return "codex --version" diff --git a/src/skillevaluator/tier3/harbor/report.py b/src/skillevaluator/tier3/harbor/report.py index fdd30100..ed4049ce 100644 --- a/src/skillevaluator/tier3/harbor/report.py +++ b/src/skillevaluator/tier3/harbor/report.py @@ -701,7 +701,7 @@ def _generate_suggestions_structured( try: from skillevaluator.tier3.eval_core.llm_judge import _extract_json, call_public_llm - content, error = call_public_llm(prompt, max_tokens=1536) + content, error = call_public_llm(prompt, max_tokens=8192) if error: logger.warning("LLM suggestion generation failed: %s", error) return [{"suggestion": s, "dimension": "", "evidence_refs": []} for s in _fallback_suggestions(findings)] diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py index 647a796f..13d58a01 100644 --- a/src/skillevaluator/tier3/harbor/runner.py +++ b/src/skillevaluator/tier3/harbor/runner.py @@ -40,14 +40,19 @@ from skillevaluator.provider_config import ( CHAT_DEFAULT_ANTHROPIC, CHAT_DEFAULT_NVIDIA, + CHAT_DEFAULT_OPENAI, GATEWAY_AGENT_DEFAULT_MODELS, + OPENCODE_OPENAI_DEFAULT_MODEL, ProviderConfig, ProviderConfigurationError, _normalize_anthropic_base_url, + effective_reasoning_effort, + is_claude_5_5_or_later, resolve_llm_provider, ) from skillevaluator.source_identity import normalized_evaluated_source from skillevaluator.tier3.case_ids import validate_case_ids +from skillevaluator.tier3.eval_core.llm_judge import JUDGE_REASONING_EFFORT from skillevaluator.tier3.evals_config import ( MAX_HARBOR_TIMEOUT_MULTIPLIER, EvalsConfigError, @@ -936,12 +941,25 @@ def _harbor_bin() -> str: # no file outside the allowlisted environment can add credentials or settings. # These controls are applied at launch, outside the secret-tracked environment. _HARBOR_LAUNCH_ENV = MappingProxyType({"HARBOR_TELEMETRY": "0", "PYTHON_DOTENV_DISABLED": "1"}) +# Harbor installs the latest agent CLI unless given a ``version``. Pin the +# releases SkillEvaluator was validated with, so trials do not change when a +# new CLI ships. Local mode runs the operator's installed CLI instead, and its +# wrappers check a minimum version for the selected model. +_CODEX_VERSION = "0.161.0" +_CLAUDE_CODE_VERSION = "2.1.292" # Harbor 0.22's Codex agent ran with ``model_reasoning_effort=high``; later # releases defer to Codex's own default. Pin the effort SkillEvaluator was # validated with so scores and cost stay comparable across Harbor upgrades. _HARBOR_AGENT_KWARGS: Mapping[str, Mapping[str, str]] = MappingProxyType( - {"codex": MappingProxyType({"reasoning_effort": "high"})} + { + "codex": MappingProxyType({"reasoning_effort": "high", "version": _CODEX_VERSION}), + "claude-code": MappingProxyType({"version": _CLAUDE_CODE_VERSION}), + } ) +# Claude Code's default effort depends on the model, and Claude Opus 5 ran at +# ``high``. Claude 5.5 or later runs at ``medium``, set explicitly so the effort +# is visible in the Harbor command. Older models keep Claude Code's default. +_CLAUDE_CODE_REASONING_EFFORT = "medium" # Allowlisted host variables that name a file or directory. A relative value keeps # its meaning by being anchored to the operator's working directory. _HARBOR_HOST_PATH_ENV_VARS = frozenset( @@ -1177,6 +1195,16 @@ def _validated_pass_threshold(value: object) -> float: return normalized +def _harbor_agent_kwargs(agent: str, model: str | None, env_mode: str) -> dict[str, str]: + """Return the Harbor agent kwargs SkillEvaluator pins for ``agent`` running ``model``.""" + kwargs = dict(_HARBOR_AGENT_KWARGS.get(agent, {})) + if env_mode == ENV_MODE_LOCAL: + kwargs.pop("version", None) + if agent == "claude-code" and model and is_claude_5_5_or_later(model): + kwargs["reasoning_effort"] = _CLAUDE_CODE_REASONING_EFFORT + return kwargs + + def build_harbor_run_command( *, dataset_path: str | Path, @@ -1269,7 +1297,7 @@ def build_harbor_run_command( command.extend(["--env", SECURE_DOCKER_ENV_IMPORT_PATH]) else: command.extend(["--agent", agent_import_path or agent, "--env", harbor_environment_type(env_mode)]) - for name, value in sorted(_HARBOR_AGENT_KWARGS.get(agent, {}).items()): + for name, value in sorted(_harbor_agent_kwargs(agent, model, env_mode).items()): command.extend(["--ak", encode_environment_kwarg(name, value)]) for name, value in sorted(validated_environment_kwargs.items()): command.extend(["--ek", encode_environment_kwarg(name, value)]) @@ -2002,6 +2030,9 @@ def _gateway_anthropic_agent_credentials(provider: ProviderConfig) -> dict[str, A standalone Anthropic key keeps its native endpoint, so a native key is never silently sent to the shared gateway. An explicit base URL without a separate key selects another API root on the operator's gateway. + + On the gateway route Claude Code stops sending pre-release request fields, + which gateways that forward to another provider reject with HTTP 400. """ credentials = _independent_anthropic_agent_credentials() if credentials.get("ANTHROPIC_API_KEY", "").strip(): @@ -2011,7 +2042,11 @@ def _gateway_anthropic_agent_credentials(provider: ProviderConfig) -> dict[str, base_url = _normalize_anthropic_base_url(provider.base_url or "", variable="SKILL_EVAL_LLM_BASE_URL") if not base_url: raise ProviderConfigurationError("SKILL_EVAL_LLM_BASE_URL is required for the Claude gateway route.") - return {"ANTHROPIC_API_KEY": provider.api_key or "", "ANTHROPIC_BASE_URL": base_url} + return { + "ANTHROPIC_API_KEY": provider.api_key or "", + "ANTHROPIC_BASE_URL": base_url, + "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS": "1", + } def _judge_model_config( @@ -2024,21 +2059,20 @@ def _judge_model_config( return {"enabled": False} for name in ("LLM_JUDGE_MODEL", "SKILL_EVAL_JUDGE_MODEL"): if model := provider_env.get(name): - return { - "enabled": True, - "provider": provider.provider, - "model": model, - "source": name, - "override_applied": True, - } + source = name + override_applied = True + break + else: + model = provider.model + source = "SKILL_EVAL_LLM_MODEL" if os.environ.get("SKILL_EVAL_LLM_MODEL", "").strip() else "provider default" + override_applied = False return { "enabled": True, "provider": provider.provider, - "model": provider.model, - "source": ( - "SKILL_EVAL_LLM_MODEL" if os.environ.get("SKILL_EVAL_LLM_MODEL", "").strip() else "provider default" - ), - "override_applied": False, + "model": model, + "source": source, + "override_applied": override_applied, + "reasoning_effort": effective_reasoning_effort(provider.provider, model, JUDGE_REASONING_EFFORT), } @@ -2417,6 +2451,15 @@ def _model_for_agent( credentials = _independent_anthropic_agent_credentials() if credentials.get("ANTHROPIC_API_KEY", "").strip() and not credentials.get("ANTHROPIC_BASE_URL"): selected, source = CHAT_DEFAULT_ANTHROPIC, "native Anthropic agent default" + elif ( + agent == "opencode" + and provider.provider == "openai" + and provider.model == CHAT_DEFAULT_OPENAI + and not os.environ.get("SKILL_EVAL_LLM_MODEL", "").strip() + ): + # The OpenAI chat default needs the Responses API for tool calls; + # an explicit SKILL_EVAL_LLM_MODEL still reaches OpenCode. + selected, source = OPENCODE_OPENAI_DEFAULT_MODEL, "native OpenAI agent default" else: selected, source = provider.model, "public provider default" if ( @@ -2435,7 +2478,7 @@ def _model_for_agent( "openai": "openai", "openai-compatible": "openai", }.get(provider.provider) - if namespace and source in {"public provider default", "openai-compatible agent default"}: + if namespace and source not in {"CLI", "evals/config.yml"}: selected = f"{namespace}/{selected}" return selected, source diff --git a/src/skillevaluator/tier3/harbor/templates/eval.py b/src/skillevaluator/tier3/harbor/templates/eval.py index 63f4ab74..1f110ac2 100644 --- a/src/skillevaluator/tier3/harbor/templates/eval.py +++ b/src/skillevaluator/tier3/harbor/templates/eval.py @@ -118,9 +118,20 @@ def _env_path(name, default): OPENAI_CHAT_URL = "https://api.openai.com/v1/chat/completions" NVIDIA_BUILD_CHAT_URL = "https://integrate.api.nvidia.com/v1/chat/completions" -# Keep in sync with skillevaluator.provider_config.CHAT_DEFAULT_OPENAI -# (sandbox template cannot import the package — see drift test). -DEFAULT_JUDGE_MODEL = "gpt-5.6-sol" +# Last-resort judge per provider when no model is configured. Keep in sync with +# skillevaluator.provider_config.CHAT_DEFAULT_MODELS (sandbox template cannot +# import the package — see drift tests). +DEFAULT_JUDGE_MODELS = { + "openai": "gpt-6.1-sol", + "anthropic": "claude-opus-5-5", + "nv_build": "nvidia/nemotron-3-super-120b-a12b", + "bedrock": "us.anthropic.claude-opus-5-5", + "openai-compatible": "nvidia/nvidia/nemotron-3-super-120b-long-ctx", +} +JUDGE_REASONING_EFFORT = "medium" +REASONING_MAX_COMPLETION_TOKENS = 32768 +ANTHROPIC_REFUSAL_FALLBACK_MODEL = "claude-opus-4-8" +ANTHROPIC_SERVER_SIDE_FALLBACK_BETA = "server-side-fallback-2026-07-01" _ANTHROPIC_DNS_LABEL_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$") _ANTHROPIC_INTERNAL_LABEL_RE = re.compile(r"^[A-Za-z0-9_](?:[A-Za-z0-9_-]{0,61}[A-Za-z0-9_])?$") _ANTHROPIC_IPV6_ZONE_RE = re.compile(r"^[A-Za-z0-9._~-]+$") @@ -1493,28 +1504,54 @@ def _should_try_fallback(error): ) +# Model capability helpers. Keep in sync with skillevaluator.provider_config (drift test). + + def _model_leaf(model): - # Keep in sync with skillevaluator.tier3.eval_core.llm_judge (drift test). leaf = str(model or "").strip().casefold().rsplit("/", 1)[-1] + leaf = re.sub(r"^(?:[a-z]+-)+?(?=(?:claude|gpt)-)", "", leaf, count=1) return re.sub(r"^(?:(?:[a-z]{2}|global)\.)?anthropic\.", "", leaf, count=1) -def _supports_custom_temperature(model): - # Keep in sync with skillevaluator.tier3.eval_core.llm_judge (drift test). - leaf = _model_leaf(model) - if leaf.startswith("gpt-5") or leaf == "claude-mythos-preview": - return False +def _claude_version(model): match = re.fullmatch( r"claude-[a-z][a-z-]*-(?P\d+)" r"(?:-(?P\d{1,2}))?" r"(?:-(?:\d{8}|latest))?" r"(?:-v\d+)?(?::\d+)?", - leaf, + _model_leaf(model), ) if match is None: - return True - version = (int(match.group("major")), int(match.group("minor") or 0)) - return version < (4, 7) + return None + return int(match.group("major")), int(match.group("minor") or 0) + + +def _is_openai_reasoning_model(model): + return _model_leaf(model).startswith(("gpt-5", "gpt-6")) + + +def _is_claude_5_5_or_later(model): + version = _claude_version(model) + return version is not None and version >= (5, 5) + + +def _supports_custom_temperature(model): + if _is_openai_reasoning_model(model) or _model_leaf(model) == "claude-mythos-preview": + return False + version = _claude_version(model) + return version is None or version < (4, 7) + + +def _completion_token_limit(model, max_tokens): + return max(max_tokens, REASONING_MAX_COMPLETION_TOKENS) if _is_openai_reasoning_model(model) else max_tokens + + +def _effective_reasoning_effort(provider, model, effort): + if provider == "anthropic": + supported = _is_claude_5_5_or_later(model) + else: + supported = provider != "bedrock" and _model_leaf(model).startswith("gpt-6") + return effort if supported else None def _is_native_openai_chat_url(provider, request_url): @@ -1582,18 +1619,19 @@ def _chat_completion_payload( resolved_request_url = _resolve_url(resolved_provider) if request_url is None else request_url token_key = ( "max_completion_tokens" - if _model_leaf(model).startswith("gpt-5") - and _is_native_openai_chat_url(resolved_provider, resolved_request_url) + if _is_openai_reasoning_model(model) and _is_native_openai_chat_url(resolved_provider, resolved_request_url) else "max_tokens" ) payload = { "model": model, - token_key: max_tokens, + token_key: _completion_token_limit(model, max_tokens), "messages": [{"role": "user", "content": prompt}], "stream": False, } if temperature is not None and _supports_custom_temperature(model): payload["temperature"] = temperature + if effort := _effective_reasoning_effort(resolved_provider, model, JUDGE_REASONING_EFFORT): + payload["reasoning_effort"] = effort if response_schema is not None: payload["response_format"] = _build_openai_response_format(response_schema, schema_name) return payload @@ -1790,6 +1828,7 @@ def _anthropic_url(): # verifier timeout. Reserve one minute for deterministic checks and artifacts. _JUDGE_WALL_TIME_BUDGET_SEC = 180.0 _ACTIVE_JUDGE_DEADLINE: ContextVar[float | None] = ContextVar("active_judge_deadline", default=None) +_ACTIVE_JUDGE_CALLS: ContextVar[list[dict[str, Any]] | None] = ContextVar("active_judge_calls", default=None) class EvalRetryConfig(NamedTuple): @@ -1971,7 +2010,10 @@ def _urlopen_with_retry(request, timeout=90): "disallowed", ) -_SCHEMA_OPTION_PATTERN = r"(?:response_format|response format|output_config|json_schema|structured[_ ]outputs?)" +# Gateways can turn a schema into a forced tool call, so a tool_choice rejection is a schema rejection too. +_SCHEMA_OPTION_PATTERN = ( + r"(?:response_format|response format|output_config|json_schema|structured[_ ]outputs?|tool_choice)" +) _SCHEMA_REJECTION_REASON = ( r"(?:unsupported|not supported|not permitted|not allowed|disallowed|" r"unknown (?:parameter|field|argument)|unrecognized (?:request argument|parameter)|" @@ -2029,6 +2071,10 @@ def _is_schema_unsupported_http_error(error): _SCHEMA_UNSUPPORTED_TARGETS: set[SchemaTargetKey] = set() +class _JudgeRefusedError(RuntimeError): + """The judge model declined the request (a safety-classifier refusal or content filter).""" + + def _urlopen_with_schema_fallback(build_request, *, target_key, use_schema, timeout=90): """Open URL with retry, falling back to prompt-only on confirmed schema capability errors.""" normalized_key = target_key if isinstance(target_key, SchemaTargetKey) else SchemaTargetKey(*target_key) @@ -2049,13 +2095,18 @@ def _urlopen_with_schema_fallback(build_request, *, target_key, use_schema, time raise -def _call_anthropic(prompt, model, max_tokens, temperature, response_schema=None): +def _call_anthropic(prompt, model, max_tokens, temperature, response_schema=None, provenance=None): + """Call the Messages API; ``provenance["model"]`` becomes the model that served the reply.""" api_key = os.environ.get("ANTHROPIC_API_KEY", "") if not api_key: return None, "ANTHROPIC_API_KEY is required for the anthropic provider" target_url = _anthropic_url() target_key = SchemaTargetKey(provider="anthropic", base_url=target_url, model=model) use_schema = response_schema is not None and target_key not in _SCHEMA_UNSUPPORTED_TARGETS + effort = _effective_reasoning_effort("anthropic", model, JUDGE_REASONING_EFFORT) + # Server-side fallback (beta, native Claude API only): a safety-classifier + # refusal is retried on the fallback model within the same call. + server_side_fallback = _is_claude_5_5_or_later(model) and urlparse(target_url).hostname == "api.anthropic.com" def _build_request(include_schema): payload = { @@ -2066,17 +2117,20 @@ def _build_request(include_schema): } if temperature is not None and _supports_custom_temperature(model): payload["temperature"] = temperature - if include_schema: - payload["output_config"] = _build_anthropic_output_config(response_schema) - return urllib.request.Request( - target_url, - data=json.dumps(payload).encode(), - headers={ - "Content-Type": "application/json", - "x-api-key": api_key, - "anthropic-version": "2023-06-01", - }, - ) + output_config = _build_anthropic_output_config(response_schema) if include_schema else {} + if effort is not None: + output_config["effort"] = effort + if output_config: + payload["output_config"] = output_config + headers = { + "Content-Type": "application/json", + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + } + if server_side_fallback: + payload["fallbacks"] = [{"model": ANTHROPIC_REFUSAL_FALLBACK_MODEL}] + headers["anthropic-beta"] = ANTHROPIC_SERVER_SIDE_FALLBACK_BETA + return urllib.request.Request(target_url, data=json.dumps(payload).encode(), headers=headers) # _anthropic_url() validates the configured base URL before this request. raw_response = _urlopen_with_schema_fallback( @@ -2086,11 +2140,14 @@ def _build_request(include_schema): timeout=90, ) body = json.loads(raw_response) - content = "".join( - str(block.get("text", "")) - for block in body.get("content", []) - if isinstance(block, dict) and block.get("type") == "text" - ) + if body.get("stop_reason") == "refusal": + raise _JudgeRefusedError("judge model declined the request (stop_reason=refusal)") + blocks = [block for block in body.get("content", []) if isinstance(block, dict)] + for block in blocks: + served = block.get("to") if block.get("type") == "fallback" else None + if provenance is not None and isinstance(served, dict) and isinstance(served.get("model"), str): + provenance["model"] = served["model"] + content = "".join(str(block.get("text", "")) for block in blocks if block.get("type") == "text") return content.strip(), None @@ -2163,7 +2220,16 @@ def _classify_bedrock_retry_error(error): return is_network, type_name, None -def _call_bedrock(prompt, model, max_tokens, temperature, timeout=90): +def _bedrock_refusal_fallback_model(model): + """Return Opus 4.8 in the same Bedrock geography as a Claude 5.5 or later model, else None.""" + prefix = re.match(r"(?:(?:[a-z]{2}|global)\.)?anthropic\.", str(model or "")) + if prefix is None or not _is_claude_5_5_or_later(model): + return None + return prefix.group(0) + ANTHROPIC_REFUSAL_FALLBACK_MODEL + + +def _call_bedrock(prompt, model, max_tokens, temperature, timeout=90, provenance=None): + """Call Bedrock Converse; ``provenance["model"]`` becomes the model that served the reply.""" try: import boto3 except ImportError: @@ -2191,47 +2257,58 @@ def _call_bedrock(prompt, model, max_tokens, temperature, timeout=90): if temperature is not None and _supports_custom_temperature(model): inference_config["temperature"] = temperature - attempt = 0 - while True: - request_timeout = _remaining_judge_timeout(timeout) - endpoint = getattr(client, "_endpoint", None) - if endpoint is not None and hasattr(endpoint, "timeout"): - endpoint.timeout = request_timeout - try: - response = client.converse( - modelId=model, - messages=[{"role": "user", "content": [{"text": prompt}]}], - inferenceConfig=inference_config, - ) - break - except Exception as error: - is_retriable, status_label, retry_after_str = _classify_bedrock_retry_error(error) - if attempt >= retry_config.max_retries or not is_retriable: - raise - - sleep_duration = _compute_bounded_retry_delay( - retry_after_str, - attempt=attempt, - base_delay=retry_config.base_delay, - max_delay=retry_config.max_delay, - error=error, - ) - logger.warning( - "LLM judge transient error (%s). Retrying in %.2fs (attempt %d/%d)...", - status_label, - sleep_duration, - attempt + 1, - retry_config.max_retries, - ) - time.sleep(sleep_duration) - attempt += 1 - + def _converse(model_id): + attempt = 0 + while True: + request_timeout = _remaining_judge_timeout(timeout) + endpoint = getattr(client, "_endpoint", None) + if endpoint is not None and hasattr(endpoint, "timeout"): + endpoint.timeout = request_timeout + try: + return client.converse( + modelId=model_id, + messages=[{"role": "user", "content": [{"text": prompt}]}], + inferenceConfig=inference_config, + ) + except Exception as error: + is_retriable, status_label, retry_after_str = _classify_bedrock_retry_error(error) + if attempt >= retry_config.max_retries or not is_retriable: + raise + + sleep_duration = _compute_bounded_retry_delay( + retry_after_str, + attempt=attempt, + base_delay=retry_config.base_delay, + max_delay=retry_config.max_delay, + error=error, + ) + logger.warning( + "LLM judge transient error (%s). Retrying in %.2fs (attempt %d/%d)...", + status_label, + sleep_duration, + attempt + 1, + retry_config.max_retries, + ) + time.sleep(sleep_duration) + attempt += 1 + + response = _converse(model) + fallback_model = _bedrock_refusal_fallback_model(model) + if response.get("stopReason") == "content_filtered" and fallback_model: + # Bedrock has no server-side fallback, so retry a classifier refusal here. + response = _converse(fallback_model) + if provenance is not None: + provenance["model"] = fallback_model + if response.get("stopReason") == "content_filtered": + raise _JudgeRefusedError("judge model declined the request (stopReason=content_filtered)") content = "".join( str(block.get("text", "")) for block in response.get("output", {}).get("message", {}).get("content", []) if isinstance(block, dict) ) return content.strip(), None + except _JudgeRefusedError: + raise except Exception as exc: return None, f"Bedrock request failed: {exc}" @@ -2242,14 +2319,82 @@ def _selected_judge_model(model=None): or os.environ.get("LLM_JUDGE_MODEL") or os.environ.get("SKILL_EVAL_JUDGE_MODEL") or os.environ.get("SKILL_EVAL_LLM_MODEL") - or DEFAULT_JUDGE_MODEL + or DEFAULT_JUDGE_MODELS.get(_public_provider(), DEFAULT_JUDGE_MODELS["openai"]) + ) + + +def _call_openai_compatible( + prompt, provider, model, max_tokens, temperature, response_schema=None, schema_name="judge_response" +): + api_key = ( + os.environ.get("NVIDIA_API_KEY", "") + if provider == "nv_build" + else os.environ.get("SKILL_EVAL_LLM_API_KEY", "") or os.environ.get("OPENAI_API_KEY", "") ) + if not api_key: + return None, f"No API key configured for {provider}" + request_url = _resolve_url(provider) + target_key = SchemaTargetKey(provider=provider, base_url=request_url, model=model) + use_schema = response_schema is not None and target_key not in _SCHEMA_UNSUPPORTED_TARGETS + + def _build_request(include_schema): + payload = _chat_completion_payload( + model, + prompt, + max_tokens, + temperature, + provider=provider, + request_url=request_url, + response_schema=response_schema if include_schema else None, + schema_name=schema_name, + ) + return urllib.request.Request( + request_url, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}, + ) + + # request_url was validated by _resolve_url() before this request. + raw_response = _urlopen_with_schema_fallback( + _build_request, + target_key=target_key, + use_schema=use_schema, + timeout=90, + ) + body = json.loads(raw_response) + choices = body.get("choices") or [{}] + first_choice = choices[0] if choices and isinstance(choices[0], dict) else {} + message = first_choice.get("message") + refusal = message.get("refusal") if isinstance(message, dict) else None + if first_choice.get("finish_reason") == "content_filter" or (isinstance(refusal, str) and refusal): + raise _JudgeRefusedError( + f"judge model declined the request (finish_reason={first_choice.get('finish_reason')})" + ) + content = message.get("content", "") if isinstance(message, dict) else "" + if content is None: + content = "" + return content.strip(), None + + +def _record_judge_call(provenance, requested_model, sent_model): + """Record the model that answered for the required judge in progress (see _call_required_judge).""" + calls = _ACTIVE_JUDGE_CALLS.get() + if calls is not None: + calls.append( + { + "model": provenance["model"], + "reasoning_effort": _effective_reasoning_effort( + provenance["provider"], sent_model, JUDGE_REASONING_EFFORT + ), + "fallback_used": provenance["model"] != requested_model, + } + ) def _call_public_llm_with_provenance( prompt, model=None, - max_tokens=1024, + max_tokens=4096, temperature=0.0, allow_model_fallback=True, response_schema=None, @@ -2273,76 +2418,45 @@ def _call_public_llm_with_provenance( max_tokens, temperature, response_schema=response_schema, + provenance=provenance, ) - if error: - return None, _redact_configured_credentials(error), provenance - return content, None, provenance - if provider == "bedrock": - content, error = _call_bedrock(prompt, candidate_model, max_tokens, temperature) - if error: - return None, _redact_configured_credentials(error), provenance - return content, None, provenance - - api_key = ( - os.environ.get("NVIDIA_API_KEY", "") - if provider == "nv_build" - else os.environ.get("SKILL_EVAL_LLM_API_KEY", "") or os.environ.get("OPENAI_API_KEY", "") - ) - if not api_key: - return None, f"No API key configured for {provider}", provenance - request_url = _resolve_url(provider) - target_key = SchemaTargetKey(provider=provider, base_url=request_url, model=candidate_model) - use_schema = response_schema is not None and target_key not in _SCHEMA_UNSUPPORTED_TARGETS - - def _build_oai_request( - include_schema, - *, - _url=request_url, - _model=candidate_model, - _key=api_key, - ): - return urllib.request.Request( - _url, - data=json.dumps( - _chat_completion_payload( - _model, - prompt, - max_tokens, - temperature, - provider=provider, - request_url=_url, - response_schema=response_schema if include_schema else None, - schema_name=schema_name, - ) - ).encode(), - headers={"Content-Type": "application/json", "Authorization": f"Bearer {_key}"}, + elif provider == "bedrock": + content, error = _call_bedrock(prompt, candidate_model, max_tokens, temperature, provenance=provenance) + else: + content, error = _call_openai_compatible( + prompt, + provider, + candidate_model, + max_tokens, + temperature, + response_schema=response_schema, + schema_name=schema_name, ) - - # request_url was validated by _resolve_url() before this request. - raw_response = _urlopen_with_schema_fallback( - _build_oai_request, - target_key=target_key, - use_schema=use_schema, - timeout=90, - ) - body = json.loads(raw_response) - choices = body.get("choices") or [{}] - first_choice = choices[0] if choices and isinstance(choices[0], dict) else {} - message = first_choice.get("message") - content = message.get("content", "") if isinstance(message, dict) else "" - if content is None: - content = "" - if candidate_model != requested_model: - logger.warning("LLM judge model %s failed; using fallback model %s", requested_model, candidate_model) - return content.strip(), None, provenance - except urllib.error.HTTPError as error: - detail, should_try_fallback = _format_http_error_with_fallback(error) + except _JudgeRefusedError as refusal: + # A classifier refusal is fallback-eligible like a missing model. + errors.append(f"{candidate_model}: {refusal}") + if not allow_model_fallback: + return None, str(refusal), provenance + continue + except urllib.error.HTTPError as http_error: + detail, should_try_fallback = _format_http_error_with_fallback(http_error) errors.append(f"{candidate_model}: {detail}") if not allow_model_fallback or not should_try_fallback: return None, detail, provenance + continue except Exception as exc: detail = f"Public provider call failed for {candidate_model}: {exc}" return None, _redact_configured_credentials(detail), provenance + if error: + return None, _redact_configured_credentials(error), provenance + if not content and candidate_model != models[-1]: + # So is an empty reply, for example reasoning that used up the output budget. + errors.append(f"{candidate_model}: empty response") + continue + if candidate_model != requested_model: + logger.warning("LLM judge model %s failed; using fallback model %s", requested_model, candidate_model) + _record_judge_call(provenance, requested_model, candidate_model) + return content, None, provenance detail = "LLM judge model fallback exhausted: " + " | ".join(errors) return None, _redact_configured_credentials(detail), last_provenance @@ -2350,7 +2464,7 @@ def _build_oai_request( def call_public_llm( prompt, model=None, - max_tokens=1024, + max_tokens=4096, temperature=0.0, allow_model_fallback=True, response_schema=None, @@ -7966,7 +8080,8 @@ def score_skill_execution( return {"score": round(avg, 4), "details": checks} -STRUCTURED_JUDGE_MAX_TOKENS = 4096 +# Keep in sync with skillevaluator.provider_config.MAX_COMPLETION_TOKENS (drift test). +STRUCTURED_JUDGE_MAX_TOKENS = 16384 _JUDGE_RETRY_REMINDER = ( "\n\nIMPORTANT: Your previous reply could not be parsed or validated. Respond with ONLY the " @@ -8188,8 +8303,11 @@ def judge_goal_accuracy(question, ground_truth, agent_text, tool_summary=""): def _ragas_goal_accuracy_enabled(): - """RAGAS is an OpenAI-only optimization, never an agent-key fallback.""" - if _public_provider() != "openai": + """RAGAS is an OpenAI-only optimization, never an agent-key fallback. + + Reasoning judges reject the sampling settings RAGAS sends, so they use the prompt judge. + """ + if _public_provider() != "openai" or _is_openai_reasoning_model(_selected_judge_model()): return False try: request_url = _resolve_url("openai") @@ -8463,11 +8581,18 @@ def _normalize_required_judge_result(metric, result): def _call_required_judge(metric, judge, *args, **kwargs): - """Run a required LLM judge under a bounded wall-time deadline and normalize its result.""" + """Run a required LLM judge under a bounded wall-time deadline and normalize its result. + + The result records the model that produced it (``judge_model_used``), the + reasoning effort sent, and whether a fallback model answered, so lift can + leave out cases whose two arms were judged by different models. + """ previous_deadline = _ACTIVE_JUDGE_DEADLINE.get() own_deadline = time.monotonic() + _resolve_judge_wall_time_budget() deadline = min(previous_deadline, own_deadline) if previous_deadline is not None else own_deadline token = _ACTIVE_JUDGE_DEADLINE.set(deadline) + calls = [] + calls_token = _ACTIVE_JUDGE_CALLS.set(calls) alarm_armed = False previous_alarm_handler = None @@ -8501,7 +8626,13 @@ def _raise_judge_timeout(_signum, _frame): result = _judge_error(f"Required {metric} judge raised {type(exc).__name__}: {exc}") finally: _ACTIVE_JUDGE_DEADLINE.reset(token) - return _normalize_required_judge_result(metric, result) + _ACTIVE_JUDGE_CALLS.reset(calls_token) + normalized = _normalize_required_judge_result(metric, result) + if calls: + normalized["judge_model_used"] = calls[-1]["model"] + normalized["judge_reasoning_effort"] = calls[-1]["reasoning_effort"] + normalized["judge_fallback_used"] = any(call["fallback_used"] for call in calls) + return normalized def _numeric_reward_payload(result, overall): diff --git a/src/skillevaluator/utils/tool_runner.py b/src/skillevaluator/utils/tool_runner.py index 3c2e98d3..5af0a4e8 100644 --- a/src/skillevaluator/utils/tool_runner.py +++ b/src/skillevaluator/utils/tool_runner.py @@ -178,7 +178,7 @@ def get_install_hint(self) -> str: "Install Semgrep separately with: brew install semgrep (macOS) " "or uv tool install semgrep" ), - "skillspector": "Install with: uv tool install git+https://github.com/NVIDIA/SkillSpector.git", + "skillspector": "Install with: uv tool install git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0", "skillevaluator": ( 'Install with: uv tool install "skillevaluator[all] @ git+https://github.com/NVIDIA/SkillEvaluator.git"' ), diff --git a/src/skillevaluator/validators/rubric_eval.py b/src/skillevaluator/validators/rubric_eval.py index ca734b2c..e8b7a15c 100644 --- a/src/skillevaluator/validators/rubric_eval.py +++ b/src/skillevaluator/validators/rubric_eval.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import Any -from skillevaluator.constants import RUBRIC_CRITERIA, RUBRIC_MAX_TOKENS, RUBRIC_MIN_SCORE +from skillevaluator.constants import RUBRIC_CRITERIA, RUBRIC_MAX_TOKENS, RUBRIC_MIN_SCORE, RUBRIC_REASONING_EFFORT from skillevaluator.inference.client import LLMClient from skillevaluator.logging_config import get_logger from skillevaluator.models.result import Finding, Severity, ValidationResult @@ -188,10 +188,38 @@ class _UnavailableRubricReport(dict): """Internal marker distinguishing transport fallback from model evidence.""" +_RUBRIC_CHECK_JSON_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "id": {"type": "string"}, + "criterion": {"type": "string"}, + "pass": {"type": "boolean"}, + "score": {"type": "integer"}, + "notes": {"type": "string"}, + }, + "required": ["id", "criterion", "pass", "score", "notes"], + "additionalProperties": False, +} +RUBRIC_JSON_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "overall_pass": {"type": "boolean"}, + "score": {"type": "integer"}, + "summary": {"type": "string"}, + "checks": {"type": "array", "items": _RUBRIC_CHECK_JSON_SCHEMA}, + }, + "required": ["overall_pass", "score", "summary", "checks"], + "additionalProperties": False, +} + + class RubricJudge(LLMClient): """LLM judge that scores a skill against qualitative criteria.""" default_max_tokens: int | None = RUBRIC_MAX_TOKENS + default_reasoning_effort: str | None = RUBRIC_REASONING_EFFORT + response_schema: dict[str, Any] | None = RUBRIC_JSON_SCHEMA + schema_name: str = "rubric_evaluation" _SYSTEM_PROMPT = ( "You are evaluating an AI Agent Skill documentation file (SKILL.md).\n" @@ -542,12 +570,13 @@ def _validate_single_skill(self, skill_path: Path) -> ValidationResult: ) if isinstance(report, _UnavailableRubricReport): + cause = f": {self._judge.last_failure}" if self._judge.last_failure else "" return self._record_judge_failure( result, report=report, skill_path=skill_path, manifest=manifest, - message="LLM judge unavailable; rubric evaluation did not run", + message=f"LLM judge unavailable; rubric evaluation did not run{cause}", check_name="llm_unavailable", suggestion="Configure a public LLM provider to enable LLM rubric evaluation", ) diff --git a/src/skillevaluator/validators/security.py b/src/skillevaluator/validators/security.py index a082a641..133cd9b4 100644 --- a/src/skillevaluator/validators/security.py +++ b/src/skillevaluator/validators/security.py @@ -39,7 +39,12 @@ ) from skillevaluator.logging_config import get_logger from skillevaluator.models.skill import SEMVER_RE -from skillevaluator.provider_config import ProviderConfigurationError, resolve_llm_provider +from skillevaluator.provider_config import ( + SKILLSPECTOR_DEFAULT_MODELS, + ProviderConfigurationError, + resolve_llm_provider, + skillspector_reasoning_effort, +) from skillevaluator.spdx import is_spdx_only_html_comment from skillevaluator.utils.tool_runner import Tools, parse_json_output from skillevaluator.validators.base import ( @@ -368,10 +373,23 @@ def _skillspector_process_env(environ: Mapping[str, str] | None = None) -> dict[ return _copy_selected_environment(source, _SKILLSPECTOR_PROCESS_ENV_NAMES) -def _skillspector_child_env() -> dict[str, str]: +def _skillspector_operator_env(environ: Mapping[str, str]) -> dict[str, str]: + """Return operator-set SkillSpector slot models, model registry and reasoning effort. + + ``SKILLSPECTOR_TEMPERATURE`` stays out: current reasoning models reject a + custom temperature, which would fail every LLM request. + """ + return _copy_selected_environment( + environ, + (name for name in environ if name.startswith("SKILLSPECTOR_MODEL_") or name == "SKILLSPECTOR_REASONING_EFFORT"), + ) + + +def _skillspector_provider_env() -> dict[str, str]: """Map public provider settings into an invocation-scoped SkillSpector environment.""" source = os.environ child_env = _skillspector_process_env(source) + child_env.update(_skillspector_operator_env(source)) skillspector_provider = source.get("SKILLSPECTOR_PROVIDER", "").strip().lower() skillspector_model = source.get("SKILLSPECTOR_MODEL", "").strip() @@ -407,7 +425,10 @@ def _skillspector_child_env() -> dict[str, str]: return child_env child_env["SKILLSPECTOR_PROVIDER"] = mapped_provider - child_env["SKILLSPECTOR_MODEL"] = skillspector_model or provider.model + # An explicit SKILL_EVAL_LLM_MODEL reaches SkillSpector; the chat default does not. + child_env["SKILLSPECTOR_MODEL"] = skillspector_model or ( + provider.model if "SKILL_EVAL_LLM_MODEL" in resolution_env else SKILLSPECTOR_DEFAULT_MODELS[provider.provider] + ) if provider.provider in {"nv_build", "openai-compatible"}: if provider.api_key: @@ -422,6 +443,32 @@ def _skillspector_child_env() -> dict[str, str]: return child_env +def _skillspector_default_reasoning_effort(child_env: Mapping[str, str]) -> str | None: + """Return SkillEvaluator's reasoning effort when every configured SkillSpector model takes it. + + The effort is one global SkillSpector setting, so a slot override naming a + model without one (for example a Claude model) leaves it unset. Without + ``SKILLSPECTOR_MODEL`` SkillSpector picks its own model and effort. + """ + if "SKILLSPECTOR_MODEL" not in child_env: + return None + efforts = { + skillspector_reasoning_effort(model) + for name, model in child_env.items() + if name.startswith("SKILLSPECTOR_MODEL") and name != "SKILLSPECTOR_MODEL_REGISTRY" + } + return efforts.pop() if len(efforts) == 1 else None + + +def _skillspector_child_env() -> dict[str, str]: + """Return the SkillSpector environment plus SkillEvaluator's default reasoning effort.""" + child_env = _skillspector_provider_env() + effort = _skillspector_default_reasoning_effort(child_env) + if effort is not None: + child_env.setdefault("SKILLSPECTOR_REASONING_EFFORT", effort) + return child_env + + def _tree_contains_artifact_dirs(root: Path) -> bool: """Return True when the SkillSpector scan tree needs staging.""" return any( @@ -765,6 +812,10 @@ def _run_skillspector_once( logger.info("Running %s on %s", stage_name, scan_root) child_env = _skillspector_child_env() if use_llm else _skillspector_process_env() + if use_llm: + # Requested LLM configuration; None leaves the choice to SkillSpector. + result.metadata["skillspector_model"] = child_env.get("SKILLSPECTOR_MODEL") + result.metadata["skillspector_reasoning_effort"] = child_env.get("SKILLSPECTOR_REASONING_EFFORT") tool_result = Tools.skillspector.run(args, timeout=300, env=child_env, replace_env=True) if tool_result.exit_code not in _SKILLSPECTOR_POLICY_EXIT_CODES: @@ -2746,7 +2797,8 @@ def _verify_findings_with_llm( if not verdicts: result.mark_scan_incomplete("llm-verification") - result.add_message("LLM finding verification was skipped (no verdicts returned)") + cause = f": {verifier.last_failure}" if verifier.last_failure else "" + result.add_message(f"LLM finding verification was skipped (no verdicts returned{cause})") return suppressed = 0 @@ -2758,7 +2810,8 @@ def _verify_findings_with_llm( verdict = verdict_data.get("verdict") confidence = verdict_data.get("confidence") - reasoning = verdict_data.get("reasoning") + # The prompt asks for "rationale"; replies that still say "reasoning" count too. + reasoning = verdict_data.get("rationale", verdict_data.get("reasoning")) if ( not isinstance(verdict, str) or verdict not in _LLM_VERDICTS @@ -2788,9 +2841,10 @@ def _verify_findings_with_llm( result.mark_scan_incomplete("llm-verification") noun = "finding" if unverified == 1 else "findings" verb = "was" if unverified == 1 else "were" + cause = f": {verifier.last_failure}" if verifier.last_failure else "" result.add_message( f"LLM verification returned verdicts for {verified} of {len(result.findings)} findings; " - f"{unverified} {noun} {verb} not verified" + f"{unverified} {noun} {verb} not verified{cause}" ) elif not suppressed: result.add_message("LLM verification reviewed all findings; no high-confidence false positives identified") diff --git a/tests/deduplication/intra_skill/test_intra_llm_analyzer.py b/tests/deduplication/intra_skill/test_intra_llm_analyzer.py index 82883b99..d1be7d21 100644 --- a/tests/deduplication/intra_skill/test_intra_llm_analyzer.py +++ b/tests/deduplication/intra_skill/test_intra_llm_analyzer.py @@ -10,6 +10,8 @@ import pytest from skillevaluator.deduplication.intra_skill.llm_analyzer import ( + SYSTEM_PROMPT, + VERDICT_JSON_SCHEMA, analyze_cluster, build_user_prompt, verdict_to_severity, @@ -110,6 +112,21 @@ def test_missing_verdict_raises(self, make_cluster) -> None: with pytest.raises(LLMClientError, match="unknown verdict"): analyze_cluster(mock_client, make_cluster()) + def test_requests_the_verdict_schema_and_reads_rationale(self, make_cluster) -> None: + mock_client = MagicMock(spec=LLMClient) + mock_client.extract_json_from_response.return_value = { + "verdict": "DUPLICATE", + "confidence": 0.9, + "rationale": "Same steps in both files", + "suggestion": "Keep one copy", + } + + result = analyze_cluster(mock_client, make_cluster()) + + assert result.reasoning == "Same steps in both files" + assert mock_client.extract_json_from_response.call_args.kwargs["response_schema"] == VERDICT_JSON_SCHEMA + assert '"rationale"' in SYSTEM_PROMPT + def test_extracts_all_fields(self, make_cluster) -> None: mock_client = MagicMock(spec=LLMClient) mock_client.extract_json_from_response.return_value = { diff --git a/tests/inference/test_llm_client.py b/tests/inference/test_llm_client.py index 9dcad7b9..ca852cb9 100644 --- a/tests/inference/test_llm_client.py +++ b/tests/inference/test_llm_client.py @@ -5,21 +5,38 @@ from __future__ import annotations +import json +from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest -from skillevaluator.inference import EmptyLLMResponseError, LLMClient, LLMClientError, LLMVerdict +from skillevaluator.inference import ( + EmptyLLMResponseError, + FindingVerifier, + LLMClient, + LLMClientError, + LLMClientRefusedError, + LLMClientTruncatedError, + LLMVerdict, +) +from skillevaluator.inference import client as client_mod from skillevaluator.inference.client import _is_native_openai_endpoint, _token_limit_kwargs from skillevaluator.provider_config import ( + ANTHROPIC_REFUSAL_FALLBACK_MODEL, + ANTHROPIC_SERVER_SIDE_FALLBACK_BETA, CHAT_DEFAULT_ANTHROPIC, CHAT_DEFAULT_BEDROCK, + CHAT_DEFAULT_GATEWAY, CHAT_DEFAULT_OPENAI, + MAX_COMPLETION_TOKENS, OPENAI_BASE_URL, PUBLIC_NVIDIA_BUILD_BASE_URL, + REASONING_MAX_COMPLETION_TOKENS, ProviderConfig, ) +from skillevaluator.validators.rubric_eval import RUBRIC_JSON_SCHEMA, RubricJudge def _gpt5_config( @@ -235,7 +252,7 @@ def test_requires_openai_provider_intent(self, provider: str) -> None: config = _gpt5_config(provider=provider) assert _is_native_openai_endpoint(config) is False - assert _token_limit_kwargs(config, 512) == {"max_tokens": 512} + assert _token_limit_kwargs(config, 512) == {"max_tokens": REASONING_MAX_COMPLETION_TOKENS} @pytest.mark.parametrize( "model", @@ -248,7 +265,7 @@ def test_requires_openai_provider_intent(self, provider: str) -> None: def test_provider_prefixed_gpt5_models_use_completion_tokens(self, model: str) -> None: config = _gpt5_config(model=model) - assert _token_limit_kwargs(config, 512) == {"max_completion_tokens": 512} + assert _token_limit_kwargs(config, 512) == {"max_completion_tokens": REASONING_MAX_COMPLETION_TOKENS} @pytest.mark.parametrize( "base_url", @@ -264,7 +281,7 @@ def test_accepts_only_canonical_openai_url_forms(self, base_url: str) -> None: config = _gpt5_config(provider="OPENAI", base_url=base_url) assert _is_native_openai_endpoint(config) is True - assert _token_limit_kwargs(config, 512) == {"max_completion_tokens": 512} + assert _token_limit_kwargs(config, 512) == {"max_completion_tokens": REASONING_MAX_COMPLETION_TOKENS} @pytest.mark.parametrize( "base_url", @@ -315,7 +332,7 @@ def test_noncanonical_raw_url_is_not_classified_as_native_openai(self, base_url: config = _gpt5_config(base_url=base_url) assert _is_native_openai_endpoint(config) is False - assert _token_limit_kwargs(config, 512) == {"max_tokens": 512} + assert _token_limit_kwargs(config, 512) == {"max_tokens": REASONING_MAX_COMPLETION_TOKENS} class TestCompletions: @@ -391,7 +408,7 @@ def test_openai_gpt5_uses_max_completion_tokens_without_temperature(self, monkey mock_openai.chat.completions.create.assert_called_once() call_kwargs = mock_openai.chat.completions.create.call_args.kwargs assert "max_tokens" not in call_kwargs - assert call_kwargs["max_completion_tokens"] == 512 + assert call_kwargs["max_completion_tokens"] == REASONING_MAX_COMPLETION_TOKENS assert "temperature" not in call_kwargs def test_nvidia_build_uses_max_tokens(self, monkeypatch: pytest.MonkeyPatch) -> None: @@ -438,13 +455,13 @@ def test_non_openai_provider_at_canonical_endpoint_uses_max_tokens( LLMClient(max_tokens=512).completions("system", "user") call_kwargs = mock_openai.chat.completions.create.call_args.kwargs - assert call_kwargs["max_tokens"] == 512 + assert call_kwargs["max_tokens"] == REASONING_MAX_COMPLETION_TOKENS assert "max_completion_tokens" not in call_kwargs assert "temperature" not in call_kwargs @pytest.mark.parametrize( ("max_tokens", "expected_max_tokens", "temperature"), - [(None, 4096, 0.0), (512, 512, 0.3)], + [(None, MAX_COMPLETION_TOKENS, 0.0), (512, 512, 0.3)], ) def test_anthropic_opus5_preserves_token_limit_and_omits_temperature( self, @@ -546,7 +563,7 @@ def test_openai_provider_custom_base_url_uses_max_tokens( mock_cls.assert_called_once_with(api_key="test-key", base_url="https://example.test/v1", max_retries=0) call_kwargs = mock_openai.chat.completions.create.call_args.kwargs - assert call_kwargs["max_tokens"] == 512 + assert call_kwargs["max_tokens"] == REASONING_MAX_COMPLETION_TOKENS assert "max_completion_tokens" not in call_kwargs def test_api_key_only_gpt5_uses_max_completion_tokens(self, monkeypatch: pytest.MonkeyPatch) -> None: @@ -561,7 +578,7 @@ def test_api_key_only_gpt5_uses_max_completion_tokens(self, monkeypatch: pytest. assert (client.base_url, mock_cls.call_args.kwargs["base_url"]) == (OPENAI_BASE_URL, OPENAI_BASE_URL) call_kwargs = mock_openai.chat.completions.create.call_args.kwargs assert "max_tokens" not in call_kwargs - assert call_kwargs["max_completion_tokens"] == 512 + assert call_kwargs["max_completion_tokens"] == REASONING_MAX_COMPLETION_TOKENS def test_api_key_only_gpt5_honors_ambient_custom_base_url(self, monkeypatch: pytest.MonkeyPatch) -> None: custom_base_url = "https://example.test/v1" @@ -579,7 +596,7 @@ def test_api_key_only_gpt5_honors_ambient_custom_base_url(self, monkeypatch: pyt mock_cls.call_args.kwargs["base_url"], call_kwargs.get("max_tokens"), call_kwargs.get("max_completion_tokens"), - ) == (custom_base_url, custom_base_url, 512, None) + ) == (custom_base_url, custom_base_url, REASONING_MAX_COMPLETION_TOKENS, None) @pytest.mark.parametrize( ("base_url", "expected_key"), @@ -612,7 +629,7 @@ def test_endpoint_url_controls_gpt5_token_key( client.completions("system", "user") call_kwargs = mock_openai.chat.completions.create.call_args.kwargs - assert call_kwargs[expected_key] == 512 + assert call_kwargs[expected_key] == REASONING_MAX_COMPLETION_TOKENS assert unexpected_key not in call_kwargs @pytest.mark.parametrize("base_url", [None, "https://example.test/v1"]) @@ -651,7 +668,7 @@ def test_zero_token_limit_uses_endpoint_appropriate_key( client.completions("system", "user") call_kwargs = mock_openai.chat.completions.create.call_args.kwargs - assert call_kwargs[expected_key] == 0 + assert call_kwargs[expected_key] == REASONING_MAX_COMPLETION_TOKENS assert unexpected_key not in call_kwargs def test_custom_gpt5_endpoint_uses_max_tokens(self) -> None: @@ -669,7 +686,7 @@ def test_custom_gpt5_endpoint_uses_max_tokens(self) -> None: mock_openai.chat.completions.create.assert_called_once() call_kwargs = mock_openai.chat.completions.create.call_args.kwargs - assert call_kwargs["max_tokens"] == 512 + assert call_kwargs["max_tokens"] == REASONING_MAX_COMPLETION_TOKENS assert "max_completion_tokens" not in call_kwargs @@ -709,3 +726,316 @@ def test_invalid_json_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: client = LLMClient() with pytest.raises(LLMClientError, match="invalid JSON"): client.extract_json_from_response("system", "user") + + +def _openai_choice_response(content: str | None, *, finish_reason: str = "stop", refusal: str | None = None): + return SimpleNamespace( + choices=[ + SimpleNamespace(finish_reason=finish_reason, message=SimpleNamespace(content=content, refusal=refusal)) + ] + ) + + +def _anthropic_response(text: str, *, stop_reason: str = "end_turn"): + content = [SimpleNamespace(type="text", text=text)] if text else [] + return SimpleNamespace(content=content, stop_reason=stop_reason) + + +def _use_provider(monkeypatch: pytest.MonkeyPatch, provider: str, model: str) -> None: + monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", provider) + monkeypatch.setenv("SKILL_EVAL_LLM_MODEL", model) + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-anthropic-key") + for name in ("SKILL_EVAL_LLM_BASE_URL", "OPENAI_BASE_URL", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(name, raising=False) + + +class TestReasoningModelRequests: + @pytest.mark.parametrize(("model", "effort"), [(CHAT_DEFAULT_OPENAI, "high"), ("gpt-5.6-sol", None)]) + def test_finding_verifier_sends_high_effort_only_to_gpt6( + self, monkeypatch: pytest.MonkeyPatch, model: str, effort: str | None + ) -> None: + _use_provider(monkeypatch, "openai", model) + mock_openai = MagicMock() + mock_openai.chat.completions.create.return_value = _openai_choice_response("Done") + + with patch("openai.OpenAI", return_value=mock_openai): + FindingVerifier().completions("system", "user") + + call_kwargs = mock_openai.chat.completions.create.call_args.kwargs + assert call_kwargs.get("reasoning_effort") == effort + assert call_kwargs["max_completion_tokens"] == REASONING_MAX_COMPLETION_TOKENS + assert "temperature" not in call_kwargs + + @pytest.mark.parametrize("role", ["dedup", "dimension_judge", "insights_judge"]) + def test_medium_effort_roles_send_medium_to_gpt6( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, role: str + ) -> None: + from skillevaluator.deduplication.intra_skill import intra_skill_validator + from skillevaluator.deduplication.intra_skill.semantic_clustering import ContentCluster + from skillevaluator.deduplication.utils.chunker import ContentChunk + from skillevaluator.evaluation.dimension_judge import DimensionJudge + from skillevaluator.evaluation.insights_judge import InsightsJudge + + _use_provider(monkeypatch, "openai", CHAT_DEFAULT_OPENAI) + mock_openai = MagicMock() + mock_openai.chat.completions.create.return_value = _openai_choice_response( + '{"verdict": "DISTINCT", "confidence": 0.9, "rationale": "Different steps.", "suggestion": ""}' + ) + + with patch("openai.OpenAI", return_value=mock_openai): + if role == "dedup": + skill_dir = tmp_path / "skill" + skill_dir.mkdir() + (skill_dir / "SKILL.md").write_text("## Section A\n" + "a" * 200 + "\n## Section B\n" + "b" * 200) + members = [ + ContentChunk("SKILL.md", "## A", 1, 2, "a" * 100, "markdown"), + ContentChunk("SKILL.md", "## B", 3, 4, "b" * 100, "markdown"), + ] + with ( + patch.object(intra_skill_validator, "EmbeddingClient") as embedding_client, + patch.object( + intra_skill_validator, + "build_clusters", + return_value=[ContentCluster(members, 1.0, 1.0, False, {"markdown"})], + ), + ): + embedding_client.return_value.embed.return_value = [[1.0, 0.0], [1.0, 0.0]] + intra_skill_validator.IntraSkillValidator().validate(skill_dir) + else: + judge = DimensionJudge() if role == "dimension_judge" else InsightsJudge() + judge.completions("system", "user") + + mock_openai.chat.completions.create.assert_called_once() + assert mock_openai.chat.completions.create.call_args.kwargs["reasoning_effort"] == "medium" + + def test_claude_5_5_rubric_gets_schema_effort_and_server_side_fallback( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + _use_provider(monkeypatch, "anthropic", "claude-opus-5-5") + mock_anthropic = MagicMock() + mock_anthropic.messages.create.return_value = _anthropic_response('{"checks": []}') + + with patch("anthropic.Anthropic", return_value=mock_anthropic): + report = RubricJudge().process(skill_name="demo", skill_content="# Demo") + + assert report == {"checks": []} + call_kwargs = mock_anthropic.messages.create.call_args.kwargs + assert call_kwargs["max_tokens"] == MAX_COMPLETION_TOKENS + assert call_kwargs["output_config"] == { + "format": {"type": "json_schema", "schema": RUBRIC_JSON_SCHEMA}, + "effort": "medium", + } + assert call_kwargs["extra_headers"] == {"anthropic-beta": ANTHROPIC_SERVER_SIDE_FALLBACK_BETA} + assert call_kwargs["extra_body"] == {"fallbacks": [{"model": ANTHROPIC_REFUSAL_FALLBACK_MODEL}]} + + def test_claude_opus_5_keeps_model_default_effort_and_no_fallback(self, monkeypatch: pytest.MonkeyPatch) -> None: + _use_provider(monkeypatch, "anthropic", "claude-opus-5") + mock_anthropic = MagicMock() + mock_anthropic.messages.create.return_value = _anthropic_response("Done") + + with patch("anthropic.Anthropic", return_value=mock_anthropic): + FindingVerifier().completions("system", "user") + + call_kwargs = mock_anthropic.messages.create.call_args.kwargs + assert {"output_config", "extra_headers", "extra_body"}.isdisjoint(call_kwargs) + + def test_anthropic_gateway_gets_effort_without_server_side_fallback(self, monkeypatch: pytest.MonkeyPatch) -> None: + _use_provider(monkeypatch, "anthropic", "claude-opus-5-5") + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://gateway.example") + mock_anthropic = MagicMock() + mock_anthropic.messages.create.return_value = _anthropic_response("Done") + + with patch("anthropic.Anthropic", return_value=mock_anthropic): + FindingVerifier().completions("system", "user") + + call_kwargs = mock_anthropic.messages.create.call_args.kwargs + assert call_kwargs["output_config"] == {"effort": "high"} + assert {"extra_headers", "extra_body"}.isdisjoint(call_kwargs) + + def test_schema_downgrade_keeps_anthropic_effort(self, monkeypatch: pytest.MonkeyPatch) -> None: + class SchemaRejected(Exception): + status_code = 400 + + _use_provider(monkeypatch, "anthropic", "claude-sonnet-5-5") + monkeypatch.setattr(client_mod, "_SCHEMA_UNSUPPORTED_TARGETS", set()) + mock_anthropic = MagicMock() + mock_anthropic.messages.create.side_effect = [ + SchemaRejected("output_config.format: Extra inputs are not permitted"), + _anthropic_response("{}"), + ] + + with patch("anthropic.Anthropic", return_value=mock_anthropic): + LLMClient(reasoning_effort="medium").completions("system", "user", response_schema={"type": "object"}) + + first, second = (call.kwargs for call in mock_anthropic.messages.create.call_args_list) + assert first["output_config"]["format"]["schema"] == {"type": "object"} + assert second["output_config"] == {"effort": "medium"} + + @pytest.mark.filterwarnings("ignore::DeprecationWarning") + @pytest.mark.parametrize("model", ["claude-opus-4-1-20250805", "claude-opus-4-0"]) + def test_anthropic_max_tokens_stay_under_the_sdk_non_streaming_limit( + self, monkeypatch: pytest.MonkeyPatch, model: str + ) -> None: + """The real SDK refuses Opus 4/4.1 non-streaming requests above 8,192 tokens before sending.""" + import httpx + + requests: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "Done"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + _use_provider(monkeypatch, "anthropic", model) + verifier = FindingVerifier(http_client=httpx.Client(transport=httpx.MockTransport(handler))) + + assert verifier.completions("system", "user") == "Done" + assert requests[0]["max_tokens"] == 8192 + + @pytest.mark.parametrize( + ("base_url", "provider_default"), + [(None, CHAT_DEFAULT_OPENAI), ("https://gateway.example/v1", CHAT_DEFAULT_GATEWAY)], + ) + def test_explicit_credentials_use_the_provider_default_model( + self, monkeypatch: pytest.MonkeyPatch, base_url: str | None, provider_default: str + ) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + + assert LLMClient(api_key="test-key", base_url=base_url).model == provider_default + + +class TestRefusalAndTruncation: + @pytest.mark.parametrize( + "response", + [ + _openai_choice_response("", finish_reason="content_filter"), + _openai_choice_response(None, refusal="I can't help with that."), + ], + ) + def test_openai_refusal_is_a_typed_error_and_not_retried(self, monkeypatch: pytest.MonkeyPatch, response) -> None: + _use_provider(monkeypatch, "openai", CHAT_DEFAULT_OPENAI) + mock_openai = MagicMock() + mock_openai.chat.completions.create.return_value = response + + with patch("openai.OpenAI", return_value=mock_openai), pytest.raises(LLMClientRefusedError): + LLMClient().completions("system", "user") + + assert mock_openai.chat.completions.create.call_count == 1 + + def test_anthropic_refusal_is_a_typed_error(self, monkeypatch: pytest.MonkeyPatch) -> None: + _use_provider(monkeypatch, "anthropic", "claude-opus-5-5") + mock_anthropic = MagicMock() + mock_anthropic.messages.create.return_value = _anthropic_response("", stop_reason="refusal") + + with patch("anthropic.Anthropic", return_value=mock_anthropic), pytest.raises(LLMClientRefusedError): + LLMClient().completions("system", "user") + + def test_bedrock_content_filter_is_a_typed_error(self, monkeypatch: pytest.MonkeyPatch) -> None: + _use_provider(monkeypatch, "bedrock", "us.anthropic.claude-opus-5-5") + + with ( + patch("litellm.completion", return_value=_openai_choice_response("", finish_reason="content_filter")), + pytest.raises(LLMClientRefusedError), + ): + LLMClient().completions("system", "user") + + @pytest.mark.parametrize( + ("provider", "model", "response"), + [ + ("openai", CHAT_DEFAULT_OPENAI, _openai_choice_response('{"partial": ', finish_reason="length")), + ("anthropic", CHAT_DEFAULT_ANTHROPIC, _anthropic_response('{"partial": ', stop_reason="max_tokens")), + ], + ) + def test_truncation_is_a_typed_error_that_keeps_partial_text( + self, monkeypatch: pytest.MonkeyPatch, provider: str, model: str, response + ) -> None: + _use_provider(monkeypatch, provider, model) + sdk = MagicMock() + sdk.chat.completions.create.return_value = response + sdk.messages.create.return_value = response + + with ( + patch("openai.OpenAI", return_value=sdk), + patch("anthropic.Anthropic", return_value=sdk), + pytest.raises(LLMClientTruncatedError) as exc_info, + ): + LLMClient().completions("system", "user") + + assert exc_info.value.content == '{"partial":' + + def test_process_records_the_refusal_cause(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _use_provider(monkeypatch, "openai", CHAT_DEFAULT_OPENAI) + mock_openai = MagicMock() + mock_openai.chat.completions.create.return_value = _openai_choice_response("", finish_reason="content_filter") + verifier = FindingVerifier() + + with patch("openai.OpenAI", return_value=mock_openai): + assert verifier.process(findings=[], skill_path=tmp_path) == {} + + assert verifier.last_failure is not None + assert "declined the request" in verifier.last_failure + + @pytest.mark.parametrize( + ("provider", "model", "make_response"), + [ + ("openai", CHAT_DEFAULT_OPENAI, lambda text: _openai_choice_response(text, finish_reason="length")), + ("anthropic", CHAT_DEFAULT_ANTHROPIC, lambda text: _anthropic_response(text, stop_reason="max_tokens")), + ], + ) + def test_verifier_keeps_complete_verdicts_from_a_truncated_reply( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, provider: str, model: str, make_response + ) -> None: + _use_provider(monkeypatch, provider, model) + reply = ( + '{"index": 0, "verdict": "false_positive", "confidence": "high", "rationale": "Test card."}\n' + '{"index": 1, "verdict": "true_positive", "confidence": "high", "rationale": "Real key."}\n' + '{"index": 2, "verdict": "false_pos' + ) + sdk = MagicMock() + sdk.chat.completions.create.return_value = make_response(reply) + sdk.messages.create.return_value = make_response(reply) + verifier = FindingVerifier() + + with patch("openai.OpenAI", return_value=sdk), patch("anthropic.Anthropic", return_value=sdk): + verdicts = verifier.process(findings=[], skill_path=tmp_path) + + assert sorted(verdicts) == [0, 1] + assert verdicts[0]["verdict"] == "false_positive" + assert verifier.last_failure is not None + assert "output-token limit" in verifier.last_failure + + def test_truncated_reply_uses_the_fallback_without_salvage(self, monkeypatch: pytest.MonkeyPatch) -> None: + class Client(LLMClient): + def get_system_prompt(self) -> str: + return "system" + + def create_user_prompt(self, **_kwargs) -> str: + return "user" + + def parse_response(self, response_text: str, **_kwargs) -> str: + return response_text + + def get_fallback_response(self, **_kwargs) -> str: + return "fallback" + + _use_provider(monkeypatch, "openai", CHAT_DEFAULT_OPENAI) + mock_openai = MagicMock() + mock_openai.chat.completions.create.return_value = _openai_choice_response("partial", finish_reason="length") + + with patch("openai.OpenAI", return_value=mock_openai): + client = Client() + assert client.process() == "fallback" + + assert client.last_failure is not None diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index 24baa4c7..84673c07 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -181,7 +181,7 @@ def test_security_keeps_gitleaks_always_on_and_skips_only_nonessential_jobs() -> assert "vars.ENABLE_GITHUB_ADVANCED_SECURITY == 'true'" in codeql_if -def test_gitleaks_synthetic_harbor_allowlists_are_exactly_scoped() -> None: +def test_gitleaks_single_value_allowlists_are_exactly_scoped() -> None: config = tomllib.loads(GITLEAKS_CONFIG.read_text(encoding="utf-8")) allowlists = {entry["description"]: entry for entry in config["allowlists"]} @@ -198,6 +198,10 @@ def test_gitleaks_synthetic_harbor_allowlists_are_exactly_scoped() -> None: "^cw-key-123456$", r"^tests/test_harbor_environment_contract\.py$", ), + "Bedrock model ID that follows a credential name in SkillSpector bridge tests": ( + r"^us\.anthropic\.claude-opus-5$", + r"^tests/validators/test_security\.py$", + ), } for description, (regex, path) in expected.items(): entry = allowlists[description] diff --git a/tests/test_gateway_defaults.py b/tests/test_gateway_defaults.py index f176a5d5..547ebc44 100644 --- a/tests/test_gateway_defaults.py +++ b/tests/test_gateway_defaults.py @@ -87,8 +87,8 @@ def test_gateway_rejects_blank_model_overrides(gateway, resolver, variable): @pytest.mark.parametrize( "agent,expected", [ - ("codex", "openai/openai/gpt-5.6-sol"), - ("claude-code", "aws/anthropic/bedrock-claude-opus-5"), + ("codex", "openai/openai/gpt-6.1-sol"), + ("claude-code", "aws/anthropic/bedrock-claude-opus-5-5"), ("opencode", "openai/nvidia/nvidia/nemotron-3-super-120b-long-ctx"), ], ) @@ -131,7 +131,12 @@ def test_gateway_shared_credentials_are_scoped_per_agent(gateway): assert claude.provider.base_url == "https://gateway.example/team" assert claude.subprocess_env["ANTHROPIC_API_KEY"] == "gateway-key" assert claude.subprocess_env["ANTHROPIC_BASE_URL"] == "https://gateway.example/team" - assert set(claude.staged_env) == {"ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"} + assert claude.subprocess_env["CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS"] == "1" + assert set(claude.staged_env) == { + "ANTHROPIC_API_KEY", + "ANTHROPIC_BASE_URL", + "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS", + } for agent in ("codex", "opencode"): assert plans[agent].provider.base_url == gateway["SKILL_EVAL_LLM_BASE_URL"] assert set(plans[agent].staged_env) == {"OPENAI_API_KEY", "OPENAI_BASE_URL"} @@ -154,13 +159,16 @@ def test_explicit_claude_route_takes_precedence(gateway, monkeypatch, base, key, assert plans["claude-code"].provider.base_url == expected_base assert plans["claude-code"].provider.api_key == expected_key assert plans["codex"].provider.api_key == "gateway-key" + # Only the shared gateway key drops pre-release fields; a separate Claude route is the operator's. + disables_betas = "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS" in plans["claude-code"].staged_env + assert disables_betas is (expected_key == "gateway-key") def test_independent_native_claude_key_uses_native_model_default(gateway, monkeypatch): monkeypatch.setenv("ANTHROPIC_API_KEY", "native-key") provider = resolve_llm_provider(gateway) assert _model_for_agent("claude-code", cli_model=None, config_agents={}, provider=provider) == ( - "claude-opus-5", + "claude-opus-5-5", "native Anthropic agent default", ) assert _model_for_agent("claude-code", cli_model="exact/override", config_agents={}, provider=provider) == ( diff --git a/tests/test_harbor_metrics_truth.py b/tests/test_harbor_metrics_truth.py index 1098acdd..11488113 100644 --- a/tests/test_harbor_metrics_truth.py +++ b/tests/test_harbor_metrics_truth.py @@ -19,6 +19,7 @@ _compute_lift, _condition_execution_summary, _count_derived_pass_rate_delta, + _logical_attempt_rewards, _mcnemar_exact_p_value, _mcnemar_exact_probability, _minimum_attainable_mcnemar_p_value, @@ -258,6 +259,57 @@ def test_lift_omits_unpaired_metrics_and_incomplete_overall() -> None: } +def _judged_reward(case_id: str, accuracy: float, judge_model: str) -> dict: + reward = { + "entry_id": case_id, + "_trial_name": f"{case_id}__attempt1", + "metric_set": DEFAULT_METRIC_SET, + **dict.fromkeys(DEFAULT_METRICS, 0.5), + "accuracy": accuracy, + } + reward["details"] = {"accuracy": {"score": accuracy, "judge_model_used": judge_model}} + return reward + + +def test_lift_leaves_out_cases_whose_arms_used_different_judges() -> None: + with_rewards = [_judged_reward("case-1", 1.0, "claude-opus-4-8"), _judged_reward("case-2", 0.8, "judge")] + without_rewards = [_judged_reward("case-1", 0.0, "judge"), _judged_reward("case-2", 0.4, "judge")] + with_scores = dict.fromkeys(DEFAULT_METRICS, 0.5) | {"accuracy": 0.9} + without_scores = dict.fromkeys(DEFAULT_METRICS, 0.5) | {"accuracy": 0.2} + + lift = _compute_lift(with_scores, without_scores, with_rewards, without_rewards, ["case-1", "case-2"]) + + assert lift["accuracy"] == {"with_skill": 0.8, "without_skill": 0.4, "delta": 0.4, "direction": "up"} + assert lift["judge_mixed_cases"] == {"accuracy": 1} + assert lift["overall"]["delta"] == round(0.4 / len(DEFAULT_METRICS), 4) + assert "judge_mixed_cases" not in _compute_lift(with_scores, without_scores, with_rewards[1:], without_rewards) + + +def test_lift_leaves_out_multistep_cases_whose_steps_used_a_fallback_judge() -> None: + def steps(case_id: str, accuracy: float, step_judges: tuple[str, ...]) -> list[dict]: + rows = [] + for index, judge_model in enumerate(step_judges, start=1): + row = _judged_reward(case_id, accuracy, judge_model) + row |= {"_trial_root_name": f"{case_id}__attempt1", "_step_name": f"step-{index}"} + rows.append(row) + return rows + + with_rewards = _logical_attempt_rewards( + [*steps("case-1", 1.0, ("judge", "claude-opus-4-8")), *steps("case-2", 0.8, ("judge", "judge"))] + ) + without_rewards = _logical_attempt_rewards( + [*steps("case-1", 0.0, ("judge", "judge")), *steps("case-2", 0.4, ("judge", "judge"))] + ) + with_scores = dict.fromkeys(DEFAULT_METRICS, 0.5) | {"accuracy": 0.9} + without_scores = dict.fromkeys(DEFAULT_METRICS, 0.5) | {"accuracy": 0.2} + + assert "details" not in with_rewards[0] + lift = _compute_lift(with_scores, without_scores, with_rewards, without_rewards, ["case-1", "case-2"]) + + assert lift["accuracy"] == {"with_skill": 0.8, "without_skill": 0.4, "delta": 0.4, "direction": "up"} + assert lift["judge_mixed_cases"] == {"accuracy": 1} + + def test_pass_summary_marks_incomplete_reward_unscored() -> None: reward = { "entry_id": "case-001", diff --git a/tests/test_harbor_runner_environment.py b/tests/test_harbor_runner_environment.py index 3a3a4b5c..e1382205 100644 --- a/tests/test_harbor_runner_environment.py +++ b/tests/test_harbor_runner_environment.py @@ -553,9 +553,29 @@ def test_judge_model_config_distinguishes_provider_default_from_explicit_model( "model": "test-model", "source": expected_source, "override_applied": False, + "reasoning_effort": None, } +@pytest.mark.parametrize( + ("provider_name", "judge_model", "effort"), + [ + ("openai", "gpt-6.1-sol", "medium"), + ("anthropic", "claude-opus-5-5", "medium"), + ("openai", "gpt-5.6-sol", None), + ("bedrock", "us.anthropic.claude-opus-5-5", None), + ], +) +def test_judge_model_config_records_the_judge_reasoning_effort( + provider_name: str, judge_model: str, effort: str | None +) -> None: + provider = _provider(provider_name) + + config = runner._judge_model_config(provider, {"LLM_JUDGE_MODEL": judge_model}, "default") + + assert config["reasoning_effort"] == effort + + def test_mixed_agents_receive_disjoint_credentials(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( runner.os, @@ -1236,6 +1256,7 @@ def launch(**kwargs): "model": "legacy-judge-model", "source": "LLM_JUDGE_MODEL", "override_applied": True, + "reasoning_effort": None, "catalog_verification": "degraded", } diff --git a/tests/test_llm_judge.py b/tests/test_llm_judge.py index b523cb17..564e3638 100644 --- a/tests/test_llm_judge.py +++ b/tests/test_llm_judge.py @@ -14,18 +14,16 @@ import pytest from tests.conftest import MockUrllibResponse, load_harbor_eval_template -from skillevaluator.provider_config import CHAT_CHEAP_OPENAI, CHAT_DEFAULT_OPENAI +from skillevaluator.provider_config import ( + CHAT_CHEAP_OPENAI, + CHAT_DEFAULT_OPENAI, + REASONING_MAX_COMPLETION_TOKENS, + _supports_custom_temperature, +) from skillevaluator.tier3.eval_core import llm_judge -@pytest.mark.parametrize( - "model", - [ - CHAT_DEFAULT_OPENAI, - f"openai/{CHAT_DEFAULT_OPENAI}", - f"openai/openai/{CHAT_DEFAULT_OPENAI}", - ], -) +@pytest.mark.parametrize("model", ["gpt-5.6-sol", "openai/gpt-5.6-sol", "openai/openai/gpt-5.6-sol"]) def test_native_openai_gpt5_uses_max_completion_tokens_without_temperature(model: str) -> None: """Verify native OpenAI gpt-5 requests use max_completion_tokens without temperature.""" payload = llm_judge._chat_completion_payload( @@ -39,7 +37,7 @@ def test_native_openai_gpt5_uses_max_completion_tokens_without_temperature(model assert payload == { "model": model, - "max_completion_tokens": 321, + "max_completion_tokens": REASONING_MAX_COMPLETION_TOKENS, "messages": [{"role": "user", "content": "Judge this response"}], "stream": False, } @@ -58,15 +56,15 @@ def test_native_openai_gpt5_uses_max_completion_tokens_without_temperature(model ) def test_gpt5_family_rejects_custom_temperature(model: str) -> None: """Verify gpt-5 models do not support custom temperature.""" - assert not llm_judge._supports_custom_temperature(model) + assert not _supports_custom_temperature(model) def test_older_models_accept_custom_temperature() -> None: """Verify legacy model families support custom temperature.""" - assert llm_judge._supports_custom_temperature("gpt-4.1-mini") - assert llm_judge._supports_custom_temperature("claude-opus-4-6") - assert llm_judge._supports_custom_temperature("claude-opus-4-20250514") - assert llm_judge._supports_custom_temperature("claude-3-5-sonnet-20241022") + assert _supports_custom_temperature("gpt-4.1-mini") + assert _supports_custom_temperature("claude-opus-4-6") + assert _supports_custom_temperature("claude-opus-4-20250514") + assert _supports_custom_temperature("claude-3-5-sonnet-20241022") @pytest.mark.parametrize( @@ -90,7 +88,52 @@ def test_older_models_accept_custom_temperature() -> None: ) def test_newer_claude_models_reject_custom_temperature(model: str) -> None: """Verify newer Claude models do not accept custom temperature.""" - assert not llm_judge._supports_custom_temperature(model) + assert not _supports_custom_temperature(model) + + +@pytest.mark.parametrize("model", ["gpt-6.1-sol", "openai/openai/gpt-6.1-sol"]) +def test_gpt6_judge_payload_sends_judge_reasoning_effort(model: str) -> None: + payload = llm_judge._chat_completion_payload( + model=model, + prompt="Judge this response", + max_tokens=321, + temperature=0.0, + provider="openai", + request_url=llm_judge.OPENAI_CHAT_URL, + ) + + assert payload["reasoning_effort"] == llm_judge.JUDGE_REASONING_EFFORT + assert payload["max_completion_tokens"] == REASONING_MAX_COMPLETION_TOKENS + assert "temperature" not in payload + + +def test_call_public_llm_returns_partial_text_from_a_truncated_reply(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build") + monkeypatch.setenv("NVIDIA_API_KEY", "test-key") + monkeypatch.delenv("SKILL_EVAL_LLM_MODEL", raising=False) + mock_openai = MagicMock() + mock_openai.chat.completions.create.return_value.choices = [ + MagicMock(finish_reason="length", message=MagicMock(content='{"results": [', refusal=None)) + ] + + with patch("openai.OpenAI", return_value=mock_openai): + assert llm_judge.call_public_llm("Judge this response") == ('{"results": [', None) + + +def test_call_public_llm_reports_a_refusal_as_an_error(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "nv_build") + monkeypatch.setenv("NVIDIA_API_KEY", "test-key") + monkeypatch.delenv("SKILL_EVAL_LLM_MODEL", raising=False) + mock_openai = MagicMock() + mock_openai.chat.completions.create.return_value.choices = [ + MagicMock(finish_reason="content_filter", message=MagicMock(content="", refusal=None)) + ] + + with patch("openai.OpenAI", return_value=mock_openai): + content, error = llm_judge.call_public_llm("Judge this response") + + assert content is None + assert error is not None and "declined the request" in error def test_call_public_llm_uses_production_gpt5_payload_without_temperature( @@ -110,7 +153,7 @@ def test_call_public_llm_uses_production_gpt5_payload_without_temperature( assert (content, error) == ("Done", None) call_kwargs = mock_openai.chat.completions.create.call_args.kwargs - assert call_kwargs["max_completion_tokens"] == 4096 + assert call_kwargs["max_completion_tokens"] == REASONING_MAX_COMPLETION_TOKENS assert "temperature" not in call_kwargs @@ -159,7 +202,7 @@ def test_shared_structured_judges_retry_empty_sdk_response( assert result["score"] == 1.0 assert mock_openai.chat.completions.create.call_count == 2 calls = mock_openai.chat.completions.create.call_args_list - assert [call.kwargs["max_tokens"] for call in calls] == [4096, 4096] + assert [call.kwargs["max_tokens"] for call in calls] == [llm_judge.STRUCTURED_JUDGE_MAX_TOKENS] * 2 assert "previous reply could not be parsed or validated" in calls[1].kwargs["messages"][1]["content"] @@ -194,7 +237,7 @@ def test_non_native_gpt5_requests_keep_max_tokens(provider: str, request_url: st request_url=request_url, ) - assert payload["max_tokens"] == 321 + assert payload["max_tokens"] == REASONING_MAX_COMPLETION_TOKENS assert "max_completion_tokens" not in payload assert "temperature" not in payload @@ -229,11 +272,24 @@ def test_completion_token_payload_resolves_provider_and_url_when_omitted( temperature=0.0, ) - assert payload["max_completion_tokens"] == 321 + assert payload["max_completion_tokens"] == REASONING_MAX_COMPLETION_TOKENS assert "max_tokens" not in payload assert "temperature" not in payload +def _judge_token_key(provider: str, request_url: str) -> str: + payload = llm_judge._chat_completion_payload( + model=CHAT_DEFAULT_OPENAI, + prompt="Judge this response", + max_tokens=321, + temperature=0.0, + provider=provider, + request_url=request_url, + ) + (token_key,) = {"max_tokens", "max_completion_tokens"} & payload.keys() + return token_key + + @pytest.mark.parametrize( "request_url", [ @@ -244,7 +300,7 @@ def test_completion_token_payload_resolves_provider_and_url_when_omitted( ], ) def test_native_openai_completion_token_url_accepts_only_canonical_variants(request_url: str) -> None: - assert llm_judge._is_native_openai_chat_url("OPENAI", request_url) + assert _judge_token_key("OPENAI", request_url) == "max_completion_tokens" @pytest.mark.parametrize( @@ -273,7 +329,7 @@ def test_native_openai_completion_token_url_accepts_only_canonical_variants(requ ], ) def test_deceptive_openai_urls_keep_max_tokens(provider: str, request_url: str) -> None: - assert not llm_judge._is_native_openai_chat_url(provider, request_url) + assert _judge_token_key(provider, request_url) == "max_tokens" @pytest.mark.parametrize( @@ -1122,6 +1178,7 @@ def fake_urlopen(req, timeout=90): "Unrecognized request argument supplied: response_format", "response_format is an unknown parameter", "output_config.format is an unexpected argument", + "tool_choice is not supported with this model", ], ) def test_schema_rejection_grammar_accepts_direct_option_errors(message: str) -> None: diff --git a/tests/test_oss_boundary.py b/tests/test_oss_boundary.py index c8a7eeb8..8308113e 100644 --- a/tests/test_oss_boundary.py +++ b/tests/test_oss_boundary.py @@ -96,7 +96,7 @@ def test_scanner_source_does_not_match_its_constructed_deny_expressions() -> Non ( "NVIDIA_API_KEY=nvapi-public-placeholder", "https://integrate.api.nvidia.com/v1", - "https://github.com/NVIDIA/SkillSpector.git", + "https://github.com/NVIDIA/SkillSpector.git@v2.12.0", 'harbor_viewer = {"job_url": "https://viewer.example/jobs/1"}', "opentelemetry-api>=1.37.0", "OTEL_EXPORTER_OTLP_ENDPOINT=https://collector.example.test", diff --git a/tests/test_oss_packaging.py b/tests/test_oss_packaging.py index ef0f214e..dd412406 100644 --- a/tests/test_oss_packaging.py +++ b/tests/test_oss_packaging.py @@ -22,6 +22,7 @@ from packaging.version import Version from skillevaluator.cli import cli +from skillevaluator.provider_config import CHAT_DEFAULT_ANTHROPIC, CHAT_DEFAULT_OPENAI from skillevaluator.tier3_environments import ( HARBOR_ENVIRONMENT_ALIAS_KWARGS, HARBOR_ENVIRONMENT_EXTRAS, @@ -792,9 +793,9 @@ def test_public_docs_show_external_nvidia_build_harness_paths_only() -> None: tier3 = (REPO_ROOT / "docs" / "tier3-live-evaluation.mdx").read_text(encoding="utf-8") public_docs = f"{readme}\n{configuration}\n{tier3}" - assert "gpt-5.6-sol" in public_docs + assert f"`{CHAT_DEFAULT_OPENAI}`" in public_docs assert "gpt-5.4-mini" in public_docs - assert "claude-opus-5" in public_docs + assert f"`{CHAT_DEFAULT_ANTHROPIC}`" in public_docs assert "nvidia/nemotron-3-super-120b-a12b" in public_docs assert "nvidia/nvidia/nemotron-3-super-120b-a12b" in public_docs assert "Nemotron Super" in public_docs @@ -887,7 +888,7 @@ def test_launch_docs_address_scanner_and_naming_ambiguities() -> None: normalized_quickstart = " ".join(quickstart.split()) assert "brew install semgrep gitleaks" in quickstart - assert "uv tool install git+https://github.com/NVIDIA/SkillSpector.git" in quickstart + assert "uv tool install git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0" in quickstart assert "Semgrep, SkillSpector, and Gitleaks" in quickstart assert "missing scanner evidence leaves the result `INCOMPLETE` and exits `1`" in normalized_quickstart assert "most often Gitleaks" not in ci @@ -897,6 +898,23 @@ def test_launch_docs_address_scanner_and_naming_ambiguities() -> None: assert "are not interchangeable" in environment +def test_skillspector_install_instructions_pin_one_release() -> None: + sources = [ + REPO_ROOT / "README.md", + *(REPO_ROOT / "docs").rglob("*.mdx"), + *(REPO_ROOT / "src" / "skillevaluator").rglob("*.py"), + ] + install_specs = { + spec + for path in sources + for spec in re.findall( + r"git\+https://github\.com/NVIDIA/SkillSpector\.git[^\s`\"]*", path.read_text(encoding="utf-8") + ) + } + + assert install_specs == {"git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0"} + + def test_ci_sarif_merge_uses_uv_tool_python() -> None: ci = (REPO_ROOT / "docs" / "ci-integration.mdx").read_text(encoding="utf-8") diff --git a/tests/test_provider_agent_routing.py b/tests/test_provider_agent_routing.py index 8a0b3070..e14802ca 100644 --- a/tests/test_provider_agent_routing.py +++ b/tests/test_provider_agent_routing.py @@ -7,7 +7,12 @@ import pytest -from skillevaluator.provider_config import ProviderConfig +from skillevaluator.provider_config import ( + CHAT_DEFAULT_OPENAI, + OPENCODE_OPENAI_DEFAULT_MODEL, + ProviderConfig, + resolve_llm_provider, +) from skillevaluator.tier3.harbor.runner import _model_for_agent, _validate_agent_provider_credentials @@ -61,6 +66,33 @@ def test_public_provider_defaults_are_normalized_for_each_agent( ) +@pytest.mark.parametrize( + ("configured_model", "expected"), + [ + (None, (f"openai/{OPENCODE_OPENAI_DEFAULT_MODEL}", "native OpenAI agent default")), + (CHAT_DEFAULT_OPENAI, (f"openai/{CHAT_DEFAULT_OPENAI}", "public provider default")), + ], +) +def test_native_openai_opencode_keeps_its_default_unless_a_chat_model_is_set( + monkeypatch: pytest.MonkeyPatch, + configured_model: str | None, + expected: tuple[str, str], +) -> None: + environ = {"SKILL_EVAL_LLM_PROVIDER": "openai", "OPENAI_API_KEY": "provider-key"} + if configured_model: + environ["SKILL_EVAL_LLM_MODEL"] = configured_model + monkeypatch.setenv("SKILL_EVAL_LLM_MODEL", configured_model) + else: + monkeypatch.delenv("SKILL_EVAL_LLM_MODEL", raising=False) + provider = resolve_llm_provider(environ) + + assert _model_for_agent("opencode", cli_model=None, config_agents={}, provider=provider) == expected + assert _model_for_agent("codex", cli_model=None, config_agents={}, provider=provider) == ( + CHAT_DEFAULT_OPENAI, + "public provider default", + ) + + @pytest.mark.parametrize( ("cli_model", "config_agents", "expected", "source"), [ diff --git a/tests/test_provider_config.py b/tests/test_provider_config.py index 780fd057..7d2cd52f 100644 --- a/tests/test_provider_config.py +++ b/tests/test_provider_config.py @@ -10,7 +10,19 @@ import pytest -from skillevaluator.provider_config import ProviderConfigurationError, resolve_embedding_provider, resolve_llm_provider +from skillevaluator.provider_config import ( + MAX_COMPLETION_TOKENS, + REASONING_MAX_COMPLETION_TOKENS, + ProviderConfigurationError, + _supports_custom_temperature, + completion_token_limit, + effective_reasoning_effort, + is_claude_5_5_or_later, + is_openai_reasoning_model, + resolve_embedding_provider, + resolve_llm_provider, + skillspector_reasoning_effort, +) PROVIDER_CONTRACT = Path(__file__).parent / "fixtures" / "public_provider_contract.json" @@ -34,25 +46,25 @@ def test_openai_provider_uses_standard_openai_credentials() -> None: } -def test_openai_provider_uses_gpt_5_5_by_default() -> None: +def test_openai_provider_uses_gpt_6_1_sol_by_default() -> None: config = resolve_llm_provider({"SKILL_EVAL_LLM_PROVIDER": "openai", "OPENAI_API_KEY": "test-openai-key"}) - assert config.model == "gpt-5.6-sol" - assert config.litellm_model == "openai/gpt-5.6-sol" + assert config.model == "gpt-6.1-sol" + assert config.litellm_model == "openai/gpt-6.1-sol" -def test_anthropic_provider_uses_claude_opus_5_by_default() -> None: +def test_anthropic_provider_uses_claude_opus_5_5_by_default() -> None: config = resolve_llm_provider({"SKILL_EVAL_LLM_PROVIDER": "anthropic", "ANTHROPIC_API_KEY": "test-anthropic-key"}) - assert config.model == "claude-opus-5" - assert config.litellm_model == "anthropic/claude-opus-5" + assert config.model == "claude-opus-5-5" + assert config.litellm_model == "anthropic/claude-opus-5-5" -def test_bedrock_provider_uses_claude_opus_5_us_profile_by_default() -> None: +def test_bedrock_provider_uses_claude_opus_5_5_us_profile_by_default() -> None: config = resolve_llm_provider({"SKILL_EVAL_LLM_PROVIDER": "bedrock"}) - assert config.model == "us.anthropic.claude-opus-5" - assert config.litellm_model == "bedrock/us.anthropic.claude-opus-5" + assert config.model == "us.anthropic.claude-opus-5-5" + assert config.litellm_model == "bedrock/us.anthropic.claude-opus-5-5" def test_chat_default_models_are_the_single_source_of_truth() -> None: @@ -65,15 +77,91 @@ def test_chat_default_models_are_the_single_source_of_truth() -> None: CHAT_DEFAULT_NVIDIA, CHAT_DEFAULT_OPENAI, ) - from skillevaluator.tier3.eval_core.llm_judge import DEFAULT_JUDGE_MODEL - assert CHAT_DEFAULT_MODELS["openai"] == CHAT_DEFAULT_OPENAI == "gpt-5.6-sol" - assert CHAT_DEFAULT_MODELS["anthropic"] == CHAT_DEFAULT_ANTHROPIC == "claude-opus-5" - assert CHAT_DEFAULT_MODELS["bedrock"] == CHAT_DEFAULT_BEDROCK == "us.anthropic.claude-opus-5" + assert CHAT_DEFAULT_MODELS["openai"] == CHAT_DEFAULT_OPENAI == "gpt-6.1-sol" + assert CHAT_DEFAULT_MODELS["anthropic"] == CHAT_DEFAULT_ANTHROPIC == "claude-opus-5-5" + assert CHAT_DEFAULT_MODELS["bedrock"] == CHAT_DEFAULT_BEDROCK == "us.anthropic.claude-opus-5-5" assert CHAT_DEFAULT_MODELS["nv_build"] == CHAT_DEFAULT_NVIDIA == "nvidia/nemotron-3-super-120b-a12b" assert CHAT_CHEAP_OPENAI == "gpt-5.4-mini" assert DIMENSION_JUDGE_MODEL == CHAT_DEFAULT_OPENAI - assert DEFAULT_JUDGE_MODEL == CHAT_DEFAULT_OPENAI + + +def test_skillspector_defaults_cover_every_provider_and_keep_supported_claude_models() -> None: + from skillevaluator.provider_config import CHAT_DEFAULT_MODELS, SKILLSPECTOR_DEFAULT_MODELS + + assert SKILLSPECTOR_DEFAULT_MODELS.keys() == CHAT_DEFAULT_MODELS.keys() + # Literals, re-validated against the pinned SkillSpector release before they change. + assert SKILLSPECTOR_DEFAULT_MODELS == { + "openai": "gpt-6.1-sol", + "anthropic": "claude-opus-5", + "bedrock": "us.anthropic.claude-opus-5", + "nv_build": "nvidia/nemotron-3-super-120b-a12b", + "openai-compatible": "nvidia/nvidia/nemotron-3-super-120b-long-ctx", + } + + +@pytest.mark.parametrize( + ("model", "effort"), + [ + ("gpt-6.1-sol", "medium"), + ("openai/openai/gpt-6.1-sol", "medium"), + ("gpt-5.6-sol", None), + ("claude-opus-5-5", None), + ("us.anthropic.claude-opus-5-5", None), + ("aws/anthropic/bedrock-claude-opus-5-5", None), + ("nvidia/nemotron-3-super-120b-a12b", None), + ], +) +def test_skillspector_reasoning_effort_is_set_only_for_gpt_6_models(model: str, effort: str | None) -> None: + assert skillspector_reasoning_effort(model) == effort + + +@pytest.mark.parametrize( + ("model", "reasoning", "claude_5_5", "custom_temperature"), + [ + ("gpt-5.6-sol", True, False, False), + ("openai/openai/gpt-5.6-sol", True, False, False), + ("gpt-6.1-sol", True, False, False), + ("openai/openai/gpt-6.1-sol", True, False, False), + ("gpt-4.1-mini", False, False, True), + ("claude-opus-5", False, False, False), + ("claude-opus-5-5", False, True, False), + ("claude-sonnet-5-5", False, True, False), + ("us.anthropic.claude-opus-5-5", False, True, False), + ("azure/anthropic/claude-opus-5-5", False, True, False), + # Gateway catalogs prefix vendor words; the old leaf check sent these a temperature. + ("aws/anthropic/bedrock-claude-opus-5", False, False, False), + ("aws/anthropic/bedrock-claude-opus-5-5", False, True, False), + ("claude-3-5-sonnet-20241022", False, False, True), + ("nvidia/nemotron-3-super-120b-a12b", False, False, True), + ], +) +def test_model_capability_helpers(model: str, reasoning: bool, claude_5_5: bool, custom_temperature: bool) -> None: + assert is_openai_reasoning_model(model) is reasoning + assert is_claude_5_5_or_later(model) is claude_5_5 + assert _supports_custom_temperature(model) is custom_temperature + expected_limit = REASONING_MAX_COMPLETION_TOKENS if reasoning else MAX_COMPLETION_TOKENS + assert completion_token_limit(model, MAX_COMPLETION_TOKENS) == expected_limit + + +@pytest.mark.parametrize( + ("provider", "model", "effort"), + [ + ("openai", "gpt-6.1-sol", "high"), + ("openai-compatible", "openai/openai/gpt-6.1-sol", "high"), + ("anthropic", "claude-opus-5-5", "high"), + # Other models keep their default effort, and Bedrock and gateway Claude routes get none. + ("openai", "gpt-5.6-sol", None), + ("anthropic", "claude-opus-5", None), + ("bedrock", "us.anthropic.claude-opus-5-5", None), + ("openai-compatible", "aws/anthropic/bedrock-claude-opus-5-5", None), + ("nv_build", "nvidia/nemotron-3-super-120b-a12b", None), + ], +) +def test_reasoning_effort_is_sent_only_where_the_route_accepts_it( + provider: str, model: str, effort: str | None +) -> None: + assert effective_reasoning_effort(provider, model, "high") == effort @pytest.mark.parametrize( diff --git a/tests/test_tier3_progress.py b/tests/test_tier3_progress.py index 38d40b2c..2586592f 100644 --- a/tests/test_tier3_progress.py +++ b/tests/test_tier3_progress.py @@ -1510,6 +1510,7 @@ def succeed(selected_provider): "model": "host-judge-model", "source": override_name, "override_applied": True, + "reasoning_effort": None, "catalog_verification": "verified", } @@ -2080,7 +2081,7 @@ def fail_staging(*_args, **_kwargs): { "labels": ["codex"], "provider": "openai-compatible", - "model": "openai/openai/gpt-5.6-sol", + "model": "openai/openai/gpt-6.1-sol", "status": "degraded", "detail": "model catalog access does not verify runtime credentials for this endpoint", }, diff --git a/tests/test_tier3_public_runtime.py b/tests/test_tier3_public_runtime.py index 4bf440da..2be92816 100644 --- a/tests/test_tier3_public_runtime.py +++ b/tests/test_tier3_public_runtime.py @@ -2614,9 +2614,9 @@ def test_write_task_toml_forwards_retry_env(tmp_path: Path) -> None: ("daytona", "skillevaluator.tier3.harbor.local_agents:SkillEvaluatorGatewayCodex"), ], ) -def test_codex_runs_pin_harbor_022_reasoning_effort(env_mode: str, agent_import_path: str | None) -> None: - from harbor.cli.utils import parse_kwargs - +def test_codex_runs_pin_harbor_022_reasoning_effort_and_cli_version( + env_mode: str, agent_import_path: str | None +) -> None: command = build_harbor_run_command( dataset_path="/tmp/dataset", agent="codex", @@ -2625,13 +2625,74 @@ def test_codex_runs_pin_harbor_022_reasoning_effort(env_mode: str, agent_import_ agent_import_path=agent_import_path, ) - agent_kwargs = [command[index + 1] for index, value in enumerate(command) if value == "--ak"] - assert parse_kwargs(agent_kwargs) == {"reasoning_effort": "high"} + # Local mode runs the installed CLI, so only Harbor-installed runs pin a version. + expected = ( + {"reasoning_effort": "high"} if env_mode == "local" else {"reasoning_effort": "high", "version": "0.161.0"} + ) + assert _harbor_agent_kwargs(command) == expected + + +def _harbor_agent_kwargs(command: list[str]) -> dict[str, object]: + from harbor.cli.utils import parse_kwargs + + return parse_kwargs([command[index + 1] for index, value in enumerate(command) if value == "--ak"]) + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + (None, {"version": "2.1.292"}), + ("claude-opus-5", {"version": "2.1.292"}), + ("us.anthropic.claude-opus-5", {"version": "2.1.292"}), + ("aws/anthropic/bedrock-claude-opus-5", {"version": "2.1.292"}), + ("claude-opus-5-5", {"reasoning_effort": "medium", "version": "2.1.292"}), + ("us.anthropic.claude-opus-5-5", {"reasoning_effort": "medium", "version": "2.1.292"}), + ("aws/anthropic/bedrock-claude-opus-5-5", {"reasoning_effort": "medium", "version": "2.1.292"}), + ("anthropic/claude-sonnet-5-5-20261001", {"reasoning_effort": "medium", "version": "2.1.292"}), + ], +) +def test_claude_code_runs_pin_cli_version_and_claude_5_5_effort(model: str | None, expected: dict[str, str]) -> None: + command = build_harbor_run_command( + dataset_path="/tmp/dataset", agent="claude-code", job_name="effort", env_mode="docker", model=model + ) + + assert _harbor_agent_kwargs(command) == expected + + +def test_local_claude_code_runs_keep_effort_without_a_cli_version() -> None: + command = build_harbor_run_command( + dataset_path="/tmp/dataset", agent="claude-code", job_name="effort", env_mode="local", model="claude-opus-5-5" + ) + + assert _harbor_agent_kwargs(command) == {"reasoning_effort": "medium"} + + +def test_documented_agent_cli_versions_match_the_pins() -> None: + from skillevaluator.tier3.harbor import local_agents, runner + + docs = Path(__file__).resolve().parents[1] / "docs" + pages = { + page: " ".join((docs / page).read_text(encoding="utf-8").split()) + for page in ("agents-and-sandboxes.mdx", "tier3-live-evaluation.mdx") + } + for page, text in pages.items(): + assert f"Codex {runner._CODEX_VERSION}" in text, page + assert f"Claude Code {runner._CLAUDE_CODE_VERSION}" in text, page + for name, minimum in ( + ("Claude Code", local_agents._CLAUDE_CODE_MIN_VERSION_FOR_CLAUDE_5_5), + ("Codex", local_agents._CODEX_MIN_VERSION_FOR_GPT_6), + ): + assert f"{name} {'.'.join(map(str, minimum))} or newer" in pages["agents-and-sandboxes.mdx"] -@pytest.mark.parametrize("agent", ["claude-code", "opencode"]) -def test_non_codex_runs_pass_no_agent_kwargs(agent: str) -> None: - command = build_harbor_run_command(dataset_path="/tmp/dataset", agent=agent, job_name="effort", env_mode="docker") +def test_opencode_runs_pass_no_agent_kwargs() -> None: + command = build_harbor_run_command( + dataset_path="/tmp/dataset", + agent="opencode", + job_name="effort", + env_mode="docker", + model="anthropic/claude-opus-5-5", + ) assert "--ak" not in command @@ -2649,3 +2710,21 @@ def test_codex_agents_render_the_pinned_reasoning_effort(tmp_path: Path) -> None ): agent = agent_class(logs_dir=tmp_path, model_name="openai/gpt-5", reasoning_effort="high") assert "-c model_reasoning_effort=high" in agent.build_cli_flags(), agent_class.__name__ + + +def test_claude_code_agents_accept_the_pinned_version_and_effort(tmp_path: Path) -> None: + from harbor.agents.installed.claude_code import ClaudeCode + + from skillevaluator.tier3.harbor import local_agents + + for agent_class in ( + ClaudeCode, + local_agents.SkillEvaluatorLocalClaudeCode, + local_agents.SkillEvaluatorNvidiaBuildClaudeCode, + local_agents.SkillEvaluatorLocalNvidiaBuildClaudeCode, + ): + agent = agent_class( + logs_dir=tmp_path, model_name="anthropic/claude-opus-5-5", reasoning_effort="medium", version="2.1.292" + ) + assert "--effort medium" in agent.build_cli_flags(), agent_class.__name__ + assert agent.version() == "2.1.292", agent_class.__name__ diff --git a/tests/test_verifier_error_redaction.py b/tests/test_verifier_error_redaction.py index 5dc18995..fe692dea 100644 --- a/tests/test_verifier_error_redaction.py +++ b/tests/test_verifier_error_redaction.py @@ -223,7 +223,7 @@ def fail_request(*_args, **_kwargs): assert error is not None assert credential not in error assert error == ( - f"Public provider call failed for {verifier_module.DEFAULT_JUDGE_MODEL}: " + f"Public provider call failed for {verifier_module.DEFAULT_JUDGE_MODELS['nv_build']}: " "transport included [REDACTED] in diagnostics" ) diff --git a/tests/tier3/test_gateway_codex.py b/tests/tier3/test_gateway_codex.py index 0283653c..1ae55700 100644 --- a/tests/tier3/test_gateway_codex.py +++ b/tests/tier3/test_gateway_codex.py @@ -39,7 +39,7 @@ def test_gateway_adapter_selection_preserves_other_routes(provider, mode, agent) @pytest.mark.parametrize( ("provider", "configured_model", "cli_model", "expected_model", "expected_source"), [ - ("openai-compatible", None, None, "openai/openai/gpt-5.6-sol", "openai-compatible agent default"), + ("openai-compatible", None, None, "openai/openai/gpt-6.1-sol", "openai-compatible agent default"), ("openai-compatible", "openai/openai/gpt-5.6-sol", None, "openai/openai/gpt-5.6-sol", "evals/config.yml"), ("openai-compatible", "openai/openai/gpt-5.6-sol", "azure/custom/deployment", "azure/custom/deployment", "CLI"), ("openai-compatible", None, "custom-deployment", "custom-deployment", "CLI"), diff --git a/tests/tier3/test_harbor_local_agents.py b/tests/tier3/test_harbor_local_agents.py index 9c1f08a1..4dcc5f0a 100644 --- a/tests/tier3/test_harbor_local_agents.py +++ b/tests/tier3/test_harbor_local_agents.py @@ -374,6 +374,84 @@ async def fake_parent_exec(self, environment, command, env=None, **kwargs): assert instruction not in run_command +def _install_with_cli_output(monkeypatch: pytest.MonkeyPatch, agent: object, stdout: str) -> list[str]: + commands: list[str] = [] + + async def fake_parent_exec(self, environment, command, **kwargs): + commands.append(command) + return ExecResult(stdout=stdout, return_code=0) + + monkeypatch.setattr("harbor.agents.installed.base.BaseInstalledAgent.exec_as_agent", fake_parent_exec) + asyncio.run(agent.install(object())) + return commands + + +@pytest.mark.parametrize( + ("agent_class", "model", "stdout", "version"), + [ + (SkillEvaluatorLocalClaudeCode, "claude-opus-5-5", "2.1.280 (Claude Code)\n", "2.1.280"), + (SkillEvaluatorLocalClaudeCode, "aws/anthropic/bedrock-claude-opus-5-5", "2.1.292 (Claude Code)\n", "2.1.292"), + (SkillEvaluatorLocalClaudeCode, "claude-opus-5", "2.1.118 (Claude Code)\n", "2.1.118"), + (SkillEvaluatorLocalCodex, "openai/gpt-6.1-sol", "codex-cli 0.159.1\n", "0.159.1"), + (SkillEvaluatorLocalCodex, "openai/openai/gpt-6.1-sol", "codex-cli 0.161.0\n", "0.161.0"), + (SkillEvaluatorLocalCodex, "openai/gpt-5.6-sol", "codex-cli 0.124.0\n", "0.124.0"), + ], +) +def test_local_install_records_a_cli_version_that_supports_the_model( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + agent_class: type, + model: str, + stdout: str, + version: str, +) -> None: + agent = agent_class(logs_dir=tmp_path, model_name=model) + + commands = _install_with_cli_output(monkeypatch, agent, stdout) + + assert len(commands) == 1 and commands[0].endswith("--version") + assert agent.version() == version + + +@pytest.mark.parametrize( + ("agent_class", "model", "stdout", "message"), + [ + ( + SkillEvaluatorLocalClaudeCode, + "claude-opus-5-5", + "2.1.279 (Claude Code)\n", + "claude-opus-5-5 needs claude 2.1.280 or newer, but the installed claude reports version '2.1.279'", + ), + ( + SkillEvaluatorLocalClaudeCode, + "us.anthropic.claude-sonnet-5-5", + "claude dev build\n", + "needs claude 2.1.280 or newer, but the installed claude reports version 'claude dev build'", + ), + ( + SkillEvaluatorLocalCodex, + "openai/openai/gpt-6.1-sol", + "codex-cli 0.159.0\n", + "openai/openai/gpt-6.1-sol needs codex 0.159.1 or newer, but the installed codex reports version '0.159.0'", + ), + ], +) +def test_local_install_rejects_a_cli_too_old_for_the_model( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + agent_class: type, + model: str, + stdout: str, + message: str, +) -> None: + agent = agent_class(logs_dir=tmp_path, model_name=model) + + with pytest.raises(RuntimeError, match=message) as excinfo: + _install_with_cli_output(monkeypatch, agent, stdout) + + assert "--env-mode docker" in str(excinfo.value) + + def test_nvidia_build_codex_final_config_keeps_dynamic_bridge_over_user_and_runtime_values( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/tier3/test_judge_failure_artifacts.py b/tests/tier3/test_judge_failure_artifacts.py index 38e2317e..c3e34b6d 100644 --- a/tests/tier3/test_judge_failure_artifacts.py +++ b/tests/tier3/test_judge_failure_artifacts.py @@ -432,8 +432,8 @@ def goal_call(prompt: str, **kwargs): assert rich["details"]["goal_accuracy"]["score"] == 1.0 assert rich["details"]["goal_accuracy"]["model"] == "retry-model" assert numeric["accuracy"] == numeric["goal_accuracy"] == numeric["behavior_check"] == 1.0 - assert [kwargs["max_tokens"] for _, kwargs in pair_calls] == [4096, 4096, 4096] - assert [kwargs["max_tokens"] for _, kwargs in goal_calls] == [4096, 4096] + assert [kwargs["max_tokens"] for _, kwargs in pair_calls] == [verifier.STRUCTURED_JUDGE_MAX_TOKENS] * 3 + assert [kwargs["max_tokens"] for _, kwargs in goal_calls] == [verifier.STRUCTURED_JUDGE_MAX_TOKENS] * 2 assert "previous reply could not be parsed or validated" in pair_calls[1][0] assert "previous reply could not be parsed or validated" in goal_calls[1][0] @@ -606,7 +606,7 @@ def goal_call(prompt: str, **kwargs): assert len(pair_calls) == 3 accuracy_attempts = [call for call in pair_calls if "SKILL_IDENTIFIED" in call[0]] assert len(accuracy_attempts) == 2 - assert [kwargs["max_tokens"] for _, kwargs in accuracy_attempts] == [4096, 4096] + assert [kwargs["max_tokens"] for _, kwargs in accuracy_attempts] == [verifier.STRUCTURED_JUDGE_MAX_TOKENS] * 2 assert len(goal_calls) == 1 assert "previous reply could not be parsed or validated" in pair_calls[1][0] assert "previous reply could not be parsed or validated" not in pair_calls[2][0] diff --git a/tests/tier3/test_judge_parse_robustness.py b/tests/tier3/test_judge_parse_robustness.py index 5bd9fa3f..db96a688 100644 --- a/tests/tier3/test_judge_parse_robustness.py +++ b/tests/tier3/test_judge_parse_robustness.py @@ -36,6 +36,7 @@ import pytest +from skillevaluator import provider_config from skillevaluator.tier3.eval_core import llm_judge _TEMPLATE = ( @@ -470,26 +471,60 @@ def test_template_behavior_judge_max_tokens_matches_eval_core_constant(): assert llm_judge.BEHAVIOR_JUDGE_MAX_TOKENS >= 4096 -def test_template_default_judge_model_matches_central_constant(): - from skillevaluator.provider_config import CHAT_DEFAULT_OPENAI +def test_template_default_judge_models_match_provider_chat_defaults(): + assert eval_template.DEFAULT_JUDGE_MODELS == provider_config.CHAT_DEFAULT_MODELS - assert eval_template.DEFAULT_JUDGE_MODEL == llm_judge.DEFAULT_JUDGE_MODEL - assert llm_judge.DEFAULT_JUDGE_MODEL == CHAT_DEFAULT_OPENAI - assert CHAT_DEFAULT_OPENAI == "gpt-5.6-sol" +@pytest.mark.parametrize("provider", sorted(provider_config.CHAT_DEFAULT_MODELS)) +def test_template_last_resort_judge_follows_the_selected_provider(judge_env, provider): + judge_env.setenv("SKILL_EVAL_LLM_PROVIDER", provider) -def test_template_gpt5_temperature_guard_matches_eval_core(): - for model in ( + assert eval_template._selected_judge_model() == provider_config.CHAT_DEFAULT_MODELS[provider] + + +@pytest.mark.parametrize( + "model", + [ "gpt-5.6-sol", "openai/gpt-5.6-sol", "openai/openai/gpt-5.6-sol", "gpt-5.4-mini", + "gpt-6.1-sol", + "openai/openai/gpt-6.1-sol", "gpt-4.1-mini", + "gpt-oss-120b", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-5-5", + "us.anthropic.claude-opus-5-5", + "aws/anthropic/bedrock-claude-opus-5", + "aws/anthropic/bedrock-claude-opus-5-5", + "claude-3-5-sonnet-20241022", "claude-mythos-preview", "anthropic/claude-mythos-preview", - ): - assert eval_template._model_leaf(model) == llm_judge._model_leaf(model) - assert eval_template._supports_custom_temperature(model) == llm_judge._supports_custom_temperature(model) + "nvidia/nemotron-3-super-120b-a12b", + ], +) +def test_template_model_capabilities_match_provider_config(model): + assert eval_template._model_leaf(model) == provider_config._model_leaf(model) + assert eval_template._supports_custom_temperature(model) == provider_config._supports_custom_temperature(model) + assert eval_template._is_openai_reasoning_model(model) == provider_config.is_openai_reasoning_model(model) + assert eval_template._is_claude_5_5_or_later(model) == provider_config.is_claude_5_5_or_later(model) + assert eval_template._completion_token_limit(model, 512) == provider_config.completion_token_limit(model, 512) + for provider in ("openai", "openai-compatible", "nv_build", "anthropic", "bedrock"): + assert eval_template._effective_reasoning_effort( + provider, model, "medium" + ) == provider_config.effective_reasoning_effort(provider, model, "medium") + + +def test_template_budget_effort_and_fallback_constants_match_package(): + assert eval_template.REASONING_MAX_COMPLETION_TOKENS == provider_config.REASONING_MAX_COMPLETION_TOKENS + assert eval_template.STRUCTURED_JUDGE_MAX_TOKENS == llm_judge.STRUCTURED_JUDGE_MAX_TOKENS + assert llm_judge.STRUCTURED_JUDGE_MAX_TOKENS == provider_config.MAX_COMPLETION_TOKENS + assert eval_template.JUDGE_REASONING_EFFORT == llm_judge.JUDGE_REASONING_EFFORT + assert eval_template.ANTHROPIC_REFUSAL_FALLBACK_MODEL == provider_config.ANTHROPIC_REFUSAL_FALLBACK_MODEL + assert eval_template.ANTHROPIC_SERVER_SIDE_FALLBACK_BETA == provider_config.ANTHROPIC_SERVER_SIDE_FALLBACK_BETA @pytest.mark.parametrize( @@ -607,7 +642,7 @@ def fake_urlopen(request, timeout): assert result["score"] == 1.0 assert len(requests) == 2 - assert [request["max_tokens"] for request in requests] == [4096, 4096] + assert [request["max_tokens"] for request in requests] == [eval_template.REASONING_MAX_COMPLETION_TOKENS] * 2 assert "previous reply could not be parsed or validated" in requests[1]["messages"][0]["content"] @@ -686,3 +721,179 @@ def test_template_salvage_score_matches_eval_core_on_partial_recovery(monkeypatc assert template == shared assert shared["score"] == round(3 / 7, 4) assert "3/7" in template["reason"] + + +# --------------------------------------------------------------------------- +# Reasoning-model request shape, refusals and judge provenance +# --------------------------------------------------------------------------- + +VALID_GOAL_RESPONSE = json.dumps( + {"user_goal": "do it", "end_state": "done", "achieved": True, "score": 1.0, "reason": "met"} +) + + +@pytest.fixture +def judge_env(monkeypatch): + for name in ( + "LLM_JUDGE_MODEL", + "SKILL_EVAL_JUDGE_MODEL", + "SKILL_EVAL_LLM_MODEL", + "LLM_JUDGE_FALLBACK_MODELS", + "SKILL_EVAL_LLM_BASE_URL", + "ANTHROPIC_BASE_URL", + "OPENAI_BASE_URL", + ): + monkeypatch.delenv(name, raising=False) + return monkeypatch + + +def _anthropic_body(text, *, stop_reason="end_turn", served_by=None): + content = [] + if served_by: + content.append({"type": "fallback", "from": {"model": "requested"}, "to": {"model": served_by}}) + if text: + content.append({"type": "text", "text": text}) + return {"content": content, "stop_reason": stop_reason} + + +def _replay(monkeypatch, replies): + requests = [] + pending = iter(replies) + + def fake_urlopen(request, timeout): + requests.append(request) + return _JSONResponse(next(pending)) + + monkeypatch.setattr(eval_template.urllib.request, "urlopen", fake_urlopen) + return requests + + +@pytest.mark.parametrize(("model", "claude_5_5"), [("claude-opus-5-5", True), ("claude-opus-5", False)]) +def test_template_anthropic_claude_5_5_sends_effort_and_server_side_fallback(judge_env, model, claude_5_5): + judge_env.setenv("ANTHROPIC_API_KEY", "test-key") + requests = _replay(judge_env, [_anthropic_body("Done")]) + + assert eval_template._call_anthropic("prompt", model, 4096, 0.0) == ("Done", None) + + payload = json.loads(requests[0].data) + if claude_5_5: + assert payload["output_config"] == {"effort": "medium"} + assert payload["fallbacks"] == [{"model": "claude-opus-4-8"}] + assert requests[0].get_header("Anthropic-beta") == "server-side-fallback-2026-07-01" + else: + assert {"output_config", "fallbacks"}.isdisjoint(payload) + assert requests[0].get_header("Anthropic-beta") is None + + +def test_template_anthropic_gateway_gets_effort_without_server_side_fallback(judge_env): + judge_env.setenv("ANTHROPIC_API_KEY", "test-key") + judge_env.setenv("ANTHROPIC_BASE_URL", "https://gateway.example") + requests = _replay(judge_env, [_anthropic_body("Done")]) + + eval_template._call_anthropic("prompt", "claude-opus-5-5", 4096, 0.0) + + payload = json.loads(requests[0].data) + assert payload["output_config"] == {"effort": "medium"} + assert "fallbacks" not in payload + + +def test_template_required_judge_records_the_model_that_answered(judge_env): + judge_env.setenv("SKILL_EVAL_LLM_PROVIDER", "anthropic") + judge_env.setenv("ANTHROPIC_API_KEY", "test-key") + judge_env.setenv("LLM_JUDGE_MODEL", "claude-opus-5-5") + _replay(judge_env, [_anthropic_body(VALID_GOAL_RESPONSE, served_by="claude-opus-4-8")]) + + result = eval_template._call_required_judge( + "goal_accuracy", eval_template._judge_goal_accuracy_custom, "q", "gt", "agent", "" + ) + + assert result["score"] == 1.0 + assert result["judge_model_used"] == "claude-opus-4-8" + assert result["judge_reasoning_effort"] == "medium" + assert result["judge_fallback_used"] is True + + +def test_template_anthropic_refusal_tries_the_configured_fallback_model(judge_env): + judge_env.setenv("SKILL_EVAL_LLM_PROVIDER", "anthropic") + judge_env.setenv("ANTHROPIC_API_KEY", "test-key") + judge_env.setenv("LLM_JUDGE_FALLBACK_MODELS", "claude-opus-4-8") + requests = _replay(judge_env, [_anthropic_body("", stop_reason="refusal"), _anthropic_body("Done")]) + + content, error, provenance = eval_template._call_public_llm_with_provenance("prompt", model="claude-opus-5-5") + + assert (content, error) == ("Done", None) + assert provenance == {"provider": "anthropic", "model": "claude-opus-4-8"} + assert [json.loads(request.data)["model"] for request in requests] == ["claude-opus-5-5", "claude-opus-4-8"] + + +def test_template_refusal_without_a_fallback_is_a_judge_error(judge_env): + judge_env.setenv("SKILL_EVAL_LLM_PROVIDER", "anthropic") + judge_env.setenv("ANTHROPIC_API_KEY", "test-key") + _replay(judge_env, [_anthropic_body("", stop_reason="refusal")]) + + content, error = eval_template.call_public_llm("prompt", model="claude-opus-5-5") + + assert content is None + assert "declined the request" in error + + +@pytest.mark.parametrize( + "first_reply", + [ + pytest.param({"choices": [{"finish_reason": "content_filter", "message": {"content": ""}}]}, id="filter"), + pytest.param({"choices": [{"message": {"content": None, "refusal": "I can't help."}}]}, id="refusal"), + pytest.param({"choices": [{"finish_reason": "length", "message": {"content": ""}}]}, id="empty"), + ], +) +def test_template_openai_refusal_or_empty_reply_tries_the_fallback_model(judge_env, first_reply): + judge_env.setenv("SKILL_EVAL_LLM_PROVIDER", "openai") + judge_env.setenv("OPENAI_API_KEY", "test-key") + judge_env.setenv("SKILL_EVAL_LLM_BASE_URL", "https://gateway.example/v1") + judge_env.setenv("LLM_JUDGE_FALLBACK_MODELS", "fallback-model") + _replay(judge_env, [first_reply, {"choices": [{"message": {"content": "Done"}}]}]) + + content, error, provenance = eval_template._call_public_llm_with_provenance("prompt", model="primary-model") + + assert (content, error, provenance["model"]) == ("Done", None, "fallback-model") + + +@pytest.mark.parametrize("model", ["us.anthropic.claude-opus-5-5", "eu.anthropic.claude-sonnet-5-5"]) +def test_template_bedrock_classifier_refusal_retries_on_opus_4_8(monkeypatch, model): + import boto3 + + client = MagicMock() + client.converse.side_effect = [ + {"stopReason": "content_filtered", "output": {"message": {"content": []}}}, + {"stopReason": "end_turn", "output": {"message": {"content": [{"text": "Done"}]}}}, + ] + monkeypatch.setattr(boto3, "client", lambda *_args, **_kwargs: client) + provenance = {"provider": "bedrock", "model": model} + + content, error = eval_template._call_bedrock("prompt", model, 4096, 0.0, provenance=provenance) + + fallback = model.split("anthropic.", 1)[0] + "anthropic.claude-opus-4-8" + assert (content, error) == ("Done", None) + assert [call.kwargs["modelId"] for call in client.converse.call_args_list] == [model, fallback] + assert provenance["model"] == fallback + + +def test_template_bedrock_refusal_without_a_fallback_route_is_reported(monkeypatch): + import boto3 + + client = MagicMock() + client.converse.return_value = {"stopReason": "content_filtered", "output": {"message": {"content": []}}} + monkeypatch.setattr(boto3, "client", lambda *_args, **_kwargs: client) + + with pytest.raises(eval_template._JudgeRefusedError): + eval_template._call_bedrock("prompt", "us.anthropic.claude-opus-5", 4096, 0.0) + + assert client.converse.call_count == 1 + + +@pytest.mark.parametrize(("model", "enabled"), [("gpt-5.6-sol", False), ("gpt-6.1-sol", False), ("gpt-4.1-mini", True)]) +def test_template_ragas_goal_scorer_skips_reasoning_judges(judge_env, model, enabled): + judge_env.setenv("SKILL_EVAL_LLM_PROVIDER", "openai") + judge_env.setenv("OPENAI_API_KEY", "test-key") + judge_env.setenv("LLM_JUDGE_MODEL", model) + + assert eval_template._ragas_goal_accuracy_enabled() is enabled diff --git a/tests/tier3/test_judge_result_semantics.py b/tests/tier3/test_judge_result_semantics.py index 070baee8..c2a2f1b5 100644 --- a/tests/tier3/test_judge_result_semantics.py +++ b/tests/tier3/test_judge_result_semantics.py @@ -126,7 +126,7 @@ def test_accuracy_retries_invalid_response_once_and_recovers( assert result["score"] == 0.6 assert result["criteria"] == _CRITERIA - assert [call["kwargs"]["max_tokens"] for call in calls] == [4096, 4096] + assert [call["kwargs"]["max_tokens"] for call in calls] == [judge_module.STRUCTURED_JUDGE_MAX_TOKENS] * 2 assert "previous reply could not be parsed or validated" in calls[1]["prompt"] @@ -234,7 +234,7 @@ def test_goal_retries_invalid_response_once_and_recovers( assert result["score"] == 1.0 assert result["user_goal"] == "complete the task" - assert [call["kwargs"]["max_tokens"] for call in calls] == [4096, 4096] + assert [call["kwargs"]["max_tokens"] for call in calls] == [judge_module.STRUCTURED_JUDGE_MAX_TOKENS] * 2 assert "previous reply could not be parsed or validated" in calls[1]["prompt"] if judge_module is eval_template: assert result["provider"] == "retry-provider" diff --git a/tests/utils/test_tool_runner.py b/tests/utils/test_tool_runner.py index 29d44607..ebe88420 100644 --- a/tests/utils/test_tool_runner.py +++ b/tests/utils/test_tool_runner.py @@ -353,7 +353,10 @@ def test_skillspector_install_hint_uses_public_source_distribution(self): with patch("shutil.which", return_value=None): tool = ExternalTool("SkillSpector", "skillspector") - assert tool.get_install_hint() == "Install with: uv tool install git+https://github.com/NVIDIA/SkillSpector.git" + assert ( + tool.get_install_hint() + == "Install with: uv tool install git+https://github.com/NVIDIA/SkillSpector.git@v2.12.0" + ) assert "[security]" not in tool.get_install_hint() def test_bundled_scanner_hints_point_at_security_extra(self): diff --git a/tests/validators/test_security.py b/tests/validators/test_security.py index db575e3f..244142b9 100644 --- a/tests/validators/test_security.py +++ b/tests/validators/test_security.py @@ -12,6 +12,7 @@ import pytest from skillevaluator.config import load_pii_patterns +from skillevaluator.provider_config import CHAT_DEFAULT_MODELS from skillevaluator.reporting import CLIReporter, HTMLReporter, JSONReporter, MarkdownReporter from skillevaluator.utils.tool_runner import ToolResult, Tools from skillevaluator.validators.base import Finding, Severity, ValidationResult @@ -3909,10 +3910,11 @@ def test_external_skillspector_receives_public_nvidia_build_openai_contract(self assert "NVIDIA_API_KEY" not in child_env @pytest.mark.parametrize( - ("provider", "credential", "model"), + ("provider", "credential", "model", "effort"), ( - ("openai", "OPENAI_API_KEY", "gpt-5.6-sol"), - ("anthropic", "ANTHROPIC_API_KEY", "claude-opus-5"), + ("openai", "OPENAI_API_KEY", "gpt-6.1-sol", "medium"), + ("anthropic", "ANTHROPIC_API_KEY", "claude-opus-5", None), + ("bedrock", "AWS_BEARER_TOKEN_BEDROCK", "us.anthropic.claude-opus-5", None), ), ) def test_skillspector_child_environment_maps_supported_public_provider_defaults( @@ -3921,17 +3923,21 @@ def test_skillspector_child_environment_maps_supported_public_provider_defaults( provider: str, credential: str, model: str, + effort: str | None, ) -> None: monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", provider) monkeypatch.setenv(credential, "provider-test-key") + monkeypatch.delenv("SKILL_EVAL_LLM_MODEL", raising=False) monkeypatch.delenv("SKILLSPECTOR_PROVIDER", raising=False) monkeypatch.delenv("SKILLSPECTOR_MODEL", raising=False) + monkeypatch.delenv("SKILLSPECTOR_REASONING_EFFORT", raising=False) child_env = _skillspector_child_env() assert child_env is not None assert child_env["SKILLSPECTOR_PROVIDER"] == provider assert child_env["SKILLSPECTOR_MODEL"] == model + assert child_env.get("SKILLSPECTOR_REASONING_EFFORT") == effort def test_skillspector_bedrock_environment_keeps_only_aws_chain(self, monkeypatch) -> None: monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "bedrock") @@ -3991,6 +3997,150 @@ def test_skillspector_child_environment_explicit_compatible_config_overrides_amb assert os.environ["OPENAI_API_KEY"] == "ambient-openai-key" assert os.environ["OPENAI_BASE_URL"] == "https://ambient.example.test/v1" + @pytest.mark.parametrize( + ("provider", "credentials", "expected"), + ( + ("openai", {"OPENAI_API_KEY": "provider-test-key"}, "gpt-6.1-sol"), + ("anthropic", {"ANTHROPIC_API_KEY": "provider-test-key"}, "claude-opus-5"), + ("bedrock", {"AWS_BEARER_TOKEN_BEDROCK": "provider-test-key"}, "us.anthropic.claude-opus-5"), + ("nv_build", {"NVIDIA_API_KEY": "provider-test-key"}, "nvidia/nemotron-3-super-120b-a12b"), + ( + "openai-compatible", + { + "SKILL_EVAL_LLM_API_KEY": "provider-test-key", + "SKILL_EVAL_LLM_BASE_URL": "https://llm.example.test/v1", + }, + "nvidia/nvidia/nemotron-3-super-120b-long-ctx", + ), + ), + ) + def test_skillspector_model_does_not_follow_a_changed_chat_default( + self, monkeypatch, provider: str, credentials: dict[str, str], expected: str + ) -> None: + monkeypatch.setitem(CHAT_DEFAULT_MODELS, provider, "changed-chat-default") + monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", provider) + for name, value in credentials.items(): + monkeypatch.setenv(name, value) + for name in ("SKILL_EVAL_LLM_MODEL", "SKILLSPECTOR_PROVIDER", "SKILLSPECTOR_MODEL"): + monkeypatch.delenv(name, raising=False) + + assert _skillspector_child_env()["SKILLSPECTOR_MODEL"] == expected + + @pytest.mark.parametrize( + ("skillspector_model", "expected"), + ((None, "claude-opus-5-5"), ("claude-opus-5", "claude-opus-5")), + ) + def test_explicit_models_reach_skillspector_in_precedence_order( + self, + monkeypatch, + skillspector_model: str | None, + expected: str, + ) -> None: + monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "provider-test-key") + monkeypatch.setenv("SKILL_EVAL_LLM_MODEL", "claude-opus-5-5") + monkeypatch.delenv("SKILLSPECTOR_PROVIDER", raising=False) + if skillspector_model is None: + monkeypatch.delenv("SKILLSPECTOR_MODEL", raising=False) + else: + monkeypatch.setenv("SKILLSPECTOR_MODEL", skillspector_model) + + assert _skillspector_child_env()["SKILLSPECTOR_MODEL"] == expected + + @pytest.mark.parametrize("skillspector_provider", (None, "openai")) + def test_operator_skillspector_model_settings_are_forwarded_without_temperature( + self, + monkeypatch, + skillspector_provider: str | None, + ) -> None: + monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "openai") + monkeypatch.setenv("OPENAI_API_KEY", "provider-test-key") + if skillspector_provider is None: + monkeypatch.delenv("SKILLSPECTOR_PROVIDER", raising=False) + else: + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", skillspector_provider) + monkeypatch.setenv("SKILLSPECTOR_MODEL_META_ANALYZER", "gpt-5.6-sol") + monkeypatch.setenv("SKILLSPECTOR_MODEL_REGISTRY", "/etc/skillspector/model_registry.yaml") + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "high") + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") + + child_env = _skillspector_child_env() + + assert child_env["SKILLSPECTOR_MODEL_META_ANALYZER"] == "gpt-5.6-sol" + assert child_env["SKILLSPECTOR_MODEL_REGISTRY"] == "/etc/skillspector/model_registry.yaml" + assert child_env["SKILLSPECTOR_REASONING_EFFORT"] == "high" + assert "SKILLSPECTOR_TEMPERATURE" not in child_env + + @pytest.mark.parametrize( + ("environment", "effort"), + ( + ({}, None), + ({"SKILL_EVAL_LLM_MODEL": "openai/openai/gpt-6.1-sol"}, "medium"), + ({"SKILL_EVAL_LLM_MODEL": "aws/anthropic/bedrock-claude-opus-5-5"}, None), + ({"SKILLSPECTOR_MODEL": "gpt-6.1-sol"}, "medium"), + ({"SKILLSPECTOR_MODEL": "gpt-5.6-sol"}, None), + ({"SKILLSPECTOR_MODEL": "gpt-6.1-sol", "SKILLSPECTOR_REASONING_EFFORT": "high"}, "high"), + ({"SKILLSPECTOR_MODEL": "gpt-6.1-sol", "SKILLSPECTOR_MODEL_META_ANALYZER": "gpt-6.1-sol"}, "medium"), + ({"SKILLSPECTOR_MODEL": "gpt-6.1-sol", "SKILLSPECTOR_MODEL_META_ANALYZER": "claude-opus-5-5"}, None), + ({"SKILLSPECTOR_MODEL": "gpt-6.1-sol", "SKILLSPECTOR_MODEL_REGISTRY": "/registry.yaml"}, "medium"), + ({"SKILLSPECTOR_PROVIDER": "openai", "SKILLSPECTOR_MODEL": "gpt-6.1-sol"}, "medium"), + ({"SKILLSPECTOR_PROVIDER": "openai"}, None), + ), + ) + def test_skillspector_reasoning_effort_defaults_to_medium_only_for_gpt_6_models( + self, + monkeypatch, + environment: dict[str, str], + effort: str | None, + ) -> None: + monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "openai-compatible") + monkeypatch.setenv("SKILL_EVAL_LLM_API_KEY", "compatible-test-key") + monkeypatch.setenv("SKILL_EVAL_LLM_BASE_URL", "https://llm.example.test/v1") + for name in ( + "SKILL_EVAL_LLM_MODEL", + "SKILLSPECTOR_PROVIDER", + "SKILLSPECTOR_MODEL", + "SKILLSPECTOR_MODEL_META_ANALYZER", + "SKILLSPECTOR_MODEL_REGISTRY", + "SKILLSPECTOR_REASONING_EFFORT", + ): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + + assert _skillspector_child_env().get("SKILLSPECTOR_REASONING_EFFORT") == effort + + def test_llm_stage_records_requested_skillspector_model_and_effort( + self, + monkeypatch, + sample_skill_dir: Path, + ) -> None: + monkeypatch.setenv("SKILL_EVAL_LLM_PROVIDER", "openai") + monkeypatch.setenv("OPENAI_API_KEY", "provider-test-key") + monkeypatch.setenv("SKILL_EVAL_LLM_MODEL", "gpt-6.1-sol") + for name in ("SKILLSPECTOR_PROVIDER", "SKILLSPECTOR_MODEL", "SKILLSPECTOR_REASONING_EFFORT"): + monkeypatch.delenv(name, raising=False) + reports = (_skillspector_json_report(), _skillspector_json_report(llm_requested=True, llm_available=True)) + + with ( + patch.object(Tools.skillspector, "_path", "/usr/bin/skillspector"), + patch.object( + Tools.skillspector, + "run", + side_effect=[ + ToolResult(success=True, stdout=json.dumps(report), stderr="", exit_code=0) for report in reports + ], + ) as mock_run, + ): + result = SecurityValidator(use_llm=True).validate_security_only(sample_skill_dir) + + static_env, child_env = (call.kwargs["env"] for call in mock_run.call_args_list) + assert "SKILLSPECTOR_REASONING_EFFORT" not in static_env + assert child_env["SKILLSPECTOR_REASONING_EFFORT"] == "medium" + assert result.metadata["skillspector_model"] == "gpt-6.1-sol" + assert result.metadata["skillspector_reasoning_effort"] == "medium" + assert result.metadata["skillspector_version"] == "2.10.0" + def test_partial_llm_verdicts_are_reported_as_partial(self, tmp_path: Path) -> None: """A verifier response covering only some findings must not claim full confirmation.""" result = ValidationResult() @@ -4019,6 +4169,43 @@ def test_partial_llm_verdicts_are_reported_as_partial(self, tmp_path: Path) -> N assert not any("confirmed all findings" in message.lower() for message in result.messages) assert any("1 of 2 findings" in message and "not verified" in message for message in result.messages) + @pytest.mark.parametrize("explanation_key", ["rationale", "reasoning"]) + def test_llm_verdict_explanation_reads_rationale_or_legacy_reasoning( + self, tmp_path: Path, explanation_key: str + ) -> None: + """The prompt asks for "rationale"; a reply using the older "reasoning" key still counts.""" + result = ValidationResult() + result.add_structured_finding( + Finding(category="PII", severity=Severity.HIGH, check_name="email", message="Email", file_path="SKILL.md"), + is_error=True, + ) + + with patch("skillevaluator.inference.FindingVerifier") as verifier_cls: + verifier_cls.return_value.verify.return_value = { + 0: {"verdict": "false_positive", "confidence": "high", explanation_key: "Documentation example."} + } + SecurityValidator(verify_llm=True)._verify_findings_with_llm(result, tmp_path) + + assert result.findings[0].metadata["llm_reasoning"] == "Documentation example." + assert result.findings[0].severity == Severity.INFO + assert "incomplete_scans" not in result.metadata + + def test_llm_verification_without_verdicts_reports_the_cause(self, tmp_path: Path) -> None: + result = ValidationResult() + result.add_structured_finding( + Finding(category="PII", severity=Severity.HIGH, check_name="email", message="Email", file_path="SKILL.md"), + is_error=True, + ) + cause = "The model declined the request (safety refusal)." + + with patch("skillevaluator.inference.FindingVerifier") as verifier_cls: + verifier_cls.return_value.verify.return_value = {} + verifier_cls.return_value.last_failure = cause + SecurityValidator(verify_llm=True)._verify_findings_with_llm(result, tmp_path) + + assert result.metadata["incomplete_scans"] == ["llm-verification"] + assert f"LLM finding verification was skipped (no verdicts returned: {cause})" in result.messages + def test_validate_security_only(self, sample_skill_dir: Path): """Test validate_security_only method runs only skillspector scan.""" validator = SecurityValidator()