From 0e989581316a8483d3db41900e28698599a760a7 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Tue, 22 Sep 2026 17:33:19 +0200 Subject: [PATCH 01/11] Add native harness artifact conformance checks and onboarding guidance Signed-off-by: Frederic Lafrance --- .../observability/harness-conformance.mdx | 32 + nemo_gym/cli/main.py | 39 + nemo_gym/harness_capabilities/README.md | 192 +++++ nemo_gym/harness_capabilities/__init__.py | 14 + nemo_gym/harness_capabilities/__main__.py | 7 + nemo_gym/harness_capabilities/checker.py | 620 ++++++++++++++++ nemo_gym/harness_capabilities/cli.py | 189 +++++ nemo_gym/harness_capabilities/reader.py | 113 +++ nemo_gym/harness_capabilities/schemas.py | 142 ++++ pyproject.toml | 5 + .../tests/test_observability.py | 11 + .../fixtures/miniswe.json | 675 ++++++++++++++++++ .../fixtures/opencode.json | 241 +++++++ .../harness_capabilities/test_checker.py | 190 +++++ .../harness_capabilities/test_gym_cli.py | 166 +++++ .../harness_capabilities/test_producers.py | 121 ++++ uv.lock | 2 + 17 files changed, 2759 insertions(+) create mode 100644 fern/versions/latest/pages/observability/harness-conformance.mdx create mode 100644 nemo_gym/harness_capabilities/README.md create mode 100644 nemo_gym/harness_capabilities/__init__.py create mode 100644 nemo_gym/harness_capabilities/__main__.py create mode 100644 nemo_gym/harness_capabilities/checker.py create mode 100644 nemo_gym/harness_capabilities/cli.py create mode 100644 nemo_gym/harness_capabilities/reader.py create mode 100644 nemo_gym/harness_capabilities/schemas.py create mode 100644 tests/unit_tests/harness_capabilities/fixtures/miniswe.json create mode 100644 tests/unit_tests/harness_capabilities/fixtures/opencode.json create mode 100644 tests/unit_tests/harness_capabilities/test_checker.py create mode 100644 tests/unit_tests/harness_capabilities/test_gym_cli.py create mode 100644 tests/unit_tests/harness_capabilities/test_producers.py diff --git a/fern/versions/latest/pages/observability/harness-conformance.mdx b/fern/versions/latest/pages/observability/harness-conformance.mdx new file mode 100644 index 00000000000..04188d88592 --- /dev/null +++ b/fern/versions/latest/pages/observability/harness-conformance.mdx @@ -0,0 +1,32 @@ +--- +title: "Harness Conformance" +description: "Check retained harness evidence and onboard a custom agent's observability." +position: 3 +--- + +# Harness Conformance + +Harness conformance checks whether collected rollouts retain model-call payloads, +token counts, invocation ownership, semantic turns, and tool execution evidence. +Run it after collecting an evaluation with observability enabled: + +```bash +gym eval conformance \ + --bundle results/my-harness/rollouts.jsonl \ + --profile gym-artifacts-p0/v1 \ + --output results/my-harness/capabilities +``` + +The command writes `capability_summary.json`, `capability_results.jsonl`, and +`capability_report.md` under an immutable report directory. Exit codes are 0 for a +met gate, 1 for an unmet gate, and 2 for a checker/input error. + +Follow the [harness owner onboarding README](https://github.com/NVIDIA-NeMo/Gym/blob/main/nemo_gym/harness_capabilities/README.md) +for capture configuration, invocation and tool contracts, profiles, troubleshooting, +and CI usage. Start with [Rollout Evidence](/main/observability/rollout-evidence) +if your harness does not yet emit agent observations. + +These are retained-artifact checks. Independent behavioral qualification and a live +probe runner are not implemented. P0 is the initial usable gate; P1 and All include +incomplete validators and currently cannot pass. Run +[rollout health checks](/main/evaluation/rollout-health) separately to assess health. diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index 043340079a9..eb36a72d253 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -649,6 +649,17 @@ def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: dispatch(target, overrides) +def _eval_conformance(args: argparse.Namespace, overrides: list[str]) -> None: + expected_overrides = ["+verbose=true"] if args.verbose else [] + if overrides != expected_overrides: + args._parser.error("conformance does not accept Hydra overrides") + from nemo_gym.harness_capabilities.cli import run_inspection + + sys.exit( + run_inspection(bundle=args.bundle, output=args.output, profile=args.profile, capture_dir=args.capture_dir) + ) + + def _eval_health_check(args: argparse.Namespace, overrides: list[str]) -> None: expected_overrides = ["+verbose=true"] if args.verbose else [] if args.json: @@ -1078,6 +1089,34 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: ), ), ), + "eval conformance": Command( + target=_eval_conformance, + summary="Check retained harness capability evidence in rollout artifacts.", + flags=( + Flag( + register=lambda p: p.add_argument( + "--bundle", required=True, type=Path, help="Rollout JSONL file or run directory." + ) + ), + Flag( + register=lambda p: p.add_argument( + "--output", required=True, type=Path, help="Directory for immutable capability reports." + ) + ), + Flag( + register=lambda p: p.add_argument( + "--capture-dir", type=Path, help="Original shared model-call capture directory, if needed." + ) + ), + Flag( + register=lambda p: p.add_argument( + "--profile", + default="gym-artifacts-p0/v1", + help="Artifact gate: gym-artifacts-p0/v1 (default), gym-artifacts-p1/v1, or gym-artifacts-all/v1.", + ) + ), + ), + ), "eval health-check": Command( target=_eval_health_check, summary="Verify rollout quality for an existing run directory.", diff --git a/nemo_gym/harness_capabilities/README.md b/nemo_gym/harness_capabilities/README.md new file mode 100644 index 00000000000..746d0c89d3a --- /dev/null +++ b/nemo_gym/harness_capabilities/README.md @@ -0,0 +1,192 @@ +# Harness capability conformance + +Check whether a Gym harness retains the evidence needed to inspect model calls, +token usage, invocation ownership, semantic turns, and tool execution. Harness +owners run this checker against their collected rollouts during onboarding and +after changes to their harness or its export path. + +## Run the checker + +Install Gym using the repository's development setup (`uv sync --extra dev`), +then inspect a completed evaluation: + +```bash +gym eval conformance \ + --bundle results/my-harness/rollouts.jsonl \ + --profile gym-artifacts-p0/v1 \ + --output results/my-harness/capabilities +``` + +`--bundle` accepts any rollout JSONL file. It also accepts a directory containing +exactly one `rollouts.jsonl` or `evaluator_rollouts.jsonl`, at its root or under +`artifacts/`. Pass the file explicitly if the directory contains multiple candidates. +Files are streamed one rollout at a time. No model endpoint is needed for inspection. + +The module interface exposes the same checks: + +```bash +python -m nemo_gym.harness_capabilities inspect \ + --bundle results/my-harness \ + --output results/my-harness/capabilities +``` + +Collection normally retains full payloads in `ng_trajectory.model_calls`. If you +also have the original shared model capture directory, add +`--capture-dir /absolute/path/to/model-calls`. This restores missing payloads and +checks exact call identity, conflicting representations, missing/extra calls, +and incomplete capture markers. Preserve the original capture directory until +inspection finishes; use a separate capture directory for each evaluation. + +Each inspection publishes a content-addressed directory containing: + +| File | Contents | +| --- | --- | +| `capability_summary.json` | Requested gate, C0–C11 verdicts, counts, source/checker/schema hashes, qualification limits | +| `capability_results.jsonl` | Per-rollout verdicts and findings with assertion IDs and source field locations | +| `capability_report.md` | Capability matrix and passing-record counts | + +Exit codes are **0** for a met gate, **1** for an unmet gate, and **2** for a +checker/input error. A gate requires every inspected record to pass each required +capability. An empty input is an error. Report files are published together only +after inspection completes and input hashes are rechecked. Repeating an identical +inspection reuses its report directory. Prompts and response bodies are omitted +from reports; source paths and field locations are retained. + +## Onboard a harness + +### 1. Enable model-call capture + +Add these top-level settings to the shared configuration used by your agent, +model servers, and collector: + +```yaml +observability_enabled: true +model_call_capture_dir: /absolute/shared/path/my-harness/model-calls +``` + +Route every model request through Gym's correlated model URL +`/ng-rollout//v1/...`. Preserve that URL when constructing an SDK +client, forwarding requests, spawning child agents, or retrying a call. Calls +sent directly to a provider do not reach Gym's capture middleware. See +[Model-call Capture](https://docs.nvidia.com/nemo/gym/main/model-server/model-call-capture) +for the custom-agent URL helpers and a runnable collection example. + +Use Gym's capture records for request/response bodies, HTTP or transport outcomes, +timing, and provider usage. Preserve absent token values as absent; do not replace +them with zero. P0 requires prompt, completion, reasoning, total, and cached token +counts on successful calls. A provider that omits one cannot meet this strict gate. + +### 2. Return invocation evidence + +Return `ng_agent_observations` using `AgentObservationBundle` and `AgentInvocation` +from `nemo_gym.rollout_observability`. Give each root or child invocation a stable +ID, retain the parent relationship, and record its ordered conversation. + +Associate every captured attempt with exactly one invocation. `ModelCallRef` +supports either `model_call_id` or the exact pair of `model_ref` and `response_id`. +All supplied reference fields must agree. For failures without response IDs, +propagate the invocation ID as `x-session-id` to the Gym model server; the collector +uses captured `client_session_id` to join those attempts to the invocation. +Model names, timestamps, and list order cannot establish ownership. Retries are +separate attempts and need separate references, even if their request bodies match. + +For a successful response, an invocation can include: + +```python +from nemo_gym.rollout_observability import AgentInvocation, AgentObservationBundle, ModelCallRef + +observations = AgentObservationBundle( + source="my_harness", + records=[ + AgentInvocation( + invocation_id=invocation_id, + model_calls=[ModelCallRef(model_ref=model_ref, response_id=response.id)], + conversation=conversation, + ) + ], +) +``` + +Here `invocation_id`, `model_ref`, `response`, and `conversation` come from the +actual harness execution. Return the bundle alongside the normal `/run` response; +the collector joins captures and produces `ng_trajectory`. + +### 3. Add turn and tool evidence + +For semantic turns, supply explicit `TrajectoryTurn` entries in a producer +`ng_trajectory`, including task/rollout/invocation identity, decision number, +timestamp, input, answer, executed-step count, resolution state, and exact call +references. The collector preserves these entries. Keep `resolved=None` until an +authoritative result exists; do not propagate a final reward to intermediate turns. + +For tools, put requests and results in the invocation conversation and measured +execution intervals in `ToolCallObservation`. Match both by invocation and +tool-call ID. Record start/end/duration, tool name, timing source, and terminal +status, including failures, timeouts, cancellation, and final submission. A model +request to execute a tool is insufficient evidence that execution occurred. + +See [Rollout Evidence](https://docs.nvidia.com/nemo/gym/main/observability/rollout-evidence) +and `nemo_gym/rollout_observability.py` for the producer contracts. + +### 4. Collect, inspect, and fix gaps + +Run a small representative evaluation with your harness configuration and capture +enabled, using normal `gym env start` / `gym eval run` commands. Wait for collection +to finish, then run `gym eval conformance` on the output JSONL. Inspect failing +assertions in `capability_results.jsonl`, fix the producer, and collect a fresh run. +Editing the report or fabricating missing evidence cannot qualify the harness. + +Keep regression coverage for successful calls, tool failures, HTTP errors, retries, +missing usage, rejected model output, and cancellation as supported by your harness. +Include concurrent/child invocation cases if supported. Check the actual emitted +records, including ownership of attempts that never produce an accepted answer. + +Run `gym eval health-check RUN_DIR --rollouts-file PATH` separately to assess +rollout health. Conformance reports and `quality_summary.json` answer different +questions; conformance does not rewrite health findings or assert health readiness. +For onboarding review, retain the harness version, run configuration, native +rollouts/captures, conformance reports, and representative rollout/health evidence. +Attach reports to your normal CI artifacts if using the exit status as a gate. + +## Profiles and supported scope + +These profiles check **retained artifacts**. Their verdicts do not independently +certify retry/cancellation scenario coverage, capture completeness, or execution +behavior. No live probe runner is included. Reports state +`is_behavioral_qualification: false` and zero witnessed scenario coverage. + +| Capability | Required in | Current check | +| --- | --- | --- | +| C0: records and identity | P0, P1, All | Readable native records, task/rollout identity, trajectory version, reader integrity | +| C1: calls and statuses | P0, P1, All | Unique call IDs, model/dialect, timing, terminal HTTP/transport evidence | +| C2: five token counts | P0, P1, All | Nonnegative counts, faithful absence/zero, retained usage agreement, subset/total semantics | +| C3: semantic turns | All | Explicit turn fields and resolvable invocation-owned call references | +| C4: model-visible history | P0, P1, All | Ordered retained requests/responses; unresolved media or opaque reasoning fails | +| C5: tool execution | P1, All | Executed tool IDs, request/result joins, terminal status and separate timing | +| C6: independent parallel timing | P1, All | Validator incomplete; always unfulfilled | +| C7: resource lifecycle | All | Validator incomplete; always unfulfilled | +| C8: verifier provenance | All | Validator incomplete; always unfulfilled | +| C9: call payloads | P0, P1, All | Retained request and response/error, explicit no-response transport failure | +| C10: invocation ownership | P0, P1, All | Exactly one explicit owner per captured attempt, consistent references and parent graph | +| C11: complete turn-call accounting | All | Validator incomplete; reference defects diagnosed, completeness never certified | + +Profile names are `gym-artifacts-p0/v1`, `gym-artifacts-p1/v1`, and +`gym-artifacts-all/v1`. P1 and All currently always fail their gates because they +require incomplete validators. All C0–C11 verdicts are reported even when selecting +P0, so owners can inspect C3/C5 independently. `onboarding-p0/v1` is not an available +profile. The supported input is native Gym trajectory schema `1.0`; ATIF exports +and normalized third-party records need separate readers. + +## Develop the checker + +```bash +python -m pytest tests/unit_tests/harness_capabilities -q +``` + +Contract fixtures use synthetic inputs through Gym's mini-SWE/OpenCode producers; +they are not live harness qualifications. Mutation tests reject missing payloads, +fabricated token counts, broken ownership, duplicate attempts, and missing tool +execution. Integration tests exercise current Gym producer/collector code and the +native CLI. Schema predicates are versioned in `schemas.py`; profile requirements +are in `checker.py`. Keep regression tests and documentation aligned when changing +either contract. diff --git a/nemo_gym/harness_capabilities/__init__.py b/nemo_gym/harness_capabilities/__init__.py new file mode 100644 index 00000000000..9b7db91f275 --- /dev/null +++ b/nemo_gym/harness_capabilities/__init__.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Portable artifact contracts for harness observability. + +Readers supply native Gym JSON; +artifact conformance is deliberately separate from behavioral qualification. +""" + +from .checker import inspect_record + + +__all__ = ["inspect_record"] +__version__ = "0.1.1" diff --git a/nemo_gym/harness_capabilities/__main__.py b/nemo_gym/harness_capabilities/__main__.py new file mode 100644 index 00000000000..1da38ba7de3 --- /dev/null +++ b/nemo_gym/harness_capabilities/__main__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from .cli import main + + +raise SystemExit(main()) diff --git a/nemo_gym/harness_capabilities/checker.py b/nemo_gym/harness_capabilities/checker.py new file mode 100644 index 00000000000..b1c3b41920d --- /dev/null +++ b/nemo_gym/harness_capabilities/checker.py @@ -0,0 +1,620 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Deterministic checks over native Gym JSON; no producer runtime imports.""" + +import base64 +import binascii +from collections import Counter +from dataclasses import asdict, dataclass +from typing import Iterable, TypeGuard + +from jsonschema import Draft202012Validator + +from .schemas import CALL_REF, SCHEMAS + + +NAMES = { + "C0": "readable records and identity", + "C1": "model calls and statuses", + "C2": "five token counts", + "C3": "canonical semantic turns", + "C4": "model-visible history", + "C5": "tool execution evidence", + "C6": "independent parallel timing", + "C7": "resource lifecycle", + "C8": "verifier resolution and rewards", + "C9": "model-call payloads", + "C10": "invocation ownership", + "C11": "complete turn-call accounting", +} +P0 = ("C0", "C1", "C2", "C4", "C9", "C10") +PENDING_VALIDATORS = frozenset({"C6", "C7", "C8", "C11"}) +PROFILES = { + "gym-artifacts-p0/v1": P0, + "gym-artifacts-p1/v1": (*P0, "C5", "C6"), + "gym-artifacts-all/v1": tuple(NAMES), +} + + +@dataclass(frozen=True) +class Finding: + capability: str + assertion: str + location: str + reason: str + + +def _schema_issues(capability: str, value: object, location: str) -> list[Finding]: + # Never put error.message in reports: jsonschema embeds source payloads. + return [ + Finding( + capability, + "schema." + str(error.validator), + location + "/" + "/".join(map(str, error.absolute_path)), + "required artifact contract is not satisfied", + ) + for error in Draft202012Validator(SCHEMAS[capability]).iter_errors(value) + ] + + +def _objects(value: object) -> list[dict]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def _mapping(value: object) -> dict: + return value if isinstance(value, dict) else {} + + +def _number(value: object) -> TypeGuard[int | float]: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _successful(call: dict) -> bool: + code = call.get("status_code") + return isinstance(code, int) and 200 <= code < 300 and not call.get("error_category") + + +def _missing_content(value: object) -> bool: + """Reject unresolved media/opaque references; validate inline image data.""" + if isinstance(value, list): + return any(_missing_content(item) for item in value) + if isinstance(value, dict): + unavailable = any(value.get(key) for key in ("file_id", "file_url", "encrypted_content")) + if value.get("image_url"): + image = value["image_url"] + image = image.get("url") if isinstance(image, dict) else image + if not isinstance(image, str) or ";base64," not in image or not image.startswith("data:"): + unavailable = True + else: + try: + if not base64.b64decode(image.split(";base64,", 1)[1], validate=True): + unavailable = True + except (ValueError, binascii.Error): + unavailable = True + return bool(unavailable) or any(_missing_content(item) for item in value.values()) + return False + + +def _resolve(reference: dict, calls: list[dict]) -> list[int]: + if not Draft202012Validator(CALL_REF).is_valid({k: v for k, v in reference.items() if v is not None}): + return [] + return [ + index + for index, call in enumerate(calls) + if all( + call.get(key) == reference[key] + for key in ("model_call_id", "model_ref", "response_id") + if reference.get(key) is not None + ) + ] + + +class _RecordInspector: + """Shared evidence and findings for capability-specific artifact checks.""" + + def __init__(self, record: dict, source: str) -> None: + self.record = record + self.source = source + self.findings: list[Finding] = [] + self.trajectory = _mapping(self.record.get("ng_trajectory")) + self.capture = _mapping(self.record.get("ng_model_call_capture")) + self.bundle = _mapping(self.record.get("ng_agent_observations")) + self.calls = _objects(self.capture.get("calls")) + self.observations = _objects(self.bundle.get("records")) + self.invocations = [r for r in self.observations if r.get("kind") == "agent_invocation"] + if not self.invocations: + self.invocations = _objects(self.trajectory.get("invocations")) + self.turns = _objects(self.trajectory.get("turns")) + self.tools = _objects(self.trajectory.get("tool_calls")) or [ + r for r in self.observations if r.get("kind") == "tool_call" + ] + self.owners: dict[int, list[str]] = {i: [] for i in range(len(self.calls))} + self.invocation_ids = [i.get("invocation_id") for i in self.invocations] + + def _fail(self, capability: str, assertion: str, location: str, reason: str) -> None: + self.findings.append(Finding(capability, assertion, self.source + location, reason)) + + def check_identity(self) -> None: + """C0: readable records, supported schema and consistent identity.""" + rollout_id = self.record.get("_ng_rollout_id") or self.capture.get("rollout_id") + if not isinstance(rollout_id, str) or not rollout_id: + self._fail( + "C0", + "identity.rollout", + "", + "explicit rollout identity is required", + ) + if self.record.get("_ng_task_index") is None and not self.trajectory.get("task_id"): + self._fail("C0", "identity.task", "", "explicit task identity is required") + if self.trajectory and self.trajectory.get("schema_version") != "1.0": + self._fail( + "C0", + "schema.version", + "/ng_trajectory/schema_version", + "unsupported trajectory version", + ) + if self.trajectory.get("rollout_id") and self.trajectory["rollout_id"] != rollout_id: + self._fail( + "C0", + "identity.conflict", + "/ng_trajectory/rollout_id", + "trajectory and capture identities differ", + ) + if not isinstance(self.capture.get("calls"), list) or len(self.calls) != len(self.capture["calls"]): + self._fail( + "C0", + "capture.shape", + "/ng_model_call_capture/calls", + "a list of call objects is required", + ) + for issue in self.record.get("_capability_reader_issues", []): + self._fail("C0", "reader.integrity", "", issue) + + def check_capture_presence(self) -> None: + """Missing captures invalidate every capability that depends on calls.""" + for capability in ("C1", "C2", "C4", "C9", "C10", "C11"): + if not self.calls: + self._fail( + capability, + "calls.required", + "/ng_model_call_capture/calls", + "no captured calls", + ) + + def check_model_calls(self) -> None: + """C1: unique call identities, model identity, status and timing.""" + call_ids = [c.get("model_call_id") for c in self.calls] + if len(set(call_ids)) != len(call_ids): + self._fail( + "C1", + "identity.unique", + "/ng_model_call_capture/calls", + "duplicate call identity", + ) + + for index, call in enumerate(self.calls): + location = f"/ng_model_call_capture/calls/{index}" + self.findings.extend(_schema_issues("C1", call, self.source + location)) + start, end = call.get("started_at"), call.get("completed_at") + if _number(start) and _number(end) and end < start: + self._fail("C1", "timing.order", location, "completion precedes start") + if not call.get("model") and not _mapping(call.get("model_ref")).get("name"): + self._fail( + "C1", + "model.identity", + location, + "model identity unavailable", + ) + + def check_token_counts(self) -> None: + """C2: complete token counts consistent with retained provider usage.""" + if not any(_successful(c) for c in self.calls): + self._fail( + "C2", + "usage.eligible", + "/ng_model_call_capture/calls", + "no successful calls with token evidence", + ) + + for index, call in enumerate(self.calls): + location = f"/ng_model_call_capture/calls/{index}" + response = call.get("response") + if _successful(call): + self.findings.extend(_schema_issues("C2", call, self.source + location)) + prompt, cached = call.get("tokens_in"), call.get("cached_tokens") + if _number(prompt) and _number(cached) and cached > prompt: + self._fail( + "C2", + "usage.cache_subset", + location, + "cached count exceeds prompt count", + ) + # Responses/chat count reasoning inside completion. Anthropic has + # different cache semantics and does not supply reasoning counts. + if call.get("dialect") in ("responses", "chat"): + completion, reasoning, total = ( + call.get(k) + for k in ( + "tokens_out", + "tokens_reasoning", + "tokens_total", + ) + ) + if _number(reasoning) and _number(completion) and reasoning > completion: + self._fail( + "C2", + "usage.reasoning_subset", + location, + "reasoning count exceeds completion count", + ) + if _number(prompt) and _number(completion) and _number(total) and total != prompt + completion: + self._fail( + "C2", + "usage.total", + location, + "total differs from prompt plus completion", + ) + usage = _mapping(_mapping(response).get("usage")) + if call.get("dialect") in ("responses", "chat"): + is_responses = call.get("dialect") == "responses" + input_key, output_key = ( + ("input_tokens", "output_tokens") if is_responses else ("prompt_tokens", "completion_tokens") + ) + expected = { + "tokens_in": usage.get(input_key), + "tokens_out": usage.get(output_key), + "tokens_total": usage.get("total_tokens"), + "tokens_reasoning": _mapping(usage.get(output_key + "_details")).get("reasoning_tokens"), + "cached_tokens": _mapping(usage.get(input_key + "_details")).get("cached_tokens"), + } + for key, value in expected.items(): + if call.get(key) != value: + self._fail( + "C2", + "usage.preservation", + location + "/" + key, + "count differs from captured provider usage, including absence", + ) + + def check_content_references(self) -> None: + """C4/C9: reject unavailable media and opaque payload dependencies.""" + for index, call in enumerate(self.calls): + location = f"/ng_model_call_capture/calls/{index}" + request, response = call.get("request"), call.get("response") + if _missing_content(request) or _missing_content(response): + for capability in ("C4", "C9"): + self._fail( + capability, + "payload.content_reference", + location, + "external, encrypted or invalid content is not resolved by this reader", + ) + + def check_history(self) -> None: + """C4: ordered model request and response history.""" + for index, call in enumerate(self.calls): + location = f"/ng_model_call_capture/calls/{index}" + request, response = call.get("request"), call.get("response") + request_key = "input" if call.get("dialect") == "responses" else "messages" + if not isinstance(request, dict) or not isinstance(request.get(request_key), (list, str)): + self._fail( + "C4", + "request.history", + location + "/request", + "ordered model input is unavailable", + ) + if _successful(call): + response_key = { + "responses": "output", + "chat": "choices", + "messages": "content", + }.get(str(call.get("dialect")), "") + if not isinstance(response, dict) or not isinstance(response.get(response_key), list): + self._fail( + "C4", + "response.history", + location + "/response", + "model output is unavailable", + ) + + def check_payloads(self) -> None: + """C9: retained payloads or explicit no-response transport failures.""" + for index, call in enumerate(self.calls): + location = f"/ng_model_call_capture/calls/{index}" + request, response = call.get("request"), call.get("response") + request_present = isinstance(request, dict) or isinstance(call.get("request_raw"), str) + response_present = isinstance(response, dict) or isinstance(call.get("response_raw"), str) + if not request_present: + self._fail( + "C9", + "payload.request", + location, + "request payload unavailable", + ) + if not response_present and (call.get("status_code") is not None or not call.get("error_category")): + self._fail( + "C9", + "payload.response", + location, + "response body unavailable; not an explicit no-response failure", + ) + + def check_ownership(self) -> None: + """C10: each captured attempt resolves to one explicit invocation owner.""" + if not self.invocations or len(set(self.invocation_ids)) != len(self.invocation_ids): + self._fail( + "C10", + "invocations.required_unique", + "/ng_agent_observations", + "unique invocations are required", + ) + parents = {i.get("invocation_id"): i.get("parent_invocation_id") for i in self.invocations} + for index, invocation in enumerate(self.invocations): + location = f"/invocations/{index}" + self.findings.extend(_schema_issues("C10", invocation, self.source + location)) + parent = invocation.get("parent_invocation_id") + seen = {invocation.get("invocation_id")} + while parent is not None and parent not in seen and parent in parents: + seen.add(parent) + parent = parents[parent] + if parent is not None: + self._fail( + "C10", + "invocation.parent", + location, + "parent is missing or cyclic", + ) + for reference in _objects(invocation.get("model_calls")): + matches = _resolve(reference, self.calls) + if len(matches) != 1: + self._fail( + "C10", + "ownership.reference", + location, + "call reference does not resolve uniquely", + ) + elif isinstance(invocation.get("invocation_id"), str): + self.owners[matches[0]].append(invocation["invocation_id"]) + for index, assigned in self.owners.items(): + if len(assigned) != 1: + self._fail( + "C10", + "ownership.exactly_once", + f"/ng_model_call_capture/calls/{index}", + "call must have exactly one explicit owner", + ) + elif self.calls[index].get("client_session_id") and self.calls[index]["client_session_id"] != assigned[0]: + self._fail( + "C10", + "ownership.session", + f"/ng_model_call_capture/calls/{index}", + "owner conflicts with captured client session", + ) + + def _check_records(self, capability: str, records: list[dict]) -> None: + if not records: + self._fail( + capability, + "records.required", + "/ng_trajectory", + "required records are absent", + ) + for index, item in enumerate(records): + self.findings.extend(_schema_issues(capability, item, f"{self.source}/{capability}/{index}")) + + def check_turns(self) -> Counter: + """C3: canonical turns; also report C11 reference defects and count joins.""" + self._check_records("C3", self.turns) + turn_refs: Counter = Counter() + turn_keys = [(t.get("invocation_id"), t.get("turn_no")) for t in self.turns] + if len(set(turn_keys)) != len(turn_keys): + self._fail( + "C3", + "turn.unique", + "/ng_trajectory/turns", + "duplicate semantic turn", + ) + for turn in self.turns: + if turn.get("invocation_id") not in self.invocation_ids or any( + turn.get(k) != self.trajectory.get(k) for k in ("task_id", "rollout_id") + ): + self._fail( + "C3", + "turn.identity", + "/ng_trajectory/turns", + "turn identity does not match trajectory/invocation", + ) + for reference in _objects(turn.get("model_calls")): + matches = _resolve(reference, self.calls) + if len(matches) != 1 or self.owners[matches[0]] != [turn.get("invocation_id")]: + self._fail( + "C3", + "turn.reference", + "/ng_trajectory/turns", + "turn call is unmatched or owned by another invocation", + ) + self._fail( + "C11", + "turn.reference", + "/ng_trajectory/turns", + "turn call is unmatched or owned by another invocation", + ) + else: + turn_refs[matches[0]] += 1 + return turn_refs + + def check_accounting(self, turn_refs: Counter) -> None: + """C11: turn-call accounting and the missing independent closure evidence.""" + if not self.turns or any(turn_refs[i] != 1 for i in range(len(self.calls))): + self._fail( + "C11", + "accounting.exactly_once", + "/ng_trajectory/turns", + "calls are not assigned to turns exactly once", + ) + # Gym 1.0 has neither closure/witness records nor auxiliary-purpose buckets. + self._fail( + "C11", + "accounting.closure", + "/ng_trajectory", + "independent closed-scope ledger and auxiliary accounting required", + ) + + def check_tools(self) -> None: + """C5: unique executed tools, request joins, outcomes and intervals.""" + self._check_records("C5", self.tools) + conversations = { + ( + invocation.get("invocation_id"), + item.get("call_id"), + item.get("type"), + ): item + for invocation in self.invocations + for item in _objects(invocation.get("conversation")) + if item.get("call_id") + } + tool_keys = [(t.get("invocation_id"), t.get("tool_call_id")) for t in self.tools] + if len(set(tool_keys)) != len(tool_keys): + self._fail( + "C5", + "tool.unique", + "/ng_trajectory/tool_calls", + "duplicate execution identity", + ) + for index, tool in enumerate(self.tools): + location = f"/tool_calls/{index}" + tool_key = (tool.get("invocation_id"), tool.get("tool_call_id")) + request = conversations.get((*tool_key, "function_call"), {}) + output = conversations.get((*tool_key, "function_call_output"), {}).get("output", tool.get("output")) + if not request.get("arguments") or request.get("name") != tool.get("tool_name"): + self._fail( + "C5", + "tool.request", + location, + "execution cannot be joined to tool name and arguments", + ) + if output is None and not tool.get("error_type"): + self._fail( + "C5", + "tool.outcome", + location, + "execution has neither output nor error evidence", + ) + if ( + _number(tool.get("started_at")) + and _number(tool.get("completed_at")) + and tool["completed_at"] < tool["started_at"] + ): + self._fail( + "C5", + "tool.interval", + location, + "completion precedes start", + ) + + def check_parallel_timing(self) -> None: + """C6: independent timing witnesses remain unavailable.""" + self._fail( + "C6", + "timing.witness", + "/tool_calls", + "overlap needs independent monotonic-clock tool witnesses", + ) + + def check_lifecycle(self) -> None: + """C7: full resource lifecycle evidence remains unavailable.""" + self._fail( + "C7", + "sandbox.lifecycle", + "/ng_agent_observations", + "session identity alone does not provide setup, verification and cleanup evidence", + ) + + def check_verifier(self) -> None: + """C8: authoritative verifier provenance remains unavailable.""" + self._fail( + "C8", + "reward.provenance", + "/reward", + "reward alone does not provide authoritative verifier scope and version", + ) + + def check_gaps(self) -> None: + """Propagate explicit producer gaps to dependent capabilities.""" + gaps: Iterable[dict] = ( + *_objects(self.capture.get("gaps")), + *_objects(self.bundle.get("gaps")), + *_objects(self.trajectory.get("gaps")), + ) + for gap in gaps: + code = str(gap.get("code", "")) + if code.startswith("model_call_capture") or code == "agent_observation_join_failed": + for capability in ("C1", "C4", "C9", "C10", "C11"): + self._fail( + capability, + "capture.gap", + "/gaps", + "producer declares missing or unreadable capture evidence", + ) + if code.startswith("model_call_reference") or code == "model_call_ownership_unavailable": + self._fail( + "C10", + "ownership.gap", + "/gaps", + "producer declares unresolved ownership", + ) + + def result(self) -> dict: + """Produce payload-free verdicts and propagate C0 integrity failures.""" + failed = {f.capability for f in self.findings} + if "C0" in failed: + for capability in NAMES: + if capability != "C0": + self._fail( + capability, + "record.integrity", + "", + "record identity or reader integrity failed", + ) + failed = set(NAMES) + return { + "source": self.source, + "capabilities": { + capability: { + "name": name, + "verdict": ("not_fulfilled" if capability in failed else "fulfilled"), + "basis": "retained_artifacts", + "validator_status": ("incomplete" if capability in PENDING_VALIDATORS else "artifact_subset"), + "behavioral_qualification": False, + } + for capability, name in NAMES.items() + }, + "findings": [asdict(f) for f in self.findings], + } + + +def inspect_record(record: dict, *, source: str = "record") -> dict: + """Check artifact availability/correctness; never qualify unseen scenarios. + + Full per-call payloads may be hydrated by the reader. Missing artifacts fail + their capabilities rather than being silently excluded from the denominator. + Reports contain field locations and codes, never prompts or response bodies. + """ + checks = _RecordInspector(record, source) + checks.check_identity() + checks.check_capture_presence() + checks.check_model_calls() + checks.check_token_counts() + checks.check_content_references() + checks.check_history() + checks.check_payloads() + checks.check_ownership() + turn_refs = checks.check_turns() + checks.check_accounting(turn_refs) + checks.check_tools() + checks.check_parallel_timing() + checks.check_lifecycle() + checks.check_verifier() + checks.check_gaps() + return checks.result() diff --git a/nemo_gym/harness_capabilities/cli.py b/nemo_gym/harness_capabilities/cli.py new file mode 100644 index 00000000000..6368c9582ea --- /dev/null +++ b/nemo_gym/harness_capabilities/cli.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Standalone command: python -m nemo_gym.harness_capabilities inspect --bundle FILE.""" + +import argparse +import hashlib +import json +import os +import shutil +import tempfile +from pathlib import Path + +from . import __version__ +from .checker import NAMES, PENDING_VALIDATORS, PROFILES, inspect_record +from .reader import digest_file, hydrate_record, json_rows +from .schemas import SCHEMA_VERSION, SCHEMAS + + +def _json(value: object) -> str: + return json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + "\n" + + +def inspect_bundle( + bundle: Path, + *, + output: Path, + profile: str, + capture_dir: Path | None = None, +) -> tuple[Path, dict]: + """Write an immutable content-addressed report set after a complete read. + + A checker error never replaces an earlier report or leaves a current-looking + summary. Files are hashed before and after reading to detect changing input. + """ + if profile not in PROFILES: + raise ValueError("unknown artifact profile") + if bundle.is_dir(): + candidates = [ + bundle / "rollouts.jsonl", + bundle / "artifacts/rollouts.jsonl", + bundle / "evaluator_rollouts.jsonl", + bundle / "artifacts/evaluator_rollouts.jsonl", + ] + matches = [path for path in candidates if path.is_file()] + if len(matches) != 1: + raise ValueError( + "bundle must contain exactly one rollouts.jsonl or evaluator_rollouts.jsonl; pass a file explicitly" + ) + bundle = matches[0] + sources = [bundle] + if capture_dir is not None: + if not capture_dir.is_dir(): + raise ValueError("capture directory does not exist") + sources.extend(sorted(capture_dir.glob("*.capture.*"))) + hashes = {str(path.resolve()): digest_file(path) for path in sources} + registry_hash = hashlib.sha256(_json({"schemas": SCHEMAS, "profiles": PROFILES}).encode()).hexdigest() + checker_hash = hashlib.sha256( + "".join(digest_file(p) for p in sorted(Path(__file__).parent.glob("*.py"))).encode() + ).hexdigest() + manifest = { + "sources": hashes, + "registry_sha256": registry_hash, + "checker_sha256": checker_hash, + "profile": profile, + } + report_id = hashlib.sha256(_json(manifest).encode()).hexdigest() + output.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=".capabilities-", dir=output)) + totals = {capability: {"observed_records": 0, "passed_records": 0} for capability in NAMES} + identities: set[object] = set() + count = 0 + try: + with (temporary / "capability_results.jsonl").open("w") as handle: + for line, raw in json_rows(bundle): + record = hydrate_record(raw, capture_dir=capture_dir) + identity = (record.get("ng_model_call_capture") or {}).get("rollout_id") or record.get( + "_ng_rollout_id" + ) + if identity in identities: + record.setdefault("_capability_reader_issues", []).append("duplicate rollout identity in input") + identities.add(identity) + result = inspect_record(record, source=f"{bundle.name}:{line}") + count += 1 + for capability, verdict in result["capabilities"].items(): + totals[capability]["observed_records"] += 1 + totals[capability]["passed_records"] += verdict["verdict"] == "fulfilled" + handle.write(json.dumps(result, allow_nan=False) + "\n") + if count == 0: + raise ValueError("no rollout records") + if any(digest_file(path) != hashes[str(path.resolve())] for path in sources): + raise ValueError("source changed during inspection") + verdicts = { + capability: { + **counts, + "verdict": ("fulfilled" if counts["passed_records"] == count else "not_fulfilled"), + "behavioral_qualification": False, + "validator_status": ("incomplete" if capability in PENDING_VALIDATORS else "artifact_subset"), + "scenario_coverage": {"observed": 0, "passed": 0}, + } + for capability, counts in totals.items() + } + passed = all(verdicts[capability]["verdict"] == "fulfilled" for capability in PROFILES[profile]) + summary = { + "schema_version": "harness-capabilities/v1", + "checker_version": __version__, + "decoder_version": SCHEMA_VERSION, + "checker_status": "completed", + "profile": profile, + "scope": "retained Gym artifacts; no behavioral qualification", + "scope_closure": "not_independently_witnessed", + "verdict": "fulfilled" if passed else "not_fulfilled", + "is_behavioral_qualification": False, + "records": count, + "capabilities": verdicts, + **manifest, + } + (temporary / "capability_summary.json").write_text(_json(summary)) + report = [ + "# Harness artifact conformance", + "", + f"Profile: `{profile}`. Gate: **{summary['verdict']}**. Records: {count}.", + "", + "These checks establish retained artifact requirements. They do not qualify the RFC's behavioral scenarios.", + "", + "| Capability | Artifact verdict | Passing records |", + "|---|---|---|", + ] + report.extend( + f"| {key}: {NAMES[key]} | {value['verdict']} | {value['passed_records']}/{count} |" + for key, value in verdicts.items() + ) + report.extend( + [ + "", + "Field-level failure locations are in `capability_results.jsonl`; payloads are never copied into this report.", + "", + ] + ) + (temporary / "capability_report.md").write_text("\n".join(report)) + destination = output / report_id + if destination.exists(): + if any((destination / path.name).read_bytes() != path.read_bytes() for path in temporary.iterdir()): + raise ValueError("existing content-addressed report differs") + shutil.rmtree(temporary) + else: + os.rename(temporary, destination) + return destination, summary + except BaseException: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + inspect = subparsers.add_parser("inspect") + inspect.add_argument("--bundle", required=True, type=Path) + inspect.add_argument("--capture-dir", type=Path) + inspect.add_argument("--output", required=True, type=Path) + inspect.add_argument("--profile", choices=PROFILES, default="gym-artifacts-p0/v1") + args = parser.parse_args(argv) + return run_inspection(bundle=args.bundle, output=args.output, profile=args.profile, capture_dir=args.capture_dir) + + +def run_inspection( + *, bundle: Path, output: Path, profile: str = "gym-artifacts-p0/v1", capture_dir: Path | None = None +) -> int: + """Inspect retained rollouts and print the report location; return 0, 1, or 2.""" + try: + destination, summary = inspect_bundle( + bundle, + output=output, + profile=profile, + capture_dir=capture_dir, + ) + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + RecursionError, + ) as exc: + # Do not print exception details: malformed input may contain secrets. + print(f"checker_error ({type(exc).__name__}); no report published") + return 2 + print(f"{summary['verdict']}: {destination / 'capability_summary.json'}") + return 0 if summary["verdict"] == "fulfilled" else 1 diff --git a/nemo_gym/harness_capabilities/reader.py b/nemo_gym/harness_capabilities/reader.py new file mode 100644 index 00000000000..68aceada962 --- /dev/null +++ b/nemo_gym/harness_capabilities/reader.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Streaming native Gym reader with explicit, loss-detecting payload joins.""" + +import copy +import hashlib +import json +import re +from pathlib import Path +from typing import Iterator + + +PAYLOADS = ("request", "response", "request_raw", "response_raw") + + +def _reject_constant(value: str) -> None: + raise ValueError("nonfinite JSON number") + + +def json_rows(path: Path) -> Iterator[tuple[int, dict]]: + """Read JSONL strictly; malformed/truncated rows are checker errors.""" + with path.open(encoding="utf-8") as handle: + for number, line in enumerate(handle, 1): + if line.strip(): + try: + value = json.loads(line, parse_constant=_reject_constant) + except (ValueError, RecursionError) as exc: + raise ValueError(f"{path.name}:{number}: invalid JSON") from exc + if not isinstance(value, dict): + raise ValueError(f"{path.name}:{number}: expected an object") + yield number, value + + +def digest_file(path: Path) -> str: + with path.open("rb") as handle: + return hashlib.file_digest(handle, "sha256").hexdigest() + + +def hydrate_record(record: dict, *, capture_dir: Path | None = None) -> dict: + """Restore payloads by exact model_call_id, never order or content hash. + + The native collector moves bodies from capture.calls into ng_trajectory. + An optional capture directory provides the retained transport exchanges. + Conflicts are preserved as reader failures instead of being overwritten. + """ + result = copy.deepcopy(record) + capture = result.get("ng_model_call_capture") or {} + calls = capture.get("calls") or [] + trajectory = result.get("ng_trajectory") or {} + issues = result.setdefault("_capability_reader_issues", []) + by_id: dict[str, dict] = {} + for call in calls: + if isinstance(call, dict) and isinstance(call.get("model_call_id"), str): + call_id = call["model_call_id"] + if call_id in by_id: + issues.append("duplicate model call identity") + by_id[call_id] = call + + def merge(items: list[dict], *, sidecar: bool) -> None: + seen: set[str] = set() + for item in items: + call_id = item.get("model_call_id") + if not isinstance(call_id, str) or call_id not in by_id or call_id in seen: + issues.append("unmatched or duplicate payload identity") + else: + seen.add(call_id) + call = by_id[call_id] + for key in PAYLOADS: + value = item.get(key) + if value is not None: + if call.get(key) is not None and call[key] != value: + issues.append("conflicting payload representations") + else: + call[key] = value + metadata = item if sidecar else item.get("response_metadata", {}) + for key in ( + "model_ref", + "response_id", + "status_code", + "dialect", + "error_category", + ): + if metadata.get(key) is not None and call.get(key) != metadata[key]: + issues.append("conflicting model-call metadata") + if not sidecar: + stats = item.get("token_stats") or {} + for canonical, captured in ( + ("prompt_tokens", "tokens_in"), + ("completion_tokens", "tokens_out"), + ("reasoning_tokens", "tokens_reasoning"), + ("total_tokens", "tokens_total"), + ("cached_tokens", "cached_tokens"), + ): + if canonical in stats and stats[canonical] != call.get(captured): + issues.append("trajectory and capture token counts disagree") + if sidecar and seen != set(by_id): + issues.append("capture sidecar and rollout have different identity sets") + + merge(trajectory.get("model_calls") or [], sidecar=False) + if capture_dir is not None: + rollout_id = capture.get("rollout_id") + if not isinstance(rollout_id, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", rollout_id): + issues.append("unsafe or missing capture identity") + else: + path = capture_dir / f"{rollout_id}.capture.jsonl" + if path.is_file(): + merge([row for _, row in json_rows(path)], sidecar=True) + else: + issues.append("requested capture sidecar is missing") + if (capture_dir / f"{rollout_id}.capture.incomplete").exists(): + issues.append("capture is marked incomplete") + return result diff --git a/nemo_gym/harness_capabilities/schemas.py b/nemo_gym/harness_capabilities/schemas.py new file mode 100644 index 00000000000..0638884a822 --- /dev/null +++ b/nemo_gym/harness_capabilities/schemas.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Versioned, intentionally stricter subsets of Gym's optional contracts. + +Pinned to Gym 26688f7d (TrajectoryRecord 1.0). Additional producer fields are +allowed: these are capability requirements, not copies of Gym's whole schema. +""" + +SCHEMA_VERSION = "gym-artifact-contracts/v1" +TEXT = {"type": "string", "minLength": 1} +NUMBER = {"type": "number"} +NONNEGATIVE = {"type": "number", "minimum": 0} +TOKENS = {"type": "integer", "minimum": 0} +TOKEN_FIELDS = ( + "tokens_in", + "tokens_out", + "tokens_reasoning", + "tokens_total", + "cached_tokens", +) + + +def object_schema(properties: dict, required: tuple | list) -> dict: + return { + "type": "object", + "properties": properties, + "required": list(required), + } + + +MODEL_REF = object_schema({"type": TEXT, "name": TEXT}, ("type", "name")) +CALL_REF = { + **object_schema( + {"model_call_id": TEXT, "model_ref": MODEL_REF, "response_id": TEXT}, + (), + ), + "anyOf": [ + {"required": ["model_call_id"]}, + {"required": ["model_ref", "response_id"]}, + ], +} +CALL = { + **object_schema( + { + "model_call_id": TEXT, + "model_ref": MODEL_REF, + # Native captures may lack a provider model on failed calls. The + # required model_ref still identifies the configured model server. + "model": {"anyOf": [TEXT, {"type": "null"}]}, + "dialect": {"enum": ["responses", "chat", "messages"]}, + "started_at": NUMBER, + "completed_at": NUMBER, + "latency_total_ms": NONNEGATIVE, + "status_code": {"type": ["integer", "null"]}, + "error_category": {"type": ["string", "null"]}, + }, + ( + "model_call_id", + "model_ref", + "dialect", + "started_at", + "completed_at", + "latency_total_ms", + ), + ), + "anyOf": [ + { + "required": ["status_code"], + "properties": { + "status_code": { + "type": "integer", + "minimum": 100, + "maximum": 599, + } + }, + }, + { + "required": ["error_category"], + "properties": {"error_category": TEXT}, + }, + ], +} +USAGE = object_schema({key: TOKENS for key in TOKEN_FIELDS}, TOKEN_FIELDS) +INVOCATION = object_schema( + { + "invocation_id": TEXT, + "model_calls": {"type": "array", "items": {"type": "object"}}, + "conversation": {"type": "array", "items": {"type": "object"}}, + }, + ("invocation_id", "model_calls"), +) +TOOL = object_schema( + { + "invocation_id": TEXT, + "tool_call_id": TEXT, + "tool_name": TEXT, + "started_at": NUMBER, + "completed_at": NUMBER, + "duration_ms": NONNEGATIVE, + "timing_source": {"enum": ["executor", "artifact", "harness"]}, + "status": {"enum": ["completed", "failed", "timeout", "cancelled"]}, + }, + ( + "invocation_id", + "tool_call_id", + "tool_name", + "started_at", + "completed_at", + "duration_ms", + "timing_source", + "status", + ), +) +TURN = object_schema( + { + "invocation_id": TEXT, + "task_id": TEXT, + "rollout_id": TEXT, + "turn_no": {"type": "integer", "minimum": 1}, + "timestamp": NUMBER, + "question": {"not": {"type": "null"}}, + "answer": {"not": {"type": "null"}}, + "step_count": {"type": "integer", "minimum": 0}, + "resolved": {"type": ["boolean", "null"]}, + "model_calls": {"type": "array", "minItems": 1}, + }, + ( + "invocation_id", + "task_id", + "rollout_id", + "turn_no", + "timestamp", + "question", + "answer", + "step_count", + "resolved", + "model_calls", + ), +) + +SCHEMAS = {"C1": CALL, "C2": USAGE, "C3": TURN, "C5": TOOL, "C10": INVOCATION} diff --git a/pyproject.toml b/pyproject.toml index 20782064bb8..19f6ae87c67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,11 @@ dependencies = [ "pydantic_core", "devtools", + # JSON Schema: Validates retained harness capability evidence. + # Updated Tue Sep 22, 2026 with jsonschema>=4.18,<5 + # License: MIT https://github.com/python-jsonschema/jsonschema/blob/main/COPYING + "jsonschema>=4.18,<5", + # Packaging: Validates SPDX license expressions in environment manifests. # Updated Tue Aug 11, 2026 with packaging==25.0 # License: Apache-2.0 OR BSD-2-Clause https://github.com/pypa/packaging/tree/25.0 diff --git a/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py b/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py index 37d5e68ef85..643e357e715 100644 --- a/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py +++ b/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py @@ -19,6 +19,7 @@ install_model_call_capture, merge_model_call_capture_into_record, ) +from nemo_gym.harness_capabilities.cli import inspect_bundle from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming from nemo_gym.rollout_collection import _attach_trajectory_record from nemo_gym.rollout_health import run_health_checks @@ -248,3 +249,13 @@ async def query(params): assert coverage[key]["evaluated"] == 1, (key, result.summary) if scenario == "http_error": assert result.summary["run"]["issues"]["model_call_failed"] == 1 + + # Inspect the emitted records, not a reconstructed copy of the expected evidence. + _, conformance = inspect_bundle( + path, output=tmp_path / "capabilities", profile="gym-artifacts-p0/v1", capture_dir=capture_dir + ) + expected = "not_fulfilled" if scenario in {"missing_usage", "missing_details"} else "fulfilled" + assert conformance["verdict"] == expected + for capability in ("C1", "C3", "C4", "C5", "C9", "C10"): + assert conformance["capabilities"][capability]["verdict"] == "fulfilled", (scenario, capability) + assert conformance["is_behavioral_qualification"] is False diff --git a/tests/unit_tests/harness_capabilities/fixtures/miniswe.json b/tests/unit_tests/harness_capabilities/fixtures/miniswe.json new file mode 100644 index 00000000000..d182259ac7c --- /dev/null +++ b/tests/unit_tests/harness_capabilities/fixtures/miniswe.json @@ -0,0 +1,675 @@ +{ + "_ng_task_index": 0, + "_ng_rollout_index": 0, + "reward": 0.0, + "response": { + "id": "resp_df81b682747645199ada18f21b22d6f6", + "created_at": 1789726149.0, + "error": null, + "incomplete_details": null, + "instructions": null, + "metadata": null, + "model": "model", + "object": "response", + "output": [ + { + "arguments": "{\"command\": \"inspect\"}", + "call_id": "tool-1", + "name": "bash", + "type": "function_call", + "id": null, + "status": null, + "namespace": null + }, + { + "call_id": "tool-1", + "output": "0\ninspected", + "type": "function_call_output", + "id": null, + "status": null + }, + { + "arguments": "{\"command\": \"submit\"}", + "call_id": "tool-2", + "name": "bash", + "type": "function_call", + "id": null, + "status": null, + "namespace": null + }, + { + "call_id": "tool-2", + "output": "0\nCOMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished", + "type": "function_call_output", + "id": null, + "status": null + } + ], + "parallel_tool_calls": true, + "temperature": null, + "tool_choice": "auto", + "tools": [], + "top_p": null, + "background": null, + "completed_at": null, + "conversation": null, + "max_output_tokens": null, + "max_tool_calls": null, + "moderation": null, + "previous_response_id": null, + "prompt": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": null, + "safety_identifier": null, + "service_tier": null, + "status": null, + "text": null, + "top_logprobs": null, + "truncation": null, + "usage": { + "input_tokens": 20, + "input_tokens_details": { + "cached_tokens": 2 + }, + "output_tokens": 10, + "output_tokens_details": { + "reasoning_tokens": 6 + }, + "total_tokens": 30 + }, + "user": null + }, + "ng_agent_observations": { + "source": "miniswe", + "records": [ + { + "kind": "agent_invocation", + "invocation_id": "invocation", + "parent_invocation_id": null, + "spawned_by_tool_call_id": null, + "status": "completed", + "duration_ms": 66.45141704939306, + "error_type": null, + "model_calls": [ + { + "model_call_id": "3a80e87fe7ad4a9bb9b203c8d1e2ffe5", + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "response_id": "response-1" + }, + { + "model_call_id": "8dce6de0d1df4f2998054919cc3eec06", + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "response_id": "response-2" + } + ], + "conversation": [ + { + "content": "You are a helpful assistant that can interact with a computer.", + "role": "system", + "type": "message", + "phase": null + }, + { + "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```", + "role": "user", + "type": "message", + "phase": null + }, + { + "arguments": "{\"command\": \"inspect\"}", + "call_id": "tool-1", + "name": "bash", + "type": "function_call", + "id": null, + "status": null, + "namespace": null + }, + { + "call_id": "tool-1", + "output": "0\ninspected", + "type": "function_call_output", + "id": null, + "status": null + }, + { + "arguments": "{\"command\": \"submit\"}", + "call_id": "tool-2", + "name": "bash", + "type": "function_call", + "id": null, + "status": null, + "namespace": null + }, + { + "call_id": "tool-2", + "output": "0\nCOMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished", + "type": "function_call_output", + "id": null, + "status": null + } + ] + }, + { + "kind": "tool_call", + "invocation_id": "invocation", + "tool_call_id": "tool-1", + "sandbox_id": null, + "tool_name": "bash", + "started_at": 1789726149.893095, + "completed_at": 1789726149.8931599, + "duration_ms": 0.06045890040695667, + "timing_source": "executor", + "status": "completed", + "error_type": null + }, + { + "kind": "tool_call", + "invocation_id": "invocation", + "tool_call_id": "tool-2", + "sandbox_id": null, + "tool_name": "bash", + "started_at": 1789726149.901033, + "completed_at": 1789726149.9011168, + "duration_ms": 0.07075001485645771, + "timing_source": "executor", + "status": "completed", + "error_type": null + } + ], + "gaps": [] + }, + "ng_trajectory": { + "schema_version": "1.0", + "task_id": "0", + "rollout_id": "0-0", + "invocations": [ + { + "kind": "agent_invocation", + "invocation_id": "invocation", + "parent_invocation_id": null, + "spawned_by_tool_call_id": null, + "status": "completed", + "duration_ms": 66.45141704939306, + "error_type": null, + "model_calls": [ + { + "model_call_id": "3a80e87fe7ad4a9bb9b203c8d1e2ffe5", + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "response_id": "response-1" + }, + { + "model_call_id": "8dce6de0d1df4f2998054919cc3eec06", + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "response_id": "response-2" + } + ], + "conversation": [ + { + "content": "You are a helpful assistant that can interact with a computer.", + "role": "system", + "type": "message", + "phase": null + }, + { + "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```", + "role": "user", + "type": "message", + "phase": null + }, + { + "arguments": "{\"command\": \"inspect\"}", + "call_id": "tool-1", + "name": "bash", + "type": "function_call", + "id": null, + "status": null, + "namespace": null + }, + { + "call_id": "tool-1", + "output": "0\ninspected", + "type": "function_call_output", + "id": null, + "status": null + }, + { + "arguments": "{\"command\": \"submit\"}", + "call_id": "tool-2", + "name": "bash", + "type": "function_call", + "id": null, + "status": null, + "namespace": null + }, + { + "call_id": "tool-2", + "output": "0\nCOMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished", + "type": "function_call_output", + "id": null, + "status": null + } + ] + } + ], + "turns": [ + { + "invocation_id": "invocation", + "task_id": "0", + "rollout_id": "0-0", + "turn_no": 1, + "timestamp": 1789726149.8928242, + "question": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + } + ], + "answer": [ + { + "arguments": "{\"command\": \"inspect\"}", + "call_id": "tool-1", + "name": "bash", + "type": "function_call", + "id": null, + "status": null, + "namespace": null + } + ], + "reasoning_content": null, + "resolved": null, + "step_count": 1, + "model_calls": [ + { + "model_call_id": null, + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "response_id": "response-1" + } + ] + }, + { + "invocation_id": "invocation", + "task_id": "0", + "rollout_id": "0-0", + "turn_no": 2, + "timestamp": 1789726149.900903, + "question": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "arguments": "{\"command\": \"inspect\"}", + "call_id": "tool-1", + "name": "bash", + "type": "function_call" + }, + { + "type": "function_call_output", + "call_id": "tool-1", + "output": "0\ninspected" + } + ], + "answer": [ + { + "arguments": "{\"command\": \"submit\"}", + "call_id": "tool-2", + "name": "bash", + "type": "function_call", + "id": null, + "status": null, + "namespace": null + } + ], + "reasoning_content": null, + "resolved": null, + "step_count": 2, + "model_calls": [ + { + "model_call_id": null, + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "response_id": "response-2" + } + ] + } + ], + "model_calls": [ + { + "model_call_id": "3a80e87fe7ad4a9bb9b203c8d1e2ffe5", + "started_at": 1789726149.874946, + "completed_at": 1789726149.876183, + "duration_ms": 1.24, + "request": { + "input": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + } + ], + "parallel_tool_calls": true, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "name": "bash", + "description": "Execute a bash command", + "parameters": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The bash command to execute" + } + }, + "required": [ + "command" + ] + }, + "strict": false + } + ] + }, + "response": { + "id": "response-1", + "object": "response", + "model": "controlled", + "created_at": 0, + "status": "completed", + "parallel_tool_calls": false, + "tools": [], + "tool_choice": "auto", + "output": [ + { + "type": "function_call", + "call_id": "tool-1", + "name": "bash", + "arguments": "{\"command\": \"inspect\"}" + } + ], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": { + "cached_tokens": 2 + }, + "output_tokens_details": { + "reasoning_tokens": 3 + } + } + }, + "response_metadata": { + "response_id": "response-1", + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "model": "controlled", + "dialect": "responses", + "status_code": 200, + "response_status": "completed", + "finish_reason": null, + "error_category": null, + "latency_ttft_ms": 1.23 + }, + "token_stats": { + "prompt_tokens": 10, + "completion_tokens": 5, + "reasoning_tokens": 3, + "total_tokens": 15, + "cached_tokens": 2 + } + }, + { + "model_call_id": "8dce6de0d1df4f2998054919cc3eec06", + "started_at": 1789726149.895183, + "completed_at": 1789726149.895329, + "duration_ms": 0.15, + "request": { + "input": [ + { + "role": "system", + "content": "You are a helpful assistant that can interact with a computer." + }, + { + "role": "user", + "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" + }, + { + "arguments": "{\"command\": \"inspect\"}", + "call_id": "tool-1", + "name": "bash", + "type": "function_call" + }, + { + "type": "function_call_output", + "call_id": "tool-1", + "output": "0\ninspected" + } + ], + "parallel_tool_calls": true, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "name": "bash", + "description": "Execute a bash command", + "parameters": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The bash command to execute" + } + }, + "required": [ + "command" + ] + }, + "strict": false + } + ] + }, + "response": { + "id": "response-2", + "object": "response", + "model": "controlled", + "created_at": 0, + "status": "completed", + "parallel_tool_calls": false, + "tools": [], + "tool_choice": "auto", + "output": [ + { + "type": "function_call", + "call_id": "tool-2", + "name": "bash", + "arguments": "{\"command\": \"submit\"}" + } + ], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens_details": { + "reasoning_tokens": 3 + } + } + }, + "response_metadata": { + "response_id": "response-2", + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "model": "controlled", + "dialect": "responses", + "status_code": 200, + "response_status": "completed", + "finish_reason": null, + "error_category": null, + "latency_ttft_ms": 0.14 + }, + "token_stats": { + "prompt_tokens": 10, + "completion_tokens": 5, + "reasoning_tokens": 3, + "total_tokens": 15, + "cached_tokens": 0 + } + } + ], + "tool_calls": [ + { + "kind": "tool_call", + "invocation_id": "invocation", + "tool_call_id": "tool-1", + "sandbox_id": null, + "tool_name": "bash", + "started_at": 1789726149.893095, + "completed_at": 1789726149.8931599, + "duration_ms": 0.06045890040695667, + "timing_source": "executor", + "status": "completed", + "error_type": null, + "output": "0\ninspected" + }, + { + "kind": "tool_call", + "invocation_id": "invocation", + "tool_call_id": "tool-2", + "sandbox_id": null, + "tool_name": "bash", + "started_at": 1789726149.901033, + "completed_at": 1789726149.9011168, + "duration_ms": 0.07075001485645771, + "timing_source": "executor", + "status": "completed", + "error_type": null, + "output": "0\nCOMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished" + } + ], + "gaps": [] + }, + "ng_model_call_capture": { + "rollout_id": "0-0", + "metrics": { + "tokens_in": 20, + "tokens_out": 10, + "tokens_reasoning": 6, + "tokens_total": 30, + "cached_tokens": 2, + "latency_total_ms": 1.39, + "num_calls": 2 + }, + "calls": [ + { + "model_call_id": "3a80e87fe7ad4a9bb9b203c8d1e2ffe5", + "response_id": "response-1", + "client_session_id": "invocation", + "call_index": 0, + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "model": "controlled", + "dialect": "responses", + "status_code": 200, + "response_status": "completed", + "finish_reason": null, + "started_at": 1789726149.874946, + "completed_at": 1789726149.876183, + "tokens_in": 10, + "tokens_out": 5, + "tokens_reasoning": 3, + "tokens_total": 15, + "tool_calls": [ + { + "call_id": "tool-1", + "name": "bash", + "arguments": { + "command": "inspect" + } + } + ], + "reasoning_content": null, + "cache_hit": true, + "cached_tokens": 2, + "cache_creation_tokens": null, + "error_category": null, + "latency_total_ms": 1.24, + "latency_ttft_ms": 1.23 + }, + { + "model_call_id": "8dce6de0d1df4f2998054919cc3eec06", + "response_id": "response-2", + "client_session_id": "invocation", + "call_index": 1, + "model_ref": { + "type": "responses_api_models", + "name": "model" + }, + "model": "controlled", + "dialect": "responses", + "status_code": 200, + "response_status": "completed", + "finish_reason": null, + "started_at": 1789726149.895183, + "completed_at": 1789726149.895329, + "tokens_in": 10, + "tokens_out": 5, + "tokens_reasoning": 3, + "tokens_total": 15, + "tool_calls": [ + { + "call_id": "tool-2", + "name": "bash", + "arguments": { + "command": "submit" + } + } + ], + "reasoning_content": null, + "cache_hit": false, + "cached_tokens": 0, + "cache_creation_tokens": null, + "error_category": null, + "latency_total_ms": 0.15, + "latency_ttft_ms": 0.14 + } + ] + } +} diff --git a/tests/unit_tests/harness_capabilities/fixtures/opencode.json b/tests/unit_tests/harness_capabilities/fixtures/opencode.json new file mode 100644 index 00000000000..e4ac11dc898 --- /dev/null +++ b/tests/unit_tests/harness_capabilities/fixtures/opencode.json @@ -0,0 +1,241 @@ +{ + "_ng_task_index": 0, + "_ng_rollout_index": 0, + "ng_model_call_capture": { + "rollout_id": "0-0", + "calls": [ + { + "model_call_id": "call-1", + "response_id": "response-1", + "client_session_id": "root", + "call_index": 0, + "model_ref": { + "type": "responses_api_models", + "name": "policy" + }, + "model": "model", + "dialect": "responses", + "status_code": 200, + "response_status": "completed", + "finish_reason": null, + "started_at": 1.0, + "completed_at": 2.0, + "tokens_in": 10, + "tokens_out": 5, + "tokens_reasoning": 2, + "tokens_total": 15, + "tool_calls": [ + { + "call_id": "tool-1", + "name": "bash", + "arguments": { + "command": "printf ok" + } + } + ], + "reasoning_content": null, + "cache_hit": true, + "cached_tokens": 3, + "cache_creation_tokens": null, + "error_category": null, + "latency_total_ms": 1000.0, + "latency_ttft_ms": null + } + ] + }, + "ng_agent_observations": { + "source": "opencode", + "records": [ + { + "kind": "agent_invocation", + "invocation_id": "root", + "parent_invocation_id": null, + "spawned_by_tool_call_id": null, + "status": "completed", + "duration_ms": null, + "error_type": null, + "model_calls": [ + { + "model_call_id": "call-1", + "model_ref": { + "type": "responses_api_models", + "name": "policy" + }, + "response_id": "response-1" + } + ], + "conversation": [ + { + "arguments": "{\"command\": \"printf ok\"}", + "call_id": "tool-1", + "name": "bash", + "type": "function_call", + "id": "tool-1", + "status": "completed", + "namespace": null + }, + { + "call_id": "tool-1", + "output": "ok", + "type": "function_call_output", + "id": null, + "status": "completed" + } + ] + }, + { + "kind": "tool_call", + "invocation_id": "root", + "tool_call_id": "tool-1", + "sandbox_id": null, + "tool_name": "bash", + "started_at": 2.0, + "completed_at": 3.0, + "duration_ms": 1000.0, + "timing_source": "artifact", + "status": "completed", + "error_type": null + } + ], + "gaps": [] + }, + "ng_trajectory": { + "schema_version": "1.0", + "task_id": "0", + "rollout_id": "0-0", + "invocations": [ + { + "kind": "agent_invocation", + "invocation_id": "root", + "parent_invocation_id": null, + "spawned_by_tool_call_id": null, + "status": "completed", + "duration_ms": null, + "error_type": null, + "model_calls": [ + { + "model_call_id": "call-1", + "model_ref": { + "type": "responses_api_models", + "name": "policy" + }, + "response_id": "response-1" + } + ], + "conversation": [ + { + "arguments": "{\"command\": \"printf ok\"}", + "call_id": "tool-1", + "name": "bash", + "type": "function_call", + "id": "tool-1", + "status": "completed", + "namespace": null + }, + { + "call_id": "tool-1", + "output": "ok", + "type": "function_call_output", + "id": null, + "status": "completed" + } + ] + } + ], + "turns": [], + "model_calls": [ + { + "model_call_id": "call-1", + "started_at": 1.0, + "completed_at": 2.0, + "duration_ms": 1000.0, + "request": { + "model": "model", + "input": [ + { + "role": "user", + "content": "Print ok" + } + ], + "tools": [ + { + "type": "function", + "name": "bash", + "parameters": { + "type": "object" + } + } + ] + }, + "response": { + "id": "response-1", + "model": "model", + "status": "completed", + "output": [ + { + "type": "function_call", + "call_id": "tool-1", + "name": "bash", + "arguments": "{\"command\":\"printf ok\"}" + } + ], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": { + "cached_tokens": 3 + }, + "output_tokens_details": { + "reasoning_tokens": 2 + } + } + }, + "response_metadata": { + "response_id": "response-1", + "model_ref": { + "type": "responses_api_models", + "name": "policy" + }, + "model": "model", + "dialect": "responses", + "status_code": 200, + "response_status": "completed", + "finish_reason": null, + "error_category": null, + "latency_ttft_ms": null + }, + "token_stats": { + "prompt_tokens": 10, + "completion_tokens": 5, + "reasoning_tokens": 2, + "total_tokens": 15, + "cached_tokens": 3 + } + } + ], + "tool_calls": [ + { + "kind": "tool_call", + "invocation_id": "root", + "tool_call_id": "tool-1", + "sandbox_id": null, + "tool_name": "bash", + "started_at": 2.0, + "completed_at": 3.0, + "duration_ms": 1000.0, + "timing_source": "artifact", + "status": "completed", + "error_type": null, + "output": "ok" + } + ], + "gaps": [ + { + "code": "turns_unavailable", + "invocation_id": null, + "detail": null + } + ] + } +} diff --git a/tests/unit_tests/harness_capabilities/test_checker.py b/tests/unit_tests/harness_capabilities/test_checker.py new file mode 100644 index 00000000000..be858c881d2 --- /dev/null +++ b/tests/unit_tests/harness_capabilities/test_checker.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Mutation tests: schema-shaped lies must not become passing artifacts.""" + +import copy +import json +from pathlib import Path + +import pytest + +from nemo_gym.harness_capabilities.checker import P0, inspect_record +from nemo_gym.harness_capabilities.cli import inspect_bundle, main +from nemo_gym.harness_capabilities.reader import hydrate_record + + +@pytest.fixture(params=["opencode", "miniswe"]) +def record(request): + return json.loads((Path(__file__).parent / f"fixtures/{request.param}.json").read_text()) + + +def verdict(record, capability): + return inspect_record(hydrate_record(record))["capabilities"][capability]["verdict"] + + +def test_opencode_native_projection_passes_p0(record): + result = inspect_record(hydrate_record(record)) + assert all(result["capabilities"][c]["verdict"] == "fulfilled" for c in P0), result["findings"] + assert result["capabilities"]["C5"]["verdict"] == "fulfilled" + assert not any(v["behavioral_qualification"] for v in result["capabilities"].values()) + + +@pytest.mark.parametrize( + "field", + [ + "tokens_in", + "tokens_out", + "tokens_reasoning", + "tokens_total", + "cached_tokens", + ], +) +@pytest.mark.parametrize("value", [None, -1, True, 1.5]) +def test_bad_token_values_fail(record, field, value): + record["ng_model_call_capture"]["calls"][0][field] = value + assert verdict(record, "C2") == "not_fulfilled" + + +def test_omitted_usage_cannot_become_zero(record): + call = record["ng_model_call_capture"]["calls"][0] + call["cached_tokens"] = 0 + del record["ng_trajectory"]["model_calls"][0]["response"]["usage"]["input_tokens_details"]["cached_tokens"] + assert verdict(record, "C2") == "not_fulfilled" + + +def test_legitimate_zero_is_available(record): + record["ng_model_call_capture"]["calls"][0]["cached_tokens"] = 0 + record["ng_trajectory"]["model_calls"][0]["token_stats"]["cached_tokens"] = 0 + record["ng_trajectory"]["model_calls"][0]["response"]["usage"]["input_tokens_details"]["cached_tokens"] = 0 + assert verdict(record, "C2") == "fulfilled" + + +def test_missing_response_usage_cannot_be_invented(record): + record["ng_trajectory"]["model_calls"][0]["response"]["usage"] = None + assert verdict(record, "C2") == "not_fulfilled" + + +def test_external_media_is_not_a_complete_payload(record): + record["ng_trajectory"]["model_calls"][0]["request"]["input"] = [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": "https://example.invalid/missing.png", + } + ], + } + ] + assert verdict(record, "C4") == "not_fulfilled" + assert verdict(record, "C9") == "not_fulfilled" + + +@pytest.mark.parametrize( + "mutation", + ["missing", "duplicate", "wrong_session", "bad_reference", "cycle"], +) +def test_ownership_is_not_guessed(record, mutation): + bundle = record["ng_agent_observations"] + invocation = next(r for r in bundle["records"] if r["kind"] == "agent_invocation") + if mutation == "missing": + invocation["model_calls"] = [] + elif mutation == "duplicate": + invocation["model_calls"] *= 2 + elif mutation == "wrong_session": + invocation["invocation_id"] = "other" + elif mutation == "bad_reference": + invocation["model_calls"][0]["response_id"] = "different" + else: + invocation["parent_invocation_id"] = invocation["invocation_id"] + assert verdict(record, "C10") == "not_fulfilled" + + +def test_failed_attempt_without_response_id_still_requires_owner(record): + failed = copy.deepcopy(record["ng_model_call_capture"]["calls"][0]) + failed.update( + model_call_id="failure", + response_id=None, + status_code=None, + error_category="timeout", + response=None, + ) + failed["request"] = {"input": []} + record["ng_model_call_capture"]["calls"].append(failed) + assert verdict(record, "C10") == "not_fulfilled" + record["ng_agent_observations"]["records"][0]["model_calls"].append({"model_call_id": "failure"}) + assert verdict(record, "C10") == "fulfilled" + assert verdict(record, "C9") == "fulfilled" + + +def test_payload_removal_fails_even_with_complete_conversation(record): + record["ng_trajectory"]["model_calls"][0]["request"] = None + assert verdict(record, "C4") == "not_fulfilled" + assert verdict(record, "C9") == "not_fulfilled" + + +def test_schema_valid_duplicate_attempt_fails(record): + record["ng_model_call_capture"]["calls"] *= 2 + assert verdict(record, "C1") == "not_fulfilled" + + +def test_nullable_provider_model_keeps_explicit_server_identity(record): + call = record["ng_model_call_capture"]["calls"][0] + call["model"] = None + assert verdict(record, "C1") == "fulfilled" + call["model_ref"] = None + assert verdict(record, "C1") == "not_fulfilled" + + +def test_sidecar_does_not_launder_payload_conflict(record, tmp_path): + call = record["ng_model_call_capture"]["calls"][0] + sidecar = {**call, "request": {"input": "different"}} + (tmp_path / "0-0.capture.jsonl").write_text(json.dumps(sidecar) + "\n") + hydrated = hydrate_record(record, capture_dir=tmp_path) + assert inspect_record(hydrated)["capabilities"]["C0"]["verdict"] == "not_fulfilled" + + +def test_tool_request_is_not_execution(record): + record["ng_agent_observations"]["records"] = [ + r for r in record["ng_agent_observations"]["records"] if r["kind"] != "tool_call" + ] + record["ng_trajectory"]["tool_calls"] = [] + assert verdict(record, "C5") == "not_fulfilled" + + +def test_independent_witness_requirements_never_pass_from_shape(record): + for capability in ("C6", "C7", "C8", "C11"): + assert verdict(record, capability) == "not_fulfilled" + + +def test_cli_atomic_replay_and_exit_codes(record, tmp_path): + bundle = tmp_path / "input.jsonl" + bundle.write_text(json.dumps(record) + "\n") + output = tmp_path / "reports" + args = ["inspect", "--bundle", str(bundle), "--output", str(output)] + assert main(args) == 0 + assert main(args) == 0 + assert len(list(output.iterdir())) == 1 + report = next(output.glob("*/capability_summary.json")) + summary = json.loads(report.read_text()) + assert summary["checker_status"] == "completed" + assert summary["is_behavioral_qualification"] is False + record["ng_agent_observations"]["records"][0]["model_calls"] = [] + bundle.write_text(json.dumps(record) + "\n") + assert main(args) == 1 + assert len(list(output.iterdir())) == 2 + bundle.write_text('{"private prompt":') + assert main(args) == 2 + assert len(list(output.iterdir())) == 2 + assert report.is_file() + + +def test_empty_and_duplicate_rollouts_fail(record, tmp_path): + bundle = tmp_path / "input.jsonl" + bundle.write_text("") + with pytest.raises(ValueError, match="no rollout records"): + inspect_bundle(bundle, output=tmp_path / "out", profile="gym-artifacts-p0/v1") + bundle.write_text((json.dumps(record) + "\n") * 2) + _, report = inspect_bundle(bundle, output=tmp_path / "out", profile="gym-artifacts-p0/v1") + assert report["verdict"] == "not_fulfilled" diff --git a/tests/unit_tests/harness_capabilities/test_gym_cli.py b/tests/unit_tests/harness_capabilities/test_gym_cli.py new file mode 100644 index 00000000000..89438003c3d --- /dev/null +++ b/tests/unit_tests/harness_capabilities/test_gym_cli.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise native command routing, retained payload joins, and report publication.""" + +import json +import sys +from pathlib import Path + +import pytest + +from nemo_gym.cli.main import main as gym_main +from nemo_gym.harness_capabilities import cli +from nemo_gym.harness_capabilities.reader import hydrate_record + + +@pytest.fixture +def record(): + return json.loads((Path(__file__).parent / "fixtures/opencode.json").read_text()) + + +@pytest.mark.parametrize( + "relative_path", + ["rollouts.jsonl", "artifacts/rollouts.jsonl", "evaluator_rollouts.jsonl", "artifacts/evaluator_rollouts.jsonl"], +) +def test_native_cli_discovers_bundle(record, tmp_path, monkeypatch, relative_path): + path = tmp_path / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record) + "\n") + quality = tmp_path / "quality_summary.json" + quality.write_text('{"health": "unchanged"}\n') + monkeypatch.setattr( + sys, "argv", ["gym", "eval", "conformance", "--bundle", str(tmp_path), "--output", str(tmp_path / "reports")] + ) + with pytest.raises(SystemExit) as exit_info: + gym_main() + assert exit_info.value.code == 0 + (summary_path,) = (tmp_path / "reports").glob("*/capability_summary.json") + summary = json.loads(summary_path.read_text()) + assert summary["verdict"] == "fulfilled" + assert summary["sources"].keys() == {str(path.resolve())} + assert quality.read_text() == '{"health": "unchanged"}\n' + + +@pytest.mark.parametrize( + "profile,code", [("gym-artifacts-p1/v1", 1), ("gym-artifacts-all/v1", 1), ("onboarding-p0/v1", 2)] +) +def test_native_cli_gate_exit_codes(record, tmp_path, monkeypatch, profile, code): + path = tmp_path / "rollouts.jsonl" + path.write_text(json.dumps(record) + "\n") + monkeypatch.setattr( + sys, + "argv", + [ + "gym", + "eval", + "conformance", + "--bundle", + str(path), + "--output", + str(tmp_path / "reports"), + "--profile", + profile, + ], + ) + with pytest.raises(SystemExit) as exit_info: + gym_main() + assert exit_info.value.code == code + if code == 2: + assert not (tmp_path / "reports").exists() + + +def test_rejects_hydra_overrides_before_writing(record, tmp_path, monkeypatch): + path = tmp_path / "rollouts.jsonl" + path.write_text(json.dumps(record) + "\n") + monkeypatch.setattr( + sys, + "argv", + ["gym", "eval", "conformance", "--bundle", str(path), "--output", str(tmp_path / "reports"), "+model=other"], + ) + with pytest.raises(SystemExit) as exit_info: + gym_main() + assert exit_info.value.code == 2 + assert not (tmp_path / "reports").exists() + + +def test_ambiguous_directory_requires_explicit_file(record, tmp_path): + for name in ("rollouts.jsonl", "evaluator_rollouts.jsonl"): + (tmp_path / name).write_text(json.dumps(record) + "\n") + with pytest.raises(ValueError, match="exactly one"): + cli.inspect_bundle(tmp_path, output=tmp_path / "reports", profile="gym-artifacts-p0/v1") + _, summary = cli.inspect_bundle( + tmp_path / "rollouts.jsonl", output=tmp_path / "reports", profile="gym-artifacts-p0/v1" + ) + assert summary["verdict"] == "fulfilled" + + +@pytest.mark.parametrize("sidecar_state", ["complete", "missing", "incomplete", "extra_call"]) +def test_native_cli_recovers_payloads_and_validates_sidecars(record, tmp_path, monkeypatch, sidecar_state): + full = hydrate_record(record) + capture_dir = tmp_path / "model-calls" + capture_dir.mkdir() + capture_file = capture_dir / "0-0.capture.jsonl" + calls = full["ng_model_call_capture"]["calls"] + if sidecar_state == "extra_call": + calls.append({**calls[0], "model_call_id": "unexpected-attempt"}) + if sidecar_state != "missing": + capture_file.write_text("".join(json.dumps(call) + "\n" for call in calls)) + if sidecar_state == "incomplete": + (capture_dir / "0-0.capture.incomplete").touch() + for call in record["ng_trajectory"]["model_calls"]: + call["request"] = call["response"] = None + path = tmp_path / "rollouts.jsonl" + path.write_text(json.dumps(record) + "\n") + monkeypatch.setattr( + sys, + "argv", + [ + "gym", + "eval", + "conformance", + "--bundle", + str(path), + "--output", + str(tmp_path / "reports"), + "--capture-dir", + str(capture_dir), + ], + ) + with pytest.raises(SystemExit) as exit_info: + gym_main() + assert exit_info.value.code == (0 if sidecar_state == "complete" else 1) + (summary_file,) = (tmp_path / "reports").glob("*/capability_summary.json") + summary = json.loads(summary_file.read_text()) + if sidecar_state != "missing": + assert str(capture_file.resolve()) in summary["sources"] + + +def test_changing_input_publishes_no_report(record, tmp_path, monkeypatch): + path = tmp_path / "rollouts.jsonl" + path.write_text(json.dumps(record) + "\n") + original_rows = cli.json_rows + + def mutate_after_read(path): + yield from original_rows(path) + path.write_text("{}\n") + + monkeypatch.setattr(cli, "json_rows", mutate_after_read) + with pytest.raises(ValueError, match="source changed"): + cli.inspect_bundle(path, output=tmp_path / "reports", profile="gym-artifacts-p0/v1") + assert list((tmp_path / "reports").iterdir()) == [] + + +def test_reports_do_not_include_payload_values(record, tmp_path, capsys): + secret = "PRIVATE-PAYLOAD-MARKER" + call = record["ng_trajectory"]["model_calls"][0] + call["request"]["input"] = secret + call["response"]["usage"]["total_tokens"] = secret + path = tmp_path / "rollouts.jsonl" + path.write_text(json.dumps(record) + "\n") + assert cli.run_inspection(bundle=path, output=tmp_path / "reports") == 1 + for file in (tmp_path / "reports").glob("*/*"): + assert secret not in file.read_text() + path.write_text('{"' + secret + '":') + assert cli.run_inspection(bundle=path, output=tmp_path / "reports") == 2 + assert secret not in capsys.readouterr().out diff --git a/tests/unit_tests/harness_capabilities/test_producers.py b/tests/unit_tests/harness_capabilities/test_producers.py new file mode 100644 index 00000000000..e5d4780a3c1 --- /dev/null +++ b/tests/unit_tests/harness_capabilities/test_producers.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract fixture through Gym's real OpenCode parser and collector projection. + +Synthetic protocol/SQLite inputs; not a live OpenCode qualification. +""" + +import json +import sqlite3 + +from nemo_gym.base_responses_api_model import build_model_call_record +from nemo_gym.harness_capabilities.checker import P0, inspect_record +from nemo_gym.harness_capabilities.reader import hydrate_record +from nemo_gym.rollout_collection import _attach_trajectory_record +from nemo_gym.rollout_observability import join_model_call_observations +from responses_api_agents.opencode_sandboxed_agent.app import ( + parse_opencode_observations, +) + + +def test_current_opencode_projection(tmp_path): + path = tmp_path / "opencode.db" + with sqlite3.connect(path) as db: + db.execute("create table session (id text, parent_id text, time_created integer)") + db.execute("create table message (id text, session_id text, data text, time_created integer)") + db.execute("create table part (id text, message_id text, session_id text, data text, time_created integer)") + db.execute("insert into session values ('root', null, 0)") + db.execute( + "insert into message values (?, ?, ?, ?)", + ( + "m1", + "root", + json.dumps( + { + "role": "assistant", + "time": {"created": 1000, "completed": 3000}, + } + ), + 1000, + ), + ) + db.execute( + "insert into part values (?, ?, ?, ?, ?)", + ( + "p1", + "m1", + "root", + json.dumps( + { + "type": "tool", + "tool": "bash", + "callID": "tool-1", + "state": { + "status": "completed", + "input": {"command": "printf ok"}, + "output": "ok", + "time": {"start": 2000, "end": 3000}, + }, + } + ), + 2000, + ), + ) + observations = parse_opencode_observations(path, "root") + exchange = { + "model_call_id": "call-1", + "client_session_id": "root", + "model_ref": {"type": "responses_api_models", "name": "policy"}, + "dialect": "responses", + "started_at": 1.0, + "completed_at": 2.0, + "latency_ms": 1000.0, + "status_code": 200, + "request": { + "model": "model", + "input": [{"role": "user", "content": "Print ok"}], + "tools": [ + { + "type": "function", + "name": "bash", + "parameters": {"type": "object"}, + } + ], + }, + "response": { + "id": "response-1", + "model": "model", + "status": "completed", + "output": [ + { + "type": "function_call", + "call_id": "tool-1", + "name": "bash", + "arguments": '{"command":"printf ok"}', + } + ], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 3}, + "output_tokens_details": {"reasoning_tokens": 2}, + }, + }, + } + call = build_model_call_record(exchange, call_index=0) + observations = join_model_call_observations(observations, [call]) + record = { + "_ng_task_index": 0, + "_ng_rollout_index": 0, + "ng_model_call_capture": { + "rollout_id": "0-0", + "calls": [call.model_dump(mode="json")], + }, + "ng_agent_observations": observations.model_dump(mode="json"), + } + _attach_trajectory_record(record, record) + result = inspect_record(hydrate_record(record)) + assert all(result["capabilities"][key]["verdict"] == "fulfilled" for key in (*P0, "C5")), result["findings"] + assert result["capabilities"]["C3"]["verdict"] == "not_fulfilled" diff --git a/uv.lock b/uv.lock index 4b0b7f5ec12..dba6338046a 100644 --- a/uv.lock +++ b/uv.lock @@ -2618,6 +2618,7 @@ dependencies = [ { name = "hydra-core" }, { name = "itsdangerous" }, { name = "jinja2" }, + { name = "jsonschema" }, { name = "mcp" }, { name = "mlflow" }, { name = "mlflow-skinny" }, @@ -2726,6 +2727,7 @@ requires-dist = [ { name = "hydra-core" }, { name = "itsdangerous" }, { name = "jinja2", specifier = ">=3.1" }, + { name = "jsonschema", specifier = ">=4.18,<5" }, { name = "mcp", specifier = ">=1.28.1,<2" }, { name = "mlflow", specifier = ">=3.16.1" }, { name = "mlflow-skinny", specifier = ">=3.16.1" }, From a92ed5c47f93c711c4ccf2e090e2c0475bdc4a4f Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Mon, 28 Sep 2026 14:30:56 +0200 Subject: [PATCH 02/11] Move harness conformance entry point into scripts Signed-off-by: Frederic Lafrance --- .../observability/harness-conformance.mdx | 4 +- nemo_gym/cli/main.py | 39 ------------------- nemo_gym/harness_capabilities/README.md | 6 +-- scripts/check_harness_conformance.py | 12 ++++++ .../{test_gym_cli.py => test_script.py} | 35 ++++++++--------- 5 files changed, 33 insertions(+), 63 deletions(-) create mode 100644 scripts/check_harness_conformance.py rename tests/unit_tests/harness_capabilities/{test_gym_cli.py => test_script.py} (85%) diff --git a/fern/versions/latest/pages/observability/harness-conformance.mdx b/fern/versions/latest/pages/observability/harness-conformance.mdx index 04188d88592..52dae1749f7 100644 --- a/fern/versions/latest/pages/observability/harness-conformance.mdx +++ b/fern/versions/latest/pages/observability/harness-conformance.mdx @@ -8,10 +8,10 @@ position: 3 Harness conformance checks whether collected rollouts retain model-call payloads, token counts, invocation ownership, semantic turns, and tool execution evidence. -Run it after collecting an evaluation with observability enabled: +From the repository root, run it after collecting an evaluation with observability enabled: ```bash -gym eval conformance \ +python scripts/check_harness_conformance.py \ --bundle results/my-harness/rollouts.jsonl \ --profile gym-artifacts-p0/v1 \ --output results/my-harness/capabilities diff --git a/nemo_gym/cli/main.py b/nemo_gym/cli/main.py index eb36a72d253..043340079a9 100644 --- a/nemo_gym/cli/main.py +++ b/nemo_gym/cli/main.py @@ -649,17 +649,6 @@ def _eval_run(args: argparse.Namespace, overrides: list[str]) -> None: dispatch(target, overrides) -def _eval_conformance(args: argparse.Namespace, overrides: list[str]) -> None: - expected_overrides = ["+verbose=true"] if args.verbose else [] - if overrides != expected_overrides: - args._parser.error("conformance does not accept Hydra overrides") - from nemo_gym.harness_capabilities.cli import run_inspection - - sys.exit( - run_inspection(bundle=args.bundle, output=args.output, profile=args.profile, capture_dir=args.capture_dir) - ) - - def _eval_health_check(args: argparse.Namespace, overrides: list[str]) -> None: expected_overrides = ["+verbose=true"] if args.verbose else [] if args.json: @@ -1089,34 +1078,6 @@ def _dataset_download(args: argparse.Namespace, overrides: list[str]) -> None: ), ), ), - "eval conformance": Command( - target=_eval_conformance, - summary="Check retained harness capability evidence in rollout artifacts.", - flags=( - Flag( - register=lambda p: p.add_argument( - "--bundle", required=True, type=Path, help="Rollout JSONL file or run directory." - ) - ), - Flag( - register=lambda p: p.add_argument( - "--output", required=True, type=Path, help="Directory for immutable capability reports." - ) - ), - Flag( - register=lambda p: p.add_argument( - "--capture-dir", type=Path, help="Original shared model-call capture directory, if needed." - ) - ), - Flag( - register=lambda p: p.add_argument( - "--profile", - default="gym-artifacts-p0/v1", - help="Artifact gate: gym-artifacts-p0/v1 (default), gym-artifacts-p1/v1, or gym-artifacts-all/v1.", - ) - ), - ), - ), "eval health-check": Command( target=_eval_health_check, summary="Verify rollout quality for an existing run directory.", diff --git a/nemo_gym/harness_capabilities/README.md b/nemo_gym/harness_capabilities/README.md index 746d0c89d3a..59bc8eb9c83 100644 --- a/nemo_gym/harness_capabilities/README.md +++ b/nemo_gym/harness_capabilities/README.md @@ -8,10 +8,10 @@ after changes to their harness or its export path. ## Run the checker Install Gym using the repository's development setup (`uv sync --extra dev`), -then inspect a completed evaluation: +then inspect a completed evaluation from the repository root: ```bash -gym eval conformance \ +python scripts/check_harness_conformance.py \ --bundle results/my-harness/rollouts.jsonl \ --profile gym-artifacts-p0/v1 \ --output results/my-harness/capabilities @@ -132,7 +132,7 @@ and `nemo_gym/rollout_observability.py` for the producer contracts. Run a small representative evaluation with your harness configuration and capture enabled, using normal `gym env start` / `gym eval run` commands. Wait for collection -to finish, then run `gym eval conformance` on the output JSONL. Inspect failing +to finish, then run `python scripts/check_harness_conformance.py` on the output JSONL. Inspect failing assertions in `capability_results.jsonl`, fix the producer, and collect a fresh run. Editing the report or fabricating missing evidence cannot qualify the harness. diff --git a/scripts/check_harness_conformance.py b/scripts/check_harness_conformance.py new file mode 100644 index 00000000000..341c1004585 --- /dev/null +++ b/scripts/check_harness_conformance.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Check retained harness capability evidence in rollout artifacts.""" + +import sys + +from nemo_gym.harness_capabilities.cli import main + + +if __name__ == "__main__": + raise SystemExit(main(["inspect", *sys.argv[1:]])) diff --git a/tests/unit_tests/harness_capabilities/test_gym_cli.py b/tests/unit_tests/harness_capabilities/test_script.py similarity index 85% rename from tests/unit_tests/harness_capabilities/test_gym_cli.py rename to tests/unit_tests/harness_capabilities/test_script.py index 89438003c3d..13e1c6a30ac 100644 --- a/tests/unit_tests/harness_capabilities/test_gym_cli.py +++ b/tests/unit_tests/harness_capabilities/test_script.py @@ -1,19 +1,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Exercise native command routing, retained payload joins, and report publication.""" +"""Exercise script command routing, retained payload joins, and report publication.""" import json +import runpy import sys from pathlib import Path import pytest -from nemo_gym.cli.main import main as gym_main from nemo_gym.harness_capabilities import cli from nemo_gym.harness_capabilities.reader import hydrate_record +SCRIPT = Path(__file__).resolve().parents[3] / "scripts/check_harness_conformance.py" + + @pytest.fixture def record(): return json.loads((Path(__file__).parent / "fixtures/opencode.json").read_text()) @@ -23,17 +26,15 @@ def record(): "relative_path", ["rollouts.jsonl", "artifacts/rollouts.jsonl", "evaluator_rollouts.jsonl", "artifacts/evaluator_rollouts.jsonl"], ) -def test_native_cli_discovers_bundle(record, tmp_path, monkeypatch, relative_path): +def test_script_discovers_bundle(record, tmp_path, monkeypatch, relative_path): path = tmp_path / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(record) + "\n") quality = tmp_path / "quality_summary.json" quality.write_text('{"health": "unchanged"}\n') - monkeypatch.setattr( - sys, "argv", ["gym", "eval", "conformance", "--bundle", str(tmp_path), "--output", str(tmp_path / "reports")] - ) + monkeypatch.setattr(sys, "argv", [str(SCRIPT), "--bundle", str(tmp_path), "--output", str(tmp_path / "reports")]) with pytest.raises(SystemExit) as exit_info: - gym_main() + runpy.run_path(str(SCRIPT), run_name="__main__") assert exit_info.value.code == 0 (summary_path,) = (tmp_path / "reports").glob("*/capability_summary.json") summary = json.loads(summary_path.read_text()) @@ -45,16 +46,14 @@ def test_native_cli_discovers_bundle(record, tmp_path, monkeypatch, relative_pat @pytest.mark.parametrize( "profile,code", [("gym-artifacts-p1/v1", 1), ("gym-artifacts-all/v1", 1), ("onboarding-p0/v1", 2)] ) -def test_native_cli_gate_exit_codes(record, tmp_path, monkeypatch, profile, code): +def test_script_gate_exit_codes(record, tmp_path, monkeypatch, profile, code): path = tmp_path / "rollouts.jsonl" path.write_text(json.dumps(record) + "\n") monkeypatch.setattr( sys, "argv", [ - "gym", - "eval", - "conformance", + str(SCRIPT), "--bundle", str(path), "--output", @@ -64,7 +63,7 @@ def test_native_cli_gate_exit_codes(record, tmp_path, monkeypatch, profile, code ], ) with pytest.raises(SystemExit) as exit_info: - gym_main() + runpy.run_path(str(SCRIPT), run_name="__main__") assert exit_info.value.code == code if code == 2: assert not (tmp_path / "reports").exists() @@ -76,10 +75,10 @@ def test_rejects_hydra_overrides_before_writing(record, tmp_path, monkeypatch): monkeypatch.setattr( sys, "argv", - ["gym", "eval", "conformance", "--bundle", str(path), "--output", str(tmp_path / "reports"), "+model=other"], + [str(SCRIPT), "--bundle", str(path), "--output", str(tmp_path / "reports"), "+model=other"], ) with pytest.raises(SystemExit) as exit_info: - gym_main() + runpy.run_path(str(SCRIPT), run_name="__main__") assert exit_info.value.code == 2 assert not (tmp_path / "reports").exists() @@ -96,7 +95,7 @@ def test_ambiguous_directory_requires_explicit_file(record, tmp_path): @pytest.mark.parametrize("sidecar_state", ["complete", "missing", "incomplete", "extra_call"]) -def test_native_cli_recovers_payloads_and_validates_sidecars(record, tmp_path, monkeypatch, sidecar_state): +def test_script_recovers_payloads_and_validates_sidecars(record, tmp_path, monkeypatch, sidecar_state): full = hydrate_record(record) capture_dir = tmp_path / "model-calls" capture_dir.mkdir() @@ -116,9 +115,7 @@ def test_native_cli_recovers_payloads_and_validates_sidecars(record, tmp_path, m sys, "argv", [ - "gym", - "eval", - "conformance", + str(SCRIPT), "--bundle", str(path), "--output", @@ -128,7 +125,7 @@ def test_native_cli_recovers_payloads_and_validates_sidecars(record, tmp_path, m ], ) with pytest.raises(SystemExit) as exit_info: - gym_main() + runpy.run_path(str(SCRIPT), run_name="__main__") assert exit_info.value.code == (0 if sidecar_state == "complete" else 1) (summary_file,) = (tmp_path / "reports").glob("*/capability_summary.json") summary = json.loads(summary_file.read_text()) From 24b4c49f6e8b6b2e93928d827d11a6086ad06505 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Mon, 28 Sep 2026 14:51:04 +0200 Subject: [PATCH 03/11] Align P0 harness evidence checks and add four-harness coverage Signed-off-by: Frederic Lafrance --- .../observability/harness-conformance.mdx | 122 +++- nemo_gym/harness_capabilities/README.md | 192 +----- nemo_gym/harness_capabilities/__init__.py | 2 +- nemo_gym/harness_capabilities/checker.py | 628 +++++++++++------- nemo_gym/harness_capabilities/cli.py | 196 ++++-- nemo_gym/harness_capabilities/reader.py | 33 +- nemo_gym/harness_capabilities/schemas.py | 131 +--- .../tests/test_observability.py | 8 +- scripts/check_harness_conformance.py | 3 +- .../harness_capabilities/fixtures/codex.jsonl | 1 + .../fixtures/hermes.jsonl | 1 + .../fixtures/opencode.jsonl | 1 + .../harness_capabilities/fixtures/pi.jsonl | 1 + .../harness_capabilities/producer_fixtures.py | 280 ++++++++ .../harness_capabilities/test_checker.py | 278 +++++++- .../harness_capabilities/test_producers.py | 140 +--- .../harness_capabilities/test_script.py | 40 +- 17 files changed, 1299 insertions(+), 758 deletions(-) create mode 100644 tests/unit_tests/harness_capabilities/fixtures/codex.jsonl create mode 100644 tests/unit_tests/harness_capabilities/fixtures/hermes.jsonl create mode 100644 tests/unit_tests/harness_capabilities/fixtures/opencode.jsonl create mode 100644 tests/unit_tests/harness_capabilities/fixtures/pi.jsonl create mode 100644 tests/unit_tests/harness_capabilities/producer_fixtures.py diff --git a/fern/versions/latest/pages/observability/harness-conformance.mdx b/fern/versions/latest/pages/observability/harness-conformance.mdx index 52dae1749f7..01280c4f30f 100644 --- a/fern/versions/latest/pages/observability/harness-conformance.mdx +++ b/fern/versions/latest/pages/observability/harness-conformance.mdx @@ -1,32 +1,120 @@ --- title: "Harness Conformance" -description: "Check retained harness evidence and onboard a custom agent's observability." +description: "Check P0 trajectory evidence and compare harness artifact coverage." position: 3 --- # Harness Conformance -Harness conformance checks whether collected rollouts retain model-call payloads, -token counts, invocation ownership, semantic turns, and tool execution evidence. -From the repository root, run it after collecting an evaluation with observability enabled: +Check the evidence retained by a harness/benchmark pair after collecting an evaluation: ```bash python scripts/check_harness_conformance.py \ --bundle results/my-harness/rollouts.jsonl \ - --profile gym-artifacts-p0/v1 \ - --output results/my-harness/capabilities + --output results/my-harness/evidence ``` -The command writes `capability_summary.json`, `capability_results.jsonl`, and -`capability_report.md` under an immutable report directory. Exit codes are 0 for a -met gate, 1 for an unmet gate, and 2 for a checker/input error. +The `gym-p0/v1` profile checks the currently supported P0 trajectory evidence +(TE-1–TE-9). Every applicable TE-1–TE-7 must pass, plus **TE-8 or TE-9** on +every rollout. It replaces the earlier experimental C0–C11 profiles. -Follow the [harness owner onboarding README](https://github.com/NVIDIA-NeMo/Gym/blob/main/nemo_gym/harness_capabilities/README.md) -for capture configuration, invocation and tool contracts, profiles, troubleshooting, -and CI usage. Start with [Rollout Evidence](/main/observability/rollout-evidence) -if your harness does not yet emit agent observations. +| Evidence | Persisted contract | +| --- | --- | +| TE-1: model call status | Exact call/model identity, HTTP or transport outcome, response status and termination metadata preserved from the response. Optional call clocks are checked when present. | +| TE-2: token counts | Supplied prompt, completion, reasoning, total and cached counts preserved on successful and failed attempts. Provider omission remains unknown. Availability is reported separately. | +| TE-3: steps | Task, repeat, rollout and step identity; timestamp; model-visible prompt; answer/tool or reasoning fields; resolution or explicit unknown. No step is inferred from an HTTP call. | +| TE-4: history | Ordered saved model inputs and outputs, including tool results and changed context. Missing media, opaque content and unavailable prior-response history fail. | +| TE-5: tool record | Execution id/name/status, originating request with dispatched arguments, and output/error. Tool clocks are outside P0. A tool request alone does not prove execution. | +| TE-6: verifier outcome | The current Gym verification surface: finite rollout reward and correctly scoped terminal resolution when present. Explicitly unavailable verification cannot be a numeric grade. Extended verifier scope/version and judge provenance are not certified by this profile. | +| TE-7: payloads | Saved request and response/error bodies. Empty returned bodies and explicit no-response transport failures remain distinct. | +| TE-8: run join | Exactly one explicit invocation owner per captured attempt, including attempts without response ids; consistent parent relationships and exact references. | +| TE-9: step join | Every retained policy attempt belongs to exactly one identified step. Retries remain attempts on that step. Explicit compaction-helper refs are excluded from policy steps. | -These are retained-artifact checks. Independent behavioral qualification and a live -probe runner are not implemented. P0 is the initial usable gate; P1 and All include -incomplete validators and currently cannot pass. Run -[rollout health checks](/main/evaluation/rollout-health) separately to assess health. +TE-2 distinguishes preservation from availability: an absent provider metric is +not an available zero or a five-count availability pass. Derived totals follow +Gym's documented prompt-plus-completion convention. Anthropic prompt totals +include cache reads and cache creation. Missing usage remains visible to health. + +The profile covers retained artifacts. It does not certify live failure/retry +scenario coverage, closed-run completeness, health, accuracy, or deployment +readiness. Independent completeness required by some health consumers is separate +from TE-9's retained reference check. Infra taxonomy (TE-10) and P1 efficiency +requirements are outside this profile. + +## Applicability and delivery + +By default the pair has policy steps, tools, and a verifier. Use `--no-tools`, +`--no-verifier`, or `--no-steps` only when that feature does not apply to the +pair. Missing evidence never automatically becomes N/A. Contradictory retained +evidence rejects an exclusion. A step-free pair still needs TE-8. + +`--bundle` accepts a JSONL file, or a directory with exactly one `rollouts.jsonl` +or `evaluator_rollouts.jsonl`, at its root or under `artifacts/`. + +The delivery surface is the rollout JSONL, including `ng_trajectory` payloads. +An optional `--capture-dir /absolute/path/to/model-calls` cross-checks exact +identity sets, payloads, metadata and incomplete markers. It cannot repair a +missing JSONL payload for a passing delivery verdict. + +Each inspection atomically publishes an immutable directory containing: + +- `evidence_summary.json`: gate, per-TE counts, token availability, applicability, + source/checker/schema hashes and qualification limits. +- `evidence_results.jsonl`: per-rollout findings and field locations. +- `evidence_report.md`: evidence table. + +Exit codes are **0** for a met gate, **1** for unmet evidence, and **2** for an +input/checker error. An empty input is an error. Reports omit payload contents. +Identical inputs and checker versions reuse the same report directory. + +## Compare harnesses + +Use the same task and capture settings when comparing harnesses: + +```bash +python scripts/check_harness_conformance.py matrix \ + --harness opencode=results/opencode/rollouts.jsonl \ + --harness pi=results/pi/rollouts.jsonl \ + --harness codex=results/codex/rollouts.jsonl \ + --harness hermes=results/hermes/rollouts.jsonl \ + --output results/harness-evidence +``` + +The command writes per-harness reports plus `harness_evidence.md` and +`harness_evidence.json`. Applicability flags apply to every input pair in that +comparison. A matrix FAIL identifies missing or inconsistent evidence; it is not +a benchmark accuracy result. + +Regression coverage exercises the current OpenCode SQLite parser, Pi event +parser, Codex exec parser, and Hermes observation hooks through Gym collection. +Controlled protocol/capture inputs and a controlled verifier reward make these +reproducible producer tests, not live harness qualifications. The captured model +columns depend on the deployed harness actually routing calls through capture. +The emitted fixtures are under `tests/unit_tests/harness_capabilities/fixtures/`; +they can be used with the same matrix command. + +## Onboard a producer + +Enable `observability_enabled: true` and set `model_call_capture_dir` to a shared +absolute directory. Route requests through Gym's correlated model URL and retain +all attempts. See [Model-call Capture](/main/model-server/model-call-capture). + +Return `ng_agent_observations` with `AgentInvocation` and exact `ModelCallRef` +keys (`model_call_id`, or unique `model_ref` plus `response_id`). Forward the +invocation id as `x-session-id` for failed-call joins through the collector. +A timestamp or shared model name never establishes ownership. + +Supply explicit `TrajectoryTurn` records with the actual prompt and decision +boundaries. Preserve tool request/result pairs and `ToolCallObservation` outcomes, +including failures and terminal submission. Keep resolution unknown on +intermediate steps. Retain the resources server's verification response. +See [Rollout Evidence](/main/observability/rollout-evidence). + +Inspect a small collected evaluation, fix producer gaps, and collect again. +Run [rollout health checks](/main/evaluation/rollout-health) separately. Evidence +checks neither rewrite `quality_summary.json` nor treat a truthfully recorded +failed model call as missing evidence. + +```bash +python -m pytest tests/unit_tests/harness_capabilities -q +``` diff --git a/nemo_gym/harness_capabilities/README.md b/nemo_gym/harness_capabilities/README.md index 59bc8eb9c83..94c895cf55d 100644 --- a/nemo_gym/harness_capabilities/README.md +++ b/nemo_gym/harness_capabilities/README.md @@ -1,192 +1,16 @@ -# Harness capability conformance +# Harness trajectory evidence -Check whether a Gym harness retains the evidence needed to inspect model calls, -token usage, invocation ownership, semantic turns, and tool execution. Harness -owners run this checker against their collected rollouts during onboarding and -after changes to their harness or its export path. - -## Run the checker - -Install Gym using the repository's development setup (`uv sync --extra dev`), -then inspect a completed evaluation from the repository root: +Check P0 evidence from a collected evaluation: ```bash python scripts/check_harness_conformance.py \ --bundle results/my-harness/rollouts.jsonl \ - --profile gym-artifacts-p0/v1 \ - --output results/my-harness/capabilities + --output results/my-harness/evidence ``` -`--bundle` accepts any rollout JSONL file. It also accepts a directory containing -exactly one `rollouts.jsonl` or `evaluator_rollouts.jsonl`, at its root or under -`artifacts/`. Pass the file explicitly if the directory contains multiple candidates. -Files are streamed one rollout at a time. No model endpoint is needed for inspection. - -The module interface exposes the same checks: - -```bash -python -m nemo_gym.harness_capabilities inspect \ - --bundle results/my-harness \ - --output results/my-harness/capabilities -``` - -Collection normally retains full payloads in `ng_trajectory.model_calls`. If you -also have the original shared model capture directory, add -`--capture-dir /absolute/path/to/model-calls`. This restores missing payloads and -checks exact call identity, conflicting representations, missing/extra calls, -and incomplete capture markers. Preserve the original capture directory until -inspection finishes; use a separate capture directory for each evaluation. - -Each inspection publishes a content-addressed directory containing: - -| File | Contents | -| --- | --- | -| `capability_summary.json` | Requested gate, C0–C11 verdicts, counts, source/checker/schema hashes, qualification limits | -| `capability_results.jsonl` | Per-rollout verdicts and findings with assertion IDs and source field locations | -| `capability_report.md` | Capability matrix and passing-record counts | - -Exit codes are **0** for a met gate, **1** for an unmet gate, and **2** for a -checker/input error. A gate requires every inspected record to pass each required -capability. An empty input is an error. Report files are published together only -after inspection completes and input hashes are rechecked. Repeating an identical -inspection reuses its report directory. Prompts and response bodies are omitted -from reports; source paths and field locations are retained. - -## Onboard a harness - -### 1. Enable model-call capture - -Add these top-level settings to the shared configuration used by your agent, -model servers, and collector: - -```yaml -observability_enabled: true -model_call_capture_dir: /absolute/shared/path/my-harness/model-calls -``` - -Route every model request through Gym's correlated model URL -`/ng-rollout//v1/...`. Preserve that URL when constructing an SDK -client, forwarding requests, spawning child agents, or retrying a call. Calls -sent directly to a provider do not reach Gym's capture middleware. See -[Model-call Capture](https://docs.nvidia.com/nemo/gym/main/model-server/model-call-capture) -for the custom-agent URL helpers and a runnable collection example. - -Use Gym's capture records for request/response bodies, HTTP or transport outcomes, -timing, and provider usage. Preserve absent token values as absent; do not replace -them with zero. P0 requires prompt, completion, reasoning, total, and cached token -counts on successful calls. A provider that omits one cannot meet this strict gate. - -### 2. Return invocation evidence - -Return `ng_agent_observations` using `AgentObservationBundle` and `AgentInvocation` -from `nemo_gym.rollout_observability`. Give each root or child invocation a stable -ID, retain the parent relationship, and record its ordered conversation. - -Associate every captured attempt with exactly one invocation. `ModelCallRef` -supports either `model_call_id` or the exact pair of `model_ref` and `response_id`. -All supplied reference fields must agree. For failures without response IDs, -propagate the invocation ID as `x-session-id` to the Gym model server; the collector -uses captured `client_session_id` to join those attempts to the invocation. -Model names, timestamps, and list order cannot establish ownership. Retries are -separate attempts and need separate references, even if their request bodies match. - -For a successful response, an invocation can include: - -```python -from nemo_gym.rollout_observability import AgentInvocation, AgentObservationBundle, ModelCallRef - -observations = AgentObservationBundle( - source="my_harness", - records=[ - AgentInvocation( - invocation_id=invocation_id, - model_calls=[ModelCallRef(model_ref=model_ref, response_id=response.id)], - conversation=conversation, - ) - ], -) -``` - -Here `invocation_id`, `model_ref`, `response`, and `conversation` come from the -actual harness execution. Return the bundle alongside the normal `/run` response; -the collector joins captures and produces `ng_trajectory`. - -### 3. Add turn and tool evidence - -For semantic turns, supply explicit `TrajectoryTurn` entries in a producer -`ng_trajectory`, including task/rollout/invocation identity, decision number, -timestamp, input, answer, executed-step count, resolution state, and exact call -references. The collector preserves these entries. Keep `resolved=None` until an -authoritative result exists; do not propagate a final reward to intermediate turns. - -For tools, put requests and results in the invocation conversation and measured -execution intervals in `ToolCallObservation`. Match both by invocation and -tool-call ID. Record start/end/duration, tool name, timing source, and terminal -status, including failures, timeouts, cancellation, and final submission. A model -request to execute a tool is insufficient evidence that execution occurred. - -See [Rollout Evidence](https://docs.nvidia.com/nemo/gym/main/observability/rollout-evidence) -and `nemo_gym/rollout_observability.py` for the producer contracts. - -### 4. Collect, inspect, and fix gaps - -Run a small representative evaluation with your harness configuration and capture -enabled, using normal `gym env start` / `gym eval run` commands. Wait for collection -to finish, then run `python scripts/check_harness_conformance.py` on the output JSONL. Inspect failing -assertions in `capability_results.jsonl`, fix the producer, and collect a fresh run. -Editing the report or fabricating missing evidence cannot qualify the harness. - -Keep regression coverage for successful calls, tool failures, HTTP errors, retries, -missing usage, rejected model output, and cancellation as supported by your harness. -Include concurrent/child invocation cases if supported. Check the actual emitted -records, including ownership of attempts that never produce an accepted answer. - -Run `gym eval health-check RUN_DIR --rollouts-file PATH` separately to assess -rollout health. Conformance reports and `quality_summary.json` answer different -questions; conformance does not rewrite health findings or assert health readiness. -For onboarding review, retain the harness version, run configuration, native -rollouts/captures, conformance reports, and representative rollout/health evidence. -Attach reports to your normal CI artifacts if using the exit status as a gate. - -## Profiles and supported scope - -These profiles check **retained artifacts**. Their verdicts do not independently -certify retry/cancellation scenario coverage, capture completeness, or execution -behavior. No live probe runner is included. Reports state -`is_behavioral_qualification: false` and zero witnessed scenario coverage. - -| Capability | Required in | Current check | -| --- | --- | --- | -| C0: records and identity | P0, P1, All | Readable native records, task/rollout identity, trajectory version, reader integrity | -| C1: calls and statuses | P0, P1, All | Unique call IDs, model/dialect, timing, terminal HTTP/transport evidence | -| C2: five token counts | P0, P1, All | Nonnegative counts, faithful absence/zero, retained usage agreement, subset/total semantics | -| C3: semantic turns | All | Explicit turn fields and resolvable invocation-owned call references | -| C4: model-visible history | P0, P1, All | Ordered retained requests/responses; unresolved media or opaque reasoning fails | -| C5: tool execution | P1, All | Executed tool IDs, request/result joins, terminal status and separate timing | -| C6: independent parallel timing | P1, All | Validator incomplete; always unfulfilled | -| C7: resource lifecycle | All | Validator incomplete; always unfulfilled | -| C8: verifier provenance | All | Validator incomplete; always unfulfilled | -| C9: call payloads | P0, P1, All | Retained request and response/error, explicit no-response transport failure | -| C10: invocation ownership | P0, P1, All | Exactly one explicit owner per captured attempt, consistent references and parent graph | -| C11: complete turn-call accounting | All | Validator incomplete; reference defects diagnosed, completeness never certified | - -Profile names are `gym-artifacts-p0/v1`, `gym-artifacts-p1/v1`, and -`gym-artifacts-all/v1`. P1 and All currently always fail their gates because they -require incomplete validators. All C0–C11 verdicts are reported even when selecting -P0, so owners can inspect C3/C5 independently. `onboarding-p0/v1` is not an available -profile. The supported input is native Gym trajectory schema `1.0`; ATIF exports -and normalized third-party records need separate readers. - -## Develop the checker - -```bash -python -m pytest tests/unit_tests/harness_capabilities -q -``` +Use `matrix --harness NAME=PATH` to compare multiple harnesses. The checker uses +TE-1–TE-9 and the `gym-p0/v1` profile. Reports are `evidence_summary.json`, +`evidence_results.jsonl`, and `evidence_report.md`. -Contract fixtures use synthetic inputs through Gym's mini-SWE/OpenCode producers; -they are not live harness qualifications. Mutation tests reject missing payloads, -fabricated token counts, broken ownership, duplicate attempts, and missing tool -execution. Integration tests exercise current Gym producer/collector code and the -native CLI. Schema predicates are versioned in `schemas.py`; profile requirements -are in `checker.py`. Keep regression tests and documentation aligned when changing -either contract. +See [Harness Conformance](../../fern/versions/latest/pages/observability/harness-conformance.mdx) +for contracts, applicability, matrix usage, producer onboarding, and limitations. diff --git a/nemo_gym/harness_capabilities/__init__.py b/nemo_gym/harness_capabilities/__init__.py index 9b7db91f275..f907cf6da0d 100644 --- a/nemo_gym/harness_capabilities/__init__.py +++ b/nemo_gym/harness_capabilities/__init__.py @@ -11,4 +11,4 @@ __all__ = ["inspect_record"] -__version__ = "0.1.1" +__version__ = "0.2.0" diff --git a/nemo_gym/harness_capabilities/checker.py b/nemo_gym/harness_capabilities/checker.py index b1c3b41920d..2766797c628 100644 --- a/nemo_gym/harness_capabilities/checker.py +++ b/nemo_gym/harness_capabilities/checker.py @@ -5,41 +5,50 @@ import base64 import binascii +import math from collections import Counter from dataclasses import asdict, dataclass from typing import Iterable, TypeGuard from jsonschema import Draft202012Validator -from .schemas import CALL_REF, SCHEMAS +from .schemas import CALL_REF, SCHEMAS, TOKEN_FIELDS NAMES = { - "C0": "readable records and identity", - "C1": "model calls and statuses", - "C2": "five token counts", - "C3": "canonical semantic turns", - "C4": "model-visible history", - "C5": "tool execution evidence", - "C6": "independent parallel timing", - "C7": "resource lifecycle", - "C8": "verifier resolution and rewards", - "C9": "model-call payloads", - "C10": "invocation ownership", - "C11": "complete turn-call accounting", -} -P0 = ("C0", "C1", "C2", "C4", "C9", "C10") -PENDING_VALIDATORS = frozenset({"C6", "C7", "C8", "C11"}) -PROFILES = { - "gym-artifacts-p0/v1": P0, - "gym-artifacts-p1/v1": (*P0, "C5", "C6"), - "gym-artifacts-all/v1": tuple(NAMES), + "TE-1": "model_call_status", + "TE-2": "token_counts", + "TE-3": "steps", + "TE-4": "history", + "TE-5": "tool_record", + "TE-6": "verifier_outcome", + "TE-7": "payloads", + "TE-8": "run_join", + "TE-9": "step_join", } +PROFILE = "gym-p0/v1" +P0 = ("TE-1", "TE-2", "TE-3", "TE-4", "TE-5", "TE-6", "TE-7") + + +@dataclass(frozen=True) +class EvidenceScope: + """Applicability of a harness/benchmark pair; absence never implies N/A.""" + + tools: bool = True + verifier: bool = True + steps: bool = True + + +def gate_passes(evidence: dict) -> bool: + """All applicable P0 rows and at least one exact call join must pass.""" + return all(evidence[key]["verdict"] in {"fulfilled", "not_applicable"} for key in P0) and any( + evidence[key]["verdict"] == "fulfilled" for key in ("TE-8", "TE-9") + ) @dataclass(frozen=True) class Finding: - capability: str + evidence: str assertion: str location: str reason: str @@ -67,12 +76,17 @@ def _mapping(value: object) -> dict: def _number(value: object) -> TypeGuard[int | float]: - return isinstance(value, (int, float)) and not isinstance(value, bool) + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) def _successful(call: dict) -> bool: code = call.get("status_code") - return isinstance(code, int) and 200 <= code < 300 and not call.get("error_category") + return ( + type(code) is int + and 200 <= code < 300 + and not call.get("error_category") + and call.get("response_status") not in {"failed", "error", "cancelled"} + ) def _missing_content(value: object) -> bool: @@ -110,11 +124,48 @@ def _resolve(reference: dict, calls: list[dict]) -> list[int]: ] +def _provider_usage(usage: dict) -> dict: + def first(*values): + return next((v for v in values if v is not None), None) + + prompt = first(usage.get("input_tokens"), usage.get("prompt_tokens")) + completion = first(usage.get("output_tokens"), usage.get("completion_tokens")) + cache_read, cache_write = usage.get("cache_read_input_tokens"), usage.get("cache_creation_input_tokens") + if all(v is None or _number(v) for v in (prompt, cache_read, cache_write)) and ( + cache_read is not None or cache_write is not None + ): + if prompt is not None or (cache_read or 0) + (cache_write or 0) > 0: + prompt = (prompt or 0) + (cache_read or 0) + (cache_write or 0) + total = usage.get("total_tokens") + if total is None and _number(prompt) and _number(completion): + total = prompt + completion + return { + "tokens_in": prompt, + "tokens_out": completion, + "tokens_total": total, + "tokens_reasoning": first( + _mapping(usage.get("output_tokens_details")).get("reasoning_tokens"), + _mapping(usage.get("completion_tokens_details")).get("reasoning_tokens"), + usage.get("reasoning_output_tokens"), + ), + "cached_tokens": first( + _mapping(usage.get("input_tokens_details")).get("cached_tokens"), + _mapping(usage.get("prompt_tokens_details")).get("cached_tokens"), + _mapping(usage.get("input_tokens_details")).get("cached_input_tokens"), + _mapping(usage.get("prompt_tokens_details")).get("cached_input_tokens"), + cache_read, + usage.get("cached_input_tokens"), + ), + } + + class _RecordInspector: """Shared evidence and findings for capability-specific artifact checks.""" - def __init__(self, record: dict, source: str) -> None: + def __init__(self, record: dict, source: str, scope: EvidenceScope) -> None: self.record = record + self.scope = scope + self.not_applicable: set[str] = set() self.source = source self.findings: list[Finding] = [] self.trajectory = _mapping(self.record.get("ng_trajectory")) @@ -136,45 +187,54 @@ def _fail(self, capability: str, assertion: str, location: str, reason: str) -> self.findings.append(Finding(capability, assertion, self.source + location, reason)) def check_identity(self) -> None: - """C0: readable records, supported schema and consistent identity.""" - rollout_id = self.record.get("_ng_rollout_id") or self.capture.get("rollout_id") + """Record integrity: readable records, supported schema and consistent identity.""" + rollout_id = ( + self.record.get("_ng_rollout_id") or self.capture.get("rollout_id") or self.trajectory.get("rollout_id") + ) if not isinstance(rollout_id, str) or not rollout_id: self._fail( - "C0", + "record", "identity.rollout", "", "explicit rollout identity is required", ) if self.record.get("_ng_task_index") is None and not self.trajectory.get("task_id"): - self._fail("C0", "identity.task", "", "explicit task identity is required") + self._fail("record", "identity.task", "", "explicit task identity is required") if self.trajectory and self.trajectory.get("schema_version") != "1.0": self._fail( - "C0", + "record", "schema.version", "/ng_trajectory/schema_version", "unsupported trajectory version", ) if self.trajectory.get("rollout_id") and self.trajectory["rollout_id"] != rollout_id: self._fail( - "C0", + "record", "identity.conflict", "/ng_trajectory/rollout_id", "trajectory and capture identities differ", ) - if not isinstance(self.capture.get("calls"), list) or len(self.calls) != len(self.capture["calls"]): + if "calls" in self.capture and ( + not isinstance(self.capture["calls"], list) or len(self.calls) != len(self.capture["calls"]) + ): self._fail( - "C0", + "record", "capture.shape", "/ng_model_call_capture/calls", "a list of call objects is required", ) + for owner, fields in ((self.trajectory, ("turns", "tool_calls", "invocations")), (self.bundle, ("records",))): + for field in fields: + values = owner.get(field) + if values is not None and (not isinstance(values, list) or len(_objects(values)) != len(values)): + self._fail("record", "records.shape", "/" + field, "evidence records must be a list of objects") for issue in self.record.get("_capability_reader_issues", []): - self._fail("C0", "reader.integrity", "", issue) + self._fail("record", "reader.integrity", "", issue) def check_capture_presence(self) -> None: """Missing captures invalidate every capability that depends on calls.""" - for capability in ("C1", "C2", "C4", "C9", "C10", "C11"): - if not self.calls: + for capability in ("TE-1", "TE-2", "TE-4", "TE-7", "TE-8", "TE-9"): + if not self.calls and (capability != "TE-9" or self.scope.steps): self._fail( capability, "calls.required", @@ -183,107 +243,97 @@ def check_capture_presence(self) -> None: ) def check_model_calls(self) -> None: - """C1: unique call identities, model identity, status and timing.""" + """TE-1: faithful terminal outcomes, including failure and cancellation.""" call_ids = [c.get("model_call_id") for c in self.calls] if len(set(call_ids)) != len(call_ids): - self._fail( - "C1", - "identity.unique", - "/ng_model_call_capture/calls", - "duplicate call identity", - ) - + self._fail("TE-1", "identity.unique", "/ng_model_call_capture/calls", "duplicate call identity") for index, call in enumerate(self.calls): location = f"/ng_model_call_capture/calls/{index}" - self.findings.extend(_schema_issues("C1", call, self.source + location)) + self.findings.extend(_schema_issues("TE-1", call, self.source + location)) + response = _mapping(call.get("response")) + status = call.get("status_code") + http_success = type(status) is int and 200 <= status < 300 + if status is None and not call.get("error_category"): + self._fail("TE-1", "outcome.required", location, "HTTP outcome or transport error is required") + if type(status) is int and status < 200: + self._fail("TE-1", "outcome.terminal", location, "informational HTTP status is not a terminal outcome") + if _successful(call) and not call.get("response_id"): + self._fail("TE-1", "response.identity", location, "a returned successful response needs identity") + if response.get("id") is not None and call.get("response_id") != response["id"]: + self._fail("TE-1", "response.identity", location, "response identity differs from retained body") + expected_status = response.get("status") + choices = _objects(response.get("choices")) + finish = next( + ( + v + for v in ( + response.get("stop_reason"), + choices[0].get("finish_reason") if choices else None, + _mapping(response.get("incomplete_details")).get("reason"), + ) + if isinstance(v, str) + ), + None, + ) + for field, expected in (("response_status", expected_status), ("finish_reason", finish)): + if response and call.get(field) != expected: + self._fail( + "TE-1", + "outcome.preservation", + location + "/" + field, + "terminal metadata differs from the retained response", + ) + if http_success and not call.get("error_category"): + terminal = ( + expected_status in {"completed", "incomplete", "failed", "error", "cancelled"} + if call.get("dialect") == "responses" + else bool(finish) + ) + if not terminal or (expected_status == "incomplete" and not finish): + self._fail("TE-1", "outcome.terminal", location, "response termination evidence is missing") + if response.get("error") and expected_status not in {"failed", "error", "cancelled"}: + self._fail("TE-1", "outcome.error", location, "response error is recorded as success") + # Clocks are optional for equivalent capture paths. Validate them when retained. start, end = call.get("started_at"), call.get("completed_at") if _number(start) and _number(end) and end < start: - self._fail("C1", "timing.order", location, "completion precedes start") - if not call.get("model") and not _mapping(call.get("model_ref")).get("name"): - self._fail( - "C1", - "model.identity", - location, - "model identity unavailable", - ) + self._fail("TE-1", "timing.order", location, "completion precedes start") def check_token_counts(self) -> None: - """C2: complete token counts consistent with retained provider usage.""" - if not any(_successful(c) for c in self.calls): - self._fail( - "C2", - "usage.eligible", - "/ng_model_call_capture/calls", - "no successful calls with token evidence", - ) - + """TE-2: preserve supplied usage on all attempts; report availability separately.""" for index, call in enumerate(self.calls): location = f"/ng_model_call_capture/calls/{index}" - response = call.get("response") - if _successful(call): - self.findings.extend(_schema_issues("C2", call, self.source + location)) - prompt, cached = call.get("tokens_in"), call.get("cached_tokens") - if _number(prompt) and _number(cached) and cached > prompt: + self.findings.extend(_schema_issues("TE-2", call, self.source + location)) + usage = _mapping(_mapping(call.get("response")).get("usage")) + expected = _provider_usage(usage) + self.findings.extend(_schema_issues("TE-2", expected, self.source + location + "/response/usage")) + for key, value in expected.items(): + if call.get(key) != value: self._fail( - "C2", - "usage.cache_subset", - location, - "cached count exceeds prompt count", - ) - # Responses/chat count reasoning inside completion. Anthropic has - # different cache semantics and does not supply reasoning counts. - if call.get("dialect") in ("responses", "chat"): - completion, reasoning, total = ( - call.get(k) - for k in ( - "tokens_out", - "tokens_reasoning", - "tokens_total", - ) + "TE-2", + "usage.preservation", + location + "/" + key, + "count differs from provider usage or documented derived total, including absence", ) - if _number(reasoning) and _number(completion) and reasoning > completion: - self._fail( - "C2", - "usage.reasoning_subset", - location, - "reasoning count exceeds completion count", - ) - if _number(prompt) and _number(completion) and _number(total) and total != prompt + completion: - self._fail( - "C2", - "usage.total", - location, - "total differs from prompt plus completion", - ) - usage = _mapping(_mapping(response).get("usage")) - if call.get("dialect") in ("responses", "chat"): - is_responses = call.get("dialect") == "responses" - input_key, output_key = ( - ("input_tokens", "output_tokens") if is_responses else ("prompt_tokens", "completion_tokens") - ) - expected = { - "tokens_in": usage.get(input_key), - "tokens_out": usage.get(output_key), - "tokens_total": usage.get("total_tokens"), - "tokens_reasoning": _mapping(usage.get(output_key + "_details")).get("reasoning_tokens"), - "cached_tokens": _mapping(usage.get(input_key + "_details")).get("cached_tokens"), - } - for key, value in expected.items(): - if call.get(key) != value: - self._fail( - "C2", - "usage.preservation", - location + "/" + key, - "count differs from captured provider usage, including absence", - ) + prompt, completion, reasoning, total, cached = (call.get(k) for k in TOKEN_FIELDS) + if _number(prompt) and _number(cached) and cached > prompt: + self._fail("TE-2", "usage.cache_subset", location, "cached count exceeds prompt count") + if ( + call.get("dialect") in {"responses", "chat"} + and _number(reasoning) + and _number(completion) + and reasoning > completion + ): + self._fail("TE-2", "usage.reasoning_subset", location, "reasoning count exceeds completion count") + if all(_number(v) for v in (prompt, completion, total)) and total != prompt + completion: + self._fail("TE-2", "usage.total", location, "total differs from prompt plus completion") def check_content_references(self) -> None: - """C4/C9: reject unavailable media and opaque payload dependencies.""" + """TE-4/TE-7: reject unavailable media and opaque payload dependencies.""" for index, call in enumerate(self.calls): location = f"/ng_model_call_capture/calls/{index}" request, response = call.get("request"), call.get("response") if _missing_content(request) or _missing_content(response): - for capability in ("C4", "C9"): + for capability in ("TE-4", "TE-7"): self._fail( capability, "payload.content_reference", @@ -292,18 +342,26 @@ def check_content_references(self) -> None: ) def check_history(self) -> None: - """C4: ordered model request and response history.""" + """TE-4: ordered model request and response history.""" for index, call in enumerate(self.calls): location = f"/ng_model_call_capture/calls/{index}" request, response = call.get("request"), call.get("response") request_key = "input" if call.get("dialect") == "responses" else "messages" if not isinstance(request, dict) or not isinstance(request.get(request_key), (list, str)): self._fail( - "C4", + "TE-4", "request.history", location + "/request", "ordered model input is unavailable", ) + previous = _mapping(request).get("previous_response_id") + if previous and not any( + c.get("response_id") == previous and c.get("request") is not None and c.get("response") is not None + for c in self.calls[:index] + ): + self._fail( + "TE-4", "request.previous_response", location, "referenced prior response history is not retained" + ) if _successful(call): response_key = { "responses": "output", @@ -312,14 +370,14 @@ def check_history(self) -> None: }.get(str(call.get("dialect")), "") if not isinstance(response, dict) or not isinstance(response.get(response_key), list): self._fail( - "C4", + "TE-4", "response.history", location + "/response", "model output is unavailable", ) def check_payloads(self) -> None: - """C9: retained payloads or explicit no-response transport failures.""" + """TE-7: retained payloads or explicit no-response transport failures.""" for index, call in enumerate(self.calls): location = f"/ng_model_call_capture/calls/{index}" request, response = call.get("request"), call.get("response") @@ -327,24 +385,24 @@ def check_payloads(self) -> None: response_present = isinstance(response, dict) or isinstance(call.get("response_raw"), str) if not request_present: self._fail( - "C9", + "TE-7", "payload.request", location, "request payload unavailable", ) if not response_present and (call.get("status_code") is not None or not call.get("error_category")): self._fail( - "C9", + "TE-7", "payload.response", location, "response body unavailable; not an explicit no-response failure", ) def check_ownership(self) -> None: - """C10: each captured attempt resolves to one explicit invocation owner.""" + """TE-8: each captured attempt resolves to one explicit invocation owner.""" if not self.invocations or len(set(self.invocation_ids)) != len(self.invocation_ids): self._fail( - "C10", + "TE-8", "invocations.required_unique", "/ng_agent_observations", "unique invocations are required", @@ -352,7 +410,7 @@ def check_ownership(self) -> None: parents = {i.get("invocation_id"): i.get("parent_invocation_id") for i in self.invocations} for index, invocation in enumerate(self.invocations): location = f"/invocations/{index}" - self.findings.extend(_schema_issues("C10", invocation, self.source + location)) + self.findings.extend(_schema_issues("TE-8", invocation, self.source + location)) parent = invocation.get("parent_invocation_id") seen = {invocation.get("invocation_id")} while parent is not None and parent not in seen and parent in parents: @@ -360,7 +418,7 @@ def check_ownership(self) -> None: parent = parents[parent] if parent is not None: self._fail( - "C10", + "TE-8", "invocation.parent", location, "parent is missing or cyclic", @@ -369,7 +427,7 @@ def check_ownership(self) -> None: matches = _resolve(reference, self.calls) if len(matches) != 1: self._fail( - "C10", + "TE-8", "ownership.reference", location, "call reference does not resolve uniquely", @@ -379,14 +437,14 @@ def check_ownership(self) -> None: for index, assigned in self.owners.items(): if len(assigned) != 1: self._fail( - "C10", + "TE-8", "ownership.exactly_once", f"/ng_model_call_capture/calls/{index}", "call must have exactly one explicit owner", ) elif self.calls[index].get("client_session_id") and self.calls[index]["client_session_id"] != assigned[0]: self._fail( - "C10", + "TE-8", "ownership.session", f"/ng_model_call_capture/calls/{index}", "owner conflicts with captured client session", @@ -403,67 +461,121 @@ def _check_records(self, capability: str, records: list[dict]) -> None: for index, item in enumerate(records): self.findings.extend(_schema_issues(capability, item, f"{self.source}/{capability}/{index}")) - def check_turns(self) -> Counter: - """C3: canonical turns; also report C11 reference defects and count joins.""" - self._check_records("C3", self.turns) - turn_refs: Counter = Counter() - turn_keys = [(t.get("invocation_id"), t.get("turn_no")) for t in self.turns] - if len(set(turn_keys)) != len(turn_keys): - self._fail( - "C3", - "turn.unique", - "/ng_trajectory/turns", - "duplicate semantic turn", - ) + def check_turns(self) -> None: + """TE-3 contains step fields; TE-9 independently checks call linkage.""" + if not self.scope.steps: + if self.turns: + self._fail( + "TE-3", "scope.contradiction", "/ng_trajectory/turns", "steps exist in a declared step-free pair" + ) + self._fail( + "TE-9", "scope.contradiction", "/ng_trajectory/turns", "steps exist in a declared step-free pair" + ) + else: + self.not_applicable.update(("TE-3", "TE-9")) + return + if self.record.get("_ng_rollout_index") is None: + self._fail("TE-3", "identity.repeat", "", "explicit repeat identity is required") + self._check_records("TE-3", self.turns) + keys = [(t.get("invocation_id"), t.get("turn_no")) for t in self.turns] + if len(set(keys)) != len(keys): + self._fail("TE-3", "turn.unique", "/ng_trajectory/turns", "duplicate step identity") for turn in self.turns: - if turn.get("invocation_id") not in self.invocation_ids or any( - turn.get(k) != self.trajectory.get(k) for k in ("task_id", "rollout_id") - ): + if any(turn.get(k) != self.trajectory.get(k) for k in ("task_id", "rollout_id")): + self._fail("TE-3", "turn.identity", "/ng_trajectory/turns", "step identity differs from rollout") + if "answer" not in turn and "reasoning_content" not in turn: self._fail( - "C3", - "turn.identity", - "/ng_trajectory/turns", - "turn identity does not match trajectory/invocation", + "TE-3", "turn.content", "/ng_trajectory/turns", "answer/tool or reasoning field is required" ) - for reference in _objects(turn.get("model_calls")): + + def check_step_join(self) -> None: + """TE-9: exact policy-attempt membership, independent of invocation refs.""" + if not self.scope.steps: + return + if any( + f.evidence == "TE-8" and f.assertion in {"ownership.reference", "invocation.parent", "ownership.session"} + for f in self.findings + ): + self._fail( + "TE-9", + "ownership.conflict", + "/ng_agent_observations", + "contradictory run refs cannot be bypassed by a step join", + ) + auxiliary: set[int] = set() + for observation in self.observations: + if observation.get("kind") == "context_compaction": + for reference in _objects(observation.get("model_calls")): + matches = _resolve(reference, self.calls) + if len(matches) != 1: + self._fail( + "TE-9", + "auxiliary.reference", + "/ng_agent_observations", + "helper reference does not resolve uniquely", + ) + else: + auxiliary.add(matches[0]) + keys = [(t.get("invocation_id"), t.get("turn_no")) for t in self.turns] + if len(set(keys)) != len(keys) or any( + not isinstance(k[0], str) or not k[0] or type(k[1]) is not int or k[1] < 1 for k in keys + ): + self._fail("TE-9", "step.identity", "/ng_trajectory/turns", "unique explicit step identities are required") + refs: Counter = Counter() + for turn in self.turns: + references = turn.get("model_calls") + if not isinstance(references, list) or len(_objects(references)) != len(references): + self._fail("TE-9", "turn.references", "/ng_trajectory/turns", "step references must be objects") + for reference in _objects(references): matches = _resolve(reference, self.calls) - if len(matches) != 1 or self.owners[matches[0]] != [turn.get("invocation_id")]: - self._fail( - "C3", - "turn.reference", - "/ng_trajectory/turns", - "turn call is unmatched or owned by another invocation", - ) + if len(matches) != 1: self._fail( - "C11", + "TE-9", "turn.reference", "/ng_trajectory/turns", - "turn call is unmatched or owned by another invocation", + "step call reference does not resolve uniquely", ) else: - turn_refs[matches[0]] += 1 - return turn_refs - - def check_accounting(self, turn_refs: Counter) -> None: - """C11: turn-call accounting and the missing independent closure evidence.""" - if not self.turns or any(turn_refs[i] != 1 for i in range(len(self.calls))): + index = matches[0] + owner = turn.get("invocation_id") + session = self.calls[index].get("client_session_id") + if (self.owners[index] and self.owners[index] != [owner]) or (session and session != owner): + self._fail( + "TE-9", + "turn.owner", + "/ng_trajectory/turns", + "step owner conflicts with explicit call ownership", + ) + refs[index] += 1 + policy = set(range(len(self.calls))) - auxiliary + if not self.turns or not policy or any(refs[i] != 1 for i in policy) or any(refs[i] for i in auxiliary): self._fail( - "C11", + "TE-9", "accounting.exactly_once", "/ng_trajectory/turns", - "calls are not assigned to turns exactly once", + "each retained policy attempt must belong to exactly one step; helpers must not be policy steps", ) - # Gym 1.0 has neither closure/witness records nor auxiliary-purpose buckets. - self._fail( - "C11", - "accounting.closure", - "/ng_trajectory", - "independent closed-scope ledger and auxiliary accounting required", - ) def check_tools(self) -> None: - """C5: unique executed tools, request joins, outcomes and intervals.""" - self._check_records("C5", self.tools) + """TE-5: unique executed tools, request joins and terminal outcomes.""" + if not self.scope.tools: + has_tool_items = any( + item.get("type") in {"function_call", "function_call_output"} + for inv in self.invocations + for item in _objects(inv.get("conversation")) + ) + has_tool_items = has_tool_items or any( + item.get("type") == "function_call_output" + for item in _objects(_mapping(self.record.get("response")).get("output")) + ) + if self.tools or has_tool_items: + self._fail( + "TE-5", "scope.contradiction", "/tool_calls", "tool evidence exists in a declared tool-free pair" + ) + else: + self.not_applicable.add("TE-5") + return + self._check_records("TE-5", self.tools) conversations = { ( invocation.get("invocation_id"), @@ -477,26 +589,31 @@ def check_tools(self) -> None: tool_keys = [(t.get("invocation_id"), t.get("tool_call_id")) for t in self.tools] if len(set(tool_keys)) != len(tool_keys): self._fail( - "C5", + "TE-5", "tool.unique", "/ng_trajectory/tool_calls", "duplicate execution identity", ) + for invocation_id, call_id, kind in conversations: + if kind == "function_call_output" and (invocation_id, call_id) not in tool_keys: + self._fail("TE-5", "tool.execution", "/tool_calls", "a tool result has no execution record") for index, tool in enumerate(self.tools): location = f"/tool_calls/{index}" tool_key = (tool.get("invocation_id"), tool.get("tool_call_id")) request = conversations.get((*tool_key, "function_call"), {}) output = conversations.get((*tool_key, "function_call_output"), {}).get("output", tool.get("output")) - if not request.get("arguments") or request.get("name") != tool.get("tool_name"): + if request.get("arguments") is None or request.get("name") != tool.get("tool_name"): self._fail( - "C5", + "TE-5", "tool.request", location, "execution cannot be joined to tool name and arguments", ) + if tool.get("output") is not None and output != tool["output"]: + self._fail("TE-5", "tool.output", location, "execution output differs from the model-visible result") if output is None and not tool.get("error_type"): self._fail( - "C5", + "TE-5", "tool.outcome", location, "execution has neither output nor error evidence", @@ -507,38 +624,56 @@ def check_tools(self) -> None: and tool["completed_at"] < tool["started_at"] ): self._fail( - "C5", + "TE-5", "tool.interval", location, "completion precedes start", ) - def check_parallel_timing(self) -> None: - """C6: independent timing witnesses remain unavailable.""" - self._fail( - "C6", - "timing.witness", - "/tool_calls", - "overlap needs independent monotonic-clock tool witnesses", - ) - - def check_lifecycle(self) -> None: - """C7: full resource lifecycle evidence remains unavailable.""" - self._fail( - "C7", - "sandbox.lifecycle", - "/ng_agent_observations", - "session identity alone does not provide setup, verification and cleanup evidence", - ) - def check_verifier(self) -> None: - """C8: authoritative verifier provenance remains unavailable.""" - self._fail( - "C8", - "reward.provenance", - "/reward", - "reward alone does not provide authoritative verifier scope and version", - ) + """TE-6's shipped Gym surface: rollout verification reward and terminal resolution.""" + reward = self.record.get("reward") + if not self.scope.verifier: + if reward is not None or any(t.get("resolved") is not None for t in self.turns): + self._fail( + "TE-6", + "scope.contradiction", + "/reward", + "verifier evidence exists in a declared verifier-free pair", + ) + else: + self.not_applicable.add("TE-6") + return + failed = self.record.get("mask_sample") is True or self.record.get("verification_error") is not None + if failed: + if reward is not None: + self._fail( + "TE-6", + "reward.unavailable", + "/reward", + "unavailable verification must not be represented as a numeric grade", + ) + if not self.record.get("failure_kind") and not self.record.get("verification_error"): + self._fail("TE-6", "verifier.error", "", "unavailable verification needs explicit failure evidence") + elif not _number(reward): + self._fail( + "TE-6", + "reward.required", + "/reward", + "finite verifier reward is required; binary resolution is not a floating grade", + ) + for invocation in {t.get("invocation_id") for t in self.turns}: + turns = sorted( + (t for t in self.turns if t.get("invocation_id") == invocation), key=lambda t: t.get("turn_no", 0) + ) + for turn in turns[:-1]: + if turn.get("resolved") is not None: + self._fail( + "TE-6", + "reward.scope", + "/ng_trajectory/turns", + "episode resolution must not be copied to intermediate steps", + ) def check_gaps(self) -> None: """Propagate explicit producer gaps to dependent capabilities.""" @@ -550,7 +685,7 @@ def check_gaps(self) -> None: for gap in gaps: code = str(gap.get("code", "")) if code.startswith("model_call_capture") or code == "agent_observation_join_failed": - for capability in ("C1", "C4", "C9", "C10", "C11"): + for capability in ("TE-1", "TE-4", "TE-7", "TE-8", "TE-9"): self._fail( capability, "capture.gap", @@ -559,49 +694,58 @@ def check_gaps(self) -> None: ) if code.startswith("model_call_reference") or code == "model_call_ownership_unavailable": self._fail( - "C10", + "TE-8", "ownership.gap", "/gaps", "producer declares unresolved ownership", ) + if self.scope.steps and code in { + "turns_unavailable", + "turn_evidence_incomplete", + "trajectory_projection_failed", + }: + self._fail("TE-3", "turn.gap", "/gaps", "producer declares unavailable step evidence") + if self.scope.steps and code == "turn_model_call_scope_incomplete": + self._fail("TE-9", "accounting.gap", "/gaps", "producer declares incomplete step-call membership") + def result(self) -> dict: - """Produce payload-free verdicts and propagate C0 integrity failures.""" - failed = {f.capability for f in self.findings} - if "C0" in failed: - for capability in NAMES: - if capability != "C0": - self._fail( - capability, - "record.integrity", - "", - "record identity or reader integrity failed", - ) - failed = set(NAMES) + """Report evidence correctness, metric availability and delivery independently.""" + failed = {f.evidence for f in self.findings} + if "record" in failed: + for key in NAMES: + self._fail(key, "record.integrity", "", "record identity or reader integrity failed") + failed.update(NAMES) + evidence = { + key: { + "name": name, + "verdict": "not_fulfilled" + if key in failed + else "not_applicable" + if key in self.not_applicable + else "fulfilled", + "basis": "retained_artifacts", + } + for key, name in NAMES.items() + } + availability = { + key: {"available": sum(c.get(key) is not None for c in self.calls), "calls": len(self.calls)} + for key in TOKEN_FIELDS + } return { "source": self.source, - "capabilities": { - capability: { - "name": name, - "verdict": ("not_fulfilled" if capability in failed else "fulfilled"), - "basis": "retained_artifacts", - "validator_status": ("incomplete" if capability in PENDING_VALIDATORS else "artifact_subset"), - "behavioral_qualification": False, - } - for capability, name in NAMES.items() - }, + "verdict": "fulfilled" if gate_passes(evidence) else "not_fulfilled", + "evidence": evidence, + "token_availability": availability, + "is_behavioral_qualification": False, + "scope_closure": "not_independently_witnessed", "findings": [asdict(f) for f in self.findings], } -def inspect_record(record: dict, *, source: str = "record") -> dict: - """Check artifact availability/correctness; never qualify unseen scenarios. - - Full per-call payloads may be hydrated by the reader. Missing artifacts fail - their capabilities rather than being silently excluded from the denominator. - Reports contain field locations and codes, never prompts or response bodies. - """ - checks = _RecordInspector(record, source) +def inspect_record(record: dict, *, source: str = "record", scope: EvidenceScope = EvidenceScope()) -> dict: + """Check TE-1–TE-9 on retained JSONL; do not infer missing evidence or applicability.""" + checks = _RecordInspector(record, source, scope) checks.check_identity() checks.check_capture_presence() checks.check_model_calls() @@ -610,11 +754,9 @@ def inspect_record(record: dict, *, source: str = "record") -> dict: checks.check_history() checks.check_payloads() checks.check_ownership() - turn_refs = checks.check_turns() - checks.check_accounting(turn_refs) + checks.check_turns() + checks.check_step_join() checks.check_tools() - checks.check_parallel_timing() - checks.check_lifecycle() checks.check_verifier() checks.check_gaps() return checks.result() diff --git a/nemo_gym/harness_capabilities/cli.py b/nemo_gym/harness_capabilities/cli.py index 6368c9582ea..1c4fc35b910 100644 --- a/nemo_gym/harness_capabilities/cli.py +++ b/nemo_gym/harness_capabilities/cli.py @@ -9,10 +9,11 @@ import os import shutil import tempfile +from dataclasses import asdict from pathlib import Path from . import __version__ -from .checker import NAMES, PENDING_VALIDATORS, PROFILES, inspect_record +from .checker import NAMES, PROFILE, EvidenceScope, inspect_record from .reader import digest_file, hydrate_record, json_rows from .schemas import SCHEMA_VERSION, SCHEMAS @@ -25,7 +26,8 @@ def inspect_bundle( bundle: Path, *, output: Path, - profile: str, + profile: str = PROFILE, + scope: EvidenceScope = EvidenceScope(), capture_dir: Path | None = None, ) -> tuple[Path, dict]: """Write an immutable content-addressed report set after a complete read. @@ -33,7 +35,7 @@ def inspect_bundle( A checker error never replaces an earlier report or leaves a current-looking summary. Files are hashed before and after reading to detect changing input. """ - if profile not in PROFILES: + if profile != PROFILE: raise ValueError("unknown artifact profile") if bundle.is_dir(): candidates = [ @@ -54,7 +56,7 @@ def inspect_bundle( raise ValueError("capture directory does not exist") sources.extend(sorted(capture_dir.glob("*.capture.*"))) hashes = {str(path.resolve()): digest_file(path) for path in sources} - registry_hash = hashlib.sha256(_json({"schemas": SCHEMAS, "profiles": PROFILES}).encode()).hexdigest() + registry_hash = hashlib.sha256(_json({"schemas": SCHEMAS, "profile": PROFILE}).encode()).hexdigest() checker_hash = hashlib.sha256( "".join(digest_file(p) for p in sorted(Path(__file__).parent.glob("*.py"))).encode() ).hexdigest() @@ -63,46 +65,57 @@ def inspect_bundle( "registry_sha256": registry_hash, "checker_sha256": checker_hash, "profile": profile, + "applicability": asdict(scope), } report_id = hashlib.sha256(_json(manifest).encode()).hexdigest() output.mkdir(parents=True, exist_ok=True) temporary = Path(tempfile.mkdtemp(prefix=".capabilities-", dir=output)) - totals = {capability: {"observed_records": 0, "passed_records": 0} for capability in NAMES} + totals = {key: {"records": 0, "fulfilled": 0, "not_fulfilled": 0, "not_applicable": 0} for key in NAMES} + token_availability = {} + passing_records = 0 identities: set[object] = set() count = 0 try: - with (temporary / "capability_results.jsonl").open("w") as handle: + with (temporary / "evidence_results.jsonl").open("w") as handle: for line, raw in json_rows(bundle): record = hydrate_record(raw, capture_dir=capture_dir) - identity = (record.get("ng_model_call_capture") or {}).get("rollout_id") or record.get( - "_ng_rollout_id" - ) + identity = (record.get("_ng_task_index"), record.get("_ng_rollout_index")) + if identity == (None, None): + identity = (record.get("ng_trajectory") or {}).get("rollout_id") if identity in identities: record.setdefault("_capability_reader_issues", []).append("duplicate rollout identity in input") identities.add(identity) - result = inspect_record(record, source=f"{bundle.name}:{line}") + result = inspect_record(record, source=f"{bundle.name}:{line}", scope=scope) count += 1 - for capability, verdict in result["capabilities"].items(): - totals[capability]["observed_records"] += 1 - totals[capability]["passed_records"] += verdict["verdict"] == "fulfilled" + passing_records += result["verdict"] == "fulfilled" + for key, verdict in result["evidence"].items(): + totals[key]["records"] += 1 + totals[key][verdict["verdict"]] += 1 + for key, values in result["token_availability"].items(): + target = token_availability.setdefault(key, {"available": 0, "calls": 0}) + for metric, value in values.items(): + target[metric] += value handle.write(json.dumps(result, allow_nan=False) + "\n") if count == 0: raise ValueError("no rollout records") if any(digest_file(path) != hashes[str(path.resolve())] for path in sources): raise ValueError("source changed during inspection") verdicts = { - capability: { + key: { **counts, - "verdict": ("fulfilled" if counts["passed_records"] == count else "not_fulfilled"), - "behavioral_qualification": False, - "validator_status": ("incomplete" if capability in PENDING_VALIDATORS else "artifact_subset"), + "verdict": "not_fulfilled" + if counts["not_fulfilled"] + else "not_applicable" + if counts["not_applicable"] == count + else "fulfilled", "scenario_coverage": {"observed": 0, "passed": 0}, } - for capability, counts in totals.items() + for key, counts in totals.items() } - passed = all(verdicts[capability]["verdict"] == "fulfilled" for capability in PROFILES[profile]) + # Apply the join alternative per rollout, not only to aggregate columns. + passed = passing_records == count summary = { - "schema_version": "harness-capabilities/v1", + "schema_version": "harness-evidence/v1", "checker_version": __version__, "decoder_version": SCHEMA_VERSION, "checker_status": "completed", @@ -112,10 +125,17 @@ def inspect_bundle( "verdict": "fulfilled" if passed else "not_fulfilled", "is_behavioral_qualification": False, "records": count, - "capabilities": verdicts, + "evidence": verdicts, + "token_availability": token_availability, + "delivery_surface": "rollout_jsonl", + "limits": [ + "retained artifacts only; no live qualification or health certification", + "TE-6 checks shipped Gym reward/resolution; extended verifier provenance is not certified", + "TE-10 and P1 evidence are outside this profile", + ], **manifest, } - (temporary / "capability_summary.json").write_text(_json(summary)) + (temporary / "evidence_summary.json").write_text(_json(summary)) report = [ "# Harness artifact conformance", "", @@ -123,21 +143,21 @@ def inspect_bundle( "", "These checks establish retained artifact requirements. They do not qualify the RFC's behavioral scenarios.", "", - "| Capability | Artifact verdict | Passing records |", + "| Evidence | Artifact verdict | Passing / applicable records |", "|---|---|---|", ] report.extend( - f"| {key}: {NAMES[key]} | {value['verdict']} | {value['passed_records']}/{count} |" + f"| {key}: {NAMES[key]} | {value['verdict']} | {value['fulfilled']}/{count - value['not_applicable']} |" for key, value in verdicts.items() ) report.extend( [ "", - "Field-level failure locations are in `capability_results.jsonl`; payloads are never copied into this report.", + "Field-level failure locations are in `evidence_results.jsonl`; payloads are never copied into this report.", "", ] ) - (temporary / "capability_report.md").write_text("\n".join(report)) + (temporary / "evidence_report.md").write_text("\n".join(report)) destination = output / report_id if destination.exists(): if any((destination / path.name).read_bytes() != path.read_bytes() for path in temporary.iterdir()): @@ -151,39 +171,125 @@ def inspect_bundle( raise +def inspect_matrix( + harnesses: dict[str, Path], *, output: Path, scope: EvidenceScope = EvidenceScope() +) -> tuple[Path, dict]: + """Check each harness bundle and publish a reproducible harness × TE matrix.""" + if not harnesses: + raise ValueError("at least one harness is required") + rows = {} + for name, bundle in harnesses.items(): + if not name or any(c not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for c in name): + raise ValueError("harness names must be alphanumeric with hyphens or underscores") + path, summary = inspect_bundle(bundle, output=output / name, scope=scope) + rows[name] = {"report": str(path / "evidence_summary.json"), **summary} + matrix = { + "schema_version": "harness-evidence-matrix/v1", + "profile": PROFILE, + "harnesses": rows, + "is_behavioral_qualification": False, + } + matrix_id = hashlib.sha256(_json(matrix).encode()).hexdigest() + destination = output / matrix_id + temporary = Path(tempfile.mkdtemp(prefix=".matrix-", dir=output)) + try: + (temporary / "harness_evidence.json").write_text(_json(matrix)) + table = [ + "# Harness × P0 trajectory evidence", + "", + "Retained artifacts only; not live harness certification.", + "", + "| Harness | " + " | ".join(NAMES) + " | P0 |", + "|---|" + "---|" * (len(NAMES) + 1), + ] + labels = {"fulfilled": "PASS", "not_fulfilled": "FAIL", "not_applicable": "N/A"} + for name, summary in rows.items(): + table.append( + "| " + + name + + " | " + + " | ".join(labels[summary["evidence"][key]["verdict"]] for key in NAMES) + + " | " + + labels[summary["verdict"]] + + " |" + ) + table.extend( + [ + "", + *[f"- {key}: {name}" for key, name in NAMES.items()], + "", + "P0 requires all applicable TE-1–TE-7 and TE-8 or TE-9 on every record.", + "TE-2 PASS means usage was preserved, not that every provider metric was available.", + "See each evidence_summary.json for input hashes, applicability, token availability and limitations.", + ] + ) + (temporary / "harness_evidence.md").write_text("\n".join(table) + "\n") + if destination.exists(): + if any((destination / p.name).read_bytes() != p.read_bytes() for p in temporary.iterdir()): + raise ValueError("existing matrix differs") + shutil.rmtree(temporary) + else: + os.rename(temporary, destination) + return destination, matrix + except BaseException: + shutil.rmtree(temporary, ignore_errors=True) + raise + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True) inspect = subparsers.add_parser("inspect") inspect.add_argument("--bundle", required=True, type=Path) - inspect.add_argument("--capture-dir", type=Path) - inspect.add_argument("--output", required=True, type=Path) - inspect.add_argument("--profile", choices=PROFILES, default="gym-artifacts-p0/v1") + inspect.add_argument( + "--capture-dir", type=Path, help="cross-check original captures; missing JSONL payloads still fail" + ) + inspect.add_argument("--profile", choices=[PROFILE], default=PROFILE) + matrix = subparsers.add_parser("matrix") + matrix.add_argument("--harness", action="append", required=True, metavar="NAME=PATH") + for command in (inspect, matrix): + command.add_argument("--output", required=True, type=Path) + command.add_argument( + "--no-tools", action="store_true", help="this harness/benchmark pair does not dispatch tools" + ) + command.add_argument("--no-verifier", action="store_true", help="this pair has no verifier") + command.add_argument("--no-steps", action="store_true", help="this pair has no policy step structure") args = parser.parse_args(argv) - return run_inspection(bundle=args.bundle, output=args.output, profile=args.profile, capture_dir=args.capture_dir) + scope = EvidenceScope(tools=not args.no_tools, verifier=not args.no_verifier, steps=not args.no_steps) + if args.command == "inspect": + return run_inspection( + bundle=args.bundle, output=args.output, profile=args.profile, capture_dir=args.capture_dir, scope=scope + ) + try: + harnesses = {} + for value in args.harness: + name, separator, path = value.partition("=") + if not separator or not path or name in harnesses: + raise ValueError("each harness needs a unique NAME=PATH") + harnesses[name] = Path(path) + destination, result = inspect_matrix(harnesses, output=args.output, scope=scope) + print(destination / "harness_evidence.md") + return 0 if all(row["verdict"] == "fulfilled" for row in result["harnesses"].values()) else 1 + except (OSError, ValueError, TypeError, KeyError, AttributeError, RecursionError) as exc: + print(f"checker_error ({type(exc).__name__}); no matrix published") + return 2 def run_inspection( - *, bundle: Path, output: Path, profile: str = "gym-artifacts-p0/v1", capture_dir: Path | None = None + *, + bundle: Path, + output: Path, + profile: str = PROFILE, + capture_dir: Path | None = None, + scope: EvidenceScope = EvidenceScope(), ) -> int: """Inspect retained rollouts and print the report location; return 0, 1, or 2.""" try: destination, summary = inspect_bundle( - bundle, - output=output, - profile=profile, - capture_dir=capture_dir, + bundle, output=output, profile=profile, capture_dir=capture_dir, scope=scope ) - except ( - OSError, - ValueError, - TypeError, - KeyError, - AttributeError, - RecursionError, - ) as exc: - # Do not print exception details: malformed input may contain secrets. + except (OSError, ValueError, TypeError, KeyError, AttributeError, RecursionError) as exc: print(f"checker_error ({type(exc).__name__}); no report published") return 2 - print(f"{summary['verdict']}: {destination / 'capability_summary.json'}") + print(f"{summary['verdict']}: {destination / 'evidence_summary.json'}") return 0 if summary["verdict"] == "fulfilled" else 1 diff --git a/nemo_gym/harness_capabilities/reader.py b/nemo_gym/harness_capabilities/reader.py index 68aceada962..383715ede1d 100644 --- a/nemo_gym/harness_capabilities/reader.py +++ b/nemo_gym/harness_capabilities/reader.py @@ -41,13 +41,40 @@ def hydrate_record(record: dict, *, capture_dir: Path | None = None) -> dict: """Restore payloads by exact model_call_id, never order or content hash. The native collector moves bodies from capture.calls into ng_trajectory. - An optional capture directory provides the retained transport exchanges. + An optional capture directory cross-checks the retained transport exchanges. + It never repairs payloads missing from the delivered JSONL. Conflicts are preserved as reader failures instead of being overwritten. """ result = copy.deepcopy(record) capture = result.get("ng_model_call_capture") or {} calls = capture.get("calls") or [] trajectory = result.get("ng_trajectory") or {} + if "ng_model_call_capture" not in result and trajectory.get("model_calls"): + # Canonical-only delivery is also a supported native surface. Copy known + # fields exactly; do not invent clocks, usage, or ownership. + calls = [] + for item in trajectory["model_calls"]: + call = { + **item.get("response_metadata", {}), + "model_call_id": item.get("model_call_id"), + "started_at": item.get("started_at"), + "completed_at": item.get("completed_at"), + } + for key in ("request", "response"): + value = item.get(key) + call[key if isinstance(value, dict) else key + "_raw"] = value + stats = item.get("token_stats") or {} + for canonical, captured in ( + ("prompt_tokens", "tokens_in"), + ("completion_tokens", "tokens_out"), + ("reasoning_tokens", "tokens_reasoning"), + ("total_tokens", "tokens_total"), + ("cached_tokens", "cached_tokens"), + ): + call[captured] = stats.get(canonical) + calls.append(call) + capture = {"rollout_id": trajectory.get("rollout_id"), "calls": calls} + result["ng_model_call_capture"] = capture issues = result.setdefault("_capability_reader_issues", []) by_id: dict[str, dict] = {} for call in calls: @@ -71,7 +98,7 @@ def merge(items: list[dict], *, sidecar: bool) -> None: if value is not None: if call.get(key) is not None and call[key] != value: issues.append("conflicting payload representations") - else: + elif not sidecar: call[key] = value metadata = item if sidecar else item.get("response_metadata", {}) for key in ( @@ -80,6 +107,8 @@ def merge(items: list[dict], *, sidecar: bool) -> None: "status_code", "dialect", "error_category", + "response_status", + "finish_reason", ): if metadata.get(key) is not None and call.get(key) != metadata[key]: issues.append("conflicting model-call metadata") diff --git a/nemo_gym/harness_capabilities/schemas.py b/nemo_gym/harness_capabilities/schemas.py index 0638884a822..d5bc8533fc0 100644 --- a/nemo_gym/harness_capabilities/schemas.py +++ b/nemo_gym/harness_capabilities/schemas.py @@ -1,93 +1,38 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Versioned, intentionally stricter subsets of Gym's optional contracts. +"""Persisted P0 trajectory evidence, independent of health and live qualification.""" -Pinned to Gym 26688f7d (TrajectoryRecord 1.0). Additional producer fields are -allowed: these are capability requirements, not copies of Gym's whole schema. -""" - -SCHEMA_VERSION = "gym-artifact-contracts/v1" +SCHEMA_VERSION = "gym-p0-evidence/v1" TEXT = {"type": "string", "minLength": 1} -NUMBER = {"type": "number"} -NONNEGATIVE = {"type": "number", "minimum": 0} -TOKENS = {"type": "integer", "minimum": 0} -TOKEN_FIELDS = ( - "tokens_in", - "tokens_out", - "tokens_reasoning", - "tokens_total", - "cached_tokens", -) +TOKENS = {"type": ["integer", "null"], "minimum": 0} +TOKEN_FIELDS = ("tokens_in", "tokens_out", "tokens_reasoning", "tokens_total", "cached_tokens") def object_schema(properties: dict, required: tuple | list) -> dict: - return { - "type": "object", - "properties": properties, - "required": list(required), - } + return {"type": "object", "properties": properties, "required": list(required)} MODEL_REF = object_schema({"type": TEXT, "name": TEXT}, ("type", "name")) CALL_REF = { - **object_schema( - {"model_call_id": TEXT, "model_ref": MODEL_REF, "response_id": TEXT}, - (), - ), - "anyOf": [ - {"required": ["model_call_id"]}, - {"required": ["model_ref", "response_id"]}, - ], + **object_schema({"model_call_id": TEXT, "model_ref": MODEL_REF, "response_id": TEXT}, ()), + "anyOf": [{"required": ["model_call_id"]}, {"required": ["model_ref", "response_id"]}], } -CALL = { - **object_schema( - { - "model_call_id": TEXT, - "model_ref": MODEL_REF, - # Native captures may lack a provider model on failed calls. The - # required model_ref still identifies the configured model server. - "model": {"anyOf": [TEXT, {"type": "null"}]}, - "dialect": {"enum": ["responses", "chat", "messages"]}, - "started_at": NUMBER, - "completed_at": NUMBER, - "latency_total_ms": NONNEGATIVE, - "status_code": {"type": ["integer", "null"]}, - "error_category": {"type": ["string", "null"]}, - }, - ( - "model_call_id", - "model_ref", - "dialect", - "started_at", - "completed_at", - "latency_total_ms", - ), - ), - "anyOf": [ - { - "required": ["status_code"], - "properties": { - "status_code": { - "type": "integer", - "minimum": 100, - "maximum": 599, - } - }, - }, - { - "required": ["error_category"], - "properties": {"error_category": TEXT}, - }, - ], -} -USAGE = object_schema({key: TOKENS for key in TOKEN_FIELDS}, TOKEN_FIELDS) -INVOCATION = object_schema( +CALL = object_schema( { - "invocation_id": TEXT, - "model_calls": {"type": "array", "items": {"type": "object"}}, - "conversation": {"type": "array", "items": {"type": "object"}}, + "model_call_id": TEXT, + "model_ref": MODEL_REF, + "dialect": {"enum": ["responses", "chat", "messages"]}, + "status_code": {"type": ["integer", "null"], "minimum": 100, "maximum": 599}, + "response_status": {"type": ["string", "null"]}, + "finish_reason": {"type": ["string", "null"]}, + "error_category": {"type": ["string", "null"]}, }, + ("model_call_id", "model_ref", "dialect"), +) +USAGE = object_schema({key: TOKENS for key in TOKEN_FIELDS}, ()) +INVOCATION = object_schema( + {"invocation_id": TEXT, "model_calls": {"type": "array", "items": {"type": "object"}}}, ("invocation_id", "model_calls"), ) TOOL = object_schema( @@ -95,22 +40,9 @@ def object_schema(properties: dict, required: tuple | list) -> dict: "invocation_id": TEXT, "tool_call_id": TEXT, "tool_name": TEXT, - "started_at": NUMBER, - "completed_at": NUMBER, - "duration_ms": NONNEGATIVE, - "timing_source": {"enum": ["executor", "artifact", "harness"]}, "status": {"enum": ["completed", "failed", "timeout", "cancelled"]}, }, - ( - "invocation_id", - "tool_call_id", - "tool_name", - "started_at", - "completed_at", - "duration_ms", - "timing_source", - "status", - ), + ("invocation_id", "tool_call_id", "tool_name", "status"), ) TURN = object_schema( { @@ -118,25 +50,10 @@ def object_schema(properties: dict, required: tuple | list) -> dict: "task_id": TEXT, "rollout_id": TEXT, "turn_no": {"type": "integer", "minimum": 1}, - "timestamp": NUMBER, + "timestamp": {"type": "number"}, "question": {"not": {"type": "null"}}, - "answer": {"not": {"type": "null"}}, - "step_count": {"type": "integer", "minimum": 0}, "resolved": {"type": ["boolean", "null"]}, - "model_calls": {"type": "array", "minItems": 1}, }, - ( - "invocation_id", - "task_id", - "rollout_id", - "turn_no", - "timestamp", - "question", - "answer", - "step_count", - "resolved", - "model_calls", - ), + ("invocation_id", "task_id", "rollout_id", "turn_no", "timestamp", "question", "resolved"), ) - -SCHEMAS = {"C1": CALL, "C2": USAGE, "C3": TURN, "C5": TOOL, "C10": INVOCATION} +SCHEMAS = {"TE-1": CALL, "TE-2": USAGE, "TE-3": TURN, "TE-5": TOOL, "TE-8": INVOCATION} diff --git a/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py b/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py index 643e357e715..2bfbde63012 100644 --- a/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py +++ b/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py @@ -252,10 +252,10 @@ async def query(params): # Inspect the emitted records, not a reconstructed copy of the expected evidence. _, conformance = inspect_bundle( - path, output=tmp_path / "capabilities", profile="gym-artifacts-p0/v1", capture_dir=capture_dir + path, output=tmp_path / "capabilities", profile="gym-p0/v1", capture_dir=capture_dir ) - expected = "not_fulfilled" if scenario in {"missing_usage", "missing_details"} else "fulfilled" + expected = "fulfilled" # Provider omission is faithful evidence; health still evaluates missing usage. assert conformance["verdict"] == expected - for capability in ("C1", "C3", "C4", "C5", "C9", "C10"): - assert conformance["capabilities"][capability]["verdict"] == "fulfilled", (scenario, capability) + for capability in ("TE-1", "TE-2", "TE-3", "TE-4", "TE-5", "TE-7", "TE-8"): + assert conformance["evidence"][capability]["verdict"] == "fulfilled", (scenario, capability) assert conformance["is_behavioral_qualification"] is False diff --git a/scripts/check_harness_conformance.py b/scripts/check_harness_conformance.py index 341c1004585..638396b5261 100644 --- a/scripts/check_harness_conformance.py +++ b/scripts/check_harness_conformance.py @@ -9,4 +9,5 @@ if __name__ == "__main__": - raise SystemExit(main(["inspect", *sys.argv[1:]])) + args = sys.argv[1:] + raise SystemExit(main(args if args and args[0] == "matrix" else ["inspect", *args])) diff --git a/tests/unit_tests/harness_capabilities/fixtures/codex.jsonl b/tests/unit_tests/harness_capabilities/fixtures/codex.jsonl new file mode 100644 index 00000000000..f7b08b71f1a --- /dev/null +++ b/tests/unit_tests/harness_capabilities/fixtures/codex.jsonl @@ -0,0 +1 @@ +{"_ng_rollout_id": "0-0", "_ng_rollout_index": 0, "_ng_task_index": 0, "ng_model_call_capture": {"calls": [{"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 0, "client_session_id": null, "completed_at": 1.5, "dialect": "responses", "error_category": null, "finish_reason": null, "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-0", "response_status": "completed", "started_at": 1.0, "status_code": 200, "tokens_in": 10, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 15, "tool_calls": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "name": "bash"}]}, {"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 1, "client_session_id": null, "completed_at": 2.5, "dialect": "responses", "error_category": null, "finish_reason": null, "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-1", "response_status": "completed", "started_at": 2.0, "status_code": 200, "tokens_in": 11, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 16, "tool_calls": []}], "rollout_id": "0-0"}, "ng_trajectory": {"gaps": [{"code": "turns_unavailable", "detail": null, "invocation_id": null}, {"code": "conversation_unavailable", "detail": null, "invocation_id": null}], "invocations": [], "model_calls": [{"completed_at": 1.5, "duration_ms": 500.0, "model_call_id": "call-0", "request": {"input": [{"content": "Print ok, then report success.", "role": "user"}], "model": "fixture-model"}, "response": {"id": "response-0", "model": "fixture-model", "output": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "name": "bash", "type": "function_call"}], "status": "completed", "usage": {"input_tokens": 10, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 2}, "total_tokens": 15}}, "response_metadata": {"dialect": "responses", "error_category": null, "finish_reason": null, "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0", "response_status": "completed", "status_code": 200}, "started_at": 1.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 10, "reasoning_tokens": 2, "total_tokens": 15}}, {"completed_at": 2.5, "duration_ms": 500.0, "model_call_id": "call-1", "request": {"input": [{"content": "Print ok, then report success.", "role": "user"}, {"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "name": "bash", "type": "function_call"}, {"call_id": "tool-1", "output": "ok", "type": "function_call_output"}], "model": "fixture-model"}, "response": {"id": "response-1", "model": "fixture-model", "output": [{"content": [{"text": "Done", "type": "output_text"}], "role": "assistant", "type": "message"}], "status": "completed", "usage": {"input_tokens": 11, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 2}, "total_tokens": 16}}, "response_metadata": {"dialect": "responses", "error_category": null, "finish_reason": null, "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1", "response_status": "completed", "status_code": 200}, "started_at": 2.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 11, "reasoning_tokens": 2, "total_tokens": 16}}], "rollout_id": "0-0", "schema_version": "1.0", "task_id": "0", "tool_calls": [], "turns": []}, "response": {"output": [{"arguments": "{\"cmd\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "exec_command", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "answer", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "usage": {"cached_input_tokens": 0, "input_tokens": 21, "output_tokens": 10, "reasoning_tokens": 0}}, "reward": 1.0} diff --git a/tests/unit_tests/harness_capabilities/fixtures/hermes.jsonl b/tests/unit_tests/harness_capabilities/fixtures/hermes.jsonl new file mode 100644 index 00000000000..e5bcdd8391e --- /dev/null +++ b/tests/unit_tests/harness_capabilities/fixtures/hermes.jsonl @@ -0,0 +1 @@ +{"_ng_rollout_id": "0-0", "_ng_rollout_index": 0, "_ng_task_index": 0, "ng_agent_observations": {"gaps": [], "records": [{"conversation": [{"content": "Print ok, then report success.", "phase": null, "role": "user", "type": "message"}, {"content": [{"annotations": [], "logprobs": null, "text": "", "type": "output_text"}], "id": "root-message-1", "phase": null, "role": "assistant", "status": "completed", "type": "message"}, {"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "root-message-3", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}, {"completed_at": 1.0, "duration_ms": 0.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "sandbox_id": null, "started_at": 1.0, "status": "completed", "timing_source": "executor", "tool_call_id": "tool-1", "tool_name": "bash"}], "source": "hermes"}, "ng_model_call_capture": {"calls": [{"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 0, "client_session_id": null, "completed_at": 1.5, "dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-0", "response_status": null, "started_at": 1.0, "status_code": 200, "tokens_in": 10, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 15, "tool_calls": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "name": "bash"}]}, {"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 1, "client_session_id": null, "completed_at": 2.5, "dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-1", "response_status": null, "started_at": 2.0, "status_code": 200, "tokens_in": 11, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 16, "tool_calls": []}], "rollout_id": "0-0"}, "ng_trajectory": {"gaps": [{"code": "turns_unavailable", "detail": null, "invocation_id": null}], "invocations": [{"conversation": [{"content": "Print ok, then report success.", "phase": null, "role": "user", "type": "message"}, {"content": [{"annotations": [], "logprobs": null, "text": "", "type": "output_text"}], "id": "root-message-1", "phase": null, "role": "assistant", "status": "completed", "type": "message"}, {"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "root-message-3", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}], "model_calls": [{"completed_at": 1.5, "duration_ms": 500.0, "model_call_id": "call-0", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "tool_calls", "index": 0, "message": {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}}], "id": "response-0", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 15}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0", "response_status": null, "status_code": 200}, "started_at": 1.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 10, "reasoning_tokens": 2, "total_tokens": 15}}, {"completed_at": 2.5, "duration_ms": 500.0, "model_call_id": "call-1", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}, {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}, {"content": "ok", "role": "tool", "tool_call_id": "tool-1"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Done", "role": "assistant"}}], "id": "response-1", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 11, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 16}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1", "response_status": null, "status_code": 200}, "started_at": 2.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 11, "reasoning_tokens": 2, "total_tokens": 16}}], "rollout_id": "0-0", "schema_version": "1.0", "task_id": "0", "tool_calls": [{"completed_at": 1.0, "duration_ms": 0.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "output": "ok", "sandbox_id": null, "started_at": 1.0, "status": "completed", "timing_source": "executor", "tool_call_id": "tool-1", "tool_name": "bash"}], "turns": []}, "reward": 1.0} diff --git a/tests/unit_tests/harness_capabilities/fixtures/opencode.jsonl b/tests/unit_tests/harness_capabilities/fixtures/opencode.jsonl new file mode 100644 index 00000000000..b2d72093744 --- /dev/null +++ b/tests/unit_tests/harness_capabilities/fixtures/opencode.jsonl @@ -0,0 +1 @@ +{"_ng_rollout_id": "0-0", "_ng_rollout_index": 0, "_ng_task_index": 0, "ng_agent_observations": {"gaps": [], "records": [{"conversation": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "message-1-1", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}, {"completed_at": 1.8, "duration_ms": 200.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "sandbox_id": null, "started_at": 1.6, "status": "completed", "timing_source": "artifact", "tool_call_id": "tool-1", "tool_name": "bash"}], "source": "opencode"}, "ng_model_call_capture": {"calls": [{"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 0, "client_session_id": "root", "completed_at": 1.5, "dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-0", "response_status": null, "started_at": 1.0, "status_code": 200, "tokens_in": 10, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 15, "tool_calls": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "name": "bash"}]}, {"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 1, "client_session_id": "root", "completed_at": 2.5, "dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-1", "response_status": null, "started_at": 2.0, "status_code": 200, "tokens_in": 11, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 16, "tool_calls": []}], "rollout_id": "0-0"}, "ng_trajectory": {"gaps": [], "invocations": [{"conversation": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "message-1-1", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}], "model_calls": [{"completed_at": 1.5, "duration_ms": 500.0, "model_call_id": "call-0", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "tool_calls", "index": 0, "message": {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}}], "id": "response-0", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 15}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0", "response_status": null, "status_code": 200}, "started_at": 1.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 10, "reasoning_tokens": 2, "total_tokens": 15}}, {"completed_at": 2.5, "duration_ms": 500.0, "model_call_id": "call-1", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}, {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}, {"content": "ok", "role": "tool", "tool_call_id": "tool-1"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Done", "role": "assistant"}}], "id": "response-1", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 11, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 16}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1", "response_status": null, "status_code": 200}, "started_at": 2.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 11, "reasoning_tokens": 2, "total_tokens": 16}}], "rollout_id": "0-0", "schema_version": "1.0", "task_id": "0", "tool_calls": [{"completed_at": 1.8, "duration_ms": 200.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "output": "ok", "sandbox_id": null, "started_at": 1.6, "status": "completed", "timing_source": "artifact", "tool_call_id": "tool-1", "tool_name": "bash"}], "turns": [{"answer": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "id": "message-0-1", "name": "bash", "type": "function_call"}], "invocation_id": "root", "model_calls": [], "question": null, "reasoning_content": null, "resolved": null, "rollout_id": "0-0", "step_count": 1, "task_id": "0", "timestamp": 1.0, "turn_no": 1}, {"answer": [{"content": [{"id": "message-1-1", "text": "Done", "type": "output_text"}], "id": "message-1", "role": "assistant", "type": "message"}], "invocation_id": "root", "model_calls": [], "question": null, "reasoning_content": null, "resolved": null, "rollout_id": "0-0", "step_count": 2, "task_id": "0", "timestamp": 2.0, "turn_no": 2}]}, "reward": 1.0} diff --git a/tests/unit_tests/harness_capabilities/fixtures/pi.jsonl b/tests/unit_tests/harness_capabilities/fixtures/pi.jsonl new file mode 100644 index 00000000000..f9553526468 --- /dev/null +++ b/tests/unit_tests/harness_capabilities/fixtures/pi.jsonl @@ -0,0 +1 @@ +{"_ng_rollout_id": "0-0", "_ng_rollout_index": 0, "_ng_task_index": 0, "ng_agent_observations": {"gaps": [{"code": "subagent_hierarchy_unavailable", "detail": null, "invocation_id": "root"}], "records": [{"conversation": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "msg-2", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}, {"completed_at": 2.0, "duration_ms": 1000.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "sandbox_id": null, "started_at": 1.0, "status": "completed", "timing_source": "harness", "tool_call_id": "tool-1", "tool_name": "bash"}], "source": "pi"}, "ng_model_call_capture": {"calls": [{"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 0, "client_session_id": null, "completed_at": 1.5, "dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-0", "response_status": null, "started_at": 1.0, "status_code": 200, "tokens_in": 10, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 15, "tool_calls": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "name": "bash"}]}, {"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 1, "client_session_id": null, "completed_at": 2.5, "dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-1", "response_status": null, "started_at": 2.0, "status_code": 200, "tokens_in": 11, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 16, "tool_calls": []}], "rollout_id": "0-0"}, "ng_trajectory": {"gaps": [{"code": "subagent_hierarchy_unavailable", "detail": null, "invocation_id": "root"}, {"code": "turns_unavailable", "detail": null, "invocation_id": null}], "invocations": [{"conversation": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "msg-2", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}], "model_calls": [{"completed_at": 1.5, "duration_ms": 500.0, "model_call_id": "call-0", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "tool_calls", "index": 0, "message": {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}}], "id": "response-0", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 15}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0", "response_status": null, "status_code": 200}, "started_at": 1.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 10, "reasoning_tokens": 2, "total_tokens": 15}}, {"completed_at": 2.5, "duration_ms": 500.0, "model_call_id": "call-1", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}, {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}, {"content": "ok", "role": "tool", "tool_call_id": "tool-1"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Done", "role": "assistant"}}], "id": "response-1", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 11, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 16}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1", "response_status": null, "status_code": 200}, "started_at": 2.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 11, "reasoning_tokens": 2, "total_tokens": 16}}], "rollout_id": "0-0", "schema_version": "1.0", "task_id": "0", "tool_calls": [{"completed_at": 2.0, "duration_ms": 1000.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "output": "ok", "sandbox_id": null, "started_at": 1.0, "status": "completed", "timing_source": "harness", "tool_call_id": "tool-1", "tool_name": "bash"}], "turns": []}, "reward": 1.0} diff --git a/tests/unit_tests/harness_capabilities/producer_fixtures.py b/tests/unit_tests/harness_capabilities/producer_fixtures.py new file mode 100644 index 00000000000..9357497e6fd --- /dev/null +++ b/tests/unit_tests/harness_capabilities/producer_fixtures.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Controlled native protocol inputs passed through real producers and collection. + +These fixtures exercise the capture-enabled delivery path. They do not establish +that a deployed harness routes every request through capture or passes live QA. +""" + +import json +import sqlite3 +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from nemo_gym.base_responses_api_model import build_model_call_record +from nemo_gym.config_types import ModelServerRef +from nemo_gym.rollout_collection import _attach_trajectory_record, _strip_capture_payloads +from nemo_gym.rollout_observability import TrajectoryRecord, join_model_call_observations +from responses_api_agents.codex_agent.app import parse_exec_jsonl +from responses_api_agents.hermes_agent.observability import HermesAgentObserver +from responses_api_agents.opencode_agent.app import _parse_opencode_session +from responses_api_agents.pi_agent.app import _build_pi_observations, parse_pi_events + + +HARNESSES = ("opencode", "pi", "codex", "hermes") +MODEL = ModelServerRef(type="responses_api_models", name="policy") +ARGS = {"command": "printf ok"} + + +def _exchanges(dialect: str, *, session: str | None) -> list[dict]: + request_messages = [{"role": "user", "content": "Print ok, then report success."}] + tool = {"id": "tool-1", "type": "function", "function": {"name": "bash", "arguments": json.dumps(ARGS)}} + tool_output = {"role": "assistant", "content": None, "tool_calls": [tool]} + result = {"role": "tool", "tool_call_id": "tool-1", "content": "ok"} + records = [] + for index in range(2): + usage = { + "prompt_tokens": 10 + index, + "completion_tokens": 5, + "total_tokens": 15 + index, + "prompt_tokens_details": {"cached_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 2}, + } + response = { + "id": f"response-{index}", + "model": "fixture-model", + "usage": usage, + "choices": [ + { + "index": 0, + "message": tool_output if index == 0 else {"role": "assistant", "content": "Done"}, + "finish_reason": "tool_calls" if index == 0 else "stop", + } + ], + } + request = { + "model": "fixture-model", + "messages": request_messages if index == 0 else [*request_messages, tool_output, result], + } + if dialect == "responses": + response = { + "id": f"response-{index}", + "model": "fixture-model", + "status": "completed", + "output": [ + {"type": "function_call", "call_id": "tool-1", "name": "bash", "arguments": json.dumps(ARGS)} + ] + if index == 0 + else [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Done"}]}], + "usage": { + "input_tokens": 10 + index, + "output_tokens": 5, + "total_tokens": 15 + index, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 2}, + }, + } + request = { + "model": "fixture-model", + "input": request_messages + if index == 0 + else [ + *request_messages, + {"type": "function_call", "call_id": "tool-1", "name": "bash", "arguments": json.dumps(ARGS)}, + {"type": "function_call_output", "call_id": "tool-1", "output": "ok"}, + ], + } + records.append( + { + "model_call_id": f"call-{index}", + "client_session_id": session, + "model_ref": MODEL.model_dump(), + "dialect": dialect, + "started_at": float(index + 1), + "completed_at": float(index + 1.5), + "latency_ms": 500.0, + "status_code": 200, + "request": request, + "response": response, + } + ) + return records + + +def _opencode(directory: Path): + path = directory / "opencode.db" + with sqlite3.connect(path) as db: + db.execute("create table session (id text, parent_id text, time_created integer)") + db.execute("create table message (id text, session_id text, data text, time_created integer)") + db.execute("create table part (id text, message_id text, session_id text, data text, time_created integer)") + db.execute("insert into session values ('root', null, 0)") + for index in range(2): + message_id = f"message-{index}" + db.execute( + "insert into message values (?, ?, ?, ?)", + ( + message_id, + "root", + json.dumps( + { + "role": "assistant", + "time": {"created": 1000 + index * 1000, "completed": 1500 + index * 1000}, + } + ), + index, + ), + ) + part = ( + { + "type": "tool", + "tool": "bash", + "callID": "tool-1", + "state": { + "status": "completed", + "input": ARGS, + "output": "ok", + "time": {"start": 1600, "end": 1800}, + }, + } + if index == 0 + else {"type": "text", "text": "Done"} + ) + for number, value in enumerate([{"type": "step-start"}, part, {"type": "step-finish"}]): + db.execute( + "insert into part values (?, ?, ?, ?, ?)", + (f"{message_id}-{number}", message_id, "root", json.dumps(value), number), + ) + trajectory = TrajectoryRecord(task_id="0", rollout_id="0-0") + return _parse_opencode_session(path, "root", trajectory), trajectory + + +def _pi(): + events = [ + { + "type": "message_end", + "message": { + "role": "assistant", + "responseId": "response-0", + "content": [{"type": "toolCall", "id": "tool-1", "name": "bash", "arguments": ARGS}], + }, + }, + {"type": "tool_execution_start", "toolCallId": "tool-1", "toolName": "bash"}, + {"type": "tool_execution_end", "toolCallId": "tool-1", "toolName": "bash", "isError": False}, + { + "type": "message_end", + "message": {"role": "toolResult", "toolCallId": "tool-1", "content": [{"type": "text", "text": "ok"}]}, + }, + { + "type": "message_end", + "message": { + "role": "assistant", + "responseId": "response-1", + "content": [{"type": "text", "text": "Done"}], + }, + }, + {"type": "agent_end", "messages": [{"role": "assistant", "stopReason": "stop"}]}, + ] + output, _ = parse_pi_events("\n".join(json.dumps(event) for event in events)) + return _build_pi_observations([(float(i), event) for i, event in enumerate(events)], "root", MODEL, output) + + +class _HermesProtocol: + """Only the upstream hook interface; Gym's instrumentation runs unmodified.""" + + def __init__(self): + self.tool_start_callback = self.tool_complete_callback = None + self._active_children = [] + self.context_compressor = SimpleNamespace(last_prompt_tokens=0) + + def _interruptible_api_call(self, params): + return SimpleNamespace(id=params["response_id"]) + + def _invoke_tool(self, name, arguments, task): + return "ok" + + def _compress_context(self, messages, system_message, **kwargs): + return messages, system_message + + +def _hermes(): + agent = _HermesProtocol() + with ( + patch("responses_api_agents.hermes_agent.observability.time", return_value=1.0), + patch("responses_api_agents.hermes_agent.observability.monotonic", return_value=1.0), + ): + observer = HermesAgentObserver(root_invocation_id="root", model_ref=MODEL).instrument(agent) + agent._interruptible_api_call({"response_id": "response-0"}) + agent.tool_start_callback("tool-1", "bash", ARGS) + agent._invoke_tool("bash", ARGS, "task") + agent.tool_complete_callback("tool-1", "bash", ARGS, "ok") + agent._interruptible_api_call({"response_id": "response-1"}) + return observer.finish( + { + "completed": True, + "messages": [ + {"role": "user", "content": "Print ok, then report success."}, + { + "role": "assistant", + "content": "", + "tool_calls": [{"id": "tool-1", "function": {"name": "bash", "arguments": ARGS}}], + }, + {"role": "tool", "tool_call_id": "tool-1", "content": "ok"}, + {"role": "assistant", "content": "Done"}, + ], + } + ) + + +def build_harness_record(harness: str, directory: Path) -> dict: + """Build a collected record using real parser/observer and collector functions.""" + directory.mkdir(parents=True, exist_ok=True) + calls = [ + build_model_call_record(exchange, call_index=i) + for i, exchange in enumerate( + _exchanges( + "responses" if harness == "codex" else "chat", session="root" if harness == "opencode" else None + ) + ) + ] + row = { + "_ng_task_index": 0, + "_ng_rollout_index": 0, + "_ng_rollout_id": "0-0", + "reward": 1.0, + "ng_model_call_capture": {"rollout_id": "0-0", "calls": [c.model_dump(mode="json") for c in calls]}, + } + if harness == "codex": + events = [ + { + "type": "item.completed", + "item": { + "id": "tool-1", + "type": "command_execution", + "command": "printf ok", + "aggregated_output": "ok", + "exit_code": 0, + "status": "completed", + }, + }, + {"type": "item.completed", "item": {"id": "answer", "type": "agent_message", "text": "Done"}}, + {"type": "turn.completed", "usage": {"input_tokens": 21, "output_tokens": 10}}, + ] + output, usage = parse_exec_jsonl("\n".join(json.dumps(e) for e in events)) + row["response"] = {"output": [item.model_dump(mode="json") for item in output], "usage": usage} + # Codex's producer returns response items, not canonical observations or turns. + else: + if harness == "opencode": + observations, trajectory = _opencode(directory) + row["ng_trajectory"] = trajectory.model_dump(mode="json") + elif harness == "pi": + observations = _pi() + elif harness == "hermes": + observations = _hermes() + else: + raise ValueError(harness) + row["ng_agent_observations"] = join_model_call_observations(observations, calls).model_dump(mode="json") + _attach_trajectory_record(row, row) + _strip_capture_payloads(row) + return row diff --git a/tests/unit_tests/harness_capabilities/test_checker.py b/tests/unit_tests/harness_capabilities/test_checker.py index be858c881d2..36100095eb5 100644 --- a/tests/unit_tests/harness_capabilities/test_checker.py +++ b/tests/unit_tests/harness_capabilities/test_checker.py @@ -9,7 +9,7 @@ import pytest -from nemo_gym.harness_capabilities.checker import P0, inspect_record +from nemo_gym.harness_capabilities.checker import EvidenceScope, inspect_record from nemo_gym.harness_capabilities.cli import inspect_bundle, main from nemo_gym.harness_capabilities.reader import hydrate_record @@ -20,14 +20,16 @@ def record(request): def verdict(record, capability): - return inspect_record(hydrate_record(record))["capabilities"][capability]["verdict"] + return inspect_record(hydrate_record(record))["evidence"][capability]["verdict"] -def test_opencode_native_projection_passes_p0(record): +def test_retained_model_evidence_and_tools_pass(record): result = inspect_record(hydrate_record(record)) - assert all(result["capabilities"][c]["verdict"] == "fulfilled" for c in P0), result["findings"] - assert result["capabilities"]["C5"]["verdict"] == "fulfilled" - assert not any(v["behavioral_qualification"] for v in result["capabilities"].values()) + assert all(result["evidence"][c]["verdict"] == "fulfilled" for c in ("TE-1", "TE-2", "TE-4", "TE-7", "TE-8")), ( + result["findings"] + ) + assert result["evidence"]["TE-5"]["verdict"] == "fulfilled" + assert result["is_behavioral_qualification"] is False @pytest.mark.parametrize( @@ -43,26 +45,26 @@ def test_opencode_native_projection_passes_p0(record): @pytest.mark.parametrize("value", [None, -1, True, 1.5]) def test_bad_token_values_fail(record, field, value): record["ng_model_call_capture"]["calls"][0][field] = value - assert verdict(record, "C2") == "not_fulfilled" + assert verdict(record, "TE-2") == "not_fulfilled" def test_omitted_usage_cannot_become_zero(record): call = record["ng_model_call_capture"]["calls"][0] call["cached_tokens"] = 0 del record["ng_trajectory"]["model_calls"][0]["response"]["usage"]["input_tokens_details"]["cached_tokens"] - assert verdict(record, "C2") == "not_fulfilled" + assert verdict(record, "TE-2") == "not_fulfilled" def test_legitimate_zero_is_available(record): record["ng_model_call_capture"]["calls"][0]["cached_tokens"] = 0 record["ng_trajectory"]["model_calls"][0]["token_stats"]["cached_tokens"] = 0 record["ng_trajectory"]["model_calls"][0]["response"]["usage"]["input_tokens_details"]["cached_tokens"] = 0 - assert verdict(record, "C2") == "fulfilled" + assert verdict(record, "TE-2") == "fulfilled" def test_missing_response_usage_cannot_be_invented(record): record["ng_trajectory"]["model_calls"][0]["response"]["usage"] = None - assert verdict(record, "C2") == "not_fulfilled" + assert verdict(record, "TE-2") == "not_fulfilled" def test_external_media_is_not_a_complete_payload(record): @@ -77,8 +79,8 @@ def test_external_media_is_not_a_complete_payload(record): ], } ] - assert verdict(record, "C4") == "not_fulfilled" - assert verdict(record, "C9") == "not_fulfilled" + assert verdict(record, "TE-4") == "not_fulfilled" + assert verdict(record, "TE-7") == "not_fulfilled" @pytest.mark.parametrize( @@ -98,7 +100,7 @@ def test_ownership_is_not_guessed(record, mutation): invocation["model_calls"][0]["response_id"] = "different" else: invocation["parent_invocation_id"] = invocation["invocation_id"] - assert verdict(record, "C10") == "not_fulfilled" + assert verdict(record, "TE-8") == "not_fulfilled" def test_failed_attempt_without_response_id_still_requires_owner(record): @@ -112,29 +114,29 @@ def test_failed_attempt_without_response_id_still_requires_owner(record): ) failed["request"] = {"input": []} record["ng_model_call_capture"]["calls"].append(failed) - assert verdict(record, "C10") == "not_fulfilled" + assert verdict(record, "TE-8") == "not_fulfilled" record["ng_agent_observations"]["records"][0]["model_calls"].append({"model_call_id": "failure"}) - assert verdict(record, "C10") == "fulfilled" - assert verdict(record, "C9") == "fulfilled" + assert verdict(record, "TE-8") == "fulfilled" + assert verdict(record, "TE-7") == "fulfilled" def test_payload_removal_fails_even_with_complete_conversation(record): record["ng_trajectory"]["model_calls"][0]["request"] = None - assert verdict(record, "C4") == "not_fulfilled" - assert verdict(record, "C9") == "not_fulfilled" + assert verdict(record, "TE-4") == "not_fulfilled" + assert verdict(record, "TE-7") == "not_fulfilled" def test_schema_valid_duplicate_attempt_fails(record): record["ng_model_call_capture"]["calls"] *= 2 - assert verdict(record, "C1") == "not_fulfilled" + assert verdict(record, "TE-1") == "not_fulfilled" def test_nullable_provider_model_keeps_explicit_server_identity(record): call = record["ng_model_call_capture"]["calls"][0] call["model"] = None - assert verdict(record, "C1") == "fulfilled" + assert verdict(record, "TE-1") == "fulfilled" call["model_ref"] = None - assert verdict(record, "C1") == "not_fulfilled" + assert verdict(record, "TE-1") == "not_fulfilled" def test_sidecar_does_not_launder_payload_conflict(record, tmp_path): @@ -142,7 +144,7 @@ def test_sidecar_does_not_launder_payload_conflict(record, tmp_path): sidecar = {**call, "request": {"input": "different"}} (tmp_path / "0-0.capture.jsonl").write_text(json.dumps(sidecar) + "\n") hydrated = hydrate_record(record, capture_dir=tmp_path) - assert inspect_record(hydrated)["capabilities"]["C0"]["verdict"] == "not_fulfilled" + assert inspect_record(hydrated)["evidence"]["TE-1"]["verdict"] == "not_fulfilled" def test_tool_request_is_not_execution(record): @@ -150,27 +152,24 @@ def test_tool_request_is_not_execution(record): r for r in record["ng_agent_observations"]["records"] if r["kind"] != "tool_call" ] record["ng_trajectory"]["tool_calls"] = [] - assert verdict(record, "C5") == "not_fulfilled" - - -def test_independent_witness_requirements_never_pass_from_shape(record): - for capability in ("C6", "C7", "C8", "C11"): - assert verdict(record, capability) == "not_fulfilled" + assert verdict(record, "TE-5") == "not_fulfilled" def test_cli_atomic_replay_and_exit_codes(record, tmp_path): bundle = tmp_path / "input.jsonl" bundle.write_text(json.dumps(record) + "\n") output = tmp_path / "reports" + record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + bundle.write_text(json.dumps(record) + "\n") args = ["inspect", "--bundle", str(bundle), "--output", str(output)] assert main(args) == 0 assert main(args) == 0 assert len(list(output.iterdir())) == 1 - report = next(output.glob("*/capability_summary.json")) + report = next(output.glob("*/evidence_summary.json")) summary = json.loads(report.read_text()) assert summary["checker_status"] == "completed" assert summary["is_behavioral_qualification"] is False - record["ng_agent_observations"]["records"][0]["model_calls"] = [] + record["ng_trajectory"]["model_calls"][0]["request"] = None bundle.write_text(json.dumps(record) + "\n") assert main(args) == 1 assert len(list(output.iterdir())) == 2 @@ -184,7 +183,222 @@ def test_empty_and_duplicate_rollouts_fail(record, tmp_path): bundle = tmp_path / "input.jsonl" bundle.write_text("") with pytest.raises(ValueError, match="no rollout records"): - inspect_bundle(bundle, output=tmp_path / "out", profile="gym-artifacts-p0/v1") + inspect_bundle(bundle, output=tmp_path / "out", profile="gym-p0/v1") bundle.write_text((json.dumps(record) + "\n") * 2) - _, report = inspect_bundle(bundle, output=tmp_path / "out", profile="gym-artifacts-p0/v1") + _, report = inspect_bundle(bundle, output=tmp_path / "out", profile="gym-p0/v1") assert report["verdict"] == "not_fulfilled" + + +def test_missing_p0_steps_or_tools_cannot_pass(record): + record["ng_trajectory"]["turns"] = [] + record["ng_trajectory"]["tool_calls"] = [] + record["ng_agent_observations"]["records"] = [ + r for r in record["ng_agent_observations"]["records"] if r["kind"] != "tool_call" + ] + result = inspect_record(hydrate_record(record)) + assert result["verdict"] == "not_fulfilled" + assert result["evidence"]["TE-3"]["verdict"] == "not_fulfilled" + assert result["evidence"]["TE-5"]["verdict"] == "not_fulfilled" + + +def test_provider_omission_is_preserved_and_reported(record): + for c in record["ng_model_call_capture"]["calls"]: + c["cached_tokens"] = c["tokens_reasoning"] = None + for c in record["ng_trajectory"]["model_calls"]: + c["token_stats"]["cached_tokens"] = c["token_stats"]["reasoning_tokens"] = None + c["response"]["usage"].pop("input_tokens_details", None) + c["response"]["usage"].pop("output_tokens_details", None) + result = inspect_record(hydrate_record(record)) + assert result["evidence"]["TE-2"]["verdict"] == "fulfilled" + assert result["token_availability"]["cached_tokens"]["available"] == 0 + assert result["token_availability"]["tokens_reasoning"]["available"] == 0 + + +@pytest.mark.parametrize("field", ["response_status", "finish_reason"]) +def test_terminal_metadata_conflict_fails(record, field): + record["ng_trajectory"]["model_calls"][0]["response_metadata"][field] = "contradiction" + assert verdict(record, "TE-1") == "not_fulfilled" + + +def test_terminal_response_metadata_required(record): + for c in record["ng_model_call_capture"]["calls"]: + c["response_status"] = c["finish_reason"] = None + for c in record["ng_trajectory"]["model_calls"]: + c["response_metadata"]["response_status"] = c["response_metadata"]["finish_reason"] = None + c["response"].pop("status", None) + c["response"].pop("incomplete_details", None) + assert verdict(record, "TE-1") == "not_fulfilled" + + +def test_truthful_failure_is_evidence_not_a_health_verdict(record): + c = record["ng_model_call_capture"]["calls"][0] + c.update(status_code=503, response_status=None, finish_reason=None) + t = record["ng_trajectory"]["model_calls"][0] + t["response_metadata"].update(status_code=503, response_status=None, finish_reason=None) + # Preserve usage reported on the failed attempt, as well as its error body. + t["response"] = {"error": {"message": "failed"}, "usage": t["response"]["usage"]} + assert verdict(record, "TE-1") == "fulfilled" + assert verdict(record, "TE-2") == "fulfilled" + c["tokens_in"] += 1 + t["token_stats"]["prompt_tokens"] = c["tokens_in"] + assert verdict(record, "TE-2") == "not_fulfilled" + + +def test_tool_record_does_not_require_p1_clocks(record): + for tool in record["ng_trajectory"]["tool_calls"] + record["ng_agent_observations"]["records"]: + if tool.get("kind") == "tool_call": + for key in ("started_at", "completed_at", "duration_ms", "timing_source"): + tool.pop(key, None) + assert verdict(record, "TE-5") == "fulfilled" + + +def test_partial_tool_execution_loss_fails(record): + invocation = record["ng_agent_observations"]["records"][0] + invocation["conversation"].append({"type": "function_call_output", "call_id": "lost-tool", "output": "ok"}) + assert verdict(record, "TE-5") == "not_fulfilled" + + +def test_na_requires_explicit_scope_and_rejects_contradictions(record): + scope = EvidenceScope(tools=False, verifier=False, steps=False) + assert inspect_record(hydrate_record(record), scope=scope)["evidence"]["TE-5"]["verdict"] == "not_fulfilled" + record["ng_trajectory"]["turns"] = [] + record["ng_trajectory"]["tool_calls"] = [] + record["ng_agent_observations"]["records"] = [ + r for r in record["ng_agent_observations"]["records"] if r["kind"] != "tool_call" + ] + for inv in record["ng_agent_observations"]["records"]: + inv["conversation"] = [] + record.pop("reward", None) + record.pop("response", None) + record["ng_trajectory"]["gaps"] = [] + result = inspect_record(hydrate_record(record), scope=scope) + assert all(result["evidence"][key]["verdict"] == "not_applicable" for key in ("TE-3", "TE-5", "TE-6", "TE-9")) + assert result["verdict"] == "fulfilled" + + +def test_step_join_can_qualify_without_invocation_refs(): + record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + for inv in record["ng_agent_observations"]["records"]: + if inv["kind"] == "agent_invocation": + inv["model_calls"] = [] + result = inspect_record(hydrate_record(record)) + assert result["evidence"]["TE-8"]["verdict"] == "not_fulfilled" + assert result["evidence"]["TE-9"]["verdict"] == "fulfilled" + assert result["verdict"] == "fulfilled" + + +def test_duplicated_step_ref_fails_without_closure_requirement(): + record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + assert verdict(record, "TE-9") == "fulfilled" + record["ng_trajectory"]["turns"][0]["model_calls"] *= 2 + assert verdict(record, "TE-9") == "not_fulfilled" + + +def test_verifier_failure_must_not_be_numeric_zero(record): + record.update(mask_sample=True, failure_kind="verifier_unavailable", reward=0.0) + assert verdict(record, "TE-6") == "not_fulfilled" + record["reward"] = None + assert verdict(record, "TE-6") == "fulfilled" + + +def test_binary_resolution_does_not_replace_reward(record): + record.pop("reward", None) + for turn in record["ng_trajectory"]["turns"]: + turn["resolved"] = True + assert verdict(record, "TE-6") == "not_fulfilled" + + +def test_canonical_only_delivery_does_not_require_capture_middleware(): + record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + del record["ng_model_call_capture"] + for call in record["ng_trajectory"]["model_calls"]: + call["started_at"] = call["completed_at"] = call["duration_ms"] = None + assert inspect_record(hydrate_record(record))["verdict"] == "fulfilled" + + +def test_missing_prior_response_history_fails(record): + record["ng_trajectory"]["model_calls"][0]["request"]["previous_response_id"] = "unretained-server-history" + assert verdict(record, "TE-4") == "not_fulfilled" + + +def test_failed_response_without_response_id_preserves_status(record): + capture = record["ng_model_call_capture"]["calls"][0] + capture.update(response_id=None, response_status="failed") + call = record["ng_trajectory"]["model_calls"][0] + call["response_metadata"].update(response_id=None, response_status="failed") + call["response"].update(id=None, status="failed", error={"code": "server_error"}) + assert verdict(record, "TE-1") == "fulfilled" + + +def test_retry_and_compaction_attempts_have_distinct_accounting(): + record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + record = hydrate_record(record) + retry = copy.deepcopy(record["ng_model_call_capture"]["calls"][0]) + retry.update(model_call_id="retry", response_id="retry-response") + record["ng_model_call_capture"]["calls"].append(retry) + record["ng_agent_observations"]["records"][0]["model_calls"].append({"model_call_id": "retry"}) + # A retry does not create a new step; its own attempt ref belongs on the existing one. + assert inspect_record(record)["evidence"]["TE-9"]["verdict"] == "not_fulfilled" + record["ng_trajectory"]["turns"][0]["model_calls"].append({"model_call_id": "retry"}) + assert inspect_record(record)["evidence"]["TE-9"]["verdict"] == "fulfilled" + helper = copy.deepcopy(retry) + helper.update(model_call_id="helper", response_id="helper-response") + record["ng_model_call_capture"]["calls"].append(helper) + record["ng_agent_observations"]["records"][0]["model_calls"].append({"model_call_id": "helper"}) + record["ng_agent_observations"]["records"].append( + { + "kind": "context_compaction", + "invocation_id": record["ng_trajectory"]["turns"][0]["invocation_id"], + "model_calls": [{"model_call_id": "helper"}], + } + ) + assert inspect_record(record)["evidence"]["TE-9"]["verdict"] == "fulfilled" + record["ng_trajectory"]["turns"][0]["model_calls"].append({"model_call_id": "helper"}) + assert inspect_record(record)["evidence"]["TE-9"]["verdict"] == "not_fulfilled" + + +def test_conflicting_run_reference_cannot_be_hidden_by_step_join(): + record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + record["ng_agent_observations"]["records"][0]["model_calls"] = [{"model_call_id": "missing"}] + assert inspect_record(hydrate_record(record))["verdict"] == "not_fulfilled" + + +def test_anthropic_cache_prompt_and_derived_total(record): + call = record["ng_model_call_capture"]["calls"][0] + trajectory = record["ng_trajectory"]["model_calls"][0] + call.update( + dialect="messages", tokens_in=15, tokens_out=5, tokens_total=20, tokens_reasoning=None, cached_tokens=3 + ) + trajectory["response_metadata"]["dialect"] = "messages" + trajectory["token_stats"].update( + prompt_tokens=15, completion_tokens=5, total_tokens=20, reasoning_tokens=None, cached_tokens=3 + ) + trajectory["response"]["usage"] = { + "input_tokens": 10, + "output_tokens": 5, + "cache_read_input_tokens": 3, + "cache_creation_input_tokens": 2, + } + assert verdict(record, "TE-2") == "fulfilled" + call["tokens_in"] = trajectory["token_stats"]["prompt_tokens"] = 10 + assert verdict(record, "TE-2") == "not_fulfilled" + + +def test_boolean_provider_usage_cannot_equal_integer_count(record): + call = record["ng_model_call_capture"]["calls"][0] + trajectory = record["ng_trajectory"]["model_calls"][0] + call["cached_tokens"] = 1 + trajectory["token_stats"]["cached_tokens"] = 1 + trajectory["response"]["usage"]["input_tokens_details"]["cached_tokens"] = True + assert verdict(record, "TE-2") == "not_fulfilled" + + +def test_response_identity_must_match_payload(record): + record["ng_model_call_capture"]["calls"][0]["response_id"] = "wrong" + record["ng_trajectory"]["model_calls"][0]["response_metadata"]["response_id"] = "wrong" + assert verdict(record, "TE-1") == "not_fulfilled" + + +def test_tool_output_conflict_fails(record): + record["ng_trajectory"]["tool_calls"][0]["output"] = "different-from-model-visible-result" + assert verdict(record, "TE-5") == "not_fulfilled" diff --git a/tests/unit_tests/harness_capabilities/test_producers.py b/tests/unit_tests/harness_capabilities/test_producers.py index e5d4780a3c1..e2cad34fa46 100644 --- a/tests/unit_tests/harness_capabilities/test_producers.py +++ b/tests/unit_tests/harness_capabilities/test_producers.py @@ -1,121 +1,39 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Contract fixture through Gym's real OpenCode parser and collector projection. - -Synthetic protocol/SQLite inputs; not a live OpenCode qualification. -""" +"""Exercise four current harness producers through the JSONL evidence checker.""" import json -import sqlite3 +from pathlib import Path + +import pytest -from nemo_gym.base_responses_api_model import build_model_call_record -from nemo_gym.harness_capabilities.checker import P0, inspect_record +from nemo_gym.harness_capabilities.checker import inspect_record +from nemo_gym.harness_capabilities.cli import inspect_matrix from nemo_gym.harness_capabilities.reader import hydrate_record -from nemo_gym.rollout_collection import _attach_trajectory_record -from nemo_gym.rollout_observability import join_model_call_observations -from responses_api_agents.opencode_sandboxed_agent.app import ( - parse_opencode_observations, -) +from tests.unit_tests.harness_capabilities.producer_fixtures import HARNESSES, build_harness_record -def test_current_opencode_projection(tmp_path): - path = tmp_path / "opencode.db" - with sqlite3.connect(path) as db: - db.execute("create table session (id text, parent_id text, time_created integer)") - db.execute("create table message (id text, session_id text, data text, time_created integer)") - db.execute("create table part (id text, message_id text, session_id text, data text, time_created integer)") - db.execute("insert into session values ('root', null, 0)") - db.execute( - "insert into message values (?, ?, ?, ?)", - ( - "m1", - "root", - json.dumps( - { - "role": "assistant", - "time": {"created": 1000, "completed": 3000}, - } - ), - 1000, - ), - ) - db.execute( - "insert into part values (?, ?, ?, ?, ?)", - ( - "p1", - "m1", - "root", - json.dumps( - { - "type": "tool", - "tool": "bash", - "callID": "tool-1", - "state": { - "status": "completed", - "input": {"command": "printf ok"}, - "output": "ok", - "time": {"start": 2000, "end": 3000}, - }, - } - ), - 2000, - ), - ) - observations = parse_opencode_observations(path, "root") - exchange = { - "model_call_id": "call-1", - "client_session_id": "root", - "model_ref": {"type": "responses_api_models", "name": "policy"}, - "dialect": "responses", - "started_at": 1.0, - "completed_at": 2.0, - "latency_ms": 1000.0, - "status_code": 200, - "request": { - "model": "model", - "input": [{"role": "user", "content": "Print ok"}], - "tools": [ - { - "type": "function", - "name": "bash", - "parameters": {"type": "object"}, - } - ], - }, - "response": { - "id": "response-1", - "model": "model", - "status": "completed", - "output": [ - { - "type": "function_call", - "call_id": "tool-1", - "name": "bash", - "arguments": '{"command":"printf ok"}', - } - ], - "usage": { - "input_tokens": 10, - "output_tokens": 5, - "total_tokens": 15, - "input_tokens_details": {"cached_tokens": 3}, - "output_tokens_details": {"reasoning_tokens": 2}, - }, - }, - } - call = build_model_call_record(exchange, call_index=0) - observations = join_model_call_observations(observations, [call]) - record = { - "_ng_task_index": 0, - "_ng_rollout_index": 0, - "ng_model_call_capture": { - "rollout_id": "0-0", - "calls": [call.model_dump(mode="json")], - }, - "ng_agent_observations": observations.model_dump(mode="json"), - } - _attach_trajectory_record(record, record) +@pytest.mark.parametrize("harness", HARNESSES) +def test_current_producer_evidence(harness, tmp_path): + record = build_harness_record(harness, tmp_path / harness) result = inspect_record(hydrate_record(record)) - assert all(result["capabilities"][key]["verdict"] == "fulfilled" for key in (*P0, "C5")), result["findings"] - assert result["capabilities"]["C3"]["verdict"] == "not_fulfilled" + passed = {key for key, value in result["evidence"].items() if value["verdict"] == "fulfilled"} + expected = {"TE-1", "TE-2", "TE-4", "TE-6", "TE-7"} + if harness != "codex": + expected |= {"TE-5", "TE-8"} + assert passed == expected, result["findings"] + assert result["verdict"] == "not_fulfilled" + # Committed regression artifacts are reproducible outputs of these producers. + fixture = Path(__file__).parent / "fixtures" / f"{harness}.jsonl" + assert json.loads(fixture.read_text()) == record + + +def test_harness_matrix_uses_individual_report_verdicts(tmp_path): + fixtures = Path(__file__).parent / "fixtures" + path, result = inspect_matrix({name: fixtures / f"{name}.jsonl" for name in HARNESSES}, output=tmp_path) + table = (path / "harness_evidence.md").read_text() + assert all(name in table for name in HARNESSES) + assert all(row["verdict"] == "not_fulfilled" for row in result["harnesses"].values()) + assert "| codex | PASS | PASS | FAIL | PASS | FAIL | PASS | PASS | FAIL | FAIL | FAIL |" in table + assert inspect_matrix({name: fixtures / f"{name}.jsonl" for name in HARNESSES}, output=tmp_path)[0] == path diff --git a/tests/unit_tests/harness_capabilities/test_script.py b/tests/unit_tests/harness_capabilities/test_script.py index 13e1c6a30ac..09759eed598 100644 --- a/tests/unit_tests/harness_capabilities/test_script.py +++ b/tests/unit_tests/harness_capabilities/test_script.py @@ -19,7 +19,7 @@ @pytest.fixture def record(): - return json.loads((Path(__file__).parent / "fixtures/opencode.json").read_text()) + return json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) @pytest.mark.parametrize( @@ -36,7 +36,7 @@ def test_script_discovers_bundle(record, tmp_path, monkeypatch, relative_path): with pytest.raises(SystemExit) as exit_info: runpy.run_path(str(SCRIPT), run_name="__main__") assert exit_info.value.code == 0 - (summary_path,) = (tmp_path / "reports").glob("*/capability_summary.json") + (summary_path,) = (tmp_path / "reports").glob("*/evidence_summary.json") summary = json.loads(summary_path.read_text()) assert summary["verdict"] == "fulfilled" assert summary["sources"].keys() == {str(path.resolve())} @@ -44,7 +44,7 @@ def test_script_discovers_bundle(record, tmp_path, monkeypatch, relative_path): @pytest.mark.parametrize( - "profile,code", [("gym-artifacts-p1/v1", 1), ("gym-artifacts-all/v1", 1), ("onboarding-p0/v1", 2)] + "profile,code", [("gym-artifacts-p1/v1", 2), ("gym-artifacts-all/v1", 2), ("onboarding-p0/v1", 2)] ) def test_script_gate_exit_codes(record, tmp_path, monkeypatch, profile, code): path = tmp_path / "rollouts.jsonl" @@ -87,15 +87,13 @@ def test_ambiguous_directory_requires_explicit_file(record, tmp_path): for name in ("rollouts.jsonl", "evaluator_rollouts.jsonl"): (tmp_path / name).write_text(json.dumps(record) + "\n") with pytest.raises(ValueError, match="exactly one"): - cli.inspect_bundle(tmp_path, output=tmp_path / "reports", profile="gym-artifacts-p0/v1") - _, summary = cli.inspect_bundle( - tmp_path / "rollouts.jsonl", output=tmp_path / "reports", profile="gym-artifacts-p0/v1" - ) + cli.inspect_bundle(tmp_path, output=tmp_path / "reports", profile="gym-p0/v1") + _, summary = cli.inspect_bundle(tmp_path / "rollouts.jsonl", output=tmp_path / "reports", profile="gym-p0/v1") assert summary["verdict"] == "fulfilled" @pytest.mark.parametrize("sidecar_state", ["complete", "missing", "incomplete", "extra_call"]) -def test_script_recovers_payloads_and_validates_sidecars(record, tmp_path, monkeypatch, sidecar_state): +def test_sidecars_validate_but_do_not_repair_missing_jsonl_payloads(record, tmp_path, monkeypatch, sidecar_state): full = hydrate_record(record) capture_dir = tmp_path / "model-calls" capture_dir.mkdir() @@ -126,8 +124,8 @@ def test_script_recovers_payloads_and_validates_sidecars(record, tmp_path, monke ) with pytest.raises(SystemExit) as exit_info: runpy.run_path(str(SCRIPT), run_name="__main__") - assert exit_info.value.code == (0 if sidecar_state == "complete" else 1) - (summary_file,) = (tmp_path / "reports").glob("*/capability_summary.json") + assert exit_info.value.code == 1 # Sidecars cannot repair the delivered JSONL surface. + (summary_file,) = (tmp_path / "reports").glob("*/evidence_summary.json") summary = json.loads(summary_file.read_text()) if sidecar_state != "missing": assert str(capture_file.resolve()) in summary["sources"] @@ -144,7 +142,7 @@ def mutate_after_read(path): monkeypatch.setattr(cli, "json_rows", mutate_after_read) with pytest.raises(ValueError, match="source changed"): - cli.inspect_bundle(path, output=tmp_path / "reports", profile="gym-artifacts-p0/v1") + cli.inspect_bundle(path, output=tmp_path / "reports", profile="gym-p0/v1") assert list((tmp_path / "reports").iterdir()) == [] @@ -161,3 +159,23 @@ def test_reports_do_not_include_payload_values(record, tmp_path, capsys): path.write_text('{"' + secret + '":') assert cli.run_inspection(bundle=path, output=tmp_path / "reports") == 2 assert secret not in capsys.readouterr().out + + +def test_script_matrix_routes_and_returns_failed_gate(tmp_path, monkeypatch): + fixtures = Path(__file__).parent / "fixtures" + args = [str(SCRIPT), "matrix", "--output", str(tmp_path)] + for name in ("opencode", "pi", "codex", "hermes"): + args.extend(["--harness", f"{name}={fixtures / (name + '.jsonl')}"]) + monkeypatch.setattr(sys, "argv", args) + with pytest.raises(SystemExit) as error: + runpy.run_path(str(SCRIPT), run_name="__main__") + assert error.value.code == 1 + (matrix,) = tmp_path.glob("*/harness_evidence.json") + assert set(json.loads(matrix.read_text())["harnesses"]) == {"opencode", "pi", "codex", "hermes"} + + +def test_matrix_error_cannot_publish_partial_matrix(tmp_path): + fixtures = Path(__file__).parent / "fixtures" + with pytest.raises(OSError): + cli.inspect_matrix({"opencode": fixtures / "opencode.jsonl", "missing": tmp_path / "missing"}, output=tmp_path) + assert not list(tmp_path.glob("*/harness_evidence.json")) From b9d28605e6cfe426ccb301ae488c8dd86eb7d26e Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Mon, 28 Sep 2026 15:18:25 +0200 Subject: [PATCH 04/11] Add scripted harness conformance runner Signed-off-by: Frederic Lafrance --- scripts/harness_conformance/README.md | 110 +++++++ scripts/harness_conformance/__init__.py | 4 + scripts/harness_conformance/episode.py | 210 ++++++++++++ scripts/harness_conformance/provider.py | 265 +++++++++++++++ scripts/harness_conformance/runner.py | 308 ++++++++++++++++++ scripts/harness_conformance/scenarios.py | 57 ++++ .../harness_conformance/tests/test_runner.py | 292 +++++++++++++++++ scripts/run_harness_conformance.py | 10 + 8 files changed, 1256 insertions(+) create mode 100644 scripts/harness_conformance/README.md create mode 100644 scripts/harness_conformance/__init__.py create mode 100644 scripts/harness_conformance/episode.py create mode 100644 scripts/harness_conformance/provider.py create mode 100644 scripts/harness_conformance/runner.py create mode 100644 scripts/harness_conformance/scenarios.py create mode 100644 scripts/harness_conformance/tests/test_runner.py create mode 100644 scripts/run_harness_conformance.py diff --git a/scripts/harness_conformance/README.md b/scripts/harness_conformance/README.md new file mode 100644 index 00000000000..e700a559741 --- /dev/null +++ b/scripts/harness_conformance/README.md @@ -0,0 +1,110 @@ +# Live harness conformance probes + +Run predetermined model replies and failures through real Gym harnesses, collect +their rollouts, and apply the existing P0 evidence checks: + +```bash +python scripts/run_harness_conformance.py \ + --harness codex --harness pi --harness opencode --harness hermes \ + --output results/conformance +``` + +Run from a Gym checkout with Gym and the selected agents' Python requirements +installed. Put the selected `codex`, `pi`, and `opencode` executables on `PATH`; +Hermes must be importable in the same Python environment. The versions pinned in +`responses_api_agents/*/configs/` are the recommended starting point. The runner +requires installed runtimes; missing dependencies are execution failures. +Omitting `--harness` selects all four. + +No model service or model credentials are needed. Each scenario starts fresh +local model, resources, agent, and compatibility environment servers on dynamically assigned loopback ports. +The existing Gym adapter runs its real harness and normal seed/verify lifecycle; +`RolloutCollectionHelper` produces the rollout and capture payloads. The runner +never supplies canonical steps, joins, or tool observations on the harness's +behalf. All implementation remains under `scripts/` for eventual CI integration. + +The OpenCode preset disables its auxiliary title and summary agents through +[agent configuration](https://opencode.ai/docs/agents/#disable), so scripted +failures reach policy requests. This configuration is saved with the run. + +The suite uses each harness's local shell tool to print a unique marker, write it +inside the output directory, and return a prescribed exit code. Local execution +must be allowed (the Codex preset uses `danger-full-access` in a fresh working +directory). These probes qualify the local adapter path; they do not exercise a +remote sandbox deployment. + +## Scenarios + +```bash +python scripts/run_harness_conformance.py --list-scenarios +python scripts/run_harness_conformance.py --harness codex \ + --scenario tool_failure --timeout 60 --output results/codex-tool-failure +``` + +| Scenario | Prescribed behavior | Evidence exercised | +| --- | --- | --- | +| `tool_success` | Two shell calls, each result carried into the next model request, then a final answer | TE-1–TE-9 | +| `tool_failure` | First command exits 7; a second command succeeds | TE-1–TE-9, including failed tool evidence | +| `usage_omitted` | Successful tool sequence with no provider usage fields | TE-1–TE-9, including unknown usage | +| `retry_429` | Two 429 replies before the tool sequence | TE-1–TE-9, including distinct attempts with identical request bodies | +| `retry_500` | One 500 reply before the tool sequence | TE-1–TE-9, including retained error payloads | +| `model_error` | Persistent 400 with no model response ID | TE-1, TE-2, TE-4, TE-7, and TE-8/TE-9 | +| `verifier_failure` | Completed tool sequence graded zero | TE-1–TE-9, including a known verifier failure | + +Both Chat Completions and Responses are supported, including streaming. Supplied +usage includes prompt, completion, reasoning, total, and cached counts. Chat +replies also include reasoning text. Requests adapt only the shell-tool name and +argument schema advertised by the harness. Unsupported tool protocols are +reported as exercise failures. + +The runner observes actual retry behavior. A runtime that does not retry the +injected error leaves that scenario incomplete. A terminal error must still +produce a collected rollout to satisfy its evidence checks; a failure sidecar +alone cannot qualify it. TE-8 and TE-9 remain alternatives per scenario. + +## Results + +Use a **new output directory** for each invocation. Existing directories are +rejected, so previous rollouts cannot accidentally qualify a new run. + +- `suite.json`: selected scenarios and runner source hashes. +- `//requests.jsonl`, `launch.json`, and `episode.log`: input, + resolved launch configuration, and process diagnostics. +- `//runtime.json`: executable version or Hermes source hash, + plus the Gym adapter source hash. +- `//rollouts.jsonl` and `capture/`: ordinary Gym outputs, + directly consumable by `scripts/check_harness_conformance.py`. +- `//witness.json`: independent endpoint attempts, actual tool + execution markers, returned tool results, and verifier outcomes. +- `//scenario_result.json` and `evidence/`: execution gaps, + input hashes, and the existing artifact checker reports. +- `conformance_summary.json` and `conformance_report.md`: harness × evidence + matrix with passing / exercised / required scenario counts. + +A scenario counts as passing only when its independent observations agree with +the retained rollout and the relevant evidence checks pass. Attempt comparison +preserves multiplicity, so dropping one of two identical retries fails. The +artifact reports keep their original retained-artifact scope; the runner's +separate report adds the scenario coverage. + +The summary is published only after every selected episode has been processed. +An interrupted run has no completed summary. A scenario subset is explicitly +marked `full_suite: false`. Timeouts stop the episode and clean up its subprocess +tree, including harnesses that start new process groups. + +Exit codes: **0** when all selected gates pass, **1** for evidence or scenario +failures, and **2** for execution, dependency, input, or checker errors. Existing +harness evidence gaps are expected to remain visible as failures. This suite +covers the current P0 checks; multimodal content, compaction, parallelism, +transport disconnects, TE-10 and P1 remain outside its qualification scope. + +Run regression checks with: + +```bash +python -m pytest scripts/harness_conformance/tests tests/unit_tests/harness_capabilities -q +``` + +To add another local harness, add its adapter class to `episode.HARNESSES`, supply +its launch settings in `episode._config`, and make its advertised shell-tool +schema consumable by `provider.Probe._tool`. Keep expectations and witnesses out +of the task input and preserve the normal Gym collection path. diff --git a/scripts/harness_conformance/__init__.py b/scripts/harness_conformance/__init__.py new file mode 100644 index 00000000000..dd6c2756e8b --- /dev/null +++ b/scripts/harness_conformance/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Experimental live harness probes; kept outside Gym's installed API.""" diff --git a/scripts/harness_conformance/episode.py b/scripts/harness_conformance/episode.py new file mode 100644 index 00000000000..591b64601b0 --- /dev/null +++ b/scripts/harness_conformance/episode.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""One disposable local Gym episode, driven through the normal /run collector.""" + +import argparse +import asyncio +import importlib +import importlib.util +import json +import os +import shutil +import socket +import subprocess +from contextlib import ExitStack, nullcontext +from pathlib import Path + +import uvicorn +import yaml + +from nemo_gym.harness_capabilities.reader import digest_file + +from .provider import Probe +from .scenarios import SCENARIOS + + +HARNESSES = {"opencode": "OpenCodeAgent", "pi": "PiAgent", "codex": "CodexAgent", "hermes": "HermesAgent"} + + +class _Server(uvicorn.Server): + # The episode process owns cancellation; the embedded servers must not + # independently install competing signal handlers. + def capture_signals(self) -> nullcontext[None]: + return nullcontext() + + +def check_runtime(harness: str) -> dict: + """Require preinstalled runtimes so a probe cannot trigger auto-installation.""" + if harness == "hermes": + if importlib.util.find_spec("run_agent") is None or importlib.util.find_spec("model_tools") is None: + raise RuntimeError("install responses_api_agents/hermes_agent/requirements.txt in this Python environment") + source = Path(importlib.util.find_spec("run_agent").origin) + return {"source": str(source), "sha256": digest_file(source)} + elif not shutil.which(harness): + raise RuntimeError(f"install the {harness} runtime and put it on PATH before running probes") + version = subprocess.run( + [harness, "--version"], capture_output=True, text=True, errors="replace", timeout=10, check=True + ) + return {"executable": shutil.which(harness), "version": version.stdout.strip()} + + +def _config(harness: str, directory: Path, ports: list[int], timeout: float) -> dict: + def server(port: int) -> dict: + return {"name": "unused", "host": "127.0.0.1", "port": port, "entrypoint": "app.py", "num_workers": 1} + + agent = { + **server(ports[2]), + "name": "probe_agent", + "concurrency": 1, + "model": "conformance-model", + "model_server": {"type": "responses_api_models", "name": "policy_model"}, + "resources_server": {"type": "resources_servers", "name": "probe_resources"}, + } + defaults_path = ( + Path(__file__).resolve().parents[2] / f"responses_api_agents/{harness}_agent/configs/{harness}_agent.yaml" + ) + defaults = yaml.safe_load(defaults_path.read_text())[f"{harness}_agent"]["responses_api_agents"][ + f"{harness}_agent" + ] + agent.update({key: value for key, value in defaults.items() if key.endswith("_version")}) + if harness in ("opencode", "pi"): + agent.update(workspace_root=str(directory / "workspace"), timeout=timeout) + if harness == "opencode": + # Auxiliary title/summary requests would consume the scripted policy replies. + agent["opencode_config"] = { + "permission": {"bash": "allow"}, + "agent": {"title": {"disable": True}, "summary": {"disable": True}}, + } + if harness == "codex": + workspace = directory / "workspace" + workspace.mkdir() + agent.update( + cwd=str(workspace), + timeout=timeout, + sandbox_mode="danger-full-access", + extra_config={"features": {"shell_snapshot": False}, "web_search": "disabled"}, + ) + if harness == "hermes": + agent.update( + max_turns=6, + enabled_toolsets=["terminal"], + terminal_backend="local", + terminal_timeout=10, + compression_enabled=False, + checkpoints_enabled=False, + api_key="conformance", + ) + return { + "head_server": {"host": "127.0.0.1", "port": ports[0]}, + "observability_enabled": True, + "model_call_capture_dir": str(directory / "capture"), + "policy_model": {"responses_api_models": {"probe": {**server(ports[0]), "name": "policy_model"}}}, + "probe_resources": {"resources_servers": {"probe": {**server(ports[1]), "name": "probe_resources"}}}, + "probe_agent": {"responses_api_agents": {f"{harness}_agent": agent}}, + "probe_environment": { + "environment_servers": { + "legacy_agent": { + **server(ports[3]), + "name": "probe_environment", + "agent_server": {"type": "responses_api_agents", "name": "probe_agent"}, + } + } + }, + } + + +async def run_episode(*, harness: str, scenario_name: str, directory: Path, timeout: float) -> None: + """Start fresh servers and let Gym write the rollout without modifying the harness.""" + runtime = check_runtime(harness) + runtime["adapter_sha256"] = digest_file( + Path(__file__).resolve().parents[2] / f"responses_api_agents/{harness}_agent/app.py" + ) + (directory / "runtime.json").write_text(json.dumps(runtime, indent=2) + "\n") + scenario = next(s for s in SCENARIOS if s.name == scenario_name) + probe = Probe(scenario, directory) + probe.save() + tasks = [] + servers = [] + with ExitStack() as stack: + sockets = [stack.enter_context(socket.socket()) for _ in range(4)] + for sock in sockets: + sock.bind(("127.0.0.1", 0)) + config = _config(harness, directory, [s.getsockname()[1] for s in sockets], timeout) + os.environ["NEMO_GYM_CONFIG_DICT"] = json.dumps(config) + (directory / "launch.json").write_text(json.dumps(config, indent=2) + "\n") + # Import after the isolated process's config has been injected, like Gym's launcher. + from environment_servers.legacy_agent.app import ( + LegacyAgentEnvironmentServer, + LegacyAgentEnvironmentServerConfig, + ) + from nemo_gym.rollout_collection import RolloutCollectionConfig, RolloutCollectionHelper + from nemo_gym.server_utils import get_global_aiohttp_client, setup_server_client + + client = setup_server_client() + module = importlib.import_module(f"responses_api_agents.{harness}_agent.app") + agent_cls = getattr(module, HARNESSES[harness]) + config_cls = getattr(module, HARNESSES[harness] + "Config") + agent = agent_cls( + config=config_cls.model_validate(config["probe_agent"]["responses_api_agents"][f"{harness}_agent"]), + server_client=client, + ) + environment = LegacyAgentEnvironmentServer( + config=LegacyAgentEnvironmentServerConfig.model_validate( + config["probe_environment"]["environment_servers"]["legacy_agent"] + ), + server_client=client, + ) + try: + apps = (probe.model_app(), probe.resources_app(), agent.setup_webserver(), environment.setup_webserver()) + for app, sock in zip(apps, sockets, strict=True): + server = _Server(uvicorn.Config(app, log_level="warning", timeout_graceful_shutdown=3)) + servers.append(server) + tasks.append(asyncio.create_task(server.serve(sockets=[sock]))) + async with asyncio.timeout(10): + while not all(server.started for server in servers): + for task in tasks: + if task.done(): + task.result() + raise RuntimeError("probe server exited during startup") + await asyncio.sleep(0.01) + inputs = directory / "requests.jsonl" + inputs.write_text(json.dumps(scenario.task()) + "\n") + async with asyncio.timeout(timeout + 10): + await RolloutCollectionHelper().run_from_config( + RolloutCollectionConfig( + agent_name="probe_agent", + input_jsonl_fpath=str(inputs), + output_jsonl_fpath=str(directory / "rollouts.jsonl"), + num_samples_in_parallel=1, + num_repeats=1, + resume_from_cache=False, + disable_aggregation=True, + disable_health_check=True, + route_failures_to_sidecar=True, + ) + ) + finally: + for server in servers: + server.should_exit = True + await asyncio.gather(*tasks, return_exceptions=True) + await get_global_aiohttp_client().close() + probe.save() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--harness", required=True, choices=HARNESSES) + parser.add_argument("--scenario", required=True, choices=[s.name for s in SCENARIOS]) + parser.add_argument("--directory", required=True, type=Path) + parser.add_argument("--timeout", required=True, type=float) + args = parser.parse_args() + asyncio.run( + run_episode( + harness=args.harness, scenario_name=args.scenario, directory=args.directory.resolve(), timeout=args.timeout + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/harness_conformance/provider.py b/scripts/harness_conformance/provider.py new file mode 100644 index 00000000000..3128ecafddd --- /dev/null +++ b/scripts/harness_conformance/provider.py @@ -0,0 +1,265 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Controlled Chat/Responses endpoints and private episode witnesses.""" + +import json +import shlex +from dataclasses import dataclass, field +from pathlib import Path +from uuid import uuid4 + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from nemo_gym.base_responses_api_model import ModelCallCaptureConfig, install_model_call_capture +from nemo_gym.chat_streaming import synthesize_chat_completion_sse +from nemo_gym.responses_streaming import ( + flatten_namespace_tools, + restore_namespace_tool_calls, + synthesize_responses_sse, +) + +from .scenarios import Scenario + + +@dataclass +class Probe: + scenario: Scenario + directory: Path + attempts: list[dict] = field(default_factory=list) + tool_calls: list[dict] = field(default_factory=list) + violations: list[str] = field(default_factory=list) + verifications: list[dict] = field(default_factory=list) + seeded: int = 0 + finished: bool = False + + def save(self) -> None: + """Persist independent observations even if collection fails.""" + witness = { + "attempts": self.attempts, + "tool_calls": self.tool_calls, + "violations": self.violations, + "verifications": self.verifications, + "seeded": self.seeded, + "finished": self.finished, + } + (self.directory / "witness.json").write_text(json.dumps(witness, indent=2) + "\n") + + def _tool(self, body: dict) -> tuple[str, dict, dict]: + tools, namespaces = flatten_namespace_tools(body.get("tools")) + for tool in tools: + function = tool.get("function", tool) + name = function.get("name", "") + bare = name.rsplit("__", 1)[-1] + properties = (function.get("parameters") or {}).get("properties", {}) + if bare in ("bash", "terminal", "shell", "shell_command", "exec_command"): + index = len(self.tool_calls) + token = f"probe_{uuid4().hex}" + marker = self.directory / f"tool-{index}.txt" + code = self.scenario.tool_exit_code if index == 0 else 0 + command = f"printf %s {shlex.quote(token)} > {shlex.quote(str(marker))}; printf %s {shlex.quote(token)}; exit {code}" + if "cmd" in properties: + args = {"cmd": command} + elif "command" in properties: + command_type = properties["command"].get("type") + args = {"command": ["sh", "-c", command] if command_type == "array" else command} + else: + continue + if "description" in (function.get("parameters") or {}).get("required", []): + args["description"] = "Run conformance shell check" + self.tool_calls.append( + { + "id": f"call_{uuid4().hex}", + "name": name, + "arguments": args, + "token": token, + "marker": str(marker), + "exit_code": code, + "result_seen": False, + } + ) + return name, args, namespaces + raise ValueError("harness did not advertise a supported shell tool") + + def _observe_tool_results(self, body: dict) -> None: + history = body.get("messages", body.get("input", [])) + for call in self.tool_calls: + results = ( + [ + item + for item in history + if isinstance(item, dict) + and ( + item.get("role") == "tool" + and item.get("tool_call_id") == call["id"] + or item.get("type") == "function_call_output" + and item.get("call_id") == call["id"] + ) + ] + if isinstance(history, list) + else [] + ) + call["result_seen"] = call["result_seen"] or any( + call["token"] in json.dumps(item.get("content", item.get("output"))) for item in results + ) + marker = Path(call["marker"]) + call["executed"] = marker.is_file() and marker.read_text() == call["token"] + + async def model(self, request: Request) -> JSONResponse | StreamingResponse: + body = await request.json() + self._observe_tool_results(body) + index = len(self.attempts) + status = 200 + if index < len(self.scenario.http_errors): + status = self.scenario.http_errors[index] + elif self.scenario.terminal_error: + status = self.scenario.http_errors[-1] + response_id = f"resp_{uuid4().hex}" + namespaces = {} + if status != 200: + payload = { + "error": {"message": "controlled model failure", "type": "conformance_error", "code": str(status)} + } + else: + try: + if self.finished: + raise ValueError("unexpected model request after the scripted final answer") + if any(not call["result_seen"] or not call.get("executed") for call in self.tool_calls): + raise ValueError("tool execution or model-visible result was not witnessed") + if len(self.tool_calls) < self.scenario.tool_steps: + name, arguments, namespaces = self._tool(body) + call = self.tool_calls[-1] + output = [ + { + "type": "function_call", + "id": call["id"], + "call_id": call["id"], + "name": name, + "arguments": json.dumps(arguments), + "status": "completed", + } + ] + else: + output = [ + { + "type": "message", + "id": f"msg_{uuid4().hex}", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "CONFORMANCE_DONE", "annotations": []}], + } + ] + self.finished = True + if request.url.path.endswith("/chat/completions"): + message = { + "role": "assistant", + "content": "CONFORMANCE_DONE" if self.finished else None, + "reasoning_content": "Perform the prescribed check.", + } + if not self.finished: + message["tool_calls"] = [ + { + "id": output[0]["call_id"], + "type": "function", + "function": {"name": name, "arguments": json.dumps(arguments)}, + } + ] + payload = { + "id": response_id, + "object": "chat.completion", + "created": 1, + "model": body["model"], + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "stop" if self.finished else "tool_calls", + } + ], + } + if self.scenario.usage: + payload["usage"] = { + "prompt_tokens": 20 + index, + "completion_tokens": 8, + "total_tokens": 28 + index, + "prompt_tokens_details": {"cached_tokens": 3}, + "completion_tokens_details": {"reasoning_tokens": 2}, + } + else: + payload = { + "id": response_id, + "object": "response", + "created_at": 1, + "status": "completed", + "error": None, + "incomplete_details": None, + "model": body["model"], + "output": restore_namespace_tool_calls(output, namespaces), + } + if self.scenario.usage: + payload["usage"] = { + "input_tokens": 20 + index, + "output_tokens": 8, + "total_tokens": 28 + index, + "input_tokens_details": {"cached_tokens": 3}, + "output_tokens_details": {"reasoning_tokens": 2}, + } + except ValueError as exc: + status = 409 + self.violations.append(str(exc)) + payload = {"error": {"message": str(exc), "type": "probe_protocol_error"}} + self.attempts.append( + {"attempt_id": f"attempt-{index}", "request": body, "response": payload, "status_code": status} + ) + self.save() + if status != 200 or not body.get("stream"): + return JSONResponse(payload, status_code=status, headers={"Retry-After": "0"}) + events = ( + synthesize_chat_completion_sse(payload, include_usage=True) + if request.url.path.endswith("/chat/completions") + else synthesize_responses_sse(payload) + ) + return StreamingResponse(events, media_type="text/event-stream") + + def model_app(self) -> FastAPI: + app = FastAPI() + install_model_call_capture( + app, + ModelCallCaptureConfig(observability_enabled=True, model_call_capture_dir=self.directory / "capture"), + model_server_name="policy_model", + ) + app.post("/v1/chat/completions", response_model=None)(self.model) + app.post("/v1/responses", response_model=None)(self.model) + return app + + def resources_app(self) -> FastAPI: + app = FastAPI() + + async def seed(request: Request) -> dict: + self.seeded += 1 + self.save() + return {} + + async def verify(request: Request) -> dict: + body = await request.json() + output = body.get("response", {}).get("output", []) + complete = any( + item.get("type") == "message" + and item.get("role") == "assistant" + and any( + part.get("text") == "CONFORMANCE_DONE" + for part in item.get("content", []) + if isinstance(part, dict) + ) + for item in output + ) + reward = float(complete and self.scenario.expected_reward == 1.0) + self.verifications.append({"reward": reward, "answer_seen": complete}) + self.save() + return {**body, "reward": reward} + + for prefix in ("", "/ng-rollout/{rollout_id}"): + app.post(prefix + "/seed_session")(seed) + app.post(prefix + "/verify")(verify) + return app diff --git a/scripts/harness_conformance/runner.py b/scripts/harness_conformance/runner.py new file mode 100644 index 00000000000..1805cdb5884 --- /dev/null +++ b/scripts/harness_conformance/runner.py @@ -0,0 +1,308 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Launch, witness, and inspect isolated episodes for the local P0 probe suite.""" + +import argparse +import hashlib +import json +import math +import os +import subprocess +import sys +import time +from collections import Counter +from pathlib import Path + +import psutil + +from nemo_gym.harness_capabilities.checker import NAMES +from nemo_gym.harness_capabilities.cli import inspect_bundle +from nemo_gym.harness_capabilities.reader import digest_file, hydrate_record, json_rows + +from .episode import HARNESSES +from .scenarios import SCENARIOS, SUITE, Scenario, suite_manifest + + +ROOT = Path(__file__).resolve().parents[2] + + +def _json(value: object) -> str: + return json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + "\n" + + +def _fingerprint(request: dict, status: int, response: dict) -> str: + # Gym's Chat SSE decoder omits the provider's optional creation clock. + response = dict(response or {}) + if response.get("object") == "chat.completion": + response.pop("created", None) + return hashlib.sha256(_json([request, status, response]).encode()).hexdigest() + + +def run_process(command: list[str], *, directory: Path, timeout: float) -> dict: + """Keep logs and reap the worker and its descendants, including new process groups.""" + with (directory / "episode.log").open("wb") as log: + proc = subprocess.Popen( + command, cwd=ROOT, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True + ) + tracked = {} + timed_out = False + try: + deadline = time.monotonic() + timeout + while proc.poll() is None: + # CLIs may start their own sessions, so killing only our process group + # would leave their children alive after a worker timeout. + try: + for child in psutil.Process(proc.pid).children(recursive=True): + tracked[(child.pid, child.create_time())] = child + except psutil.NoSuchProcess: + pass + remaining = deadline - time.monotonic() + if remaining <= 0: + timed_out = True + break + try: + proc.wait(timeout=min(0.1, remaining)) + except subprocess.TimeoutExpired: + pass + finally: + if proc.poll() is None: + proc.terminate() + for child in reversed(list(tracked.values())): + try: + child.terminate() + except psutil.NoSuchProcess: + pass + _, alive = psutil.wait_procs(list(tracked.values()), timeout=2) + for child in alive: + try: + child.kill() + except psutil.NoSuchProcess: + pass + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + return {"returncode": proc.returncode, "timed_out": timed_out} + + +def inspect_episode(scenario: Scenario, directory: Path, execution: dict) -> dict: + """Require witnessed exercise and a real rollout before counting a TE as passing.""" + issues = [] + if execution["timed_out"]: + issues.append("episode exceeded its timeout") + if execution["returncode"] != 0: + issues.append("episode process failed; see episode.log") + witness_path = directory / "witness.json" + witness = json.loads(witness_path.read_text()) if witness_path.exists() else {} + attempts = witness.get("attempts", []) + issues.extend(witness.get("violations", [])) + statuses = [attempt["status_code"] for attempt in attempts] + if witness.get("seeded") != 1: + issues.append("expected one fresh episode initialization") + if not attempts: + issues.append("the harness never reached the controlled model endpoint") + if statuses[: len(scenario.http_errors)] != list(scenario.http_errors): + issues.append("not all prescribed model failures were observed") + if len(scenario.http_errors) > 1 and len(attempts) >= len(scenario.http_errors): + if any(a["request"] != attempts[0]["request"] for a in attempts[1 : len(scenario.http_errors)]): + issues.append("retry scenario did not repeat the same request body") + if scenario.terminal_error: + if not statuses or any(status != scenario.http_errors[-1] for status in statuses): + issues.append("terminal model failure was not observed") + elif not witness.get("finished"): + issues.append("the harness did not finish the scripted model exchange") + tools = witness.get("tool_calls", []) + if len(tools) != scenario.tool_steps or any( + not tool.get("executed") or not tool.get("result_seen") for tool in tools + ): + issues.append("prescribed tool executions and their returned results were not all witnessed") + verifications = witness.get("verifications", []) + if len(verifications) != 1 or verifications[0]["reward"] != scenario.expected_reward: + issues.append("expected verifier outcome was not observed") + elif not scenario.terminal_error and not verifications[0]["answer_seen"]: + issues.append("the verifier did not receive the scripted final answer") + bundle = directory / "rollouts.jsonl" + records = list(json_rows(bundle)) if bundle.exists() else [] + summary = None + report = None + if len(records) != 1: + issues.append("expected exactly one collected rollout") + else: + record = hydrate_record(records[0][1]) + calls = record.get("ng_model_call_capture", {}).get("calls", []) + expected = Counter(_fingerprint(a["request"], a["status_code"], a["response"]) for a in attempts) + observed = Counter(_fingerprint(c.get("request"), c.get("status_code"), c.get("response")) for c in calls) + if expected != observed: + issues.append("retained model attempts differ from the independent endpoint witness") + if record.get("reward") != scenario.expected_reward: + issues.append("rollout reward differs from the verifier witness") + destination, summary = inspect_bundle(bundle, output=directory / "evidence", capture_dir=directory / "capture") + report = str(destination.relative_to(directory) / "evidence_summary.json") + exercised = not issues + evidence = {} + for key in scenario.evidence: + verdict = summary["evidence"][key]["verdict"] if summary else "not_fulfilled" + evidence[key] = { + "verdict": "fulfilled" if exercised and verdict == "fulfilled" else "not_fulfilled", + "artifact_verdict": verdict, + } + # The two join contracts remain alternatives, just as in the P0 inspector. + mandatory = [key for key in scenario.evidence if key not in ("TE-8", "TE-9")] + passed = exercised and all(evidence[key]["verdict"] == "fulfilled" for key in mandatory) + passed = passed and any(evidence[key]["verdict"] == "fulfilled" for key in ("TE-8", "TE-9")) + return { + "scenario": scenario.name, + "exercised": exercised, + "verdict": "fulfilled" if passed else "not_fulfilled", + "issues": issues, + "execution": execution, + "model_attempts": len(attempts), + "evidence": evidence, + "artifact_report": report, + "hashes": { + path.name: digest_file(path) + for path in (bundle, witness_path, directory / "runtime.json", directory / "launch.json") + if path.exists() + }, + } + + +def run_suite( + *, harnesses: list[str], scenarios: tuple[Scenario, ...], output: Path, timeout: float +) -> tuple[dict, int]: + """Run each requested harness/scenario in a fresh process and publish the completed report.""" + if not harnesses or not scenarios: + raise ValueError("at least one harness and one scenario are required") + output = output.resolve() + output.mkdir(parents=True, exist_ok=False) + manifest = suite_manifest(scenarios) + manifest.update( + harnesses=harnesses, + timeout=timeout, + python=sys.version, + runner_sources={p.name: digest_file(p) for p in sorted(Path(__file__).parent.glob("*.py"))}, + ) + (output / "suite.json").write_text(_json(manifest)) + rows = {} + execution_error = False + for harness in harnesses: + results = [] + for scenario in scenarios: + directory = output / harness / scenario.name + directory.mkdir(parents=True) + print(f"{harness}: {scenario.name}", flush=True) + command = [ + sys.executable, + "-m", + "scripts.harness_conformance.episode", + "--harness", + harness, + "--scenario", + scenario.name, + "--directory", + str(directory), + "--timeout", + str(timeout), + ] + execution = run_process(command, directory=directory, timeout=timeout + 30) + execution_error |= execution["returncode"] != 0 or execution["timed_out"] + try: + result = inspect_episode(scenario, directory, execution) + except (OSError, ValueError, TypeError, KeyError, AttributeError, RecursionError) as exc: + execution_error = True + result = { + "scenario": scenario.name, + "exercised": False, + "verdict": "not_fulfilled", + "execution": execution, + "checker_error": type(exc).__name__, + "issues": ["could not inspect this episode's artifacts"], + "artifact_report": None, + "evidence": { + key: {"verdict": "not_fulfilled", "artifact_verdict": "not_fulfilled"} + for key in scenario.evidence + }, + } + (directory / "scenario_result.json").write_text(_json(result)) + results.append(result) + counts = {} + for key in NAMES: + required = [r for r in results if key in r["evidence"]] + counts[key] = { + "required": len(required), + "observed": sum(r["exercised"] for r in required), + "passed": sum(r["evidence"][key]["verdict"] == "fulfilled" for r in required), + } + rows[harness] = { + "scenarios": results, + "evidence": counts, + "verdict": "fulfilled" if all(r["verdict"] == "fulfilled" for r in results) else "not_fulfilled", + } + passed = all(row["verdict"] == "fulfilled" for row in rows.values()) + summary = { + "schema_version": "harness-probe-report/v1", + "suite": SUITE, + "runner_status": "completed", + "full_suite": scenarios == SCENARIOS, + "verdict": "fulfilled" if passed else "not_fulfilled", + "suite_sha256": digest_file(output / "suite.json"), + "harnesses": rows, + "limits": [ + "local harness runtime and controlled Chat/Responses model only; no remote sandbox qualification", + "TE-10, P1, multimodal, compaction, parallelism and deployment health are outside this suite", + ], + } + table = [ + "# Live harness P0 probes", + "", + "Each cell is passing / exercised / required scenarios.", + "", + "| Harness | " + " | ".join(NAMES) + " | Gate |", + "|---|" + "---|" * (len(NAMES) + 1), + ] + for harness, row in rows.items(): + cells = [f"{c['passed']}/{c['observed']}/{c['required']}" for c in row["evidence"].values()] + table.append("| " + " | ".join([harness, *cells, row["verdict"]]) + " |") + table += [ + "", + "TE-8 or TE-9 is sufficient per scenario. A selected subset does not qualify the full suite.", + "See each scenario_result.json for execution gaps and the unmodified artifact-checker report.", + "", + ] + (output / "conformance_report.md").write_text("\n".join(table)) + temporary = output / ".conformance_summary.json.tmp" + temporary.write_text(_json(summary)) + os.replace(temporary, output / "conformance_summary.json") + return summary, 2 if execution_error else 0 if passed else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--harness", action="append", choices=HARNESSES, help="repeat for a matrix; defaults to all four" + ) + parser.add_argument( + "--scenario", action="append", choices=[s.name for s in SCENARIOS], help="run a diagnostic subset" + ) + parser.add_argument("--output", type=Path, help="new directory for rollouts, captures, witnesses and reports") + parser.add_argument("--timeout", type=float, default=90, help="seconds per harness episode (default: 90)") + parser.add_argument("--list-scenarios", action="store_true") + args = parser.parse_args(argv) + if args.list_scenarios: + print(_json(suite_manifest(SCENARIOS)), end="") + return 0 + if args.output is None: + parser.error("--output is required") + if not math.isfinite(args.timeout) or args.timeout <= 0: + parser.error("--timeout must be a finite positive number") + harnesses = list(dict.fromkeys(args.harness or HARNESSES)) + scenarios = tuple(s for s in SCENARIOS if not args.scenario or s.name in args.scenario) + try: + _, code = run_suite(harnesses=harnesses, scenarios=scenarios, output=args.output, timeout=args.timeout) + except (OSError, ValueError, TypeError, KeyError) as exc: + print(f"runner_error: {exc}", file=sys.stderr) + return 2 + print(args.output.resolve() / "conformance_report.md") + return code diff --git a/scripts/harness_conformance/scenarios.py b/scripts/harness_conformance/scenarios.py new file mode 100644 index 00000000000..44f069ea0f9 --- /dev/null +++ b/scripts/harness_conformance/scenarios.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Versioned, model-independent requests used by every harness preset.""" + +from dataclasses import asdict, dataclass + + +SUITE = "harness-p0-probes/v1" +ALL_EVIDENCE = tuple(f"TE-{i}" for i in range(1, 10)) + + +@dataclass(frozen=True) +class Scenario: + name: str + description: str + evidence: tuple[str, ...] = ALL_EVIDENCE + tool_steps: int = 2 + tool_exit_code: int = 0 + usage: bool = True + http_errors: tuple[int, ...] = () + terminal_error: bool = False + expected_reward: float = 1.0 + + def task(self) -> dict: + """Only the task reaches the harness; expectations stay in the runner.""" + return { + "task_id": self.name, + "responses_create_params": { + "input": [ + {"role": "user", "content": "Run the requested shell checks, then report CONFORMANCE_DONE."} + ], + }, + } + + +SCENARIOS = ( + Scenario("tool_success", "Two successful tool executions, changed history, reasoning and a final answer."), + Scenario("tool_failure", "A shell command exits 7, followed by recovery and a final answer.", tool_exit_code=7), + Scenario("usage_omitted", "The provider omits usage; counts must remain unknown.", usage=False), + Scenario("retry_429", "Two identical requests receive 429 before recovery.", http_errors=(429, 429)), + Scenario("retry_500", "A model HTTP 500 is followed by recovery.", http_errors=(500,)), + Scenario( + "model_error", + "A terminal model HTTP 400 with an error body and no response ID.", + evidence=("TE-1", "TE-2", "TE-4", "TE-7", "TE-8", "TE-9"), + tool_steps=0, + http_errors=(400,), + terminal_error=True, + expected_reward=0.0, + ), + Scenario("verifier_failure", "A completed trajectory receives a known zero reward.", expected_reward=0.0), +) + + +def suite_manifest(scenarios: tuple[Scenario, ...]) -> dict: + return {"version": SUITE, "scenarios": [asdict(scenario) for scenario in scenarios]} diff --git a/scripts/harness_conformance/tests/test_runner.py b/scripts/harness_conformance/tests/test_runner.py new file mode 100644 index 00000000000..af6d8454702 --- /dev/null +++ b/scripts/harness_conformance/tests/test_runner.py @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise wire protocols, independent witnesses, and fail-closed reporting.""" + +import copy +import json +import subprocess +import sys +from pathlib import Path + +import psutil +import pytest +from fastapi.testclient import TestClient +from scripts.harness_conformance.episode import HARNESSES, _config +from scripts.harness_conformance.provider import Probe +from scripts.harness_conformance.runner import inspect_episode, main, run_process, run_suite +from scripts.harness_conformance.scenarios import SCENARIOS + +from nemo_gym.base_responses_api_model import build_model_call_record +from nemo_gym.harness_capabilities.reader import hydrate_record, json_rows + + +SCENARIO = {s.name: s for s in SCENARIOS} +TOOL = { + "type": "function", + "function": { + "name": "bash", + "parameters": {"type": "object", "properties": {"command": {"type": "string"}}, "required": ["command"]}, + }, +} + + +def test_catalog_covers_p0_without_exposing_expectations(): + assert set().union(*(s.evidence for s in SCENARIOS)) == {f"TE-{i}" for i in range(1, 10)} + for scenario in SCENARIOS: + assert set(scenario.task()) == {"task_id", "responses_create_params"} + assert main(["--list-scenarios"]) == 0 + + +@pytest.mark.parametrize("harness", HARNESSES) +def test_launch_uses_local_endpoints_and_isolated_workspaces(harness, tmp_path): + config = _config(harness, tmp_path, [10001, 10002, 10003, 10004], 12) + from omegaconf import OmegaConf + + from nemo_gym.rollout_collection import _environment_server_for_agent, _environment_servers_by_agent + + assert ( + _environment_server_for_agent("probe_agent", _environment_servers_by_agent(OmegaConf.create(config))) + == "probe_environment" + ) + agent = config["probe_agent"]["responses_api_agents"][f"{harness}_agent"] + assert agent["model_server"] == {"type": "responses_api_models", "name": "policy_model"} + assert config["model_call_capture_dir"] == str(tmp_path / "capture") + # Validate available adapters without invoking their auto-install hooks. + if harness != "hermes": + import importlib + + module = importlib.import_module(f"responses_api_agents.{harness}_agent.app") + getattr(module, HARNESSES[harness] + "Config").model_validate(agent) + + +@pytest.mark.parametrize("scenario_name", ["tool_success", "tool_failure", "usage_omitted", "retry_429", "retry_500"]) +def test_live_chat_endpoint_executes_script_and_captures_every_attempt(tmp_path, scenario_name): + scenario = SCENARIO[scenario_name] + probe = Probe(scenario, tmp_path) + messages = [{"role": "user", "content": "Run checks"}] + body = {"model": "conformance-model", "messages": messages, "tools": [TOOL]} + with TestClient(probe.model_app()) as client: + for status in scenario.http_errors: + response = client.post("/ng-rollout/0-0/v1/chat/completions", json=body) + assert response.status_code == status + for index in range(2): + response = client.post("/ng-rollout/0-0/v1/chat/completions", json=body) + assert response.status_code == 200 + message = response.json()["choices"][0]["message"] + call = message["tool_calls"][0] + execution = subprocess.run( + json.loads(call["function"]["arguments"])["command"], + shell=True, + capture_output=True, + text=True, + check=False, + ) + assert execution.returncode == (scenario.tool_exit_code if index == 0 else 0) + messages.extend([message, {"role": "tool", "tool_call_id": call["id"], "content": execution.stdout}]) + response = client.post("/ng-rollout/0-0/v1/chat/completions", json=body) + assert response.json()["choices"][0]["message"]["content"] == "CONFORMANCE_DONE" + assert probe.finished and not probe.violations + assert all(t["executed"] and t["result_seen"] for t in probe.tool_calls) + captures = [ + build_model_call_record(row, call_index=i).model_dump() + for i, row in json_rows(tmp_path / "capture/0-0.capture.jsonl") + ] + assert len(captures) == len(scenario.http_errors) + 3 + assert len({c["model_call_id"] for c in captures}) == len(captures) + assert [c["status_code"] for c in captures] == [*scenario.http_errors, 200, 200, 200] + if not scenario.usage: + assert all(c.get("tokens_in") is None and c.get("tokens_out") is None for c in captures) + else: + assert captures[-1]["tokens_in"] == 20 + len(scenario.http_errors) + 2 + assert captures[-1]["tokens_reasoning"] == 2 + assert captures[-1]["cached_tokens"] == 3 + + +@pytest.mark.parametrize("dialect", ["chat/completions", "responses"]) +def test_streaming_tool_roundtrip_and_unknown_usage(tmp_path, dialect): + probe = Probe(SCENARIO["usage_omitted"], tmp_path) + body = {"model": "conformance-model", "stream": True} + if dialect == "responses": + body.update( + input=[{"role": "user", "content": "Run checks"}], + tools=[{"type": "namespace", "name": "functions", "tools": [{"type": "function", **TOOL["function"]}]}], + ) + else: + body.update(messages=[{"role": "user", "content": "Run checks"}], tools=[TOOL]) + with TestClient(probe.model_app()) as client: + response = client.post(f"/ng-rollout/0-0/v1/{dialect}", json=body) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + events = [json.loads(line[6:]) for line in response.text.splitlines() if line.startswith("data: {")] + if dialect == "responses": + assert events[-1]["type"] == "response.completed" + call = events[-1]["response"]["output"][0] + assert call["namespace"] == "functions" and call["name"] == "bash" + else: + assert response.text.endswith("data: [DONE]\n\n") + assert any(e["choices"] and e["choices"][0]["delta"].get("tool_calls") for e in events) + (capture,) = [ + build_model_call_record(row, call_index=i).model_dump() + for i, row in json_rows(tmp_path / "capture/0-0.capture.jsonl") + ] + assert capture["tokens_in"] is None + assert capture["tokens_out"] is None + + +def test_tool_request_without_execution_is_not_exercised(tmp_path): + probe = Probe(SCENARIO["tool_success"], tmp_path) + body = {"model": "probe", "messages": [], "tools": [TOOL]} + with TestClient(probe.model_app()) as client: + assert client.post("/v1/chat/completions", json=body).status_code == 200 + assert client.post("/v1/chat/completions", json=body).status_code == 409 + assert probe.violations and not probe.finished + + +def test_terminal_failure_does_not_turn_into_a_success_on_retry(tmp_path): + probe = Probe(SCENARIO["model_error"], tmp_path) + with TestClient(probe.model_app()) as client: + for _ in range(3): + response = client.post("/v1/responses", json={"model": "probe", "input": []}) + assert response.status_code == 400 + assert "id" not in response.json() + assert len(probe.attempts) == 3 and not probe.finished + + +@pytest.mark.parametrize("name,reward", [("tool_success", 1.0), ("verifier_failure", 0.0)]) +def test_verifier_records_actual_final_answer(tmp_path, name, reward): + probe = Probe(SCENARIO[name], tmp_path) + body = { + "response": { + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "CONFORMANCE_DONE"}], + } + ] + } + } + with TestClient(probe.resources_app()) as client: + client.post("/seed_session", json={}) + response = client.post("/ng-rollout/0-0/verify", json=body) + assert response.json()["reward"] == reward + assert probe.verifications == [{"reward": reward, "answer_seen": True}] + assert probe.seeded == 1 + + +@pytest.fixture +def retained_episode(tmp_path): + # Contract fixture only: used to prove a green artifact cannot hide a missing live probe. + fixture = Path(__file__).resolve().parents[3] / "tests/unit_tests/harness_capabilities/fixtures/miniswe.json" + raw = json.loads(fixture.read_text()) + record = hydrate_record(raw) + calls = record["ng_model_call_capture"]["calls"] + witness = { + "seeded": 1, + "finished": True, + "violations": [], + "verifications": [{"reward": 0.0, "answer_seen": True}], + "tool_calls": [{"executed": True, "result_seen": True}] * 2, + "attempts": [ + {"request": c["request"], "response": c["response"], "status_code": c["status_code"]} for c in calls + ], + } + (tmp_path / "rollouts.jsonl").write_text(json.dumps(raw) + "\n") + (tmp_path / "witness.json").write_text(json.dumps(witness)) + (tmp_path / "capture").mkdir() + (tmp_path / "capture/0-0.capture.jsonl").write_text("".join(json.dumps(c) + "\n" for c in calls)) + return tmp_path, witness + + +def test_artifact_success_requires_independent_exercise(retained_episode): + directory, witness = retained_episode + result = inspect_episode(SCENARIO["verifier_failure"], directory, {"returncode": 0, "timed_out": False}) + assert result["verdict"] == "fulfilled", result["issues"] + witness["attempts"].append(copy.deepcopy(witness["attempts"][0])) + (directory / "witness.json").write_text(json.dumps(witness)) + result = inspect_episode(SCENARIO["verifier_failure"], directory, {"returncode": 0, "timed_out": False}) + assert result["verdict"] == "not_fulfilled" + assert not result["exercised"] + assert all(v["verdict"] == "not_fulfilled" for v in result["evidence"].values()) + assert "retained model attempts differ" in " ".join(result["issues"]) + + +def test_execution_failure_cannot_pass_even_with_artifacts(retained_episode): + directory, _ = retained_episode + result = inspect_episode(SCENARIO["verifier_failure"], directory, {"returncode": 1, "timed_out": True}) + assert result["verdict"] == "not_fulfilled" and not result["exercised"] + assert "episode exceeded its timeout" in result["issues"] + + +def test_missing_runtime_is_recorded_and_no_stale_output_reused(tmp_path, monkeypatch): + monkeypatch.setenv("PATH", "") + output = tmp_path / "run" + summary, code = run_suite(harnesses=["pi", "opencode"], scenarios=(SCENARIOS[0],), output=output, timeout=1) + assert code == 2 and not summary["full_suite"] + for row in summary["harnesses"].values(): + assert row["evidence"]["TE-1"] == {"required": 1, "observed": 0, "passed": 0} + assert (output / "conformance_summary.json").is_file() + before = (output / "suite.json").read_bytes() + assert main(["--harness", "pi", "--output", str(output)]) == 2 + assert (output / "suite.json").read_bytes() == before + + +def test_timeout_reaps_descendant_in_another_process_group(tmp_path): + child_code = "import time; time.sleep(300)" + worker = ( + "import subprocess,sys,time; from pathlib import Path; " + f"p=subprocess.Popen([sys.executable,'-c',{child_code!r}],start_new_session=True); " + f"Path({str(tmp_path / 'pid')!r}).write_text(str(p.pid)); time.sleep(300)" + ) + result = run_process([sys.executable, "-c", worker], directory=tmp_path, timeout=1) + assert result["timed_out"] + pid = int((tmp_path / "pid").read_text()) + assert not psutil.pid_exists(pid) or psutil.Process(pid).status() == psutil.STATUS_ZOMBIE + + +@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf"]) +def test_invalid_timeout_rejected_before_launch(tmp_path, timeout): + with pytest.raises(SystemExit) as exc: + main(["--output", str(tmp_path / "absent"), "--timeout", timeout]) + assert exc.value.code == 2 + assert not (tmp_path / "absent").exists() + + +def test_witness_detects_changed_payload_even_when_attempt_identity_matches(retained_episode): + directory, witness = retained_episode + witness["attempts"][0]["response"]["usage"]["total_tokens"] += 1 + (directory / "witness.json").write_text(json.dumps(witness)) + result = inspect_episode(SCENARIO["verifier_failure"], directory, {"returncode": 0, "timed_out": False}) + assert result["verdict"] == "not_fulfilled" + assert "retained model attempts differ" in " ".join(result["issues"]) + + +def test_checker_error_does_not_prevent_remaining_episodes(tmp_path, monkeypatch): + from scripts.harness_conformance import runner + + inspected = [] + monkeypatch.setattr(runner, "run_process", lambda *args, **kwargs: {"returncode": 0, "timed_out": False}) + + def fail_inspection(scenario, directory, execution): + inspected.append(scenario.name) + raise ValueError("truncated rollout") + + monkeypatch.setattr(runner, "inspect_episode", fail_inspection) + summary, code = run_suite(harnesses=["pi"], scenarios=SCENARIOS[:2], output=tmp_path / "run", timeout=1) + assert code == 2 + assert inspected == ["tool_success", "tool_failure"] + assert summary["harnesses"]["pi"]["evidence"]["TE-1"] == {"required": 2, "observed": 0, "passed": 0} + + +def test_interrupted_run_does_not_publish_completed_summary(tmp_path, monkeypatch): + from scripts.harness_conformance import runner + + def interrupt(*args, **kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(runner, "run_process", interrupt) + output = tmp_path / "run" + with pytest.raises(KeyboardInterrupt): + run_suite(harnesses=["pi"], scenarios=SCENARIOS[:1], output=output, timeout=1) + assert not (output / "conformance_summary.json").exists() diff --git a/scripts/run_harness_conformance.py b/scripts/run_harness_conformance.py new file mode 100644 index 00000000000..cc3a677a8b2 --- /dev/null +++ b/scripts/run_harness_conformance.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run scripted requests through real Gym harnesses and inspect the resulting rollouts.""" + +from harness_conformance.runner import main + + +if __name__ == "__main__": + raise SystemExit(main()) From f28cb2a883d0783222b68adb3d0c94377bea934d Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Thu, 1 Oct 2026 15:07:09 +0200 Subject: [PATCH 05/11] Separate checker fixtures and gate generated trajectory evidence tables Signed-off-by: Frederic Lafrance --- .../observability/harness-conformance.mdx | 15 +- .../reference/trajectory-capabilities.mdx | 65 +- nemo_gym/harness_capabilities/README.md | 6 + scripts/harness_conformance/README.md | 31 + scripts/harness_conformance/runner.py | 11 +- scripts/harness_conformance/table.py | 235 ++++++ .../harness_conformance/tests/test_runner.py | 5 +- .../harness_conformance/tests/test_table.py | 203 ++++++ .../harness_capabilities/fixtures/codex.jsonl | 1 - .../fixtures/hermes.jsonl | 1 - .../fixtures/miniswe.json | 675 ------------------ .../fixtures/opencode.json | 241 ------- .../fixtures/opencode.jsonl | 1 - .../harness_capabilities/fixtures/pi.jsonl | 1 - .../harness_capabilities/producer_fixtures.py | 280 -------- .../harness_capabilities/synthetic.py | 125 ++++ .../harness_capabilities/test_checker.py | 28 +- .../harness_capabilities/test_producers.py | 39 - .../harness_capabilities/test_script.py | 47 +- 19 files changed, 729 insertions(+), 1281 deletions(-) create mode 100644 scripts/harness_conformance/table.py create mode 100644 scripts/harness_conformance/tests/test_table.py delete mode 100644 tests/unit_tests/harness_capabilities/fixtures/codex.jsonl delete mode 100644 tests/unit_tests/harness_capabilities/fixtures/hermes.jsonl delete mode 100644 tests/unit_tests/harness_capabilities/fixtures/miniswe.json delete mode 100644 tests/unit_tests/harness_capabilities/fixtures/opencode.json delete mode 100644 tests/unit_tests/harness_capabilities/fixtures/opencode.jsonl delete mode 100644 tests/unit_tests/harness_capabilities/fixtures/pi.jsonl delete mode 100644 tests/unit_tests/harness_capabilities/producer_fixtures.py create mode 100644 tests/unit_tests/harness_capabilities/synthetic.py delete mode 100644 tests/unit_tests/harness_capabilities/test_producers.py diff --git a/fern/versions/latest/pages/observability/harness-conformance.mdx b/fern/versions/latest/pages/observability/harness-conformance.mdx index 01280c4f30f..7b9ffc38177 100644 --- a/fern/versions/latest/pages/observability/harness-conformance.mdx +++ b/fern/versions/latest/pages/observability/harness-conformance.mdx @@ -85,13 +85,14 @@ The command writes per-harness reports plus `harness_evidence.md` and comparison. A matrix FAIL identifies missing or inconsistent evidence; it is not a benchmark accuracy result. -Regression coverage exercises the current OpenCode SQLite parser, Pi event -parser, Codex exec parser, and Hermes observation hooks through Gym collection. -Controlled protocol/capture inputs and a controlled verifier reward make these -reproducible producer tests, not live harness qualifications. The captured model -columns depend on the deployed harness actually routing calls through capture. -The emitted fixtures are under `tests/unit_tests/harness_capabilities/fixtures/`; -they can be used with the same matrix command. +Checker unit tests use hand-authored synthetic evidence contracts, independent of +harness implementations. Mutations verify missing, conflicting, and malformed +evidence without prescribing which checks a real harness should pass. + +The [Trajectory Capability Matrix](/main/reference/trajectory-capabilities) is +generated separately from fresh scripted harness runs. Its regeneration command +requires passing runner, harness adapter, and checker unit tests at a specific +Gym commit before replacing the table. ## Onboard a producer diff --git a/fern/versions/latest/pages/reference/trajectory-capabilities.mdx b/fern/versions/latest/pages/reference/trajectory-capabilities.mdx index 5cc49acbf90..e39ca783d09 100644 --- a/fern/versions/latest/pages/reference/trajectory-capabilities.mdx +++ b/fern/versions/latest/pages/reference/trajectory-capabilities.mdx @@ -4,9 +4,68 @@ description: "Rollout-evidence coverage by agent and execution path." position: 5 --- +# Trajectory Capability Matrix + Use this reference to check which evidence an agent path provides. Start with [Rollout Evidence](/main/observability/rollout-evidence) for collection and artifact relationships. +## Generated trajectory evidence (TE) + +This table runs the local harnesses against scripted model replies, tool executions, +retries, and failures, then checks the collected rollouts. The +[TE-1–TE-9 contracts](/main/observability/harness-conformance) cover model outcomes, +token preservation, steps, history, tools, verification, payloads, and call joins. +TE-2 checks that supplied usage is preserved; it does not require every provider +metric to be available. Remote sandboxes, multimodal inputs, compaction, parallel +execution, TE-10, and P1 are outside this suite. + +{/* BEGIN GENERATED TE TABLE */} + +No gated generation has been published yet. Run the command below to populate the table. + +{/* END GENERATED TE TABLE */} + +### Regenerate the TE table + +From a Gym checkout matching the full source commit SHA, install Gym's development +dependencies and the selected agents' Python dependencies and runtimes. Run: + +```bash +python -m scripts.harness_conformance.table \ + --commit <40-character-gym-commit-sha> \ + --output /absolute/path/to/new-conformance-results +``` + +The default selects OpenCode, Pi, Codex, and Hermes. Repeat `--harness NAME` to +select fewer; all others are explicitly marked **Not run**. Every selected harness +runs the full scenario suite. The probes execute local shell tools in fresh working +directories and require no model service or model credentials. See +[`scripts/harness_conformance/README.md`](https://github.com/NVIDIA-NeMo/Gym/blob/main/scripts/harness_conformance/README.md) +for runtime setup, scenarios, and the execution scope. + +Regeneration requires passing runner unit tests, each selected harness's adapter +unit tests, and checker unit tests at that commit. Failing, empty, or entirely +skipped suites stop regeneration. Skipped tests in otherwise passing suites are +reported. Source changes during testing or probing, missing runtimes, and +execution/checker errors also stop publication, preserving the existing table. +An unmet evidence requirement produces a FAIL cell and can be published. + +The command replaces only the generated block above and writes a content-addressed +JSON generation report under `fern/assets/trajectory-capabilities/`. The report +records the Gym commit, runtime versions/source hashes, test counts and log hashes, +scenario results, and artifact hashes. Full logs, captures, and rollouts remain in +`--output`; keep them when reviewing a generation. Save source changes first, then +commit the generated page and report in a subsequent commit so their source pin +remains stable. Git and jj checkouts are supported. + +## Deprecated coverage table (C-table) + + +The manually maintained C-table below is deprecated. Use the generated TE table +above for tested evidence coverage at a specific commit. The historical claims +remain here temporarily for comparison. + + ✅ **Full** means complete for the documented path; 🟡 **Partial** means some fields or paths are missing; ❌ **Unavailable** means the path does not supply that evidence. These are manually maintained coverage claims, not guarantees for every configuration. @@ -18,7 +77,7 @@ Contributors changing an agent should update its row with the config, execution evidence in the same pull request. -## Model Calls and History +### Model Calls and History Capture requires `observability_enabled`, a shared `model_call_capture_dir`, and a rollout-prefixed Gym Model Server URL. Direct-provider routes bypass capture. Token counts depend on provider usage @@ -63,7 +122,7 @@ fields. History may come from captured payloads or harness conversations. | `toolsandbox_agent` | ✅ | ✅ | ✅ | | `verifiers_agent` | ✅ | ✅ | ✅ | -## Agent Activity +### Agent Activity Turns require explicit semantic-turn records. Tool execution covers output, status, and timestamps; parallel timing requires independent tool intervals. Resource/sandbox coverage means that the @@ -85,7 +144,7 @@ benchmark path emits standardized agent evidence, not that every resource metric The other agents listed above report these four dimensions as unavailable. Model HTTP capture alone does not establish tool execution or sandbox activity. -## Path Qualifications +### Path Qualifications | Path | Qualification | | --- | --- | diff --git a/nemo_gym/harness_capabilities/README.md b/nemo_gym/harness_capabilities/README.md index 94c895cf55d..dd3e4bf9c13 100644 --- a/nemo_gym/harness_capabilities/README.md +++ b/nemo_gym/harness_capabilities/README.md @@ -12,5 +12,11 @@ Use `matrix --harness NAME=PATH` to compare multiple harnesses. The checker uses TE-1–TE-9 and the `gym-p0/v1` profile. Reports are `evidence_summary.json`, `evidence_results.jsonl`, and `evidence_report.md`. +Unit tests use hand-authored contracts in `tests/unit_tests/harness_capabilities/synthetic.py`; +they validate the checker independently of harness implementations. To measure +actual harnesses and regenerate the commit-pinned documentation table, use +`python -m scripts.harness_conformance.table --commit --output `. +See the [runner guide](../../scripts/harness_conformance/README.md) for prerequisites and test gates. + See [Harness Conformance](../../fern/versions/latest/pages/observability/harness-conformance.mdx) for contracts, applicability, matrix usage, producer onboarding, and limitations. diff --git a/scripts/harness_conformance/README.md b/scripts/harness_conformance/README.md index e700a559741..f19b6b3c73b 100644 --- a/scripts/harness_conformance/README.md +++ b/scripts/harness_conformance/README.md @@ -1,5 +1,36 @@ # Live harness conformance probes +## Regenerate the documentation table + +From a checkout matching a full Gym commit SHA, rebuild the TE table in +[`reference/trajectory-capabilities`](../../fern/versions/latest/pages/reference/trajectory-capabilities.mdx): + +```bash +python -m scripts.harness_conformance.table \ + --commit <40-character-gym-commit-sha> \ + --output /absolute/path/to/new-conformance-results +``` + +The command runs runner unit tests, each selected harness's adapter unit tests, +checker unit tests, and then the full live scenario suite. The checkout must match +the source commit before and after validation (Git and jj are supported). There is +no test-skip switch. Failed, empty, or entirely skipped test suites, dependency or +execution errors, and checker errors leave the previous table unchanged. Evidence +FAILs are valid results and appear in the generated table. + +All four harnesses are selected by default. Repeat `--harness NAME` to select a +subset; omitted harnesses show **Not run**, and no previous row is reused. Runtime +versions/source hashes, test results, scenario results, and artifact hashes are +saved in a content-addressed JSON report under `fern/assets/trajectory-capabilities/`. +Test logs and full probe artifacts are retained in the new `--output` directory. +Commit the generated page and report after the source commit to keep its pin stable. + +Checker and runner unit tests use the synthetic contracts in +`tests/unit_tests/harness_capabilities/synthetic.py`. Actual harness capabilities +are measured by probes; expected harness verdicts do not belong in library tests. + +## Run diagnostic probes + Run predetermined model replies and failures through real Gym harnesses, collect their rollouts, and apply the existing P0 evidence checks: diff --git a/scripts/harness_conformance/runner.py b/scripts/harness_conformance/runner.py index 1805cdb5884..2b3792d7119 100644 --- a/scripts/harness_conformance/runner.py +++ b/scripts/harness_conformance/runner.py @@ -42,8 +42,17 @@ def _fingerprint(request: dict, status: int, response: dict) -> str: def run_process(command: list[str], *, directory: Path, timeout: float) -> dict: """Keep logs and reap the worker and its descendants, including new process groups.""" with (directory / "episode.log").open("wb") as log: + # Probe the checkout being reported, even when another editable Gym or + # extra component root is present in the caller's environment. + env = {**os.environ, "PYTHONPATH": str(ROOT), "NEMO_GYM_EXTRA_ROOTS": str(ROOT)} proc = subprocess.Popen( - command, cwd=ROOT, stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True + command, + cwd=ROOT, + env=env, + stdin=subprocess.DEVNULL, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, ) tracked = {} timed_out = False diff --git a/scripts/harness_conformance/table.py b/scripts/harness_conformance/table.py new file mode 100644 index 00000000000..382fc4dbe8f --- /dev/null +++ b/scripts/harness_conformance/table.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regenerate the reference TE table from tested source and fresh full-suite probes. + +From a checkout matching a full commit SHA, with Gym/dev and harness dependencies: + + python -m scripts.harness_conformance.table --commit <40-character-sha> \ + --output /absolute/path/to/new-evidence-directory + +Runner, selected adapter, and checker tests must pass before any probes run. +The source tree must still match the commit after testing and probing. Evidence +failures are reportable; execution/checker errors prevent publication. Commit the +generated page and JSON separately from the source revision they describe. +""" + +import argparse +import hashlib +import json +import math +import os +import re +import subprocess +import sys +import tempfile +import xml.etree.ElementTree as ET +from pathlib import Path + +import nemo_gym +from nemo_gym.harness_capabilities import checker +from nemo_gym.harness_capabilities.checker import NAMES +from nemo_gym.harness_capabilities.reader import digest_file + +from . import episode, runner +from .episode import HARNESSES +from .runner import run_suite +from .scenarios import SCENARIOS, SUITE + + +ROOT = Path(__file__).resolve().parents[2] +PAGE = Path("fern/versions/latest/pages/reference/trajectory-capabilities.mdx") +ASSETS = Path("fern/assets/trajectory-capabilities") +BEGIN = "{/* BEGIN GENERATED TE TABLE */}" +END = "{/* END GENERATED TE TABLE */}" + + +def verify_revision(root: Path, commit: str) -> None: + """Require an existing full SHA and matching source, for Git or jj checkouts.""" + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError("--commit must be a full lowercase 40-character commit SHA") + + def read(*command: str) -> str: + return subprocess.check_output(command, cwd=root, text=True, stderr=subprocess.PIPE).strip() + + if (root / ".jj").exists(): + resolved = read("jj", "--no-pager", "log", "--no-graph", "-r", commit, "-T", "commit_id") + changes = read("jj", "--no-pager", "diff", "--from", commit, "--to", "@", "--summary") + else: + resolved = read("git", "rev-parse", "--verify", f"{commit}^{{commit}}") + changes = read("git", "diff", commit, "--name-only") + changes += read("git", "ls-files", "--others", "--exclude-standard") + if resolved != commit or changes: + raise ValueError("source tree differs from --commit; save all source changes before regenerating") + + +def run_tests(root: Path, output: Path, *, name: str, path: str) -> dict: + """Run a complete test directory; errors and empty/all-skipped suites fail closed.""" + report = output / f"{name}.xml" + log = output / f"{name}.log" + arguments = ["-m", "pytest", path, "--import-mode=importlib", "-o", "addopts=", "-q", "--junitxml", str(report)] + env = {key: value for key, value in os.environ.items() if key != "PYTEST_ADDOPTS"} + env["PYTHONPATH"] = str(root) + env["NEMO_GYM_EXTRA_ROOTS"] = str(root) + print(f"Testing {name}: {path}", flush=True) + with log.open("w") as handle: + result = subprocess.run( + [sys.executable, *arguments], cwd=root, env=env, stdout=handle, stderr=subprocess.STDOUT, check=False + ) + if result.returncode: + raise ValueError(f"{name} tests failed (exit {result.returncode}); see {log}") + cases = ET.parse(report).findall(".//testcase") + skipped = sum(case.find("skipped") is not None for case in cases) + if ( + not cases + or skipped == len(cases) + or any(case.find("failure") is not None or case.find("error") is not None for case in cases) + ): + raise ValueError(f"{name} tests did not establish a passing suite; see {report}") + return { + "path": path, + "passed": len(cases) - skipped, + "skipped": skipped, + "log_sha256": digest_file(log), + "junit_sha256": digest_file(report), + } + + +def render_table(report: dict, *, asset: str) -> str: + """Render counts and provenance without deriving capabilities from unit tests.""" + commit = report["gym_commit"] + lines = [ + BEGIN, + "", + f"Gym commit: [`{commit}`](https://github.com/NVIDIA-NeMo/Gym/commit/{commit}).", + f"Suite: `{report['suite']}`. [Generation report](../../../../assets/trajectory-capabilities/{asset}).", + "", + "Cells show **passing / exercised / required** scenarios. PASS requires every required scenario;", + "FAIL includes missing exercise or missing evidence. **Not run** makes no capability claim.", + "", + "| Harness | " + " | ".join(NAMES) + " | P0 |", + "| --- |" + " --- |" * (len(NAMES) + 1), + ] + for harness in HARNESSES: + row = report["harnesses"].get(harness) + if row is None: + cells = ["Not run"] * (len(NAMES) + 1) + else: + cells = [] + for key in NAMES: + count = row["evidence"][key] + label = "PASS" if count["passed"] == count["required"] and count["required"] else "FAIL" + cells.append(f"{label} {count['passed']}/{count['observed']}/{count['required']}") + cells.append("PASS" if row["verdict"] == "fulfilled" else "FAIL") + lines.append("| " + " | ".join([f"`{harness}`", *cells]) + " |") + lines += ["", "P0 requires TE-1–TE-7 and either TE-8 or TE-9 **per scenario**.", "", "### Tested dependencies", ""] + for harness, runtime in report["runtimes"].items(): + identity = runtime.get("version") or ("source SHA-256 " + runtime["sha256"]) + lines.append(f"- `{harness}`: `{identity.replace('`', '').replace(chr(10), ' ')}`") + lines += ["", "### Unit-test gates", "", "| Suite | Passed | Skipped |", "| --- | ---: | ---: |"] + for name, result in report["tests"].items(): + lines.append(f"| `{name}` | {result['passed']} | {result['skipped']} |") + lines += ["", END] + return "\n".join(lines) + + +def _atomic_write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, delete=False) as handle: + temporary = Path(handle.name) + try: + handle.write(content) + handle.close() + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def regenerate(*, root: Path, commit: str, output: Path, harnesses: list[str], timeout: float) -> None: + """Gate fresh full-suite results and atomically replace only the generated page block.""" + if not harnesses or len(set(harnesses)) != len(harnesses) or set(harnesses) - HARNESSES.keys(): + raise ValueError("select at least one supported harness, without duplicates") + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("timeout must be positive and finite") + verify_revision(root, commit) + page = root / PAGE + original = page.read_text() + if original.count(BEGIN) != 1 or original.count(END) != 1 or original.index(BEGIN) >= original.index(END): + raise ValueError("reference page needs exactly one ordered pair of generated-table markers") + output = output.resolve() + output.mkdir(parents=True, exist_ok=False) + test_paths = {"runner": "scripts/harness_conformance/tests"} + test_paths.update({harness: f"responses_api_agents/{harness}_agent/tests" for harness in harnesses}) + test_paths["checker"] = "tests/unit_tests/harness_capabilities" + tests = {name: run_tests(root, output, name=name, path=path) for name, path in test_paths.items()} + verify_revision(root, commit) + probes = output / "probes" + summary, code = run_suite(harnesses=harnesses, scenarios=SCENARIOS, output=probes, timeout=timeout) + if code not in (0, 1) or summary["runner_status"] != "completed" or not summary["full_suite"]: + raise ValueError(f"probe execution/checker error; reference table unchanged; see {probes}") + runtimes = {} + for harness in harnesses: + identities = [] + for scenario in SCENARIOS: + runtime = json.loads((probes / harness / scenario.name / "runtime.json").read_text()) + # Runtime locations are machine-local; versions and source hashes are portable. + identities.append({key: value for key, value in runtime.items() if key not in ("source", "executable")}) + if not identities[0] or any(identity != identities[0] for identity in identities): + raise ValueError(f"{harness} runtime changed during probing") + runtimes[harness] = identities[0] + report = { + "schema_version": "trajectory-capabilities-table/v1", + "gym_commit": commit, + "suite": SUITE, + "tests": tests, + "runtimes": runtimes, + "python": sys.version, + "harnesses": summary["harnesses"], + "limits": summary["limits"], + "suite_sha256": summary["suite_sha256"], + "summary_sha256": digest_file(probes / "conformance_summary.json"), + } + serialized = json.dumps(report, sort_keys=True, indent=2, allow_nan=False) + "\n" + asset = hashlib.sha256(serialized.encode()).hexdigest() + ".json" + block = render_table(report, asset=asset) + updated = original[: original.index(BEGIN)] + block + original[original.index(END) + len(END) :] + verify_revision(root, commit) + if page.read_text() != original: + raise ValueError("reference page changed during regeneration") + # Content-addressed reports preserve the previous page's provenance even if + # publication is interrupted between these two atomic writes. + _atomic_write(root / ASSETS / asset, serialized) + _atomic_write(page, updated) + print(f"Updated {page} for {commit}") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--commit", required=True, help="full source commit SHA; the checkout must match it") + parser.add_argument( + "--output", required=True, type=Path, help="new directory for test logs and full probe artifacts" + ) + parser.add_argument("--harness", action="append", choices=HARNESSES, help="defaults to all; others show Not run") + parser.add_argument("--timeout", type=float, default=120, help="timeout in seconds per probe") + args = parser.parse_args(argv) + try: + for module in (nemo_gym, checker, episode, runner): + if not Path(module.__file__).resolve().is_relative_to(ROOT): + raise ValueError( + f"{module.__name__} was imported from another checkout; use a clean Python environment" + ) + regenerate( + root=ROOT, + commit=args.commit, + output=args.output, + harnesses=args.harness or list(HARNESSES), + timeout=args.timeout, + ) + except (OSError, ValueError, KeyError, subprocess.CalledProcessError, ET.ParseError) as exc: + print(f"Regeneration failed: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/harness_conformance/tests/test_runner.py b/scripts/harness_conformance/tests/test_runner.py index af6d8454702..ce6627532ee 100644 --- a/scripts/harness_conformance/tests/test_runner.py +++ b/scripts/harness_conformance/tests/test_runner.py @@ -7,7 +7,6 @@ import json import subprocess import sys -from pathlib import Path import psutil import pytest @@ -19,6 +18,7 @@ from nemo_gym.base_responses_api_model import build_model_call_record from nemo_gym.harness_capabilities.reader import hydrate_record, json_rows +from tests.unit_tests.harness_capabilities.synthetic import evidence_record SCENARIO = {s.name: s for s in SCENARIOS} @@ -178,8 +178,7 @@ def test_verifier_records_actual_final_answer(tmp_path, name, reward): @pytest.fixture def retained_episode(tmp_path): # Contract fixture only: used to prove a green artifact cannot hide a missing live probe. - fixture = Path(__file__).resolve().parents[3] / "tests/unit_tests/harness_capabilities/fixtures/miniswe.json" - raw = json.loads(fixture.read_text()) + raw = evidence_record() record = hydrate_record(raw) calls = record["ng_model_call_capture"]["calls"] witness = { diff --git a/scripts/harness_conformance/tests/test_table.py b/scripts/harness_conformance/tests/test_table.py new file mode 100644 index 00000000000..cfaa9c24c92 --- /dev/null +++ b/scripts/harness_conformance/tests/test_table.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Publication gates use synthetic probe results, never preset harness verdicts.""" + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest +from scripts.harness_conformance import table + + +COMMIT = "a" * 40 + + +@pytest.fixture +def generation(tmp_path, monkeypatch): + root = tmp_path / "repo" + page = root / table.PAGE + page.parent.mkdir(parents=True) + page.write_text(f"Introduction\n{table.BEGIN}\nold table\n{table.END}\nDeprecated C-table\n") + calls = [] + monkeypatch.setattr(table, "verify_revision", lambda *args: calls.append("revision")) + + def tests(root, output, *, name, path): + calls.append(name) + return {"path": path, "passed": 2, "skipped": 0} + + monkeypatch.setattr(table, "run_tests", tests) + + def probes(*, harnesses, scenarios, output, timeout): + calls.append("probes") + assert scenarios == table.SCENARIOS + for harness in harnesses: + for scenario in scenarios: + directory = output / harness / scenario.name + directory.mkdir(parents=True) + (directory / "runtime.json").write_text(json.dumps({"version": "synthetic-runtime-v1"})) + summary = { + "runner_status": "completed", + "full_suite": True, + "suite_sha256": "suite-hash", + "limits": [], + "harnesses": { + harness: { + "verdict": "not_fulfilled", + "evidence": {key: {"passed": 0, "observed": 1, "required": 7} for key in table.NAMES}, + } + for harness in harnesses + }, + } + (output / "conformance_summary.json").write_text(json.dumps(summary)) + return summary, 1 + + monkeypatch.setattr(table, "run_suite", probes) + return ( + root, + page, + calls, + {"root": root, "commit": COMMIT, "output": tmp_path / "out", "harnesses": ["pi"], "timeout": 1}, + ) + + +@pytest.mark.parametrize("failed_gate", ["runner", "pi", "checker"]) +def test_failed_unit_gate_preserves_table_and_never_runs_probes(generation, monkeypatch, failed_gate): + root, page, calls, kwargs = generation + original = page.read_bytes() + run_tests = table.run_tests + + def fail(root, output, *, name, path): + if name == failed_gate: + raise ValueError("test failure") + return run_tests(root, output, name=name, path=path) + + monkeypatch.setattr(table, "run_tests", fail) + with pytest.raises(ValueError, match="test failure"): + table.regenerate(**kwargs) + assert "probes" not in calls + assert page.read_bytes() == original + assert not (root / table.ASSETS).exists() + + +def test_evidence_failures_publish_commit_runtime_and_test_provenance(generation): + root, page, calls, kwargs = generation + table.regenerate(**kwargs) + assert calls == ["revision", "runner", "pi", "checker", "revision", "probes", "revision"] + text = page.read_text() + assert text.startswith("Introduction\n") and text.endswith("\nDeprecated C-table\n") + assert COMMIT in text and "synthetic-runtime-v1" in text + assert "| `pi` | FAIL 0/1/7" in text + assert "| `hermes` | Not run" in text + (asset,) = (root / table.ASSETS).glob("*.json") + assert asset.name in text + report = json.loads(asset.read_text()) + assert report["gym_commit"] == COMMIT + assert set(report["tests"]) == {"runner", "pi", "checker"} + + +@pytest.mark.parametrize("problem", ["execution_error", "incomplete", "source_changed", "runtime_changed"]) +def test_failed_generation_preserves_previous_table(generation, monkeypatch, problem): + root, page, _, kwargs = generation + original = page.read_bytes() + run_suite = table.run_suite + + def fail(**args): + summary, code = run_suite(**args) + if problem == "execution_error": + code = 2 + elif problem == "incomplete": + summary["full_suite"] = False + elif problem == "runtime_changed": + runtime = args["output"] / "pi" / table.SCENARIOS[-1].name / "runtime.json" + runtime.write_text('{"version": "different"}') + else: + + def changed(*args): + raise ValueError("source changed") + + monkeypatch.setattr(table, "verify_revision", changed) + return summary, code + + monkeypatch.setattr(table, "run_suite", fail) + with pytest.raises(ValueError): + table.regenerate(**kwargs) + assert page.read_bytes() == original + assert not (root / table.ASSETS).exists() + + +@pytest.mark.parametrize( + "code,xml", + [ + (1, ""), + (5, ""), + (0, ""), + (0, ""), + (0, ""), + ], +) +def test_pytest_gate_rejects_failures_and_vacuous_success(tmp_path, monkeypatch, code, xml): + def run(command, **kwargs): + Path(command[-1]).write_text(xml) + return subprocess.CompletedProcess(command, code) + + monkeypatch.setattr(table.subprocess, "run", run) + with pytest.raises(ValueError): + table.run_tests(tmp_path, tmp_path, name="checker", path="tests") + + +def test_pytest_gate_runs_whole_directory_and_reports_skips(tmp_path, monkeypatch): + monkeypatch.setenv("PYTEST_ADDOPTS", "-k nonexistent") + + def run(command, **kwargs): + assert "PYTEST_ADDOPTS" not in kwargs["env"] + assert kwargs["env"]["NEMO_GYM_EXTRA_ROOTS"] == str(tmp_path) + assert kwargs["cwd"] == tmp_path + assert command[3] == "tests/unit_tests/harness_capabilities" + Path(command[-1]).write_text( + "" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(table.subprocess, "run", run) + result = table.run_tests(tmp_path, tmp_path, name="checker", path="tests/unit_tests/harness_capabilities") + assert result["passed"] == 1 and result["skipped"] == 1 + + +@pytest.mark.skipif(shutil.which("jj") is None, reason="jj is not installed") +@pytest.mark.parametrize("vcs", ["jj", "git"]) +def test_revision_rejects_source_changes_and_untracked_files(tmp_path, monkeypatch, vcs): + def jj(*args): + return subprocess.check_output(["jj", "--no-pager", *args], cwd=tmp_path, text=True).strip() + + jj("git", "init", "--colocate") + source = tmp_path / "source.py" + source.write_text("original\n") + commit = jj("log", "--no-graph", "-r", "@", "-T", "commit_id") + if vcs == "git": + # Exercise read-only Git verification against jj's colocated object store. + exists = Path.exists + monkeypatch.setattr(Path, "exists", lambda path: False if path == tmp_path / ".jj" else exists(path)) + table.verify_revision(tmp_path, commit) + source.write_text("edited\n") + with pytest.raises(ValueError, match="differs"): + table.verify_revision(tmp_path, commit) + source.write_text("original\n") + (tmp_path / "extra.py").write_text("untracked\n") + with pytest.raises(ValueError, match="differs"): + table.verify_revision(tmp_path, commit) + + +@pytest.mark.parametrize("commit", ["HEAD", "main", "a" * 7, "a" * 39 + ";", "A" * 40]) +def test_revision_requires_full_commit_id(tmp_path, commit): + with pytest.raises(ValueError, match="full lowercase"): + table.verify_revision(tmp_path, commit) + + +def test_cli_refuses_modules_from_another_checkout(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(table.runner, "__file__", str(tmp_path / "other/runner.py")) + assert table.main(["--commit", COMMIT, "--output", str(tmp_path / "out")]) == 2 + assert "another checkout" in capsys.readouterr().err + assert not (tmp_path / "out").exists() diff --git a/tests/unit_tests/harness_capabilities/fixtures/codex.jsonl b/tests/unit_tests/harness_capabilities/fixtures/codex.jsonl deleted file mode 100644 index f7b08b71f1a..00000000000 --- a/tests/unit_tests/harness_capabilities/fixtures/codex.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"_ng_rollout_id": "0-0", "_ng_rollout_index": 0, "_ng_task_index": 0, "ng_model_call_capture": {"calls": [{"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 0, "client_session_id": null, "completed_at": 1.5, "dialect": "responses", "error_category": null, "finish_reason": null, "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-0", "response_status": "completed", "started_at": 1.0, "status_code": 200, "tokens_in": 10, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 15, "tool_calls": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "name": "bash"}]}, {"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 1, "client_session_id": null, "completed_at": 2.5, "dialect": "responses", "error_category": null, "finish_reason": null, "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-1", "response_status": "completed", "started_at": 2.0, "status_code": 200, "tokens_in": 11, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 16, "tool_calls": []}], "rollout_id": "0-0"}, "ng_trajectory": {"gaps": [{"code": "turns_unavailable", "detail": null, "invocation_id": null}, {"code": "conversation_unavailable", "detail": null, "invocation_id": null}], "invocations": [], "model_calls": [{"completed_at": 1.5, "duration_ms": 500.0, "model_call_id": "call-0", "request": {"input": [{"content": "Print ok, then report success.", "role": "user"}], "model": "fixture-model"}, "response": {"id": "response-0", "model": "fixture-model", "output": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "name": "bash", "type": "function_call"}], "status": "completed", "usage": {"input_tokens": 10, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 2}, "total_tokens": 15}}, "response_metadata": {"dialect": "responses", "error_category": null, "finish_reason": null, "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0", "response_status": "completed", "status_code": 200}, "started_at": 1.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 10, "reasoning_tokens": 2, "total_tokens": 15}}, {"completed_at": 2.5, "duration_ms": 500.0, "model_call_id": "call-1", "request": {"input": [{"content": "Print ok, then report success.", "role": "user"}, {"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "name": "bash", "type": "function_call"}, {"call_id": "tool-1", "output": "ok", "type": "function_call_output"}], "model": "fixture-model"}, "response": {"id": "response-1", "model": "fixture-model", "output": [{"content": [{"text": "Done", "type": "output_text"}], "role": "assistant", "type": "message"}], "status": "completed", "usage": {"input_tokens": 11, "input_tokens_details": {"cached_tokens": 0}, "output_tokens": 5, "output_tokens_details": {"reasoning_tokens": 2}, "total_tokens": 16}}, "response_metadata": {"dialect": "responses", "error_category": null, "finish_reason": null, "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1", "response_status": "completed", "status_code": 200}, "started_at": 2.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 11, "reasoning_tokens": 2, "total_tokens": 16}}], "rollout_id": "0-0", "schema_version": "1.0", "task_id": "0", "tool_calls": [], "turns": []}, "response": {"output": [{"arguments": "{\"cmd\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "exec_command", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "answer", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "usage": {"cached_input_tokens": 0, "input_tokens": 21, "output_tokens": 10, "reasoning_tokens": 0}}, "reward": 1.0} diff --git a/tests/unit_tests/harness_capabilities/fixtures/hermes.jsonl b/tests/unit_tests/harness_capabilities/fixtures/hermes.jsonl deleted file mode 100644 index e5bcdd8391e..00000000000 --- a/tests/unit_tests/harness_capabilities/fixtures/hermes.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"_ng_rollout_id": "0-0", "_ng_rollout_index": 0, "_ng_task_index": 0, "ng_agent_observations": {"gaps": [], "records": [{"conversation": [{"content": "Print ok, then report success.", "phase": null, "role": "user", "type": "message"}, {"content": [{"annotations": [], "logprobs": null, "text": "", "type": "output_text"}], "id": "root-message-1", "phase": null, "role": "assistant", "status": "completed", "type": "message"}, {"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "root-message-3", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}, {"completed_at": 1.0, "duration_ms": 0.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "sandbox_id": null, "started_at": 1.0, "status": "completed", "timing_source": "executor", "tool_call_id": "tool-1", "tool_name": "bash"}], "source": "hermes"}, "ng_model_call_capture": {"calls": [{"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 0, "client_session_id": null, "completed_at": 1.5, "dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-0", "response_status": null, "started_at": 1.0, "status_code": 200, "tokens_in": 10, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 15, "tool_calls": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "name": "bash"}]}, {"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 1, "client_session_id": null, "completed_at": 2.5, "dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-1", "response_status": null, "started_at": 2.0, "status_code": 200, "tokens_in": 11, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 16, "tool_calls": []}], "rollout_id": "0-0"}, "ng_trajectory": {"gaps": [{"code": "turns_unavailable", "detail": null, "invocation_id": null}], "invocations": [{"conversation": [{"content": "Print ok, then report success.", "phase": null, "role": "user", "type": "message"}, {"content": [{"annotations": [], "logprobs": null, "text": "", "type": "output_text"}], "id": "root-message-1", "phase": null, "role": "assistant", "status": "completed", "type": "message"}, {"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "root-message-3", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}], "model_calls": [{"completed_at": 1.5, "duration_ms": 500.0, "model_call_id": "call-0", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "tool_calls", "index": 0, "message": {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}}], "id": "response-0", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 15}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0", "response_status": null, "status_code": 200}, "started_at": 1.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 10, "reasoning_tokens": 2, "total_tokens": 15}}, {"completed_at": 2.5, "duration_ms": 500.0, "model_call_id": "call-1", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}, {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}, {"content": "ok", "role": "tool", "tool_call_id": "tool-1"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Done", "role": "assistant"}}], "id": "response-1", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 11, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 16}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1", "response_status": null, "status_code": 200}, "started_at": 2.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 11, "reasoning_tokens": 2, "total_tokens": 16}}], "rollout_id": "0-0", "schema_version": "1.0", "task_id": "0", "tool_calls": [{"completed_at": 1.0, "duration_ms": 0.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "output": "ok", "sandbox_id": null, "started_at": 1.0, "status": "completed", "timing_source": "executor", "tool_call_id": "tool-1", "tool_name": "bash"}], "turns": []}, "reward": 1.0} diff --git a/tests/unit_tests/harness_capabilities/fixtures/miniswe.json b/tests/unit_tests/harness_capabilities/fixtures/miniswe.json deleted file mode 100644 index d182259ac7c..00000000000 --- a/tests/unit_tests/harness_capabilities/fixtures/miniswe.json +++ /dev/null @@ -1,675 +0,0 @@ -{ - "_ng_task_index": 0, - "_ng_rollout_index": 0, - "reward": 0.0, - "response": { - "id": "resp_df81b682747645199ada18f21b22d6f6", - "created_at": 1789726149.0, - "error": null, - "incomplete_details": null, - "instructions": null, - "metadata": null, - "model": "model", - "object": "response", - "output": [ - { - "arguments": "{\"command\": \"inspect\"}", - "call_id": "tool-1", - "name": "bash", - "type": "function_call", - "id": null, - "status": null, - "namespace": null - }, - { - "call_id": "tool-1", - "output": "0\ninspected", - "type": "function_call_output", - "id": null, - "status": null - }, - { - "arguments": "{\"command\": \"submit\"}", - "call_id": "tool-2", - "name": "bash", - "type": "function_call", - "id": null, - "status": null, - "namespace": null - }, - { - "call_id": "tool-2", - "output": "0\nCOMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished", - "type": "function_call_output", - "id": null, - "status": null - } - ], - "parallel_tool_calls": true, - "temperature": null, - "tool_choice": "auto", - "tools": [], - "top_p": null, - "background": null, - "completed_at": null, - "conversation": null, - "max_output_tokens": null, - "max_tool_calls": null, - "moderation": null, - "previous_response_id": null, - "prompt": null, - "prompt_cache_key": null, - "prompt_cache_retention": null, - "reasoning": null, - "safety_identifier": null, - "service_tier": null, - "status": null, - "text": null, - "top_logprobs": null, - "truncation": null, - "usage": { - "input_tokens": 20, - "input_tokens_details": { - "cached_tokens": 2 - }, - "output_tokens": 10, - "output_tokens_details": { - "reasoning_tokens": 6 - }, - "total_tokens": 30 - }, - "user": null - }, - "ng_agent_observations": { - "source": "miniswe", - "records": [ - { - "kind": "agent_invocation", - "invocation_id": "invocation", - "parent_invocation_id": null, - "spawned_by_tool_call_id": null, - "status": "completed", - "duration_ms": 66.45141704939306, - "error_type": null, - "model_calls": [ - { - "model_call_id": "3a80e87fe7ad4a9bb9b203c8d1e2ffe5", - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "response_id": "response-1" - }, - { - "model_call_id": "8dce6de0d1df4f2998054919cc3eec06", - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "response_id": "response-2" - } - ], - "conversation": [ - { - "content": "You are a helpful assistant that can interact with a computer.", - "role": "system", - "type": "message", - "phase": null - }, - { - "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```", - "role": "user", - "type": "message", - "phase": null - }, - { - "arguments": "{\"command\": \"inspect\"}", - "call_id": "tool-1", - "name": "bash", - "type": "function_call", - "id": null, - "status": null, - "namespace": null - }, - { - "call_id": "tool-1", - "output": "0\ninspected", - "type": "function_call_output", - "id": null, - "status": null - }, - { - "arguments": "{\"command\": \"submit\"}", - "call_id": "tool-2", - "name": "bash", - "type": "function_call", - "id": null, - "status": null, - "namespace": null - }, - { - "call_id": "tool-2", - "output": "0\nCOMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished", - "type": "function_call_output", - "id": null, - "status": null - } - ] - }, - { - "kind": "tool_call", - "invocation_id": "invocation", - "tool_call_id": "tool-1", - "sandbox_id": null, - "tool_name": "bash", - "started_at": 1789726149.893095, - "completed_at": 1789726149.8931599, - "duration_ms": 0.06045890040695667, - "timing_source": "executor", - "status": "completed", - "error_type": null - }, - { - "kind": "tool_call", - "invocation_id": "invocation", - "tool_call_id": "tool-2", - "sandbox_id": null, - "tool_name": "bash", - "started_at": 1789726149.901033, - "completed_at": 1789726149.9011168, - "duration_ms": 0.07075001485645771, - "timing_source": "executor", - "status": "completed", - "error_type": null - } - ], - "gaps": [] - }, - "ng_trajectory": { - "schema_version": "1.0", - "task_id": "0", - "rollout_id": "0-0", - "invocations": [ - { - "kind": "agent_invocation", - "invocation_id": "invocation", - "parent_invocation_id": null, - "spawned_by_tool_call_id": null, - "status": "completed", - "duration_ms": 66.45141704939306, - "error_type": null, - "model_calls": [ - { - "model_call_id": "3a80e87fe7ad4a9bb9b203c8d1e2ffe5", - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "response_id": "response-1" - }, - { - "model_call_id": "8dce6de0d1df4f2998054919cc3eec06", - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "response_id": "response-2" - } - ], - "conversation": [ - { - "content": "You are a helpful assistant that can interact with a computer.", - "role": "system", - "type": "message", - "phase": null - }, - { - "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```", - "role": "user", - "type": "message", - "phase": null - }, - { - "arguments": "{\"command\": \"inspect\"}", - "call_id": "tool-1", - "name": "bash", - "type": "function_call", - "id": null, - "status": null, - "namespace": null - }, - { - "call_id": "tool-1", - "output": "0\ninspected", - "type": "function_call_output", - "id": null, - "status": null - }, - { - "arguments": "{\"command\": \"submit\"}", - "call_id": "tool-2", - "name": "bash", - "type": "function_call", - "id": null, - "status": null, - "namespace": null - }, - { - "call_id": "tool-2", - "output": "0\nCOMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished", - "type": "function_call_output", - "id": null, - "status": null - } - ] - } - ], - "turns": [ - { - "invocation_id": "invocation", - "task_id": "0", - "rollout_id": "0-0", - "turn_no": 1, - "timestamp": 1789726149.8928242, - "question": [ - { - "role": "system", - "content": "You are a helpful assistant that can interact with a computer." - }, - { - "role": "user", - "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" - } - ], - "answer": [ - { - "arguments": "{\"command\": \"inspect\"}", - "call_id": "tool-1", - "name": "bash", - "type": "function_call", - "id": null, - "status": null, - "namespace": null - } - ], - "reasoning_content": null, - "resolved": null, - "step_count": 1, - "model_calls": [ - { - "model_call_id": null, - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "response_id": "response-1" - } - ] - }, - { - "invocation_id": "invocation", - "task_id": "0", - "rollout_id": "0-0", - "turn_no": 2, - "timestamp": 1789726149.900903, - "question": [ - { - "role": "system", - "content": "You are a helpful assistant that can interact with a computer." - }, - { - "role": "user", - "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" - }, - { - "arguments": "{\"command\": \"inspect\"}", - "call_id": "tool-1", - "name": "bash", - "type": "function_call" - }, - { - "type": "function_call_output", - "call_id": "tool-1", - "output": "0\ninspected" - } - ], - "answer": [ - { - "arguments": "{\"command\": \"submit\"}", - "call_id": "tool-2", - "name": "bash", - "type": "function_call", - "id": null, - "status": null, - "namespace": null - } - ], - "reasoning_content": null, - "resolved": null, - "step_count": 2, - "model_calls": [ - { - "model_call_id": null, - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "response_id": "response-2" - } - ] - } - ], - "model_calls": [ - { - "model_call_id": "3a80e87fe7ad4a9bb9b203c8d1e2ffe5", - "started_at": 1789726149.874946, - "completed_at": 1789726149.876183, - "duration_ms": 1.24, - "request": { - "input": [ - { - "role": "system", - "content": "You are a helpful assistant that can interact with a computer." - }, - { - "role": "user", - "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" - } - ], - "parallel_tool_calls": true, - "tool_choice": "auto", - "tools": [ - { - "type": "function", - "name": "bash", - "description": "Execute a bash command", - "parameters": { - "type": "object", - "properties": { - "command": { - "type": "string", - "description": "The bash command to execute" - } - }, - "required": [ - "command" - ] - }, - "strict": false - } - ] - }, - "response": { - "id": "response-1", - "object": "response", - "model": "controlled", - "created_at": 0, - "status": "completed", - "parallel_tool_calls": false, - "tools": [], - "tool_choice": "auto", - "output": [ - { - "type": "function_call", - "call_id": "tool-1", - "name": "bash", - "arguments": "{\"command\": \"inspect\"}" - } - ], - "usage": { - "input_tokens": 10, - "output_tokens": 5, - "total_tokens": 15, - "input_tokens_details": { - "cached_tokens": 2 - }, - "output_tokens_details": { - "reasoning_tokens": 3 - } - } - }, - "response_metadata": { - "response_id": "response-1", - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "model": "controlled", - "dialect": "responses", - "status_code": 200, - "response_status": "completed", - "finish_reason": null, - "error_category": null, - "latency_ttft_ms": 1.23 - }, - "token_stats": { - "prompt_tokens": 10, - "completion_tokens": 5, - "reasoning_tokens": 3, - "total_tokens": 15, - "cached_tokens": 2 - } - }, - { - "model_call_id": "8dce6de0d1df4f2998054919cc3eec06", - "started_at": 1789726149.895183, - "completed_at": 1789726149.895329, - "duration_ms": 0.15, - "request": { - "input": [ - { - "role": "system", - "content": "You are a helpful assistant that can interact with a computer." - }, - { - "role": "user", - "content": "Please solve this issue: inspect then submit\n\nYou can execute bash commands and edit files to implement the necessary changes.\n\n## Recommended Workflow\n\nThis workflow should be done step-by-step so that you can iterate on your changes and any possible problems.\n\n1. Analyze the codebase by finding and reading relevant files\n2. Create a script to reproduce the issue\n3. Edit the source code to resolve the issue\n4. Verify your fix works by running your script again\n5. Test edge cases to ensure your fix is robust\n6. Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\n## Command Execution Rules\n\nYou are operating in an environment where\n\n1. You issue at least one command\n2. The system executes the command(s) in a subshell\n3. You see the result(s)\n4. You write your next command(s)\n\nEach response should include:\n\n1. **Reasoning text** where you explain your analysis and plan\n2. At least one tool call with your command\n\n**CRITICAL REQUIREMENTS:**\n\n- Your response SHOULD include reasoning text explaining what you're doing\n- Your response MUST include AT LEAST ONE bash tool call\n- Directory or environment variable changes are not persistent. Every action is executed in a new subshell.\n- However, you can prefix any action with `MY_ENV_VAR=MY_VALUE cd /path/to/working/dir && ...` or write/load environment variables from files\n- Submit your changes and finish your work by issuing the following command: `echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`.\n Do not combine it with any other command. After this command, you cannot continue working on this task.\n\nExample of a CORRECT response:\n\nI need to understand the structure of the repository first. Let me check what files are in the current directory to get a better understanding of the codebase.\n\n[Makes bash tool call with {\"command\": \"ls -la\"} as arguments]\n\n\n\nLinux 6 test x86_64\n\n\n## Useful command examples\n\n### Create a new file:\n\n```bash\ncat <<'EOF' > newfile.py\nimport numpy as np\nhello = \"world\"\nprint(hello)\nEOF\n```\n\n### Edit files with sed:```bash\n# Replace all occurrences\nsed -i 's/old_string/new_string/g' filename.py\n\n# Replace only first occurrence\nsed -i 's/old_string/new_string/' filename.py\n\n# Replace first occurrence on line 1\nsed -i '1s/old_string/new_string/' filename.py\n\n# Replace all occurrences in lines 1-10\nsed -i '1,10s/old_string/new_string/g' filename.py\n```\n\n### View file content:\n\n```bash\n# View specific lines with numbers\nnl -ba filename.py | sed -n '10,20p'\n```\n\n### Any other command you want to run\n\n```bash\nanything\n```" - }, - { - "arguments": "{\"command\": \"inspect\"}", - "call_id": "tool-1", - "name": "bash", - "type": "function_call" - }, - { - "type": "function_call_output", - "call_id": "tool-1", - "output": "0\ninspected" - } - ], - "parallel_tool_calls": true, - "tool_choice": "auto", - "tools": [ - { - "type": "function", - "name": "bash", - "description": "Execute a bash command", - "parameters": { - "type": "object", - "properties": { - "command": { - "type": "string", - "description": "The bash command to execute" - } - }, - "required": [ - "command" - ] - }, - "strict": false - } - ] - }, - "response": { - "id": "response-2", - "object": "response", - "model": "controlled", - "created_at": 0, - "status": "completed", - "parallel_tool_calls": false, - "tools": [], - "tool_choice": "auto", - "output": [ - { - "type": "function_call", - "call_id": "tool-2", - "name": "bash", - "arguments": "{\"command\": \"submit\"}" - } - ], - "usage": { - "input_tokens": 10, - "output_tokens": 5, - "total_tokens": 15, - "input_tokens_details": { - "cached_tokens": 0 - }, - "output_tokens_details": { - "reasoning_tokens": 3 - } - } - }, - "response_metadata": { - "response_id": "response-2", - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "model": "controlled", - "dialect": "responses", - "status_code": 200, - "response_status": "completed", - "finish_reason": null, - "error_category": null, - "latency_ttft_ms": 0.14 - }, - "token_stats": { - "prompt_tokens": 10, - "completion_tokens": 5, - "reasoning_tokens": 3, - "total_tokens": 15, - "cached_tokens": 0 - } - } - ], - "tool_calls": [ - { - "kind": "tool_call", - "invocation_id": "invocation", - "tool_call_id": "tool-1", - "sandbox_id": null, - "tool_name": "bash", - "started_at": 1789726149.893095, - "completed_at": 1789726149.8931599, - "duration_ms": 0.06045890040695667, - "timing_source": "executor", - "status": "completed", - "error_type": null, - "output": "0\ninspected" - }, - { - "kind": "tool_call", - "invocation_id": "invocation", - "tool_call_id": "tool-2", - "sandbox_id": null, - "tool_name": "bash", - "started_at": 1789726149.901033, - "completed_at": 1789726149.9011168, - "duration_ms": 0.07075001485645771, - "timing_source": "executor", - "status": "completed", - "error_type": null, - "output": "0\nCOMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished" - } - ], - "gaps": [] - }, - "ng_model_call_capture": { - "rollout_id": "0-0", - "metrics": { - "tokens_in": 20, - "tokens_out": 10, - "tokens_reasoning": 6, - "tokens_total": 30, - "cached_tokens": 2, - "latency_total_ms": 1.39, - "num_calls": 2 - }, - "calls": [ - { - "model_call_id": "3a80e87fe7ad4a9bb9b203c8d1e2ffe5", - "response_id": "response-1", - "client_session_id": "invocation", - "call_index": 0, - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "model": "controlled", - "dialect": "responses", - "status_code": 200, - "response_status": "completed", - "finish_reason": null, - "started_at": 1789726149.874946, - "completed_at": 1789726149.876183, - "tokens_in": 10, - "tokens_out": 5, - "tokens_reasoning": 3, - "tokens_total": 15, - "tool_calls": [ - { - "call_id": "tool-1", - "name": "bash", - "arguments": { - "command": "inspect" - } - } - ], - "reasoning_content": null, - "cache_hit": true, - "cached_tokens": 2, - "cache_creation_tokens": null, - "error_category": null, - "latency_total_ms": 1.24, - "latency_ttft_ms": 1.23 - }, - { - "model_call_id": "8dce6de0d1df4f2998054919cc3eec06", - "response_id": "response-2", - "client_session_id": "invocation", - "call_index": 1, - "model_ref": { - "type": "responses_api_models", - "name": "model" - }, - "model": "controlled", - "dialect": "responses", - "status_code": 200, - "response_status": "completed", - "finish_reason": null, - "started_at": 1789726149.895183, - "completed_at": 1789726149.895329, - "tokens_in": 10, - "tokens_out": 5, - "tokens_reasoning": 3, - "tokens_total": 15, - "tool_calls": [ - { - "call_id": "tool-2", - "name": "bash", - "arguments": { - "command": "submit" - } - } - ], - "reasoning_content": null, - "cache_hit": false, - "cached_tokens": 0, - "cache_creation_tokens": null, - "error_category": null, - "latency_total_ms": 0.15, - "latency_ttft_ms": 0.14 - } - ] - } -} diff --git a/tests/unit_tests/harness_capabilities/fixtures/opencode.json b/tests/unit_tests/harness_capabilities/fixtures/opencode.json deleted file mode 100644 index e4ac11dc898..00000000000 --- a/tests/unit_tests/harness_capabilities/fixtures/opencode.json +++ /dev/null @@ -1,241 +0,0 @@ -{ - "_ng_task_index": 0, - "_ng_rollout_index": 0, - "ng_model_call_capture": { - "rollout_id": "0-0", - "calls": [ - { - "model_call_id": "call-1", - "response_id": "response-1", - "client_session_id": "root", - "call_index": 0, - "model_ref": { - "type": "responses_api_models", - "name": "policy" - }, - "model": "model", - "dialect": "responses", - "status_code": 200, - "response_status": "completed", - "finish_reason": null, - "started_at": 1.0, - "completed_at": 2.0, - "tokens_in": 10, - "tokens_out": 5, - "tokens_reasoning": 2, - "tokens_total": 15, - "tool_calls": [ - { - "call_id": "tool-1", - "name": "bash", - "arguments": { - "command": "printf ok" - } - } - ], - "reasoning_content": null, - "cache_hit": true, - "cached_tokens": 3, - "cache_creation_tokens": null, - "error_category": null, - "latency_total_ms": 1000.0, - "latency_ttft_ms": null - } - ] - }, - "ng_agent_observations": { - "source": "opencode", - "records": [ - { - "kind": "agent_invocation", - "invocation_id": "root", - "parent_invocation_id": null, - "spawned_by_tool_call_id": null, - "status": "completed", - "duration_ms": null, - "error_type": null, - "model_calls": [ - { - "model_call_id": "call-1", - "model_ref": { - "type": "responses_api_models", - "name": "policy" - }, - "response_id": "response-1" - } - ], - "conversation": [ - { - "arguments": "{\"command\": \"printf ok\"}", - "call_id": "tool-1", - "name": "bash", - "type": "function_call", - "id": "tool-1", - "status": "completed", - "namespace": null - }, - { - "call_id": "tool-1", - "output": "ok", - "type": "function_call_output", - "id": null, - "status": "completed" - } - ] - }, - { - "kind": "tool_call", - "invocation_id": "root", - "tool_call_id": "tool-1", - "sandbox_id": null, - "tool_name": "bash", - "started_at": 2.0, - "completed_at": 3.0, - "duration_ms": 1000.0, - "timing_source": "artifact", - "status": "completed", - "error_type": null - } - ], - "gaps": [] - }, - "ng_trajectory": { - "schema_version": "1.0", - "task_id": "0", - "rollout_id": "0-0", - "invocations": [ - { - "kind": "agent_invocation", - "invocation_id": "root", - "parent_invocation_id": null, - "spawned_by_tool_call_id": null, - "status": "completed", - "duration_ms": null, - "error_type": null, - "model_calls": [ - { - "model_call_id": "call-1", - "model_ref": { - "type": "responses_api_models", - "name": "policy" - }, - "response_id": "response-1" - } - ], - "conversation": [ - { - "arguments": "{\"command\": \"printf ok\"}", - "call_id": "tool-1", - "name": "bash", - "type": "function_call", - "id": "tool-1", - "status": "completed", - "namespace": null - }, - { - "call_id": "tool-1", - "output": "ok", - "type": "function_call_output", - "id": null, - "status": "completed" - } - ] - } - ], - "turns": [], - "model_calls": [ - { - "model_call_id": "call-1", - "started_at": 1.0, - "completed_at": 2.0, - "duration_ms": 1000.0, - "request": { - "model": "model", - "input": [ - { - "role": "user", - "content": "Print ok" - } - ], - "tools": [ - { - "type": "function", - "name": "bash", - "parameters": { - "type": "object" - } - } - ] - }, - "response": { - "id": "response-1", - "model": "model", - "status": "completed", - "output": [ - { - "type": "function_call", - "call_id": "tool-1", - "name": "bash", - "arguments": "{\"command\":\"printf ok\"}" - } - ], - "usage": { - "input_tokens": 10, - "output_tokens": 5, - "total_tokens": 15, - "input_tokens_details": { - "cached_tokens": 3 - }, - "output_tokens_details": { - "reasoning_tokens": 2 - } - } - }, - "response_metadata": { - "response_id": "response-1", - "model_ref": { - "type": "responses_api_models", - "name": "policy" - }, - "model": "model", - "dialect": "responses", - "status_code": 200, - "response_status": "completed", - "finish_reason": null, - "error_category": null, - "latency_ttft_ms": null - }, - "token_stats": { - "prompt_tokens": 10, - "completion_tokens": 5, - "reasoning_tokens": 2, - "total_tokens": 15, - "cached_tokens": 3 - } - } - ], - "tool_calls": [ - { - "kind": "tool_call", - "invocation_id": "root", - "tool_call_id": "tool-1", - "sandbox_id": null, - "tool_name": "bash", - "started_at": 2.0, - "completed_at": 3.0, - "duration_ms": 1000.0, - "timing_source": "artifact", - "status": "completed", - "error_type": null, - "output": "ok" - } - ], - "gaps": [ - { - "code": "turns_unavailable", - "invocation_id": null, - "detail": null - } - ] - } -} diff --git a/tests/unit_tests/harness_capabilities/fixtures/opencode.jsonl b/tests/unit_tests/harness_capabilities/fixtures/opencode.jsonl deleted file mode 100644 index b2d72093744..00000000000 --- a/tests/unit_tests/harness_capabilities/fixtures/opencode.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"_ng_rollout_id": "0-0", "_ng_rollout_index": 0, "_ng_task_index": 0, "ng_agent_observations": {"gaps": [], "records": [{"conversation": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "message-1-1", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}, {"completed_at": 1.8, "duration_ms": 200.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "sandbox_id": null, "started_at": 1.6, "status": "completed", "timing_source": "artifact", "tool_call_id": "tool-1", "tool_name": "bash"}], "source": "opencode"}, "ng_model_call_capture": {"calls": [{"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 0, "client_session_id": "root", "completed_at": 1.5, "dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-0", "response_status": null, "started_at": 1.0, "status_code": 200, "tokens_in": 10, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 15, "tool_calls": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "name": "bash"}]}, {"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 1, "client_session_id": "root", "completed_at": 2.5, "dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-1", "response_status": null, "started_at": 2.0, "status_code": 200, "tokens_in": 11, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 16, "tool_calls": []}], "rollout_id": "0-0"}, "ng_trajectory": {"gaps": [], "invocations": [{"conversation": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "message-1-1", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}], "model_calls": [{"completed_at": 1.5, "duration_ms": 500.0, "model_call_id": "call-0", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "tool_calls", "index": 0, "message": {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}}], "id": "response-0", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 15}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0", "response_status": null, "status_code": 200}, "started_at": 1.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 10, "reasoning_tokens": 2, "total_tokens": 15}}, {"completed_at": 2.5, "duration_ms": 500.0, "model_call_id": "call-1", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}, {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}, {"content": "ok", "role": "tool", "tool_call_id": "tool-1"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Done", "role": "assistant"}}], "id": "response-1", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 11, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 16}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1", "response_status": null, "status_code": 200}, "started_at": 2.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 11, "reasoning_tokens": 2, "total_tokens": 16}}], "rollout_id": "0-0", "schema_version": "1.0", "task_id": "0", "tool_calls": [{"completed_at": 1.8, "duration_ms": 200.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "output": "ok", "sandbox_id": null, "started_at": 1.6, "status": "completed", "timing_source": "artifact", "tool_call_id": "tool-1", "tool_name": "bash"}], "turns": [{"answer": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "id": "message-0-1", "name": "bash", "type": "function_call"}], "invocation_id": "root", "model_calls": [], "question": null, "reasoning_content": null, "resolved": null, "rollout_id": "0-0", "step_count": 1, "task_id": "0", "timestamp": 1.0, "turn_no": 1}, {"answer": [{"content": [{"id": "message-1-1", "text": "Done", "type": "output_text"}], "id": "message-1", "role": "assistant", "type": "message"}], "invocation_id": "root", "model_calls": [], "question": null, "reasoning_content": null, "resolved": null, "rollout_id": "0-0", "step_count": 2, "task_id": "0", "timestamp": 2.0, "turn_no": 2}]}, "reward": 1.0} diff --git a/tests/unit_tests/harness_capabilities/fixtures/pi.jsonl b/tests/unit_tests/harness_capabilities/fixtures/pi.jsonl deleted file mode 100644 index f9553526468..00000000000 --- a/tests/unit_tests/harness_capabilities/fixtures/pi.jsonl +++ /dev/null @@ -1 +0,0 @@ -{"_ng_rollout_id": "0-0", "_ng_rollout_index": 0, "_ng_task_index": 0, "ng_agent_observations": {"gaps": [{"code": "subagent_hierarchy_unavailable", "detail": null, "invocation_id": "root"}], "records": [{"conversation": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "msg-2", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}, {"completed_at": 2.0, "duration_ms": 1000.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "sandbox_id": null, "started_at": 1.0, "status": "completed", "timing_source": "harness", "tool_call_id": "tool-1", "tool_name": "bash"}], "source": "pi"}, "ng_model_call_capture": {"calls": [{"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 0, "client_session_id": null, "completed_at": 1.5, "dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-0", "response_status": null, "started_at": 1.0, "status_code": 200, "tokens_in": 10, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 15, "tool_calls": [{"arguments": {"command": "printf ok"}, "call_id": "tool-1", "name": "bash"}]}, {"cache_creation_tokens": null, "cache_hit": false, "cached_tokens": 0, "call_index": 1, "client_session_id": null, "completed_at": 2.5, "dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_total_ms": 500.0, "latency_ttft_ms": null, "model": "fixture-model", "model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "reasoning_content": null, "response_id": "response-1", "response_status": null, "started_at": 2.0, "status_code": 200, "tokens_in": 11, "tokens_out": 5, "tokens_reasoning": 2, "tokens_total": 16, "tool_calls": []}], "rollout_id": "0-0"}, "ng_trajectory": {"gaps": [{"code": "subagent_hierarchy_unavailable", "detail": null, "invocation_id": "root"}, {"code": "turns_unavailable", "detail": null, "invocation_id": null}], "invocations": [{"conversation": [{"arguments": "{\"command\": \"printf ok\"}", "call_id": "tool-1", "id": "tool-1", "name": "bash", "namespace": null, "status": "completed", "type": "function_call"}, {"call_id": "tool-1", "id": null, "output": "ok", "status": "completed", "type": "function_call_output"}, {"content": [{"annotations": [], "logprobs": null, "text": "Done", "type": "output_text"}], "id": "msg-2", "phase": null, "role": "assistant", "status": "completed", "type": "message"}], "duration_ms": null, "error_type": null, "invocation_id": "root", "kind": "agent_invocation", "model_calls": [{"model_call_id": "call-0", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0"}, {"model_call_id": "call-1", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1"}], "parent_invocation_id": null, "spawned_by_tool_call_id": null, "status": "completed"}], "model_calls": [{"completed_at": 1.5, "duration_ms": 500.0, "model_call_id": "call-0", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "tool_calls", "index": 0, "message": {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}}], "id": "response-0", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 15}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "tool_calls", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-0", "response_status": null, "status_code": 200}, "started_at": 1.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 10, "reasoning_tokens": 2, "total_tokens": 15}}, {"completed_at": 2.5, "duration_ms": 500.0, "model_call_id": "call-1", "request": {"messages": [{"content": "Print ok, then report success.", "role": "user"}, {"content": null, "role": "assistant", "tool_calls": [{"function": {"arguments": "{\"command\": \"printf ok\"}", "name": "bash"}, "id": "tool-1", "type": "function"}]}, {"content": "ok", "role": "tool", "tool_call_id": "tool-1"}], "model": "fixture-model"}, "response": {"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "Done", "role": "assistant"}}], "id": "response-1", "model": "fixture-model", "usage": {"completion_tokens": 5, "completion_tokens_details": {"reasoning_tokens": 2}, "prompt_tokens": 11, "prompt_tokens_details": {"cached_tokens": 0}, "total_tokens": 16}}, "response_metadata": {"dialect": "chat", "error_category": null, "finish_reason": "stop", "latency_ttft_ms": null, "model": "fixture-model", "model_ref": {"name": "policy", "type": "responses_api_models"}, "response_id": "response-1", "response_status": null, "status_code": 200}, "started_at": 2.0, "token_stats": {"cached_tokens": 0, "completion_tokens": 5, "prompt_tokens": 11, "reasoning_tokens": 2, "total_tokens": 16}}], "rollout_id": "0-0", "schema_version": "1.0", "task_id": "0", "tool_calls": [{"completed_at": 2.0, "duration_ms": 1000.0, "error_type": null, "invocation_id": "root", "kind": "tool_call", "output": "ok", "sandbox_id": null, "started_at": 1.0, "status": "completed", "timing_source": "harness", "tool_call_id": "tool-1", "tool_name": "bash"}], "turns": []}, "reward": 1.0} diff --git a/tests/unit_tests/harness_capabilities/producer_fixtures.py b/tests/unit_tests/harness_capabilities/producer_fixtures.py deleted file mode 100644 index 9357497e6fd..00000000000 --- a/tests/unit_tests/harness_capabilities/producer_fixtures.py +++ /dev/null @@ -1,280 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Controlled native protocol inputs passed through real producers and collection. - -These fixtures exercise the capture-enabled delivery path. They do not establish -that a deployed harness routes every request through capture or passes live QA. -""" - -import json -import sqlite3 -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import patch - -from nemo_gym.base_responses_api_model import build_model_call_record -from nemo_gym.config_types import ModelServerRef -from nemo_gym.rollout_collection import _attach_trajectory_record, _strip_capture_payloads -from nemo_gym.rollout_observability import TrajectoryRecord, join_model_call_observations -from responses_api_agents.codex_agent.app import parse_exec_jsonl -from responses_api_agents.hermes_agent.observability import HermesAgentObserver -from responses_api_agents.opencode_agent.app import _parse_opencode_session -from responses_api_agents.pi_agent.app import _build_pi_observations, parse_pi_events - - -HARNESSES = ("opencode", "pi", "codex", "hermes") -MODEL = ModelServerRef(type="responses_api_models", name="policy") -ARGS = {"command": "printf ok"} - - -def _exchanges(dialect: str, *, session: str | None) -> list[dict]: - request_messages = [{"role": "user", "content": "Print ok, then report success."}] - tool = {"id": "tool-1", "type": "function", "function": {"name": "bash", "arguments": json.dumps(ARGS)}} - tool_output = {"role": "assistant", "content": None, "tool_calls": [tool]} - result = {"role": "tool", "tool_call_id": "tool-1", "content": "ok"} - records = [] - for index in range(2): - usage = { - "prompt_tokens": 10 + index, - "completion_tokens": 5, - "total_tokens": 15 + index, - "prompt_tokens_details": {"cached_tokens": 0}, - "completion_tokens_details": {"reasoning_tokens": 2}, - } - response = { - "id": f"response-{index}", - "model": "fixture-model", - "usage": usage, - "choices": [ - { - "index": 0, - "message": tool_output if index == 0 else {"role": "assistant", "content": "Done"}, - "finish_reason": "tool_calls" if index == 0 else "stop", - } - ], - } - request = { - "model": "fixture-model", - "messages": request_messages if index == 0 else [*request_messages, tool_output, result], - } - if dialect == "responses": - response = { - "id": f"response-{index}", - "model": "fixture-model", - "status": "completed", - "output": [ - {"type": "function_call", "call_id": "tool-1", "name": "bash", "arguments": json.dumps(ARGS)} - ] - if index == 0 - else [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Done"}]}], - "usage": { - "input_tokens": 10 + index, - "output_tokens": 5, - "total_tokens": 15 + index, - "input_tokens_details": {"cached_tokens": 0}, - "output_tokens_details": {"reasoning_tokens": 2}, - }, - } - request = { - "model": "fixture-model", - "input": request_messages - if index == 0 - else [ - *request_messages, - {"type": "function_call", "call_id": "tool-1", "name": "bash", "arguments": json.dumps(ARGS)}, - {"type": "function_call_output", "call_id": "tool-1", "output": "ok"}, - ], - } - records.append( - { - "model_call_id": f"call-{index}", - "client_session_id": session, - "model_ref": MODEL.model_dump(), - "dialect": dialect, - "started_at": float(index + 1), - "completed_at": float(index + 1.5), - "latency_ms": 500.0, - "status_code": 200, - "request": request, - "response": response, - } - ) - return records - - -def _opencode(directory: Path): - path = directory / "opencode.db" - with sqlite3.connect(path) as db: - db.execute("create table session (id text, parent_id text, time_created integer)") - db.execute("create table message (id text, session_id text, data text, time_created integer)") - db.execute("create table part (id text, message_id text, session_id text, data text, time_created integer)") - db.execute("insert into session values ('root', null, 0)") - for index in range(2): - message_id = f"message-{index}" - db.execute( - "insert into message values (?, ?, ?, ?)", - ( - message_id, - "root", - json.dumps( - { - "role": "assistant", - "time": {"created": 1000 + index * 1000, "completed": 1500 + index * 1000}, - } - ), - index, - ), - ) - part = ( - { - "type": "tool", - "tool": "bash", - "callID": "tool-1", - "state": { - "status": "completed", - "input": ARGS, - "output": "ok", - "time": {"start": 1600, "end": 1800}, - }, - } - if index == 0 - else {"type": "text", "text": "Done"} - ) - for number, value in enumerate([{"type": "step-start"}, part, {"type": "step-finish"}]): - db.execute( - "insert into part values (?, ?, ?, ?, ?)", - (f"{message_id}-{number}", message_id, "root", json.dumps(value), number), - ) - trajectory = TrajectoryRecord(task_id="0", rollout_id="0-0") - return _parse_opencode_session(path, "root", trajectory), trajectory - - -def _pi(): - events = [ - { - "type": "message_end", - "message": { - "role": "assistant", - "responseId": "response-0", - "content": [{"type": "toolCall", "id": "tool-1", "name": "bash", "arguments": ARGS}], - }, - }, - {"type": "tool_execution_start", "toolCallId": "tool-1", "toolName": "bash"}, - {"type": "tool_execution_end", "toolCallId": "tool-1", "toolName": "bash", "isError": False}, - { - "type": "message_end", - "message": {"role": "toolResult", "toolCallId": "tool-1", "content": [{"type": "text", "text": "ok"}]}, - }, - { - "type": "message_end", - "message": { - "role": "assistant", - "responseId": "response-1", - "content": [{"type": "text", "text": "Done"}], - }, - }, - {"type": "agent_end", "messages": [{"role": "assistant", "stopReason": "stop"}]}, - ] - output, _ = parse_pi_events("\n".join(json.dumps(event) for event in events)) - return _build_pi_observations([(float(i), event) for i, event in enumerate(events)], "root", MODEL, output) - - -class _HermesProtocol: - """Only the upstream hook interface; Gym's instrumentation runs unmodified.""" - - def __init__(self): - self.tool_start_callback = self.tool_complete_callback = None - self._active_children = [] - self.context_compressor = SimpleNamespace(last_prompt_tokens=0) - - def _interruptible_api_call(self, params): - return SimpleNamespace(id=params["response_id"]) - - def _invoke_tool(self, name, arguments, task): - return "ok" - - def _compress_context(self, messages, system_message, **kwargs): - return messages, system_message - - -def _hermes(): - agent = _HermesProtocol() - with ( - patch("responses_api_agents.hermes_agent.observability.time", return_value=1.0), - patch("responses_api_agents.hermes_agent.observability.monotonic", return_value=1.0), - ): - observer = HermesAgentObserver(root_invocation_id="root", model_ref=MODEL).instrument(agent) - agent._interruptible_api_call({"response_id": "response-0"}) - agent.tool_start_callback("tool-1", "bash", ARGS) - agent._invoke_tool("bash", ARGS, "task") - agent.tool_complete_callback("tool-1", "bash", ARGS, "ok") - agent._interruptible_api_call({"response_id": "response-1"}) - return observer.finish( - { - "completed": True, - "messages": [ - {"role": "user", "content": "Print ok, then report success."}, - { - "role": "assistant", - "content": "", - "tool_calls": [{"id": "tool-1", "function": {"name": "bash", "arguments": ARGS}}], - }, - {"role": "tool", "tool_call_id": "tool-1", "content": "ok"}, - {"role": "assistant", "content": "Done"}, - ], - } - ) - - -def build_harness_record(harness: str, directory: Path) -> dict: - """Build a collected record using real parser/observer and collector functions.""" - directory.mkdir(parents=True, exist_ok=True) - calls = [ - build_model_call_record(exchange, call_index=i) - for i, exchange in enumerate( - _exchanges( - "responses" if harness == "codex" else "chat", session="root" if harness == "opencode" else None - ) - ) - ] - row = { - "_ng_task_index": 0, - "_ng_rollout_index": 0, - "_ng_rollout_id": "0-0", - "reward": 1.0, - "ng_model_call_capture": {"rollout_id": "0-0", "calls": [c.model_dump(mode="json") for c in calls]}, - } - if harness == "codex": - events = [ - { - "type": "item.completed", - "item": { - "id": "tool-1", - "type": "command_execution", - "command": "printf ok", - "aggregated_output": "ok", - "exit_code": 0, - "status": "completed", - }, - }, - {"type": "item.completed", "item": {"id": "answer", "type": "agent_message", "text": "Done"}}, - {"type": "turn.completed", "usage": {"input_tokens": 21, "output_tokens": 10}}, - ] - output, usage = parse_exec_jsonl("\n".join(json.dumps(e) for e in events)) - row["response"] = {"output": [item.model_dump(mode="json") for item in output], "usage": usage} - # Codex's producer returns response items, not canonical observations or turns. - else: - if harness == "opencode": - observations, trajectory = _opencode(directory) - row["ng_trajectory"] = trajectory.model_dump(mode="json") - elif harness == "pi": - observations = _pi() - elif harness == "hermes": - observations = _hermes() - else: - raise ValueError(harness) - row["ng_agent_observations"] = join_model_call_observations(observations, calls).model_dump(mode="json") - _attach_trajectory_record(row, row) - _strip_capture_payloads(row) - return row diff --git a/tests/unit_tests/harness_capabilities/synthetic.py b/tests/unit_tests/harness_capabilities/synthetic.py new file mode 100644 index 00000000000..185d057be58 --- /dev/null +++ b/tests/unit_tests/harness_capabilities/synthetic.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Hand-authored evidence contracts, independent of harnesses and collector code. + +Keep the representations separate so mutations can expose conflicting copies. +These records specify checker inputs; they make no claims about any producer. +""" + +from copy import deepcopy + + +def evidence_record() -> dict: + """Return two synthetic policy steps with calls, executed tools and zero reward.""" + model_ref = {"type": "responses_api_models", "name": "synthetic-model"} + conversation = [{"role": "user", "content": "Read two values."}] + captures, calls, turns, tools = [], [], [], [] + for number in (1, 2): + reference = {"model_call_id": f"attempt-{number}", "response_id": f"response-{number}", "model_ref": model_ref} + request = {"input": deepcopy(conversation)} + tool_request = { + "type": "function_call", + "call_id": f"tool-{number}", + "name": "read_value", + "arguments": '{"key": "example"}', + } + tool_result = {"type": "function_call_output", "call_id": f"tool-{number}", "output": "value"} + response = { + "id": reference["response_id"], + "status": "completed", + "output": [tool_request], + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"cached_tokens": 2}, + "output_tokens_details": {"reasoning_tokens": 3}, + }, + } + metadata = { + "model_ref": model_ref, + "model": "synthetic-model", + "response_id": reference["response_id"], + "dialect": "responses", + "status_code": 200, + "response_status": "completed", + "finish_reason": None, + "error_category": None, + "client_session_id": "invocation", + } + captures.append( + { + **reference, + **metadata, + "tokens_in": 10, + "tokens_out": 5, + "tokens_total": 15, + "tokens_reasoning": 3, + "cached_tokens": 2, + } + ) + calls.append( + { + "model_call_id": reference["model_call_id"], + "response_metadata": metadata, + "request": request, + "response": response, + "token_stats": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + "reasoning_tokens": 3, + "cached_tokens": 2, + }, + } + ) + turns.append( + { + "invocation_id": "invocation", + "task_id": "0", + "rollout_id": "0-0", + "turn_no": number, + "timestamp": float(number), + "question": deepcopy(conversation), + "answer": [tool_request], + "resolved": None, + "model_calls": [reference], + } + ) + tools.append( + { + "kind": "tool_call", + "invocation_id": "invocation", + "tool_call_id": f"tool-{number}", + "tool_name": "read_value", + "status": "completed", + "output": "value", + } + ) + conversation.extend([tool_request, tool_result]) + invocation = { + "kind": "agent_invocation", + "invocation_id": "invocation", + "parent_invocation_id": None, + "model_calls": [deepcopy(turn["model_calls"][0]) for turn in turns], + "conversation": conversation, + } + return { + "_ng_task_index": 0, + "_ng_rollout_index": 0, + "reward": 0.0, + "response": {"output": deepcopy(conversation[1:])}, + "ng_model_call_capture": {"rollout_id": "0-0", "calls": deepcopy(captures)}, + "ng_agent_observations": {"records": deepcopy([invocation, *tools]), "gaps": []}, + "ng_trajectory": { + "schema_version": "1.0", + "task_id": "0", + "rollout_id": "0-0", + "gaps": [], + "invocations": [deepcopy(invocation)], + "model_calls": deepcopy(calls), + "turns": deepcopy(turns), + "tool_calls": deepcopy(tools), + }, + } diff --git a/tests/unit_tests/harness_capabilities/test_checker.py b/tests/unit_tests/harness_capabilities/test_checker.py index 36100095eb5..6ca48b53fb5 100644 --- a/tests/unit_tests/harness_capabilities/test_checker.py +++ b/tests/unit_tests/harness_capabilities/test_checker.py @@ -5,18 +5,18 @@ import copy import json -from pathlib import Path import pytest from nemo_gym.harness_capabilities.checker import EvidenceScope, inspect_record from nemo_gym.harness_capabilities.cli import inspect_bundle, main from nemo_gym.harness_capabilities.reader import hydrate_record +from tests.unit_tests.harness_capabilities.synthetic import evidence_record -@pytest.fixture(params=["opencode", "miniswe"]) -def record(request): - return json.loads((Path(__file__).parent / f"fixtures/{request.param}.json").read_text()) +@pytest.fixture +def record(): + return evidence_record() def verdict(record, capability): @@ -25,10 +25,10 @@ def verdict(record, capability): def test_retained_model_evidence_and_tools_pass(record): result = inspect_record(hydrate_record(record)) - assert all(result["evidence"][c]["verdict"] == "fulfilled" for c in ("TE-1", "TE-2", "TE-4", "TE-7", "TE-8")), ( - result["findings"] - ) - assert result["evidence"]["TE-5"]["verdict"] == "fulfilled" + assert all(result["evidence"][c]["verdict"] == "fulfilled" for c in (f"TE-{i}" for i in range(1, 10))), result[ + "findings" + ] + assert result["verdict"] == "fulfilled" assert result["is_behavioral_qualification"] is False @@ -159,8 +159,6 @@ def test_cli_atomic_replay_and_exit_codes(record, tmp_path): bundle = tmp_path / "input.jsonl" bundle.write_text(json.dumps(record) + "\n") output = tmp_path / "reports" - record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) - bundle.write_text(json.dumps(record) + "\n") args = ["inspect", "--bundle", str(bundle), "--output", str(output)] assert main(args) == 0 assert main(args) == 0 @@ -277,7 +275,7 @@ def test_na_requires_explicit_scope_and_rejects_contradictions(record): def test_step_join_can_qualify_without_invocation_refs(): - record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + record = evidence_record() for inv in record["ng_agent_observations"]["records"]: if inv["kind"] == "agent_invocation": inv["model_calls"] = [] @@ -288,7 +286,7 @@ def test_step_join_can_qualify_without_invocation_refs(): def test_duplicated_step_ref_fails_without_closure_requirement(): - record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + record = evidence_record() assert verdict(record, "TE-9") == "fulfilled" record["ng_trajectory"]["turns"][0]["model_calls"] *= 2 assert verdict(record, "TE-9") == "not_fulfilled" @@ -309,7 +307,7 @@ def test_binary_resolution_does_not_replace_reward(record): def test_canonical_only_delivery_does_not_require_capture_middleware(): - record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + record = evidence_record() del record["ng_model_call_capture"] for call in record["ng_trajectory"]["model_calls"]: call["started_at"] = call["completed_at"] = call["duration_ms"] = None @@ -331,7 +329,7 @@ def test_failed_response_without_response_id_preserves_status(record): def test_retry_and_compaction_attempts_have_distinct_accounting(): - record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + record = evidence_record() record = hydrate_record(record) retry = copy.deepcopy(record["ng_model_call_capture"]["calls"][0]) retry.update(model_call_id="retry", response_id="retry-response") @@ -358,7 +356,7 @@ def test_retry_and_compaction_attempts_have_distinct_accounting(): def test_conflicting_run_reference_cannot_be_hidden_by_step_join(): - record = json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + record = evidence_record() record["ng_agent_observations"]["records"][0]["model_calls"] = [{"model_call_id": "missing"}] assert inspect_record(hydrate_record(record))["verdict"] == "not_fulfilled" diff --git a/tests/unit_tests/harness_capabilities/test_producers.py b/tests/unit_tests/harness_capabilities/test_producers.py deleted file mode 100644 index e2cad34fa46..00000000000 --- a/tests/unit_tests/harness_capabilities/test_producers.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Exercise four current harness producers through the JSONL evidence checker.""" - -import json -from pathlib import Path - -import pytest - -from nemo_gym.harness_capabilities.checker import inspect_record -from nemo_gym.harness_capabilities.cli import inspect_matrix -from nemo_gym.harness_capabilities.reader import hydrate_record -from tests.unit_tests.harness_capabilities.producer_fixtures import HARNESSES, build_harness_record - - -@pytest.mark.parametrize("harness", HARNESSES) -def test_current_producer_evidence(harness, tmp_path): - record = build_harness_record(harness, tmp_path / harness) - result = inspect_record(hydrate_record(record)) - passed = {key for key, value in result["evidence"].items() if value["verdict"] == "fulfilled"} - expected = {"TE-1", "TE-2", "TE-4", "TE-6", "TE-7"} - if harness != "codex": - expected |= {"TE-5", "TE-8"} - assert passed == expected, result["findings"] - assert result["verdict"] == "not_fulfilled" - # Committed regression artifacts are reproducible outputs of these producers. - fixture = Path(__file__).parent / "fixtures" / f"{harness}.jsonl" - assert json.loads(fixture.read_text()) == record - - -def test_harness_matrix_uses_individual_report_verdicts(tmp_path): - fixtures = Path(__file__).parent / "fixtures" - path, result = inspect_matrix({name: fixtures / f"{name}.jsonl" for name in HARNESSES}, output=tmp_path) - table = (path / "harness_evidence.md").read_text() - assert all(name in table for name in HARNESSES) - assert all(row["verdict"] == "not_fulfilled" for row in result["harnesses"].values()) - assert "| codex | PASS | PASS | FAIL | PASS | FAIL | PASS | PASS | FAIL | FAIL | FAIL |" in table - assert inspect_matrix({name: fixtures / f"{name}.jsonl" for name in HARNESSES}, output=tmp_path)[0] == path diff --git a/tests/unit_tests/harness_capabilities/test_script.py b/tests/unit_tests/harness_capabilities/test_script.py index 09759eed598..c93389204c7 100644 --- a/tests/unit_tests/harness_capabilities/test_script.py +++ b/tests/unit_tests/harness_capabilities/test_script.py @@ -12,6 +12,7 @@ from nemo_gym.harness_capabilities import cli from nemo_gym.harness_capabilities.reader import hydrate_record +from tests.unit_tests.harness_capabilities.synthetic import evidence_record SCRIPT = Path(__file__).resolve().parents[3] / "scripts/check_harness_conformance.py" @@ -19,7 +20,7 @@ @pytest.fixture def record(): - return json.loads((Path(__file__).parent / "fixtures/miniswe.json").read_text()) + return evidence_record() @pytest.mark.parametrize( @@ -161,21 +162,41 @@ def test_reports_do_not_include_payload_values(record, tmp_path, capsys): assert secret not in capsys.readouterr().out -def test_script_matrix_routes_and_returns_failed_gate(tmp_path, monkeypatch): - fixtures = Path(__file__).parent / "fixtures" - args = [str(SCRIPT), "matrix", "--output", str(tmp_path)] - for name in ("opencode", "pi", "codex", "hermes"): - args.extend(["--harness", f"{name}={fixtures / (name + '.jsonl')}"]) +def test_script_matrix_routes_and_returns_failed_gate(record, tmp_path, monkeypatch): + passing = tmp_path / "passing.jsonl" + failing = tmp_path / "failing.jsonl" + passing.write_text(json.dumps(record) + "\n") + record["ng_trajectory"]["model_calls"][0]["request"] = None + failing.write_text(json.dumps(record) + "\n") + output = tmp_path / "reports" + args = [ + str(SCRIPT), + "matrix", + "--output", + str(output), + "--harness", + f"complete={passing}", + "--harness", + f"missing-payload={failing}", + ] monkeypatch.setattr(sys, "argv", args) with pytest.raises(SystemExit) as error: runpy.run_path(str(SCRIPT), run_name="__main__") assert error.value.code == 1 - (matrix,) = tmp_path.glob("*/harness_evidence.json") - assert set(json.loads(matrix.read_text())["harnesses"]) == {"opencode", "pi", "codex", "hermes"} - - -def test_matrix_error_cannot_publish_partial_matrix(tmp_path): - fixtures = Path(__file__).parent / "fixtures" + (matrix,) = output.glob("*/harness_evidence.json") + rows = json.loads(matrix.read_text())["harnesses"] + assert rows["complete"]["verdict"] == "fulfilled" + assert rows["missing-payload"]["evidence"]["TE-7"]["verdict"] == "not_fulfilled" + table = matrix.with_suffix(".md").read_text() + assert "| complete | PASS | PASS | PASS | PASS | PASS | PASS | PASS | PASS | PASS | PASS |" in table + assert "| missing-payload | PASS | PASS | PASS | FAIL | PASS | PASS | FAIL | PASS | PASS | FAIL |" in table + # Replaying the same matrix is idempotent. + assert cli.inspect_matrix({"complete": passing, "missing-payload": failing}, output=output)[0] == matrix.parent + + +def test_matrix_error_cannot_publish_partial_matrix(record, tmp_path): + bundle = tmp_path / "complete.jsonl" + bundle.write_text(json.dumps(record) + "\n") with pytest.raises(OSError): - cli.inspect_matrix({"opencode": fixtures / "opencode.jsonl", "missing": tmp_path / "missing"}, output=tmp_path) + cli.inspect_matrix({"complete": bundle, "missing": tmp_path / "missing"}, output=tmp_path) assert not list(tmp_path.glob("*/harness_evidence.json")) From bcf36026980f7a03e9b44303d7ac63c6cfd1e0f9 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Thu, 1 Oct 2026 15:19:17 +0200 Subject: [PATCH 06/11] Regenerate trajectory evidence table from tested source Signed-off-by: Frederic Lafrance --- ...e2373b4195530934ed44be4ead0d755892c90.json | 1367 +++++++++++++++++ .../reference/trajectory-capabilities.mdx | 35 +- 2 files changed, 1399 insertions(+), 3 deletions(-) create mode 100644 fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json diff --git a/fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json b/fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json new file mode 100644 index 00000000000..28b6b24f7b0 --- /dev/null +++ b/fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json @@ -0,0 +1,1367 @@ +{ + "gym_commit": "939cace5af1436789ead920d0f0b3e663784ad73", + "harnesses": { + "codex": { + "evidence": { + "TE-1": { + "observed": 6, + "passed": 6, + "required": 7 + }, + "TE-2": { + "observed": 6, + "passed": 6, + "required": 7 + }, + "TE-3": { + "observed": 5, + "passed": 0, + "required": 6 + }, + "TE-4": { + "observed": 6, + "passed": 6, + "required": 7 + }, + "TE-5": { + "observed": 5, + "passed": 0, + "required": 6 + }, + "TE-6": { + "observed": 5, + "passed": 5, + "required": 6 + }, + "TE-7": { + "observed": 6, + "passed": 6, + "required": 7 + }, + "TE-8": { + "observed": 6, + "passed": 0, + "required": 7 + }, + "TE-9": { + "observed": 6, + "passed": 0, + "required": 7 + } + }, + "scenarios": [ + { + "artifact_report": "evidence/5aab8de881f63b27a3e99f33d5bdd1b8304290240cdde9296afe41b0a1cc5cc8/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "c50c25be678c42b37f0deebb12c2fcd7492f8ff3312cf2de16ddd48f09c6980f", + "rollouts.jsonl": "3c172fb0b59eea8d992a9cfa439c92b48a994d05cc66a25994e02445511636b1", + "runtime.json": "4437e9ed95e1e690dd5ee530dffd7119542f6e79723025c7d9159c88a06f11af", + "witness.json": "4e3c897af2316413799be4965ad4127a1fa66fc8fa75cf1016b0929ad0fe9a49" + }, + "issues": [], + "model_attempts": 3, + "scenario": "tool_success", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/8635b8e327166b8641b7e3d5ad1e91694593d38b054b7e7166d634b088e9b4be/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "7e82e16e5bd9ca890fbfb9cebf92b4abcc85a893862184d333817e90e19ae9a4", + "rollouts.jsonl": "38aed6bb934098ff7b6631027597b32fc9ce65f7afcf0d9fd20aae8a5cfa7727", + "runtime.json": "4437e9ed95e1e690dd5ee530dffd7119542f6e79723025c7d9159c88a06f11af", + "witness.json": "f4d5cdae6317c0c76b1284be863ebef6c817abb5ae049ff6bbb32a62b03f5b7a" + }, + "issues": [], + "model_attempts": 3, + "scenario": "tool_failure", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/234c0a6881303ecccd98b7b41ec220ff9e2a57c63e5c3eee4f70b74257d458fa/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "79ac7968a1a850a65be51b3a561ae2f616ecd713e893af27f58dbd2ee6efac05", + "rollouts.jsonl": "9bf72d47b09efeea5531511a57ebb8dc1bbd293447e8779806ada4c806d5aa0a", + "runtime.json": "4437e9ed95e1e690dd5ee530dffd7119542f6e79723025c7d9159c88a06f11af", + "witness.json": "8317c66824810d8b5f2f0f4c022c6c652de122a8d75001d87b3f54b2de8d1255" + }, + "issues": [], + "model_attempts": 3, + "scenario": "usage_omitted", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/c2edb5bdcfc56c0948f8cd2a9386f16961a5d8d5c0123bf19408725fde55ac4d/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "not_fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "not_fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "not_fulfilled" + }, + "TE-5": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "not_fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "not_fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": false, + "hashes": { + "launch.json": "5676898c9df7531b4f9afa02929b097880444d13652e0fa67b92045a79e52a91", + "rollouts.jsonl": "3e91f09015df7cf3bf7ce48c92c86c372580a10fa5ada2b74b16da5892341518", + "runtime.json": "4437e9ed95e1e690dd5ee530dffd7119542f6e79723025c7d9159c88a06f11af", + "witness.json": "0b9cb6c2b2f788650d26a942db4e5b5655b7aab9222877350e0f7e31fad4de1a" + }, + "issues": [ + "not all prescribed model failures were observed", + "the harness did not finish the scripted model exchange", + "prescribed tool executions and their returned results were not all witnessed", + "expected verifier outcome was not observed", + "rollout reward differs from the verifier witness" + ], + "model_attempts": 1, + "scenario": "retry_429", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/33b51fd8b86da01a70ad351365a7abaff0421d23eb4a263da21d5ac87b1c0768/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "e3154d6e2ad5b275099f9a96fdb9759d706164790a78dfb0226735a52e897ddd", + "rollouts.jsonl": "3e8316c3cfb81c0d67bbb298354e33c1a2101fc49dcca44e18634721198bd13f", + "runtime.json": "4437e9ed95e1e690dd5ee530dffd7119542f6e79723025c7d9159c88a06f11af", + "witness.json": "b070b15eb369d4d0d21fe69e2126abae4fb013877b3f726748b88db3d8a2d1a9" + }, + "issues": [], + "model_attempts": 4, + "scenario": "retry_500", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/4617eb0101f2b497c44b7e662ea37f0f57d63405b15efbde10317c1ac2f2d25a/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "2754eea61baf1e27eff742d5786c89f19c2ae59cae31afb67af6135489b7ad76", + "rollouts.jsonl": "382a4674b6b0ef178a55bc076da9c5ce4e16c099a5a01e1ead4d7f7d8866cff2", + "runtime.json": "4437e9ed95e1e690dd5ee530dffd7119542f6e79723025c7d9159c88a06f11af", + "witness.json": "4dbdcf556eb11e0c6652d1568a1072f61d480f3a7a1459028c451a374dd70820" + }, + "issues": [], + "model_attempts": 1, + "scenario": "model_error", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/55cd5c8233a0e750072609c7b55cbfe01fc8019ac36df4d779baf6753b504bfb/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "c8f9eb1b0ebd66e0eca34d89d478ee3fefbd0ae3e7b7a9313d9a4eca0fdb0c0a", + "rollouts.jsonl": "c77fe7eb61475582c14f6c30a291a7ffc8598918a7d4ea656ede7479c294f3d0", + "runtime.json": "4437e9ed95e1e690dd5ee530dffd7119542f6e79723025c7d9159c88a06f11af", + "witness.json": "501e5bf89c623aff24b9048b28e56afd6985f2cd956540e5afa634a09f798940" + }, + "issues": [], + "model_attempts": 3, + "scenario": "verifier_failure", + "verdict": "not_fulfilled" + } + ], + "verdict": "not_fulfilled" + }, + "opencode": { + "evidence": { + "TE-1": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-2": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-3": { + "observed": 6, + "passed": 0, + "required": 6 + }, + "TE-4": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-5": { + "observed": 6, + "passed": 6, + "required": 6 + }, + "TE-6": { + "observed": 6, + "passed": 6, + "required": 6 + }, + "TE-7": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-8": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-9": { + "observed": 7, + "passed": 0, + "required": 7 + } + }, + "scenarios": [ + { + "artifact_report": "evidence/e5232389b937ab01cfe070df7a03eb8b24df3f1b7f676f461f31af1d22ecfc29/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "95035022e91224f57be8bc6d129b9faf09f0cf649a8859e68309f6f6a2fc0205", + "rollouts.jsonl": "b5e26b58c3851e2fa9faebfd06f469b7aede447df8b265aa86b377adc2ad827e", + "runtime.json": "a46cccea8c3ea498e0d64e2b2cd2dc7b4be04cd32bdd538d6792e7ef45fb6b90", + "witness.json": "44497e8898f50d180975bec04eb5181604351375f2d738442d24dc3d384c9598" + }, + "issues": [], + "model_attempts": 3, + "scenario": "tool_success", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/6b0770fb7ae5965e558b9ff5b1d9f18f0b5aebdd91c404a81e139420b5f621bf/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "f96e5b6706d56d3820da2313b6100e771d3dab9811de4d0a1cdb036843d4b72d", + "rollouts.jsonl": "4c44873cb87c55e9f498262fc92ffdb92ac4a5fb76e773768a1da1582164e2c0", + "runtime.json": "a46cccea8c3ea498e0d64e2b2cd2dc7b4be04cd32bdd538d6792e7ef45fb6b90", + "witness.json": "7ea7d666dca72290da2f62f616b124e481f577d3972102feae0bf3db7404f95c" + }, + "issues": [], + "model_attempts": 3, + "scenario": "tool_failure", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/e9d5ffdfc4b4fbb1b292760c2055d99a871a797ebaac5b9069704917669f0ef5/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "b41e7e113cd6706182c3b422b935e1280e4150eec8655371444eec2b10724234", + "rollouts.jsonl": "7ab816046bc16f73121e2eb011913c1632ce86acea7657eeeaed63cf045512ef", + "runtime.json": "a46cccea8c3ea498e0d64e2b2cd2dc7b4be04cd32bdd538d6792e7ef45fb6b90", + "witness.json": "ca056e25d34559310532f0734f234bd3ebdc7102f7366bc7a800b11c516225bc" + }, + "issues": [], + "model_attempts": 3, + "scenario": "usage_omitted", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/cdf1ff8136bb5713965009ae6f09cf9a949e43e6556f8b2230995f0268ef220c/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "197f663e6a8dfe2e0e0115c031b4bc25e972fd557d28321cbef38013b06bed0f", + "rollouts.jsonl": "e933b9bc62357b9d7ebb13cd033a380b15c4442760026978f21dea3f63962583", + "runtime.json": "a46cccea8c3ea498e0d64e2b2cd2dc7b4be04cd32bdd538d6792e7ef45fb6b90", + "witness.json": "c2ebaf92715c66abc437058928030eaa5a053cf46a0cf7e100846111662c9935" + }, + "issues": [], + "model_attempts": 5, + "scenario": "retry_429", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/e0fb5dadabf66769febc3bc2cb67dca945b79dc3310c3afe005ce69a7eba2b17/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "9712561039d614925aa8984a3cf33e47dd67b51f2854e72705ed7698e1ef77a0", + "rollouts.jsonl": "d140402dbeebcbcfa5f3c59c3cbf7882fe8094a8299cf25f5e4640486512b481", + "runtime.json": "a46cccea8c3ea498e0d64e2b2cd2dc7b4be04cd32bdd538d6792e7ef45fb6b90", + "witness.json": "2a37c3b38771b2d82fff89ae4aafaf2befa29f3a0ba9d802cc4f5543fbb5fe15" + }, + "issues": [], + "model_attempts": 4, + "scenario": "retry_500", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/609152e30c9d3f2d0d2e76d16dbb37cce5dd91eefbbef1558699413b7d480843/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "9334616ebd21fad19e9dbb79dfa6c2b1f5df62aeaab74c5ad6982f20b2b70f8d", + "rollouts.jsonl": "ed32822271f1bf7fa60a78c3c6c3539bcc051f6ff97acadb2cb24941e05a3265", + "runtime.json": "a46cccea8c3ea498e0d64e2b2cd2dc7b4be04cd32bdd538d6792e7ef45fb6b90", + "witness.json": "c159595d3ab18f4406c8aa10bc1be737d9e4e58e635352139ea87e970a85c389" + }, + "issues": [], + "model_attempts": 1, + "scenario": "model_error", + "verdict": "fulfilled" + }, + { + "artifact_report": "evidence/d1c01b2c5bc34c8b3855967709ffa13369c2130c71ef2384a0ade94241c00c9f/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "b2fc75912e57bb2216a2e96739db4f8ddc68de629bbe2e2744c6a36750fe7c82", + "rollouts.jsonl": "a1e223faf33a33e468b40d01922cc7bef379311626d4942e2623b00aad9566cb", + "runtime.json": "a46cccea8c3ea498e0d64e2b2cd2dc7b4be04cd32bdd538d6792e7ef45fb6b90", + "witness.json": "b2f221214160762c7304ae1a85e55e32d6166a8905414766d05bfed3747a5ff6" + }, + "issues": [], + "model_attempts": 3, + "scenario": "verifier_failure", + "verdict": "not_fulfilled" + } + ], + "verdict": "not_fulfilled" + }, + "pi": { + "evidence": { + "TE-1": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-2": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-3": { + "observed": 6, + "passed": 0, + "required": 6 + }, + "TE-4": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-5": { + "observed": 6, + "passed": 6, + "required": 6 + }, + "TE-6": { + "observed": 6, + "passed": 6, + "required": 6 + }, + "TE-7": { + "observed": 7, + "passed": 7, + "required": 7 + }, + "TE-8": { + "observed": 7, + "passed": 4, + "required": 7 + }, + "TE-9": { + "observed": 7, + "passed": 0, + "required": 7 + } + }, + "scenarios": [ + { + "artifact_report": "evidence/0bb8cf29a2a7c14b622fa34b28b6f7da620a9461fe971e22018a484130bde6fe/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "7394b02a04cc7d2a845049e538c244c94f4498b472becf580aca7cf5e0480f4e", + "rollouts.jsonl": "a3c7b4f84fc02db3c9989f17324652571bbac4c9466b93122b74cc6d97ebe767", + "runtime.json": "92dcc997ba9de317b53c51cfee0e18a60f5b40e52d01f1783e8729a0d6ea5c29", + "witness.json": "7a0cc7d8bf6ed11d6f28fa3685b47b743a42f7c95e74c74cbf36f19940abf608" + }, + "issues": [], + "model_attempts": 3, + "scenario": "tool_success", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/2ce996aa1ca4c3e16eaf6346c7ddfddfd2d48f97b6c5983b44d8355486d7a199/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "239666fc79a08d0f2a6f9f5f6e351054c57190b33d19e3cecbc41ce8db7f0c2c", + "rollouts.jsonl": "a6ad3c41cb16326d2d9dd27d7c08d264840054ac6129a71936d6c5322f891fc7", + "runtime.json": "92dcc997ba9de317b53c51cfee0e18a60f5b40e52d01f1783e8729a0d6ea5c29", + "witness.json": "cae8dde033e4b150eb3529e56cebc7ca7e1acce0805a8116d694f7b5102079b9" + }, + "issues": [], + "model_attempts": 3, + "scenario": "tool_failure", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/059d55d2a83c6186c48ddc841a16c2d2c4b220eaaa517d97a8f06487399bcc8c/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "c8b310032dd4399433d10ed76e22a28ab69f79770ae13d4669319ae18ad11fce", + "rollouts.jsonl": "575ecc5fc96ac251adc0e5392cea1dc17c5d9e79e3b399f661547c38e5f0cc07", + "runtime.json": "92dcc997ba9de317b53c51cfee0e18a60f5b40e52d01f1783e8729a0d6ea5c29", + "witness.json": "6a4280069265417ba2f9c77bc6888898c57fabff8338773bafb4da9c73841f84" + }, + "issues": [], + "model_attempts": 3, + "scenario": "usage_omitted", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/55582c015422508f8c7c140969a76d14f8bb153377689a0b50e675369ea01f6b/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "122fb0f2c48df8e4d3db2d611bbe979a75a3c511d952e5a569daa6e47a8b0544", + "rollouts.jsonl": "6db981a0e44963ac2cebd18e95b996668728e05c4e8ad0c7c9619698abb06eb4", + "runtime.json": "92dcc997ba9de317b53c51cfee0e18a60f5b40e52d01f1783e8729a0d6ea5c29", + "witness.json": "043a09551112360f830bf19c7962daa3af27379e8fdd8db16c2aad41ea21994f" + }, + "issues": [], + "model_attempts": 5, + "scenario": "retry_429", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/1f9307f26aa79dd49a44c12885819278f115c340ae278a09334d34d6586c7892/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "d99dfa7be0f59f455cbd4a36acb9346f13d2da85a2909f23075e3f6efcdf4e8c", + "rollouts.jsonl": "c970c689ebaaf0d2600cac31e87a9be34dd9a50c27678030e5c28bc7c1300a40", + "runtime.json": "92dcc997ba9de317b53c51cfee0e18a60f5b40e52d01f1783e8729a0d6ea5c29", + "witness.json": "1349969761533812d179fd4877f5e3f397616fa011965873318285160079388a" + }, + "issues": [], + "model_attempts": 4, + "scenario": "retry_500", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/4ee86caed333a92e96f1d58021daab2683b0db1873b1ced22a71f882631cbde5/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "33dd7a4c1ac3a5f279004f76e1bd725c446a88245daabdda1368944d7d504eca", + "rollouts.jsonl": "3ae4f7e18833b39efcc1d049bb0b493fe02dabb3abd3d9d3cbc1954b063bbc00", + "runtime.json": "92dcc997ba9de317b53c51cfee0e18a60f5b40e52d01f1783e8729a0d6ea5c29", + "witness.json": "0020ca50a1e5110d20886c464fa674ed0871cb3c8a060840d1f7a317c7f370e3" + }, + "issues": [], + "model_attempts": 1, + "scenario": "model_error", + "verdict": "not_fulfilled" + }, + { + "artifact_report": "evidence/054dde5e54cd4ec58476286fcd0a58f3f30c4cc8edb9e15f526d344e14f32c0a/evidence_summary.json", + "evidence": { + "TE-1": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-2": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-3": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + }, + "TE-4": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-5": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-6": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-7": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-8": { + "artifact_verdict": "fulfilled", + "verdict": "fulfilled" + }, + "TE-9": { + "artifact_verdict": "not_fulfilled", + "verdict": "not_fulfilled" + } + }, + "execution": { + "returncode": 0, + "timed_out": false + }, + "exercised": true, + "hashes": { + "launch.json": "bffb47a9144db73f6850d82ce2ba7fc5c46a6d99253d28020746f32021b8dece", + "rollouts.jsonl": "549d97e6e6ce42eabe7b3efefdf373cc5fff8bceb003cba74846e71fb7d768a7", + "runtime.json": "92dcc997ba9de317b53c51cfee0e18a60f5b40e52d01f1783e8729a0d6ea5c29", + "witness.json": "9e4138efff5527adf79f43f5ae7afdbeae1b416d4b33e7c954c1af2dccdb3dce" + }, + "issues": [], + "model_attempts": 3, + "scenario": "verifier_failure", + "verdict": "not_fulfilled" + } + ], + "verdict": "not_fulfilled" + } + }, + "limits": [ + "local harness runtime and controlled Chat/Responses model only; no remote sandbox qualification", + "TE-10, P1, multimodal, compaction, parallelism and deployment health are outside this suite" + ], + "python": "3.13.14 (main, Aug 5 2026, 15:33:38) [Clang 22.1.3 ]", + "runtimes": { + "codex": { + "adapter_sha256": "bf55ed52f7bbb05b7a76237cc5680ef14246e4fa94b6c76a6ec3530936b0409c", + "version": "codex-cli 0.155.0-alpha.9.2" + }, + "opencode": { + "adapter_sha256": "cbb12f7a386c9d129983f0d0bf5facdb4cb57a33d430c80b92ad8fc1374a7f3d", + "version": "1.17.11" + }, + "pi": { + "adapter_sha256": "3b99e9a60de5d6e825cb16dd18c63b8bb9afd3e9161d669280d4cd121afa08e1", + "version": "0.80.2" + } + }, + "schema_version": "trajectory-capabilities-table/v1", + "suite": "harness-p0-probes/v1", + "suite_sha256": "4cb1ccd9d4a64138a34e5f8940da7c66b55b667268ea25be791c93a45376ff65", + "summary_sha256": "40fcfb2c6d4c957cc25579b91d32e24a1b380407ff42ed219c9ad9e4c5b68355", + "tests": { + "checker": { + "junit_sha256": "08b30f56476faa1acf4a270c5180372cb420aa10d2491005054258906d729aa4", + "log_sha256": "f12116fbb42fe25165fc0e9088bcdaf5cb1ce52eef7c22d4e85a29639095cd2d", + "passed": 77, + "path": "tests/unit_tests/harness_capabilities", + "skipped": 0 + }, + "codex": { + "junit_sha256": "56e3535a7b316e34f81dd1de7040c2198f5645aba75b306f9418e327d1cf2bc0", + "log_sha256": "cc9e8f3c660bf05578ff2450402f4b974c78d4986e877a9adf2347abd2b33577", + "passed": 46, + "path": "responses_api_agents/codex_agent/tests", + "skipped": 0 + }, + "opencode": { + "junit_sha256": "02509b146f5404252701f8f823d9594e7f6734563c98c6695dac54b39a749b32", + "log_sha256": "b21aafd27505c80ec99c15833125f562d6a8cb87ec597e18223c32f0d7795b98", + "passed": 82, + "path": "responses_api_agents/opencode_agent/tests", + "skipped": 0 + }, + "pi": { + "junit_sha256": "749ecd9ae6709c60547f295ab303d72cdabc00ddd840146bfb8413ea238d4ae4", + "log_sha256": "6fb5bcee3ae2c68a7bbdc839dac6ef9c040dc5398befa5c97439e3734f596c9b", + "passed": 34, + "path": "responses_api_agents/pi_agent/tests", + "skipped": 0 + }, + "runner": { + "junit_sha256": "944724921767038266dc2010cfce517c113e0063a5356e628d87e07f08742fff", + "log_sha256": "788b4e62074bb67ede85563607cab6736fa871110bffb2939f4494a88b4ab14b", + "passed": 49, + "path": "scripts/harness_conformance/tests", + "skipped": 0 + } + } +} diff --git a/fern/versions/latest/pages/reference/trajectory-capabilities.mdx b/fern/versions/latest/pages/reference/trajectory-capabilities.mdx index e39ca783d09..3ab62a7da96 100644 --- a/fern/versions/latest/pages/reference/trajectory-capabilities.mdx +++ b/fern/versions/latest/pages/reference/trajectory-capabilities.mdx @@ -21,7 +21,36 @@ execution, TE-10, and P1 are outside this suite. {/* BEGIN GENERATED TE TABLE */} -No gated generation has been published yet. Run the command below to populate the table. +Gym commit: [`939cace5af1436789ead920d0f0b3e663784ad73`](https://github.com/NVIDIA-NeMo/Gym/commit/939cace5af1436789ead920d0f0b3e663784ad73). +Suite: `harness-p0-probes/v1`. [Generation report](../../../../assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json). + +Cells show **passing / exercised / required** scenarios. PASS requires every required scenario; +FAIL includes missing exercise or missing evidence. **Not run** makes no capability claim. + +| Harness | TE-1 | TE-2 | TE-3 | TE-4 | TE-5 | TE-6 | TE-7 | TE-8 | TE-9 | P0 | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `opencode` | PASS 7/7/7 | PASS 7/7/7 | FAIL 0/6/6 | PASS 7/7/7 | PASS 6/6/6 | PASS 6/6/6 | PASS 7/7/7 | PASS 7/7/7 | FAIL 0/7/7 | FAIL | +| `pi` | PASS 7/7/7 | PASS 7/7/7 | FAIL 0/6/6 | PASS 7/7/7 | PASS 6/6/6 | PASS 6/6/6 | PASS 7/7/7 | FAIL 4/7/7 | FAIL 0/7/7 | FAIL | +| `codex` | FAIL 6/6/7 | FAIL 6/6/7 | FAIL 0/5/6 | FAIL 6/6/7 | FAIL 0/5/6 | FAIL 5/5/6 | FAIL 6/6/7 | FAIL 0/6/7 | FAIL 0/6/7 | FAIL | +| `hermes` | Not run | Not run | Not run | Not run | Not run | Not run | Not run | Not run | Not run | Not run | + +P0 requires TE-1–TE-7 and either TE-8 or TE-9 **per scenario**. + +### Tested dependencies + +- `opencode`: `1.17.11` +- `pi`: `0.80.2` +- `codex`: `codex-cli 0.155.0-alpha.9.2` + +### Unit-test gates + +| Suite | Passed | Skipped | +| --- | ---: | ---: | +| `runner` | 49 | 0 | +| `opencode` | 82 | 0 | +| `pi` | 34 | 0 | +| `codex` | 46 | 0 | +| `checker` | 77 | 0 | {/* END GENERATED TE TABLE */} @@ -73,8 +102,8 @@ claims, not guarantees for every configuration. The statuses below retain the coverage reported in [main at 88d9d453](https://github.com/NVIDIA-NeMo/Gym/blob/88d9d45385f15b1351cdd055fff396186000f84f/fern/versions/latest/pages/reference/trajectory-capabilities.mdx). That source does not record a full runtime-validation date or per-row test configuration. -Contributors changing an agent should update its row with the config, execution path, and validation -evidence in the same pull request. +New validation results belong in the generated TE table above; these historical rows are retained +for comparison. ### Model Calls and History From d3c74bf28ba49f06dcc14ffb567dc80a0872b8d0 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Thu, 1 Oct 2026 16:05:22 +0200 Subject: [PATCH 07/11] Validate conformance evidence against shared rollout models Signed-off-by: Frederic Lafrance --- nemo_gym/harness_capabilities/README.md | 24 +++ nemo_gym/harness_capabilities/checker.py | 147 ++++++++++-------- nemo_gym/harness_capabilities/cli.py | 15 +- nemo_gym/harness_capabilities/contracts.py | 61 ++++++++ nemo_gym/harness_capabilities/reader.py | 11 +- nemo_gym/harness_capabilities/schemas.py | 59 ------- .../harness_capabilities/synthetic.py | 11 +- .../harness_capabilities/test_checker.py | 144 +++++++++++++++++ 8 files changed, 341 insertions(+), 131 deletions(-) create mode 100644 nemo_gym/harness_capabilities/contracts.py delete mode 100644 nemo_gym/harness_capabilities/schemas.py diff --git a/nemo_gym/harness_capabilities/README.md b/nemo_gym/harness_capabilities/README.md index dd3e4bf9c13..1bd604d4e72 100644 --- a/nemo_gym/harness_capabilities/README.md +++ b/nemo_gym/harness_capabilities/README.md @@ -12,6 +12,30 @@ Use `matrix --harness NAME=PATH` to compare multiple harnesses. The checker uses TE-1–TE-9 and the `gym-p0/v1` profile. Reports are `evidence_summary.json`, `evidence_results.jsonl`, and `evidence_report.md`. +Evidence objects must validate Gym's shared models at these paths: + +| JSON path | Model | +|---|---| +| `ng_model_call_capture.calls[]` | `ModelCallRecord` | +| `ng_trajectory.model_calls[]` | `TrajectoryModelCall` | +| `ng_trajectory.turns[]` | `TrajectoryTurn` | +| `ng_trajectory.invocations[]` | `AgentInvocation` | +| `ng_trajectory.tool_calls[]` | `TrajectoryToolCall` | +| `ng_agent_observations.records[]` | `AgentObservationRecord` (discriminated by `kind`) | + +Validation is strict and reports the object's JSON path. Invalid objects are +record integrity failures; missing evidence is evaluated by the TE checks and +explicit applicability. Both trajectory and observation representations are +validated when present. Invocation and tool checks retain their supported +fallback between these paths. + +TE requirements apply on top of model validity. For example, TE-3 requires a +non-null `question`, a non-null `answer` or `reasoning_content`, and an explicitly +present `resolved` value (including `null` for unknown). TE-5 requires a tool name +and terminal status; TE-1 requires model/server identity and a supported dialect; +TE-2 requires nonnegative counts consistent with retained provider usage. Reports +hash the path/model registry and shared model sources as well as the checker. + Unit tests use hand-authored contracts in `tests/unit_tests/harness_capabilities/synthetic.py`; they validate the checker independently of harness implementations. To measure actual harnesses and regenerate the commit-pinned documentation table, use diff --git a/nemo_gym/harness_capabilities/checker.py b/nemo_gym/harness_capabilities/checker.py index 2766797c628..57f6c4a4068 100644 --- a/nemo_gym/harness_capabilities/checker.py +++ b/nemo_gym/harness_capabilities/checker.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Deterministic checks over native Gym JSON; no producer runtime imports.""" +"""Deterministic model and semantic checks over native Gym JSON.""" import base64 import binascii @@ -10,9 +10,11 @@ from dataclasses import asdict, dataclass from typing import Iterable, TypeGuard -from jsonschema import Draft202012Validator +from pydantic import ValidationError -from .schemas import CALL_REF, SCHEMAS, TOKEN_FIELDS +from nemo_gym.rollout_observability import ModelCallRef + +from .contracts import TOKEN_FIELDS, model_errors NAMES = { @@ -54,19 +56,6 @@ class Finding: reason: str -def _schema_issues(capability: str, value: object, location: str) -> list[Finding]: - # Never put error.message in reports: jsonschema embeds source payloads. - return [ - Finding( - capability, - "schema." + str(error.validator), - location + "/" + "/".join(map(str, error.absolute_path)), - "required artifact contract is not satisfied", - ) - for error in Draft202012Validator(SCHEMAS[capability]).iter_errors(value) - ] - - def _objects(value: object) -> list[dict]: return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] @@ -111,7 +100,14 @@ def _missing_content(value: object) -> bool: def _resolve(reference: dict, calls: list[dict]) -> list[int]: - if not Draft202012Validator(CALL_REF).is_valid({k: v for k, v in reference.items() if v is not None}): + try: + ModelCallRef.model_validate(reference, strict=True) + except ValidationError: + return [] + # TE joins require nonempty identifiers even where the model allows empty strings. + if any(reference.get(key) == "" for key in ("model_call_id", "response_id")): + return [] + if reference.get("model_ref") is not None and not reference["model_ref"].get("name"): return [] return [ index @@ -173,13 +169,28 @@ def __init__(self, record: dict, source: str, scope: EvidenceScope) -> None: self.bundle = _mapping(self.record.get("ng_agent_observations")) self.calls = _objects(self.capture.get("calls")) self.observations = _objects(self.bundle.get("records")) + self.invocation_locations = [ + f"/ng_agent_observations/records/{i}" + for i, r in enumerate(self.observations) + if r.get("kind") == "agent_invocation" + ] self.invocations = [r for r in self.observations if r.get("kind") == "agent_invocation"] if not self.invocations: self.invocations = _objects(self.trajectory.get("invocations")) + self.invocation_locations = [f"/ng_trajectory/invocations/{i}" for i in range(len(self.invocations))] self.turns = _objects(self.trajectory.get("turns")) self.tools = _objects(self.trajectory.get("tool_calls")) or [ r for r in self.observations if r.get("kind") == "tool_call" ] + self.tool_locations = ( + [f"/ng_trajectory/tool_calls/{i}" for i in range(len(self.tools))] + if _objects(self.trajectory.get("tool_calls")) + else [ + f"/ng_agent_observations/records/{i}" + for i, r in enumerate(self.observations) + if r.get("kind") == "tool_call" + ] + ) self.owners: dict[int, list[str]] = {i: [] for i in range(len(self.calls))} self.invocation_ids = [i.get("invocation_id") for i in self.invocations] @@ -214,20 +225,6 @@ def check_identity(self) -> None: "/ng_trajectory/rollout_id", "trajectory and capture identities differ", ) - if "calls" in self.capture and ( - not isinstance(self.capture["calls"], list) or len(self.calls) != len(self.capture["calls"]) - ): - self._fail( - "record", - "capture.shape", - "/ng_model_call_capture/calls", - "a list of call objects is required", - ) - for owner, fields in ((self.trajectory, ("turns", "tool_calls", "invocations")), (self.bundle, ("records",))): - for field in fields: - values = owner.get(field) - if values is not None and (not isinstance(values, list) or len(_objects(values)) != len(values)): - self._fail("record", "records.shape", "/" + field, "evidence records must be a list of objects") for issue in self.record.get("_capability_reader_issues", []): self._fail("record", "reader.integrity", "", issue) @@ -249,7 +246,13 @@ def check_model_calls(self) -> None: self._fail("TE-1", "identity.unique", "/ng_model_call_capture/calls", "duplicate call identity") for index, call in enumerate(self.calls): location = f"/ng_model_call_capture/calls/{index}" - self.findings.extend(_schema_issues("TE-1", call, self.source + location)) + self._require_text("TE-1", call, ("model_call_id", "dialect"), location) + if not call.get("model_ref") or not call["model_ref"].get("name"): + self._fail("TE-1", "model_ref.required", location, "explicit model server identity is required") + if call.get("dialect") not in {"responses", "chat", "messages"}: + self._fail("TE-1", "dialect.supported", location + "/dialect", "unsupported capture dialect") + if call.get("status_code") is not None and not 100 <= call["status_code"] <= 599: + self._fail("TE-1", "status.range", location + "/status_code", "invalid HTTP status") response = _mapping(call.get("response")) status = call.get("status_code") http_success = type(status) is int and 200 <= status < 300 @@ -302,10 +305,10 @@ def check_token_counts(self) -> None: """TE-2: preserve supplied usage on all attempts; report availability separately.""" for index, call in enumerate(self.calls): location = f"/ng_model_call_capture/calls/{index}" - self.findings.extend(_schema_issues("TE-2", call, self.source + location)) + self._check_token_values(call, location) usage = _mapping(_mapping(call.get("response")).get("usage")) expected = _provider_usage(usage) - self.findings.extend(_schema_issues("TE-2", expected, self.source + location + "/response/usage")) + self._check_token_values(expected, location + "/response/usage") for key, value in expected.items(): if call.get(key) != value: self._fail( @@ -409,8 +412,10 @@ def check_ownership(self) -> None: ) parents = {i.get("invocation_id"): i.get("parent_invocation_id") for i in self.invocations} for index, invocation in enumerate(self.invocations): - location = f"/invocations/{index}" - self.findings.extend(_schema_issues("TE-8", invocation, self.source + location)) + location = self.invocation_locations[index] + self._require_text("TE-8", invocation, ("invocation_id",), location) + if "model_calls" not in invocation: + self._fail("TE-8", "references.required", location + "/model_calls", "explicit call refs are required") parent = invocation.get("parent_invocation_id") seen = {invocation.get("invocation_id")} while parent is not None and parent not in seen and parent in parents: @@ -450,16 +455,22 @@ def check_ownership(self) -> None: "owner conflicts with captured client session", ) - def _check_records(self, capability: str, records: list[dict]) -> None: + def _require_text(self, capability: str, item: dict, fields: tuple[str, ...], location: str) -> None: + for field in fields: + if not item.get(field): + self._fail(capability, "identity.required", location + "/" + field, "nonempty value is required") + + def _check_token_values(self, values: dict, location: str) -> None: + for field in TOKEN_FIELDS: + value = values.get(field) + if value is not None and (type(value) is not int or value < 0): + self._fail( + "TE-2", "usage.count", location + "/" + field, "count must be a nonnegative integer or null" + ) + + def _check_records(self, capability: str, records: list[dict], location: str) -> None: if not records: - self._fail( - capability, - "records.required", - "/ng_trajectory", - "required records are absent", - ) - for index, item in enumerate(records): - self.findings.extend(_schema_issues(capability, item, f"{self.source}/{capability}/{index}")) + self._fail(capability, "records.required", location, "required records are absent") def check_turns(self) -> None: """TE-3 contains step fields; TE-9 independently checks call linkage.""" @@ -476,17 +487,25 @@ def check_turns(self) -> None: return if self.record.get("_ng_rollout_index") is None: self._fail("TE-3", "identity.repeat", "", "explicit repeat identity is required") - self._check_records("TE-3", self.turns) + self._check_records("TE-3", self.turns, "/ng_trajectory/turns") keys = [(t.get("invocation_id"), t.get("turn_no")) for t in self.turns] if len(set(keys)) != len(keys): self._fail("TE-3", "turn.unique", "/ng_trajectory/turns", "duplicate step identity") - for turn in self.turns: - if any(turn.get(k) != self.trajectory.get(k) for k in ("task_id", "rollout_id")): - self._fail("TE-3", "turn.identity", "/ng_trajectory/turns", "step identity differs from rollout") - if "answer" not in turn and "reasoning_content" not in turn: + for index, turn in enumerate(self.turns): + location = f"/ng_trajectory/turns/{index}" + self._require_text("TE-3", turn, ("invocation_id", "task_id", "rollout_id"), location) + if turn.get("question") is None: + self._fail( + "TE-3", "turn.question", location + "/question", "non-null model-visible prompt is required" + ) + if "resolved" not in turn: self._fail( - "TE-3", "turn.content", "/ng_trajectory/turns", "answer/tool or reasoning field is required" + "TE-3", "turn.resolved", location + "/resolved", "resolution or explicit unknown is required" ) + if any(turn.get(k) != self.trajectory.get(k) for k in ("task_id", "rollout_id")): + self._fail("TE-3", "turn.identity", location, "step identity differs from rollout") + if turn.get("answer") is None and turn.get("reasoning_content") is None: + self._fail("TE-3", "turn.content", location, "answer/tool or reasoning field is required") def check_step_join(self) -> None: """TE-9: exact policy-attempt membership, independent of invocation refs.""" @@ -575,7 +594,7 @@ def check_tools(self) -> None: else: self.not_applicable.add("TE-5") return - self._check_records("TE-5", self.tools) + self._check_records("TE-5", self.tools, "/ng_trajectory/tool_calls") conversations = { ( invocation.get("invocation_id"), @@ -598,7 +617,10 @@ def check_tools(self) -> None: if kind == "function_call_output" and (invocation_id, call_id) not in tool_keys: self._fail("TE-5", "tool.execution", "/tool_calls", "a tool result has no execution record") for index, tool in enumerate(self.tools): - location = f"/tool_calls/{index}" + location = self.tool_locations[index] + self._require_text("TE-5", tool, ("invocation_id", "tool_call_id", "tool_name"), location) + if tool.get("status") not in {"completed", "failed", "timeout", "cancelled"}: + self._fail("TE-5", "tool.terminal", location + "/status", "terminal tool outcome is required") tool_key = (tool.get("invocation_id"), tool.get("tool_call_id")) request = conversations.get((*tool_key, "function_call"), {}) output = conversations.get((*tool_key, "function_call_output"), {}).get("output", tool.get("output")) @@ -618,17 +640,6 @@ def check_tools(self) -> None: location, "execution has neither output nor error evidence", ) - if ( - _number(tool.get("started_at")) - and _number(tool.get("completed_at")) - and tool["completed_at"] < tool["started_at"] - ): - self._fail( - "TE-5", - "tool.interval", - location, - "completion precedes start", - ) def check_verifier(self) -> None: """TE-6's shipped Gym surface: rollout verification reward and terminal resolution.""" @@ -746,6 +757,14 @@ def result(self) -> dict: def inspect_record(record: dict, *, source: str = "record", scope: EvidenceScope = EvidenceScope()) -> dict: """Check TE-1–TE-9 on retained JSONL; do not infer missing evidence or applicability.""" checks = _RecordInspector(record, source, scope) + checks.findings.extend( + Finding("record", code, source + path, "object at the evidence path does not validate its Gym model") + for path, code in model_errors(record) + ) + if checks.findings: + # Semantic checks require typed identities and containers. An invalid native + # object is a record integrity failure, never partially valid evidence. + return checks.result() checks.check_identity() checks.check_capture_presence() checks.check_model_calls() diff --git a/nemo_gym/harness_capabilities/cli.py b/nemo_gym/harness_capabilities/cli.py index 1c4fc35b910..2118ec116d0 100644 --- a/nemo_gym/harness_capabilities/cli.py +++ b/nemo_gym/harness_capabilities/cli.py @@ -14,8 +14,8 @@ from . import __version__ from .checker import NAMES, PROFILE, EvidenceScope, inspect_record +from .contracts import PATH_MODELS, SCHEMA_VERSION from .reader import digest_file, hydrate_record, json_rows -from .schemas import SCHEMA_VERSION, SCHEMAS def _json(value: object) -> str: @@ -56,10 +56,15 @@ def inspect_bundle( raise ValueError("capture directory does not exist") sources.extend(sorted(capture_dir.glob("*.capture.*"))) hashes = {str(path.resolve()): digest_file(path) for path in sources} - registry_hash = hashlib.sha256(_json({"schemas": SCHEMAS, "profile": PROFILE}).encode()).hexdigest() - checker_hash = hashlib.sha256( - "".join(digest_file(p) for p in sorted(Path(__file__).parent.glob("*.py"))).encode() - ).hexdigest() + registry = {path: adapter.json_schema() for path, adapter in PATH_MODELS.items()} + registry_hash = hashlib.sha256(_json({"path_models": registry, "profile": PROFILE}).encode()).hexdigest() + # Include model validators as well as generated shapes in report identity. + gym = Path(__file__).parent.parent + checker_sources = sorted(Path(__file__).parent.glob("*.py")) + [ + gym / name + for name in ("rollout_observability.py", "base_responses_api_model.py", "config_types.py", "openai_utils.py") + ] + checker_hash = hashlib.sha256("".join(digest_file(p) for p in checker_sources).encode()).hexdigest() manifest = { "sources": hashes, "registry_sha256": registry_hash, diff --git a/nemo_gym/harness_capabilities/contracts.py b/nemo_gym/harness_capabilities/contracts.py new file mode 100644 index 00000000000..75da5e5990e --- /dev/null +++ b/nemo_gym/harness_capabilities/contracts.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Native evidence paths and their authoritative Gym models. + +Validate retained JSON strictly before applying the stronger TE semantics. Missing +collections are handled by the capability checks and explicit applicability. +""" + +from collections.abc import Iterator + +from pydantic import TypeAdapter, ValidationError + +from nemo_gym.base_responses_api_model import ModelCallRecord +from nemo_gym.rollout_observability import ( + AgentInvocation, + AgentObservationRecord, + TrajectoryModelCall, + TrajectoryToolCall, + TrajectoryTurn, +) + + +SCHEMA_VERSION = "gym-p0-evidence/v2" +TOKEN_FIELDS = ("tokens_in", "tokens_out", "tokens_reasoning", "tokens_total", "cached_tokens") +PATH_MODELS = { + "ng_model_call_capture.calls": TypeAdapter(list[ModelCallRecord]), + "ng_trajectory.model_calls": TypeAdapter(list[TrajectoryModelCall]), + "ng_trajectory.turns": TypeAdapter(list[TrajectoryTurn]), + "ng_trajectory.invocations": TypeAdapter(list[AgentInvocation]), + "ng_trajectory.tool_calls": TypeAdapter(list[TrajectoryToolCall]), + "ng_agent_observations.records": TypeAdapter(list[AgentObservationRecord]), +} + + +def model_errors(record: dict) -> Iterator[tuple[str, str]]: + """Yield JSON pointers and error codes without copying source payloads.""" + for path, adapter in PATH_MODELS.items(): + parent, field = path.split(".") + if parent not in record: + continue + owner = record[parent] + location = f"/{parent}" + if not isinstance(owner, dict): + yield location, "path.object" + continue + if field not in owner: + continue + location += f"/{field}" + try: + adapter.validate_python(owner[field], strict=True) + except ValidationError as exc: + # Neither messages nor input/context are safe: validators can embed payloads. + for error in exc.errors(include_url=False, include_context=False, include_input=False): + parts = error["loc"] + if path == "ng_agent_observations.records" and len(parts) > 1: + # Pydantic inserts the discriminated union variant after the index; + # it is not a field in the persisted object. + parts = (parts[0], *parts[2:]) + pointer = "".join("/" + str(part).replace("~", "~0").replace("/", "~1") for part in parts) + yield location + pointer, "model." + error["type"] diff --git a/nemo_gym/harness_capabilities/reader.py b/nemo_gym/harness_capabilities/reader.py index 383715ede1d..4ffe5d03153 100644 --- a/nemo_gym/harness_capabilities/reader.py +++ b/nemo_gym/harness_capabilities/reader.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import Iterator +from .contracts import model_errors + PAYLOADS = ("request", "response", "request_raw", "response_raw") @@ -46,6 +48,10 @@ def hydrate_record(record: dict, *, capture_dir: Path | None = None) -> dict: Conflicts are preserved as reader failures instead of being overwritten. """ result = copy.deepcopy(record) + # Leave malformed native objects intact for the checker to report at their + # actual paths; do not attempt payload joins through invalid containers. + if next(model_errors(result), None) is not None: + return result capture = result.get("ng_model_call_capture") or {} calls = capture.get("calls") or [] trajectory = result.get("ng_trajectory") or {} @@ -53,8 +59,11 @@ def hydrate_record(record: dict, *, capture_dir: Path | None = None) -> dict: # Canonical-only delivery is also a supported native surface. Copy known # fields exactly; do not invent clocks, usage, or ownership. calls = [] - for item in trajectory["model_calls"]: + for index, item in enumerate(trajectory["model_calls"]): call = { + # ModelCallRecord's append index is only a reader projection of + # canonical array order, never evidence of identity or causality. + "call_index": index, **item.get("response_metadata", {}), "model_call_id": item.get("model_call_id"), "started_at": item.get("started_at"), diff --git a/nemo_gym/harness_capabilities/schemas.py b/nemo_gym/harness_capabilities/schemas.py deleted file mode 100644 index d5bc8533fc0..00000000000 --- a/nemo_gym/harness_capabilities/schemas.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Persisted P0 trajectory evidence, independent of health and live qualification.""" - -SCHEMA_VERSION = "gym-p0-evidence/v1" -TEXT = {"type": "string", "minLength": 1} -TOKENS = {"type": ["integer", "null"], "minimum": 0} -TOKEN_FIELDS = ("tokens_in", "tokens_out", "tokens_reasoning", "tokens_total", "cached_tokens") - - -def object_schema(properties: dict, required: tuple | list) -> dict: - return {"type": "object", "properties": properties, "required": list(required)} - - -MODEL_REF = object_schema({"type": TEXT, "name": TEXT}, ("type", "name")) -CALL_REF = { - **object_schema({"model_call_id": TEXT, "model_ref": MODEL_REF, "response_id": TEXT}, ()), - "anyOf": [{"required": ["model_call_id"]}, {"required": ["model_ref", "response_id"]}], -} -CALL = object_schema( - { - "model_call_id": TEXT, - "model_ref": MODEL_REF, - "dialect": {"enum": ["responses", "chat", "messages"]}, - "status_code": {"type": ["integer", "null"], "minimum": 100, "maximum": 599}, - "response_status": {"type": ["string", "null"]}, - "finish_reason": {"type": ["string", "null"]}, - "error_category": {"type": ["string", "null"]}, - }, - ("model_call_id", "model_ref", "dialect"), -) -USAGE = object_schema({key: TOKENS for key in TOKEN_FIELDS}, ()) -INVOCATION = object_schema( - {"invocation_id": TEXT, "model_calls": {"type": "array", "items": {"type": "object"}}}, - ("invocation_id", "model_calls"), -) -TOOL = object_schema( - { - "invocation_id": TEXT, - "tool_call_id": TEXT, - "tool_name": TEXT, - "status": {"enum": ["completed", "failed", "timeout", "cancelled"]}, - }, - ("invocation_id", "tool_call_id", "tool_name", "status"), -) -TURN = object_schema( - { - "invocation_id": TEXT, - "task_id": TEXT, - "rollout_id": TEXT, - "turn_no": {"type": "integer", "minimum": 1}, - "timestamp": {"type": "number"}, - "question": {"not": {"type": "null"}}, - "resolved": {"type": ["boolean", "null"]}, - }, - ("invocation_id", "task_id", "rollout_id", "turn_no", "timestamp", "question", "resolved"), -) -SCHEMAS = {"TE-1": CALL, "TE-2": USAGE, "TE-3": TURN, "TE-5": TOOL, "TE-8": INVOCATION} diff --git a/tests/unit_tests/harness_capabilities/synthetic.py b/tests/unit_tests/harness_capabilities/synthetic.py index 185d057be58..a14ff323a12 100644 --- a/tests/unit_tests/harness_capabilities/synthetic.py +++ b/tests/unit_tests/harness_capabilities/synthetic.py @@ -52,6 +52,7 @@ def evidence_record() -> dict: { **reference, **metadata, + "call_index": number - 1, "tokens_in": 10, "tokens_out": 5, "tokens_total": 15, @@ -62,7 +63,7 @@ def evidence_record() -> dict: calls.append( { "model_call_id": reference["model_call_id"], - "response_metadata": metadata, + "response_metadata": {key: value for key, value in metadata.items() if key != "client_session_id"}, "request": request, "response": response, "token_stats": { @@ -80,6 +81,7 @@ def evidence_record() -> dict: "task_id": "0", "rollout_id": "0-0", "turn_no": number, + "step_count": number, "timestamp": float(number), "question": deepcopy(conversation), "answer": [tool_request], @@ -111,7 +113,12 @@ def evidence_record() -> dict: "reward": 0.0, "response": {"output": deepcopy(conversation[1:])}, "ng_model_call_capture": {"rollout_id": "0-0", "calls": deepcopy(captures)}, - "ng_agent_observations": {"records": deepcopy([invocation, *tools]), "gaps": []}, + "ng_agent_observations": { + "records": deepcopy( + [invocation, *[{key: value for key, value in tool.items() if key != "output"} for tool in tools]] + ), + "gaps": [], + }, "ng_trajectory": { "schema_version": "1.0", "task_id": "0", diff --git a/tests/unit_tests/harness_capabilities/test_checker.py b/tests/unit_tests/harness_capabilities/test_checker.py index 6ca48b53fb5..924035c34c8 100644 --- a/tests/unit_tests/harness_capabilities/test_checker.py +++ b/tests/unit_tests/harness_capabilities/test_checker.py @@ -400,3 +400,147 @@ def test_response_identity_must_match_payload(record): def test_tool_output_conflict_fails(record): record["ng_trajectory"]["tool_calls"][0]["output"] = "different-from-model-visible-result" assert verdict(record, "TE-5") == "not_fulfilled" + + +@pytest.mark.parametrize( + ("parent", "field", "index", "mutation", "error_path"), + [ + ("ng_trajectory", "turns", 0, {"step_count": -1}, "/step_count"), + ("ng_trajectory", "turns", 0, {"step_count": "1"}, "/step_count"), + ("ng_trajectory", "turns", 0, {"unexpected": "private-value"}, "/unexpected"), + ("ng_trajectory", "turns", 0, {"model_calls": [{}]}, "/model_calls/0"), + ("ng_trajectory", "invocations", 0, {"status": "invalid"}, "/status"), + ("ng_trajectory", "tool_calls", 0, {"duration_ms": -1}, "/duration_ms"), + ("ng_trajectory", "tool_calls", 0, {"started_at": 2, "completed_at": 1}, ""), + ("ng_trajectory", "model_calls", 0, {"response_metadata": []}, "/response_metadata"), + ("ng_trajectory", "model_calls", 0, {"token_stats": {"prompt_tokens": -1}}, "/token_stats/prompt_tokens"), + ("ng_model_call_capture", "calls", 0, {"call_index": "0"}, "/call_index"), + ("ng_agent_observations", "records", 0, {"status": "invalid"}, "/status"), + ("ng_agent_observations", "records", 1, {"timing_source": "invalid"}, "/timing_source"), + ("ng_agent_observations", "records", 1, {"output": "private-value"}, "/output"), + ], +) +def test_shared_models_reject_invalid_objects_at_exact_paths(record, parent, field, index, mutation, error_path): + record[parent][field][index].update(mutation) + result = inspect_record(hydrate_record(record)) + assert result["verdict"] == "not_fulfilled" + assert any( + finding["assertion"].startswith("model.") + and finding["location"] == f"record/{parent}/{field}/{index}{error_path}" + for finding in result["findings"] + ) + assert "private-value" not in json.dumps(result) + + +@pytest.mark.parametrize(("parent", "field"), [("ng_trajectory", "turns"), ("ng_model_call_capture", "calls")]) +@pytest.mark.parametrize("value", [None, {}, [None]]) +def test_invalid_evidence_collections_are_reported(record, parent, field, value, tmp_path): + record[parent][field] = value + bundle = tmp_path / "invalid.jsonl" + bundle.write_text(json.dumps(record) + "\n") + assert main(["inspect", "--bundle", str(bundle), "--output", str(tmp_path / "report")]) == 1 + result = json.loads(next((tmp_path / "report").glob("*/evidence_results.jsonl")).read_text()) + assert any(f["location"].startswith(f"invalid.jsonl:1/{parent}/{field}") for f in result["findings"]) + + +@pytest.mark.parametrize( + ("parent", "field", "capability"), + [("ng_trajectory", "turns", "TE-3"), ("ng_model_call_capture", "calls", "TE-1")], +) +def test_valid_objects_at_unrecognized_paths_do_not_count(record, parent, field, capability): + record["somewhere_else"] = record[parent].pop(field) + assert verdict(record, capability) == "not_fulfilled" + + +@pytest.mark.parametrize(("field", "assertion"), [("question", "turn.question"), ("resolved", "turn.resolved")]) +def test_optional_turn_fields_are_required_by_te3(record, field, assertion): + from nemo_gym.rollout_observability import TrajectoryTurn + + turn = record["ng_trajectory"]["turns"][0] + del turn[field] + TrajectoryTurn.model_validate(turn, strict=True) + result = inspect_record(hydrate_record(record)) + assert result["evidence"]["TE-3"]["verdict"] == "not_fulfilled" + assert any(f["assertion"] == assertion for f in result["findings"]) + + +def test_null_question_is_insufficient_even_when_turn_model_valid(record): + record["ng_trajectory"]["turns"][0]["question"] = None + assert verdict(record, "TE-3") == "not_fulfilled" + + +@pytest.mark.parametrize("answer", [None, "answer", []]) +@pytest.mark.parametrize("reasoning", [None, "reasoning"]) +def test_te3_requires_answer_or_reasoning_beyond_model_validity(record, answer, reasoning): + from nemo_gym.rollout_observability import TrajectoryTurn + + turn = record["ng_trajectory"]["turns"][0] + turn.update(answer=answer, reasoning_content=reasoning) + TrajectoryTurn.model_validate(turn, strict=True) + expected = "fulfilled" if answer is not None or reasoning is not None else "not_fulfilled" + assert verdict(record, "TE-3") == expected + + +@pytest.mark.parametrize("field", ["tool_name", "status"]) +def test_optional_tool_fields_remain_required_by_te5(record, field): + from nemo_gym.rollout_observability import TrajectoryToolCall + + tool = record["ng_trajectory"]["tool_calls"][0] + del tool[field] + TrajectoryToolCall.model_validate(tool, strict=True) + assert verdict(record, "TE-5") == "not_fulfilled" + + +def test_invocation_and_tool_fallbacks_use_observation_paths(record): + record["ng_trajectory"].pop("invocations") + record["ng_trajectory"].pop("tool_calls") + assert inspect_record(hydrate_record(record))["verdict"] == "fulfilled" + record["ng_agent_observations"]["records"][1]["status"] = "unknown" + result = inspect_record(hydrate_record(record)) + assert any( + f["assertion"] == "tool.terminal" and f["location"] == "record/ng_agent_observations/records/1/status" + for f in result["findings"] + ) + + +def test_invocations_can_be_supplied_at_trajectory_path(record): + del record["ng_agent_observations"] + assert inspect_record(hydrate_record(record))["verdict"] == "fulfilled" + + +def test_model_call_reference_rejects_extra_fields(record): + record["ng_agent_observations"]["records"][0]["model_calls"][0]["unexpected"] = True + result = inspect_record(hydrate_record(record)) + assert any( + f["assertion"] == "model.extra_forbidden" + and f["location"] == "record/ng_agent_observations/records/0/model_calls/0/unexpected" + for f in result["findings"] + ) + + +@pytest.mark.parametrize( + ("parent", "collection", "field"), + [("ng_trajectory", "turns", "step_count"), ("ng_model_call_capture", "calls", "call_index")], +) +def test_required_model_fields_cannot_be_omitted(record, parent, collection, field): + del record[parent][collection][0][field] + result = inspect_record(hydrate_record(record)) + assert any( + f["assertion"] == "model.missing" and f["location"] == f"record/{parent}/{collection}/0/{field}" + for f in result["findings"] + ) + + +@pytest.mark.parametrize( + ("field", "value", "assertion"), + [("dialect", "unknown", "dialect.supported"), ("status_code", 600, "status.range")], +) +def test_model_valid_call_still_requires_te1_semantics(record, field, value, assertion): + from nemo_gym.base_responses_api_model import ModelCallRecord + + call = record["ng_model_call_capture"]["calls"][0] + call[field] = value + record["ng_trajectory"]["model_calls"][0]["response_metadata"][field] = value + ModelCallRecord.model_validate(call, strict=True) + result = inspect_record(hydrate_record(record)) + assert any(f["evidence"] == "TE-1" and f["assertion"] == assertion for f in result["findings"]) From 6a2c1d971b12bdcd790322f3f3a8a7ddbe93f4b2 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Thu, 1 Oct 2026 23:49:42 +0200 Subject: [PATCH 08/11] Reconcile conformance tool witnesses and clarify inspector usage Signed-off-by: Frederic Lafrance --- .../observability/harness-conformance.mdx | 4 +- nemo_gym/harness_capabilities/README.md | 37 +++++-- scripts/harness_conformance/README.md | 8 +- scripts/harness_conformance/provider.py | 5 + scripts/harness_conformance/runner.py | 58 +++++++++++ .../harness_conformance/tests/test_runner.py | 97 ++++++++++++++++++- ...ance.py => inspect_harness_conformance.py} | 2 +- .../harness_capabilities/test_script.py | 2 +- 8 files changed, 200 insertions(+), 13 deletions(-) rename scripts/{check_harness_conformance.py => inspect_harness_conformance.py} (82%) diff --git a/fern/versions/latest/pages/observability/harness-conformance.mdx b/fern/versions/latest/pages/observability/harness-conformance.mdx index 7b9ffc38177..f32b7aca85a 100644 --- a/fern/versions/latest/pages/observability/harness-conformance.mdx +++ b/fern/versions/latest/pages/observability/harness-conformance.mdx @@ -9,7 +9,7 @@ position: 3 Check the evidence retained by a harness/benchmark pair after collecting an evaluation: ```bash -python scripts/check_harness_conformance.py \ +python scripts/inspect_harness_conformance.py \ --bundle results/my-harness/rollouts.jsonl \ --output results/my-harness/evidence ``` @@ -72,7 +72,7 @@ Identical inputs and checker versions reuse the same report directory. Use the same task and capture settings when comparing harnesses: ```bash -python scripts/check_harness_conformance.py matrix \ +python scripts/inspect_harness_conformance.py matrix \ --harness opencode=results/opencode/rollouts.jsonl \ --harness pi=results/pi/rollouts.jsonl \ --harness codex=results/codex/rollouts.jsonl \ diff --git a/nemo_gym/harness_capabilities/README.md b/nemo_gym/harness_capabilities/README.md index 1bd604d4e72..4a264adae81 100644 --- a/nemo_gym/harness_capabilities/README.md +++ b/nemo_gym/harness_capabilities/README.md @@ -3,7 +3,7 @@ Check P0 evidence from a collected evaluation: ```bash -python scripts/check_harness_conformance.py \ +python scripts/inspect_harness_conformance.py \ --bundle results/my-harness/rollouts.jsonl \ --output results/my-harness/evidence ``` @@ -12,6 +12,34 @@ Use `matrix --harness NAME=PATH` to compare multiple harnesses. The checker uses TE-1–TE-9 and the `gym-p0/v1` profile. Reports are `evidence_summary.json`, `evidence_results.jsonl`, and `evidence_report.md`. +For example, this excerpt from `evidence_report.md` shows retained model-call +evidence passing while one tool record is missing its name: + +```markdown +Profile: `gym-p0/v1`. Gate: **not_fulfilled**. Records: 1. + +| Evidence | Artifact verdict | Passing / applicable records | +|---|---|---| +| TE-1: model_call_status | fulfilled | 1/1 | +| TE-5: tool_record | not_fulfilled | 0/1 | +``` + +One corresponding entry in that rollout's `findings` array in +`evidence_results.jsonl` identifies the missing field: + +```json +{ + "evidence": "TE-5", + "assertion": "identity.required", + "location": "rollouts.jsonl:1/ng_trajectory/tool_calls/0/tool_name", + "reason": "nonempty value is required" +} +``` + +These verdicts describe evidence conformance, not task success. A rollout with +zero reward can have fully conforming evidence; a successful task can have +incomplete evidence. + Evidence objects must validate Gym's shared models at these paths: | JSON path | Model | @@ -36,11 +64,8 @@ and terminal status; TE-1 requires model/server identity and a supported dialect TE-2 requires nonnegative counts consistent with retained provider usage. Reports hash the path/model registry and shared model sources as well as the checker. -Unit tests use hand-authored contracts in `tests/unit_tests/harness_capabilities/synthetic.py`; -they validate the checker independently of harness implementations. To measure -actual harnesses and regenerate the commit-pinned documentation table, use -`python -m scripts.harness_conformance.table --commit --output `. -See the [runner guide](../../scripts/harness_conformance/README.md) for prerequisites and test gates. +For live harness checks and regenerating the capability table, see the +[runner guide](../../scripts/harness_conformance/README.md). See [Harness Conformance](../../fern/versions/latest/pages/observability/harness-conformance.mdx) for contracts, applicability, matrix usage, producer onboarding, and limitations. diff --git a/scripts/harness_conformance/README.md b/scripts/harness_conformance/README.md index f19b6b3c73b..d144dcb59be 100644 --- a/scripts/harness_conformance/README.md +++ b/scripts/harness_conformance/README.md @@ -104,7 +104,7 @@ rejected, so previous rollouts cannot accidentally qualify a new run. - `//runtime.json`: executable version or Hermes source hash, plus the Gym adapter source hash. - `//rollouts.jsonl` and `capture/`: ordinary Gym outputs, - directly consumable by `scripts/check_harness_conformance.py`. + directly consumable by `scripts/inspect_harness_conformance.py`. - `//witness.json`: independent endpoint attempts, actual tool execution markers, returned tool results, and verifier outcomes. - `//scenario_result.json` and `evidence/`: execution gaps, @@ -114,7 +114,11 @@ rejected, so previous rollouts cannot accidentally qualify a new run. A scenario counts as passing only when its independent observations agree with the retained rollout and the relevant evidence checks pass. Attempt comparison -preserves multiplicity, so dropping one of two identical retries fails. The +preserves multiplicity, so dropping one of two identical retries fails. Tool +comparison joins each witnessed call ID to exactly one execution and its +invocation's request/result, checking the name, arguments, prescribed exit +status, and complete model-visible output. Partial logging loss or altered +tool evidence fails even when the retained artifacts are internally consistent. The artifact reports keep their original retained-artifact scope; the runner's separate report adds the scenario coverage. diff --git a/scripts/harness_conformance/provider.py b/scripts/harness_conformance/provider.py index 3128ecafddd..3e2985dbf60 100644 --- a/scripts/harness_conformance/provider.py +++ b/scripts/harness_conformance/provider.py @@ -77,6 +77,7 @@ def _tool(self, body: dict) -> tuple[str, dict, dict]: "marker": str(marker), "exit_code": code, "result_seen": False, + "outputs": [], } ) return name, args, namespaces @@ -103,6 +104,10 @@ def _observe_tool_results(self, body: dict) -> None: call["result_seen"] = call["result_seen"] or any( call["token"] in json.dumps(item.get("content", item.get("output"))) for item in results ) + for item in results: + output = item.get("content", item.get("output")) + if output is not None and output not in call["outputs"]: + call["outputs"].append(output) marker = Path(call["marker"]) call["executed"] = marker.is_file() and marker.read_text() == call["token"] diff --git a/scripts/harness_conformance/runner.py b/scripts/harness_conformance/runner.py index 2b3792d7119..fa5628a7817 100644 --- a/scripts/harness_conformance/runner.py +++ b/scripts/harness_conformance/runner.py @@ -96,6 +96,63 @@ def run_process(command: list[str], *, directory: Path, timeout: float) -> dict: return {"returncode": proc.returncode, "timed_out": timed_out} +def _tool_witness_issues(record: dict, witnessed: list[dict]) -> list[str]: + """Join independently witnessed tools to retained executions and conversation items.""" + trajectory = record.get("ng_trajectory") or {} + observations = (record.get("ng_agent_observations") or {}).get("records", []) + # Match the inspector's supported trajectory/observation fallbacks. + tools = trajectory.get("tool_calls") or [r for r in observations if r.get("kind") == "tool_call"] + invocations = [r for r in observations if r.get("kind") == "agent_invocation"] or trajectory.get("invocations", []) + expected_ids = Counter(t["id"] for t in witnessed) + if any(count != 1 for count in expected_ids.values()) or expected_ids != Counter(t["tool_call_id"] for t in tools): + return ["retained tool identities differ from the independent tool witness"] + + issues = [] + by_id = {t["tool_call_id"]: t for t in tools} + for expected in witnessed: + tool = by_id[expected["id"]] + items = [ + item + for invocation in invocations + if invocation.get("invocation_id") == tool.get("invocation_id") + for item in invocation.get("conversation", []) + if item.get("call_id") == expected["id"] + ] + requests = [i for i in items if i.get("type") == "function_call"] + results = [i for i in items if i.get("type") == "function_call_output"] + if len(requests) != 1 or len(results) != 1: + issues.append("retained tool request/result does not join uniquely to the witnessed execution") + continue + request = requests[0] + try: + arguments = json.loads(request["arguments"]) + except (KeyError, TypeError, ValueError): + arguments = None + name = request.get("name") + if request.get("namespace"): + name = f"{request['namespace']}__{name}" + if ( + name != expected["name"] + or arguments != expected["arguments"] + or tool.get("tool_name") != request.get("name") + ): + issues.append("retained tool name or arguments differ from the independent tool witness") + # Each probe command terminates with its prescribed code. A nonzero exit + # must remain a failed execution even when its stdout was retained. + status = "failed" if expected["exit_code"] else "completed" + if tool.get("status") != status: + issues.append("retained tool status differs from the independent tool witness") + output = results[0].get("output") + outputs = expected.get("outputs", []) + if ( + not outputs + or any(observed != output for observed in outputs) + or (tool.get("output") is not None and tool["output"] != output) + ): + issues.append("retained tool output differs from the independent tool witness") + return issues + + def inspect_episode(scenario: Scenario, directory: Path, execution: dict) -> dict: """Require witnessed exercise and a real rollout before counting a TE as passing.""" issues = [] @@ -145,6 +202,7 @@ def inspect_episode(scenario: Scenario, directory: Path, execution: dict) -> dic observed = Counter(_fingerprint(c.get("request"), c.get("status_code"), c.get("response")) for c in calls) if expected != observed: issues.append("retained model attempts differ from the independent endpoint witness") + issues.extend(_tool_witness_issues(record, tools)) if record.get("reward") != scenario.expected_reward: issues.append("rollout reward differs from the verifier witness") destination, summary = inspect_bundle(bundle, output=directory / "evidence", capture_dir=directory / "capture") diff --git a/scripts/harness_conformance/tests/test_runner.py b/scripts/harness_conformance/tests/test_runner.py index ce6627532ee..5109ee2779a 100644 --- a/scripts/harness_conformance/tests/test_runner.py +++ b/scripts/harness_conformance/tests/test_runner.py @@ -88,6 +88,7 @@ def test_live_chat_endpoint_executes_script_and_captures_every_attempt(tmp_path, assert response.json()["choices"][0]["message"]["content"] == "CONFORMANCE_DONE" assert probe.finished and not probe.violations assert all(t["executed"] and t["result_seen"] for t in probe.tool_calls) + assert all(t["outputs"] == [t["token"]] for t in probe.tool_calls) captures = [ build_model_call_record(row, call_index=i).model_dump() for i, row in json_rows(tmp_path / "capture/0-0.capture.jsonl") @@ -186,7 +187,18 @@ def retained_episode(tmp_path): "finished": True, "violations": [], "verifications": [{"reward": 0.0, "answer_seen": True}], - "tool_calls": [{"executed": True, "result_seen": True}] * 2, + "tool_calls": [ + { + "id": f"tool-{number}", + "name": "read_value", + "arguments": {"key": "example"}, + "exit_code": 0, + "outputs": ["value"], + "executed": True, + "result_seen": True, + } + for number in (1, 2) + ], "attempts": [ {"request": c["request"], "response": c["response"], "status_code": c["status_code"]} for c in calls ], @@ -218,6 +230,89 @@ def test_execution_failure_cannot_pass_even_with_artifacts(retained_episode): assert "episode exceeded its timeout" in result["issues"] +@pytest.mark.parametrize( + "mutation", ["missing", "duplicate", "extra", "arguments", "name", "output", "status", "owner"] +) +def test_tool_witness_rejects_lost_or_changed_evidence(retained_episode, mutation): + directory, _ = retained_episode + bundle = directory / "rollouts.jsonl" + record = json.loads(bundle.read_text()) + trajectory = record["ng_trajectory"] + tools = trajectory["tool_calls"] + invocations = [trajectory["invocations"][0], record["ng_agent_observations"]["records"][0]] + if mutation == "missing": + tools.pop() + record["ng_agent_observations"]["records"].pop() + for invocation in invocations: + invocation["conversation"] = [i for i in invocation["conversation"] if i.get("call_id") != "tool-2"] + elif mutation in {"duplicate", "extra"}: + tool = copy.deepcopy(tools[0]) + if mutation == "extra": + tool["tool_call_id"] = "unwitnessed-tool" + tools.append(tool) + elif mutation in {"arguments", "name"}: + for invocation in invocations: + invocation["conversation"][1][mutation] = '{"key":"changed"}' if mutation == "arguments" else "other_tool" + if mutation == "name": + tools[0]["tool_name"] = "other_tool" + elif mutation == "output": + tools[0]["output"] = "changed" + for invocation in invocations: + invocation["conversation"][2]["output"] = "changed" + elif mutation == "status": + tools[0]["status"] = "failed" + else: + tools[0]["invocation_id"] = "other-invocation" + bundle.write_text(json.dumps(record) + "\n") + result = inspect_episode(SCENARIO["verifier_failure"], directory, {"returncode": 0, "timed_out": False}) + assert result["verdict"] == "not_fulfilled" and not result["exercised"] + assert any("retained tool" in issue for issue in result["issues"]) + assert not any("retained model attempts differ" in issue for issue in result["issues"]) + if mutation in {"missing", "arguments", "name", "output", "status"}: + # These artifacts remain internally consistent; the independent witness + # must expose the lost/changed evidence even when TE-5 alone passes. + assert result["evidence"]["TE-5"]["artifact_verdict"] == "fulfilled" + + +@pytest.mark.parametrize("surface", ["trajectory", "observations"]) +def test_tool_witness_supports_retained_surface_fallbacks(retained_episode, surface): + directory, _ = retained_episode + bundle = directory / "rollouts.jsonl" + record = json.loads(bundle.read_text()) + if surface == "trajectory": + del record["ng_agent_observations"] + else: + del record["ng_trajectory"]["tool_calls"] + del record["ng_trajectory"]["invocations"] + bundle.write_text(json.dumps(record) + "\n") + result = inspect_episode(SCENARIO["verifier_failure"], directory, {"returncode": 0, "timed_out": False}) + assert result["verdict"] == "fulfilled", result["issues"] + + +@pytest.mark.parametrize("failed", [True, False]) +def test_tool_witness_requires_prescribed_failure_status(retained_episode, failed): + directory, witness = retained_episode + witness["tool_calls"][0]["exit_code"] = 7 + (directory / "witness.json").write_text(json.dumps(witness)) + bundle = directory / "rollouts.jsonl" + record = json.loads(bundle.read_text()) + if failed: + record["ng_trajectory"]["tool_calls"][0]["status"] = "failed" + bundle.write_text(json.dumps(record) + "\n") + result = inspect_episode(SCENARIO["verifier_failure"], directory, {"returncode": 0, "timed_out": False}) + assert result["exercised"] == failed + assert ("retained tool status differs from the independent tool witness" in result["issues"]) != failed + + +def test_missing_output_witness_cannot_qualify_artifacts(retained_episode): + directory, witness = retained_episode + del witness["tool_calls"][0]["outputs"] + (directory / "witness.json").write_text(json.dumps(witness)) + result = inspect_episode(SCENARIO["verifier_failure"], directory, {"returncode": 0, "timed_out": False}) + assert not result["exercised"] + assert "retained tool output differs from the independent tool witness" in result["issues"] + + def test_missing_runtime_is_recorded_and_no_stale_output_reused(tmp_path, monkeypatch): monkeypatch.setenv("PATH", "") output = tmp_path / "run" diff --git a/scripts/check_harness_conformance.py b/scripts/inspect_harness_conformance.py similarity index 82% rename from scripts/check_harness_conformance.py rename to scripts/inspect_harness_conformance.py index 638396b5261..c5694dd54cc 100644 --- a/scripts/check_harness_conformance.py +++ b/scripts/inspect_harness_conformance.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Check retained harness capability evidence in rollout artifacts.""" +"""Inspect retained harness capability evidence in rollout artifacts.""" import sys diff --git a/tests/unit_tests/harness_capabilities/test_script.py b/tests/unit_tests/harness_capabilities/test_script.py index c93389204c7..40621f6b89e 100644 --- a/tests/unit_tests/harness_capabilities/test_script.py +++ b/tests/unit_tests/harness_capabilities/test_script.py @@ -15,7 +15,7 @@ from tests.unit_tests.harness_capabilities.synthetic import evidence_record -SCRIPT = Path(__file__).resolve().parents[3] / "scripts/check_harness_conformance.py" +SCRIPT = Path(__file__).resolve().parents[3] / "scripts/inspect_harness_conformance.py" @pytest.fixture From f3ba752dfcb958d37aa02a17b1a754353b280fec Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Fri, 2 Oct 2026 00:03:39 +0200 Subject: [PATCH 09/11] Fix conformance secret-detector false positives Signed-off-by: Frederic Lafrance --- .github/workflows/config/.secrets.baseline | 658 +++++++++++++++++++++ nemo_gym/harness_capabilities/cli.py | 3 +- 2 files changed, 660 insertions(+), 1 deletion(-) diff --git a/.github/workflows/config/.secrets.baseline b/.github/workflows/config/.secrets.baseline index a02468c3e52..89545e29499 100644 --- a/.github/workflows/config/.secrets.baseline +++ b/.github/workflows/config/.secrets.baseline @@ -144,6 +144,664 @@ "line_number": 3 } ], + "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json": [ + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "7621e1af379fafbde450d3bb1a96566f24a0f496", + "is_verified": false, + "line_number": 2, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "2cb635bfe54c88c1ee08088477656e79bd293831", + "is_verified": false, + "line_number": 99, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "943a7ea8d8acda62d62681e2b9b8efabe26e3d21", + "is_verified": false, + "line_number": 100, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "5c8cf377b59a3ca3c11d965eb944e3eb28f8206f", + "is_verified": false, + "line_number": 101, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "ca3f05a88f5d5692e44dc457c5e6b1c707891381", + "is_verified": false, + "line_number": 102, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "78382f7416d51d882c66c0c0612f4457038bb879", + "is_verified": false, + "line_number": 155, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "5da1f090449950b362266efa0119f0b151ac289f", + "is_verified": false, + "line_number": 156, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "85dc66a2ccdb8783338eb3dcc9dd566d508e7ae0", + "is_verified": false, + "line_number": 158, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "a1864c7d0cf2dad886af0d5db3dbe3aac1425cf4", + "is_verified": false, + "line_number": 211, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "5520c86d600cf13b331da90afef266e1debc880c", + "is_verified": false, + "line_number": 212, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "9dfab02267e38f830289c399dbe833043ced29c2", + "is_verified": false, + "line_number": 214, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "50e4e198a3f44f663ca6ba689696fd47519ba8de", + "is_verified": false, + "line_number": 267, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "11ffa535f1e23805bd8595f555b548d7db57c112", + "is_verified": false, + "line_number": 268, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "97d841cf8e281b5c92065e2192382662142e6c3e", + "is_verified": false, + "line_number": 270, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "3157fb6314aa2c3000dbb9fb1079b57489514ea3", + "is_verified": false, + "line_number": 329, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "e98df9fa20774977f79d443ca19c4459a391cd88", + "is_verified": false, + "line_number": 330, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "47567f129a519dafeeae6f75589934885614687b", + "is_verified": false, + "line_number": 332, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "ad77cad4c11176c72bef00837a6f51d5be8c6ebc", + "is_verified": false, + "line_number": 373, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "4fe50065fc938329fdbe2250e96f8492e9902472", + "is_verified": false, + "line_number": 374, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "5080c91c7fae3e64854ae42ec765b8519ff56fc0", + "is_verified": false, + "line_number": 376, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "b56f3f107107f601f7d05d15864c9ca60bd5ba13", + "is_verified": false, + "line_number": 429, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "fa789e35dfbf0640045a3fb376bbf624b71d3582", + "is_verified": false, + "line_number": 430, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "fbccc37f0fe5101aaa9e488b5073bb1c902be946", + "is_verified": false, + "line_number": 432, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "44dbf1fa7110e041d1f284a651fb1a6b4d83f094", + "is_verified": false, + "line_number": 537, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "3827210a10ca5d0ed8a59831e0ab61318a7f1bc7", + "is_verified": false, + "line_number": 538, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "ee21052ee464c820045d6e49383cef44ec194d97", + "is_verified": false, + "line_number": 539, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "83402992afcbd2f25fb96bb0c4918168c5a1979e", + "is_verified": false, + "line_number": 540, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "f37a01e095f14527a5f2e3431719901662b8be7b", + "is_verified": false, + "line_number": 593, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "06e5729cb0e63e7232654c344b67174268fa73d6", + "is_verified": false, + "line_number": 594, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "17ecbdceee35057e6c8f82099285c06619b06361", + "is_verified": false, + "line_number": 596, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "fbce04328093487cd2c210de19d28b9eefd38eb1", + "is_verified": false, + "line_number": 649, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "922222140d82d8ebd244bdc33fbf5adf43657c99", + "is_verified": false, + "line_number": 650, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "bfc485bf4825d0ff7ab94afa81433b24b3dafac4", + "is_verified": false, + "line_number": 652, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "e0b85704190ddc2a65be4cc0f4c7dd451e2c0ef5", + "is_verified": false, + "line_number": 705, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "f3e4abdd57fcd85d4de142b65408b1d503e562b8", + "is_verified": false, + "line_number": 706, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "2fdec23cc072dd19188618aaa215767969b6b988", + "is_verified": false, + "line_number": 708, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "5e2a7883a4bb2217cd772e921999bbbd5a80f36c", + "is_verified": false, + "line_number": 761, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "593b4d1df3a3a5409ff7283786d33d1e76d75028", + "is_verified": false, + "line_number": 762, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "674349a3de8b719d1e8478cf971bfac50904af92", + "is_verified": false, + "line_number": 764, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "6d6351559d6af824546685b26bc8ac0a0c5fd857", + "is_verified": false, + "line_number": 805, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "a12ef505a6ce10056ab760fb88ddbe47d2114995", + "is_verified": false, + "line_number": 806, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "32e8d313e85aa9b172e761d6da3ca5386dc2b561", + "is_verified": false, + "line_number": 808, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "296c933f363fc5e61115cfb9199173052b9755dc", + "is_verified": false, + "line_number": 861, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "5d41287f557d92bde5a961e4e7c091dfe8c091f1", + "is_verified": false, + "line_number": 862, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "65f46c5c4c7088c3b6b79ab108f84ee42fc83999", + "is_verified": false, + "line_number": 864, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "c8ecf122ac555918729b7a52bf6c66333e6c44b2", + "is_verified": false, + "line_number": 969, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "6ca21b5942a92a2838f614779cb96504d5aad9cd", + "is_verified": false, + "line_number": 970, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "37208687726b571c2149553d5de23a45fb29ec59", + "is_verified": false, + "line_number": 971, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "004f8491c40f2008369ae837dea7eb5661099370", + "is_verified": false, + "line_number": 972, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "211b23a4e45850f68f3868703270e240d6294c52", + "is_verified": false, + "line_number": 1025, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "216f883fcf94c234e61b67e8242b02c9c86808a6", + "is_verified": false, + "line_number": 1026, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "86943eb170d1fa211494a4abd0b9663354dafca3", + "is_verified": false, + "line_number": 1028, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "351d3123664839e3ef8224f74d77bef7102b092a", + "is_verified": false, + "line_number": 1081, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "c6f04c39f5acb9aef0b102951908ac20f1a1f389", + "is_verified": false, + "line_number": 1082, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "6e29ba3c1b26492a5aa4e2a8fa155a0bd12fbfc7", + "is_verified": false, + "line_number": 1084, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "5760041d10d02749a4a327d7382a792b84deea1b", + "is_verified": false, + "line_number": 1137, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "626768bf5d33249d7ea857695c27f74c07ac4532", + "is_verified": false, + "line_number": 1138, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "e9cdca7c9a7d7d3d6fdab02874d6dac6f5e6025e", + "is_verified": false, + "line_number": 1140, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "68eeab2978c2ac2e2edc42501638454730a83286", + "is_verified": false, + "line_number": 1193, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "5c54aee83eb02add7b3bc9f85aa7a691fd157c6a", + "is_verified": false, + "line_number": 1194, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "445b534875851acb90b66f8337b6885fac578ecd", + "is_verified": false, + "line_number": 1196, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "df1615fa449e1fd9dfe7e7434f88044c997a6e8f", + "is_verified": false, + "line_number": 1237, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "0705eb7f7b53aeccb8a7474091709aaf6463f23f", + "is_verified": false, + "line_number": 1238, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "6faa24f376aaf6bf2c2d71e6948e053a31055375", + "is_verified": false, + "line_number": 1240, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "769dd8c389d513f543a6196e0e833aa83510cd7a", + "is_verified": false, + "line_number": 1293, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "ec9d54a4797f0438308d5fee6d0ea3ad1c74cd63", + "is_verified": false, + "line_number": 1294, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "934052a15828db972bfd72104551880e0347a0c8", + "is_verified": false, + "line_number": 1296, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "ce6fb95074cd7d5d7a8151240708296058ab1091", + "is_verified": false, + "line_number": 1314, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "9cbb7ef9d80590010b42b56f4394eaea58561913", + "is_verified": false, + "line_number": 1318, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "399328bef989ae8b4a1fbcc76fada37d8afc548c", + "is_verified": false, + "line_number": 1322, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "3974be5dbb87af7b90fd519c5b30484ddc3258da", + "is_verified": false, + "line_number": 1328, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "13cdd22033cb4f18d0500a540115904285978484", + "is_verified": false, + "line_number": 1329, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "996d1bc7c21e545a82792b9ebf8e0234bab4ffe6", + "is_verified": false, + "line_number": 1332, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "08ee66eb4c0590cdf138e3ffab32d8b3d9c2d9a6", + "is_verified": false, + "line_number": 1333, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "667eaabba116bf7fbe4ace7db69414f8d614279c", + "is_verified": false, + "line_number": 1339, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "c84a1af16aa2a91ddcbf01122d77e8a63efa9308", + "is_verified": false, + "line_number": 1340, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "e2af7c806fb1c5384f38efb98e93bb690c3cbd7e", + "is_verified": false, + "line_number": 1346, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "2c9789dab4471ef0e0d89c7ad2de03c32a3725c9", + "is_verified": false, + "line_number": 1347, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "a1ca3b10d0670a8eed9c03377d6460eae6a1b2b8", + "is_verified": false, + "line_number": 1353, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "0261484f7138da58154920f9d11ab730d0ffa6e3", + "is_verified": false, + "line_number": 1354, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "890dc85617adc58539b34056a19c33e4fe2586a1", + "is_verified": false, + "line_number": 1360, + "is_secret": false + }, + { + "type": "Hex High Entropy String", + "filename": "fern/assets/trajectory-capabilities/425964eadebb395fea46f594177e2373b4195530934ed44be4ead0d755892c90.json", + "hashed_secret": "0a1fbb587c36074a1c1b69f2766eef562a30653b", + "is_verified": false, + "line_number": 1361, + "is_secret": false + } + ], "nemo_gym/token_id_capture/staging/conformance/golden_vectors.json": [ { "type": "Hex High Entropy String", diff --git a/nemo_gym/harness_capabilities/cli.py b/nemo_gym/harness_capabilities/cli.py index 2118ec116d0..3a690a872f8 100644 --- a/nemo_gym/harness_capabilities/cli.py +++ b/nemo_gym/harness_capabilities/cli.py @@ -11,6 +11,7 @@ import tempfile from dataclasses import asdict from pathlib import Path +from string import ascii_letters, digits from . import __version__ from .checker import NAMES, PROFILE, EvidenceScope, inspect_record @@ -184,7 +185,7 @@ def inspect_matrix( raise ValueError("at least one harness is required") rows = {} for name, bundle in harnesses.items(): - if not name or any(c not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" for c in name): + if not name or any(c not in ascii_letters + digits + "_-" for c in name): raise ValueError("harness names must be alphanumeric with hyphens or underscores") path, summary = inspect_bundle(bundle, output=output / name, scope=scope) rows[name] = {"report": str(path / "evidence_summary.json"), **summary} From 213b911bc6851f063e6c4f93217b54d71170cbe7 Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Fri, 2 Oct 2026 11:12:58 +0200 Subject: [PATCH 10/11] Accept numeric rewards on masked verification results Signed-off-by: Frederic Lafrance --- .../observability/harness-conformance.mdx | 2 +- nemo_gym/harness_capabilities/README.md | 7 ++ nemo_gym/harness_capabilities/checker.py | 34 +++++---- .../harness_capabilities/test_checker.py | 76 ++++++++++++++++++- 4 files changed, 102 insertions(+), 17 deletions(-) diff --git a/fern/versions/latest/pages/observability/harness-conformance.mdx b/fern/versions/latest/pages/observability/harness-conformance.mdx index f32b7aca85a..beee3bb8507 100644 --- a/fern/versions/latest/pages/observability/harness-conformance.mdx +++ b/fern/versions/latest/pages/observability/harness-conformance.mdx @@ -25,7 +25,7 @@ every rollout. It replaces the earlier experimental C0–C11 profiles. | TE-3: steps | Task, repeat, rollout and step identity; timestamp; model-visible prompt; answer/tool or reasoning fields; resolution or explicit unknown. No step is inferred from an HTTP call. | | TE-4: history | Ordered saved model inputs and outputs, including tool results and changed context. Missing media, opaque content and unavailable prior-response history fail. | | TE-5: tool record | Execution id/name/status, originating request with dispatched arguments, and output/error. Tool clocks are outside P0. A tool request alone does not prove execution. | -| TE-6: verifier outcome | The current Gym verification surface: finite rollout reward and correctly scoped terminal resolution when present. Explicitly unavailable verification cannot be a numeric grade. Extended verifier scope/version and judge provenance are not certified by this profile. | +| TE-6: verifier outcome | The current Gym verification surface: finite rollout reward, including for masked results, and correctly scoped terminal resolution when present. `mask_sample=true` excludes the reward from scores. Masked or explicitly incomplete verification requires nonblank `failure_kind` and `failure_reason`. Extended verifier scope/version and judge provenance are not certified by this profile. | | TE-7: payloads | Saved request and response/error bodies. Empty returned bodies and explicit no-response transport failures remain distinct. | | TE-8: run join | Exactly one explicit invocation owner per captured attempt, including attempts without response ids; consistent parent relationships and exact references. | | TE-9: step join | Every retained policy attempt belongs to exactly one identified step. Retries remain attempts on that step. Explicit compaction-helper refs are excluded from policy steps. | diff --git a/nemo_gym/harness_capabilities/README.md b/nemo_gym/harness_capabilities/README.md index 4a264adae81..7ceaf8bb030 100644 --- a/nemo_gym/harness_capabilities/README.md +++ b/nemo_gym/harness_capabilities/README.md @@ -40,6 +40,13 @@ These verdicts describe evidence conformance, not task success. A rollout with zero reward can have fully conforming evidence; a successful task can have incomplete evidence. +TE-6 requires a finite numeric `reward` even when `mask_sample=true` excludes it +from scoring. Masked results and explicitly incomplete verification require +nonblank `failure_kind` and `failure_reason`; a masked `reward=0.0` is valid +evidence. The Gym profile uses the default unmasked behavior when `mask_sample` +is omitted and checks `evaluation_completed` when supplied. Both flags must be +booleans when present. + Evidence objects must validate Gym's shared models at these paths: | JSON path | Model | diff --git a/nemo_gym/harness_capabilities/checker.py b/nemo_gym/harness_capabilities/checker.py index 57f6c4a4068..9f487117ac0 100644 --- a/nemo_gym/harness_capabilities/checker.py +++ b/nemo_gym/harness_capabilities/checker.py @@ -644,8 +644,13 @@ def check_tools(self) -> None: def check_verifier(self) -> None: """TE-6's shipped Gym surface: rollout verification reward and terminal resolution.""" reward = self.record.get("reward") + failed = ( + self.record.get("mask_sample") is True + or self.record.get("evaluation_completed") is False + or self.record.get("verification_error") is not None + ) if not self.scope.verifier: - if reward is not None or any(t.get("resolved") is not None for t in self.turns): + if reward is not None or failed or any(t.get("resolved") is not None for t in self.turns): self._fail( "TE-6", "scope.contradiction", @@ -655,24 +660,27 @@ def check_verifier(self) -> None: else: self.not_applicable.add("TE-6") return - failed = self.record.get("mask_sample") is True or self.record.get("verification_error") is not None - if failed: - if reward is not None: - self._fail( - "TE-6", - "reward.unavailable", - "/reward", - "unavailable verification must not be represented as a numeric grade", - ) - if not self.record.get("failure_kind") and not self.record.get("verification_error"): - self._fail("TE-6", "verifier.error", "", "unavailable verification needs explicit failure evidence") - elif not _number(reward): + for field in ("mask_sample", "evaluation_completed"): + if field in self.record and type(self.record[field]) is not bool: + self._fail("TE-6", "verifier.boolean", "/" + field, "verification flags must be booleans when present") + # Gym requires a numeric reward even when the mask excludes it from scoring. + if not _number(reward): self._fail( "TE-6", "reward.required", "/reward", "finite verifier reward is required; binary resolution is not a floating grade", ) + if failed: + for field in ("failure_kind", "failure_reason"): + value = self.record.get(field) + if not isinstance(value, str) or not value.strip(): + self._fail( + "TE-6", + "verifier.error", + "/" + field, + "masked or incomplete verification needs nonblank failure metadata", + ) for invocation in {t.get("invocation_id") for t in self.turns}: turns = sorted( (t for t in self.turns if t.get("invocation_id") == invocation), key=lambda t: t.get("turn_no", 0) diff --git a/tests/unit_tests/harness_capabilities/test_checker.py b/tests/unit_tests/harness_capabilities/test_checker.py index 924035c34c8..a32aba93d1a 100644 --- a/tests/unit_tests/harness_capabilities/test_checker.py +++ b/tests/unit_tests/harness_capabilities/test_checker.py @@ -292,11 +292,81 @@ def test_duplicated_step_ref_fails_without_closure_requirement(): assert verdict(record, "TE-9") == "not_fulfilled" -def test_verifier_failure_must_not_be_numeric_zero(record): - record.update(mask_sample=True, failure_kind="verifier_unavailable", reward=0.0) +@pytest.mark.parametrize("reward", [0.0, 1.0, -0.5]) +def test_masked_numeric_reward_is_valid_evidence(record, reward): + record.update(mask_sample=True, failure_kind="judge_failed", failure_reason="judge unavailable", reward=reward) + assert verdict(record, "TE-6") == "fulfilled" + + +@pytest.mark.asyncio +async def test_judge_failsafe_response_satisfies_te6(record): + from nemo_gym.base_resources_server import BaseVerifyRequest, BaseVerifyResponse + from nemo_gym.judge import JudgeError, judge_failsafe + + @judge_failsafe + async def verify(body): + raise JudgeError("judge unavailable") + + body = BaseVerifyRequest( + responses_create_params={"input": "task"}, + response={ + "id": "response-1", + "created_at": 0, + "model": "test", + "object": "response", + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + }, + ) + data = json.loads((await verify(body)).body) + verified = BaseVerifyResponse.model_validate(data) + assert verified.reward == 0.0 and verified.mask_sample is True + record.update(data) + assert verdict(record, "TE-6") == "fulfilled" + + +@pytest.mark.parametrize("masked", [False, True]) +@pytest.mark.parametrize("reward", [None, True, "0.0", float("nan"), float("inf")]) +def test_verifier_reward_must_be_numeric_even_when_masked(record, masked, reward): + record.update(mask_sample=masked, failure_kind="judge_failed", failure_reason="judge unavailable", reward=reward) assert verdict(record, "TE-6") == "not_fulfilled" - record["reward"] = None + + +@pytest.mark.parametrize( + "signal", [{"mask_sample": True}, {"evaluation_completed": False}, {"verification_error": "error"}] +) +@pytest.mark.parametrize("field", ["failure_kind", "failure_reason"]) +@pytest.mark.parametrize("value", [None, "", " \t", False]) +def test_unavailable_verification_requires_both_failure_fields(record, signal, field, value): + record.update(signal, failure_kind="judge_failed", failure_reason="judge unavailable") assert verdict(record, "TE-6") == "fulfilled" + if value is None: + record.pop(field) + else: + record[field] = value + result = inspect_record(hydrate_record(record)) + assert result["evidence"]["TE-6"]["verdict"] == "not_fulfilled" + assert any(f["assertion"] == "verifier.error" and f["location"] == "record/" + field for f in result["findings"]) + + +@pytest.mark.parametrize("field", ["mask_sample", "evaluation_completed"]) +@pytest.mark.parametrize("value", [None, 0, "false"]) +def test_verification_flags_must_be_boolean_when_supplied(record, field, value): + record[field] = value + assert verdict(record, "TE-6") == "not_fulfilled" + + +def test_unmasked_diagnostic_metadata_does_not_invalidate_reward(record): + record.update(mask_sample=False, failure_kind="judge_failed") + assert verdict(record, "TE-6") == "fulfilled" + + +def test_masked_result_cannot_be_declared_verifier_free(record): + record.update(mask_sample=True, reward=None) + result = inspect_record(hydrate_record(record), scope=EvidenceScope(verifier=False)) + assert result["evidence"]["TE-6"]["verdict"] == "not_fulfilled" def test_binary_resolution_does_not_replace_reward(record): From adc58180847a5cba479665b8bfada6f363809fff Mon Sep 17 00:00:00 2001 From: Frederic Lafrance Date: Fri, 2 Oct 2026 11:58:38 +0200 Subject: [PATCH 11/11] test(agent): document miniSWE submission evidence gap Assert the known TE-5 findings for the final submit tool call and document that native miniSWE must persist a correlated terminal tool result. Signed-off-by: Frederic Lafrance --- nemo_gym/harness_capabilities/README.md | 7 ++++++ .../miniswe_sandboxed_agent/README.md | 6 ++++- .../tests/test_observability.py | 22 ++++++++++++++++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/nemo_gym/harness_capabilities/README.md b/nemo_gym/harness_capabilities/README.md index 7ceaf8bb030..273fb4fd508 100644 --- a/nemo_gym/harness_capabilities/README.md +++ b/nemo_gym/harness_capabilities/README.md @@ -40,6 +40,13 @@ These verdicts describe evidence conformance, not task success. A rollout with zero reward can have fully conforming evidence; a successful task can have incomplete evidence. +Known producer gap: `miniswe_sandboxed_agent` retains its final submit tool call +as `incomplete`, without output or error evidence. This fails TE-5 with +`tool.terminal` and `tool.outcome`, so the P0 gate is `not_fulfilled` for those +rollouts. mini-SWE must save a terminal result correlated with the submit call +before exiting, and Gym must retain it in the projected evidence. See the +[mini-SWE observability limits](../../responses_api_agents/miniswe_sandboxed_agent/README.md#native-behavior-and-observability). + TE-6 requires a finite numeric `reward` even when `mask_sample=true` excludes it from scoring. Masked results and explicitly incomplete verification require nonblank `failure_kind` and `failure_reason`; a masked `reward=0.0` is valid diff --git a/responses_api_agents/miniswe_sandboxed_agent/README.md b/responses_api_agents/miniswe_sandboxed_agent/README.md index 206234d3bf0..e9198a6c5eb 100644 --- a/responses_api_agents/miniswe_sandboxed_agent/README.md +++ b/responses_api_agents/miniswe_sandboxed_agent/README.md @@ -66,7 +66,11 @@ Removing the custom wrapper has these limits: durations. Gym emits `tool_timing_unavailable` gaps rather than estimating them. - Submission exits before a final tool observation is saved. The decision and native submission text remain available; that tool's execution observation is - incomplete. + incomplete, with no output or error evidence. This known mini-SWE gap fails + TE-5 (`tool.terminal` and `tool.outcome`) and the `gym-p0/v1` conformance gate, + even when the task succeeds. mini-SWE needs to persist a terminal tool result + correlated with the submit call before exiting; Gym's projection must then + retain that evidence. The observability test asserts this gap until it is fixed. - Native trajectories are saved after each step. A forced kill during a model request or command can lose the active step, or leave a partially written trajectory. Gym model-server capture remains independent of that file. diff --git a/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py b/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py index 2bfbde63012..67ab786dd9a 100644 --- a/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py +++ b/responses_api_agents/miniswe_sandboxed_agent/tests/test_observability.py @@ -251,11 +251,27 @@ async def query(params): assert result.summary["run"]["issues"]["model_call_failed"] == 1 # Inspect the emitted records, not a reconstructed copy of the expected evidence. - _, conformance = inspect_bundle( + report_dir, conformance = inspect_bundle( path, output=tmp_path / "capabilities", profile="gym-p0/v1", capture_dir=capture_dir ) - expected = "fulfilled" # Provider omission is faithful evidence; health still evaluates missing usage. + # Known mini-SWE gap: submission exits before saving the final tool observation. + # Fix the native submission evidence before expecting TE-5 and the P0 gate to pass. + submitted = scenario != "http_error" + expected = "not_fulfilled" if submitted else "fulfilled" assert conformance["verdict"] == expected - for capability in ("TE-1", "TE-2", "TE-3", "TE-4", "TE-5", "TE-7", "TE-8"): + assert conformance["evidence"]["TE-5"]["verdict"] == expected + findings = json.loads((report_dir / "evidence_results.jsonl").read_text())["findings"] + tool_findings = { + (finding["assertion"], finding["location"]) for finding in findings if finding["evidence"] == "TE-5" + } + if submitted: + assert trajectory["tool_calls"][-1]["status"] == "incomplete" + assert trajectory["tool_calls"][-1]["output"] is None + location = f"evaluator_rollouts.jsonl:1/ng_trajectory/tool_calls/{len(trajectory['tool_calls']) - 1}" + assert tool_findings == {("tool.terminal", location + "/status"), ("tool.outcome", location)} + else: + assert tool_findings == set() + # Provider omission is faithful evidence; health still evaluates missing usage. + for capability in ("TE-1", "TE-2", "TE-3", "TE-4", "TE-6", "TE-7", "TE-8"): assert conformance["evidence"][capability]["verdict"] == "fulfilled", (scenario, capability) assert conformance["is_behavioral_qualification"] is False