Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions tests/test_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -665,6 +665,17 @@ def test_a_goose_read_image_source_that_isnt_a_url_is_blocked_as_the_hook_blocks
self.assertEqual([(c["verdict"], c["rule_ids"]) for c in json.loads(out)["calls"]],
[("deny", ["TK-HOOK-BLOCKED"]), ("allow", [])])

def test_a_gptme_call_its_hook_cant_read_is_blocked_as_the_hook_blocks_it(self):
"""A markdown call without an args list, and an MCP call whose content isn't JSON, stopped the whole --agent
gptme replay (a plain ValueError). gptme's hook blocks both: audit denies them and replays the rest."""
_write_jsonl(self.root / "calls.jsonl", [_line("shell", {"args": ["x"], "content": "ls"}),
_line("srv.do", {"args": [], "content": "not json"}),
_line("shell", {"args": [], "content": "ls"})])
code, out, err = _run("--calls", str(self.root / "calls.jsonl"), "--agent", "gptme", "--json")
self.assertEqual(code, 0, err)
self.assertEqual([(c["verdict"], c["rule_ids"]) for c in json.loads(out)["calls"]],
[("deny", ["TK-HOOK-BLOCKED"]), ("deny", ["TK-HOOK-BLOCKED"]), ("allow", [])])

def test_a_mapping_bug_is_not_reported_as_the_hook_blocking_the_call(self):
"""#771: only UnreadableCall is a call the hook blocks; any other ValueError from the mapping is a bug that ends
the replay with its message (exit 2)."""
Expand Down
30 changes: 0 additions & 30 deletions tests/test_bounded.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,14 @@
the first)."""
import base64
import collections
import contextlib
import importlib.util
import itertools
import json
import os
import shutil
import sys
import tempfile
import unittest
import zipfile
from unittest import mock

import test_signer_service as ts
import test_ts_verifier as tv
Expand All @@ -30,7 +27,6 @@
from tracekit.signer.quotas import Limits
from tracekit.signer.rpc_schema import RPCError
from tracekit.signer.service import STRICTNESS, _stricter
from tracekit.storage import file as storage_file
from tracekit.verify import v2

ENGINES = ("re2", "regex") if importlib.util.find_spec("re2") else ("regex",)
Expand Down Expand Up @@ -355,32 +351,6 @@ class EveryStructuralMutationOfARunInParts(EveryStructuralMutation):
# that does fails the test). Past ~200, the signer's scans of its live approvals cost more than the starts they save
RUNS_PER_SIGNER = 200
LIMITS = Limits(open_runs=2 * RUNS_PER_SIGNER, pending_approvals=2 * RUNS_PER_SIGNER, events_per_s=1e9, burst=10 ** 9)
# the modules whose syncs a signer on the file store makes: its logs, its new files, and its state and key files
SIGNER_SYNCS = {"tracekit.storage.file", "tracekit.signer.service", "tracekit.deploy.files"}


@contextlib.contextmanager
def signer_without_fsync():
"""The signer's syncs (SIGNER_SYNCS) skipped in this process while it's open: the state space checks the approval
path, and durability has tests of its own (test_storage_file's test_durability_modes and
test_a_failed_background_sync_names_the_records_not_known_durable, storage_contract's
test_torn_tail_is_set_aside_and_reported), while fsync made the space ~14x slower on windows-latest (#722). Yields
the modules of the syncs made anywhere else, which still run: a sync the signer moves to another module fails the
test instead of silently making it slow again."""
elsewhere, real = [], {name: getattr(os, name) for name in ("fsync", "fdatasync") if hasattr(os, name)}

def stub(name):
def sync(fd):
caller = sys._getframe(1).f_globals.get("__name__")
if caller not in SIGNER_SYNCS:
elsewhere.append(caller)
real[name](fd)
return sync
with contextlib.ExitStack() as stack:
stack.enter_context(mock.patch.object(storage_file, "_sync", lambda fd, full: None)) # macOS's is an fcntl
for name in real:
stack.enter_context(mock.patch.object(os, name, stub(name)))
yield elsewhere


class ApprovalStateSpace(unittest.TestCase):
Expand Down
23 changes: 13 additions & 10 deletions tracekit/agent_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,14 +415,14 @@ def _gptme_tool(name, ti):
args, content, kwargs = ti.get("args"), ti.get("content"), ti.get("kwargs")
if kwargs is not None:
if not isinstance(kwargs, dict) or args is not None or content is not None:
raise ValueError("gptme native calls must carry only a kwargs object")
raise UnreadableCall("gptme native calls must carry only a kwargs object")
elif not isinstance(args, list) or not all(isinstance(arg, str) for arg in args) or not isinstance(content, str):
raise ValueError("gptme markdown calls must carry an args list and text content")
raise UnreadableCall("gptme markdown calls must carry an args list and text content")

if name == "tmux":
operations = ti.get("operations")
if not isinstance(operations, list) or not operations or not all(isinstance(op, str) for op in operations):
raise ValueError("gptme tmux requires the parsed operations")
raise UnreadableCall("gptme tmux requires the parsed operations")
commands, pane_input = [], False
for op in operations:
parts = op.split(maxsplit=1)
Expand Down Expand Up @@ -452,7 +452,7 @@ def _gptme_tool(name, ti):
patches = ti.get("patches")
if not isinstance(patches, list) or not patches or not all(
isinstance(p, list) and len(p) == 2 and all(isinstance(v, str) and v for v in p) for p in patches):
raise ValueError("gptme patch_many requires the prepared patches")
raise UnreadableCall("gptme patch_many requires the prepared patches")
return "Edit", {"paths": [path for path, _ in patches], "patches": [patch for _, patch in patches]}

if name == "mcp":
Expand All @@ -465,10 +465,10 @@ def _gptme_tool(name, ti):
value = kwargs.get(key) if kwargs is not None else content.strip()
if name == "shell" and kwargs is None:
if args:
raise ValueError("gptme markdown shell takes no positional arguments")
raise UnreadableCall("gptme markdown shell takes no positional arguments")
value = value.removeprefix("$ ")
if not isinstance(value, str):
raise ValueError(f"gptme {name} requires text {key}")
raise UnreadableCall(f"gptme {name} requires text {key}")
if name == "ipython":
return name, {key: value}
result = {"command": value}
Expand All @@ -488,7 +488,7 @@ def _gptme_tool(name, ti):
path = path.split(" ", 1)[1]
paths = [path]
if not paths or not all(isinstance(path, str) and path for path in paths):
raise ValueError(f"gptme {name} requires a file path")
raise UnreadableCall(f"gptme {name} requires a file path")
result = {"file_path": os.path.expanduser(paths[0])}
if name == "read":
if len(paths) > 1:
Expand All @@ -498,15 +498,18 @@ def _gptme_tool(name, ti):
native_key = "edit" if name in ("morph", "hashline_edit") else key
value = kwargs.get(native_key) if kwargs is not None else content
if not isinstance(value, str):
raise ValueError(f"gptme {name} requires text {key}")
raise UnreadableCall(f"gptme {name} requires text {key}")
result[key] = value
return mapped, result

if "." in name:
server, _, command = name.partition(".")
params = kwargs if kwargs is not None else json.loads(content)
try:
params = kwargs if kwargs is not None else json.loads(content)
except ValueError:
raise UnreadableCall("gptme MCP call content isn't JSON") from None
if not server or not command or not isinstance(params, dict):
raise ValueError("gptme MCP calls require a qualified tool name and an arguments object")
raise UnreadableCall("gptme MCP calls require a qualified tool name and an arguments object")
return f"mcp__{server}__{command}", params
return name, kwargs if kwargs is not None else ti

Expand Down
Loading