From e69742f03dcc285ed6fa58596783d60fd38bf192 Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:09:40 +0000 Subject: [PATCH 1/9] Integrations: MCP big lists and input-required rounds, LangChain parallel asks, tracekit last, gateway listing errors - MCP client (#685): a list page over half a signer frame is recorded as its digest (result_field, as the gateway does), so list_tools no longer fails with quota_exceeded; mcp's own re-list inside call_tool (validate_tool_result) is no longer recorded as input the model read. - MCP input-required rounds (#718): a pending round is keyed by the asyncio task driving the call, so a server that reuses request_state can't move one call's result under another's decision; a call the task leaves between rounds completes with the input request it returned. - LangChain (#687): a call whose approval is already answered doesn't pause again, so in a tracekit_tool_node with two asks the approved call runs on resume (16 of 16 on invoke and ainvoke); the call still waiting is refused. - tracekit last: OpenCode, Kilo, goose and Antigravity runs finish at the signer's idle timeout, not when the session ends. - MCP gateway: the repeated-cursor error says what happened; a listing past its shared 600 s budget names the upstream; docs/mcp-gateway.md documents the budget. --- docs/integrations/langchain.md | 9 +- docs/integrations/mcp.md | 4 +- docs/mcp-gateway.md | 3 + tests/test_contract_mcp.py | 133 ++++++++++++++++++++++++++++ tests/test_instrument.py | 13 +++ tests/test_integration_langchain.py | 47 +++++++++- tests/test_mcp_gateway.py | 20 ++++- tracekit/cli.py | 3 + tracekit/integrations/langchain.py | 18 +++- tracekit/integrations/mcp.py | 61 ++++++++----- tracekit/mcp_gateway.py | 12 +-- 11 files changed, 286 insertions(+), 37 deletions(-) diff --git a/docs/integrations/langchain.md b/docs/integrations/langchain.md index 82cdea08a..4c2ea42be 100644 --- a/docs/integrations/langchain.md +++ b/docs/integrations/langchain.md @@ -85,7 +85,14 @@ calls with `tracekit.instrument()` or the [LLM gateway](../gateway.md). The mess On resume LangGraph re-runs the tool node, so the middleware asks the signer again, this time naming the approval. The signer, not the saved state, decides: the call runs only if that approval was granted for this exact call and these exact args, and was not used before. Otherwise the call is denied as above. The `approval_id` in the resume - value is only a hint; leaving it out does not skip the check. + value is only a hint; leaving it out does not skip the check. A call whose approval was already answered when the + node re-runs doesn't pause again: the signer decides it at once. + + A `tracekit_tool_node` runs every call of a model message in one node, and LangGraph gives that node one resume + value, to whichever call pauses first. When two calls ask, the one a person approved runs on resume and the other, + still waiting, is denied (`TK-APPROVAL-UNBOUND`, naming its approval): approve that one too, and the model's retry + runs it. With three or more, a call still waiting after a resume pauses the node again, and LangGraph's re-run then + denies a call that already ran once. `create_agent` gives each call a task of its own and has neither limit. Exceptions raised by the tool are recorded and re-raised unchanged. diff --git a/docs/integrations/mcp.md b/docs/integrations/mcp.md index a74d03f0f..998f47f0e 100644 --- a/docs/integrations/mcp.md +++ b/docs/integrations/mcp.md @@ -39,7 +39,7 @@ Every other attribute is the wrapped session's. Only calls made through the wrap | right before it is sent | `approval_consume` | the same call and arguments, and the approval id when the call waited for one | | after the server answers | `complete` | `status` `ok`, or `error` for a result with `isError: true`; a `sha256:` commitment to the result (not the result), or the exception type and message | -`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`observe`, source `mcp:/tools`, `/resources` or `/prompts`): the model reads the descriptions the server wrote, and the page the caller asked for is what is recorded. A page the signer cannot record is withheld, unless the run fails open for `mcp`, when an unreachable signer returns it with a warning; a refusal is withheld either way. A tool call or prompt that returns an `InputRequiredResult` (MCP 2.3) records each round's input requests as untrusted observed input (`observe`, source `mcp:/input_required`). When the server provides a `request_state` and the call needed no human approval, it is decided and completed once, under its initial decision, when the terminal result arrives. Rounds from a server that returns no `request_state`, or for a decision that required human approval (as the signer refuses to consume a used approval), are decided and completed as separate calls per round. +`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`observe`, source `mcp:/tools`, `/resources` or `/prompts`): the model reads the descriptions the server wrote, and the page the caller asked for is what is recorded (a page over half the signer's 1 MiB frame as its `sha256:` digest, as the [MCP gateway](../mcp-gateway.md) records one). The list mcp's `call_tool` makes itself, to check a result against the tool's output schema, never reaches the model and isn't recorded. A page the signer cannot record is withheld, unless the run fails open for `mcp`, when an unreachable signer returns it with a warning; a refusal is withheld either way. A tool call or prompt that returns an `InputRequiredResult` (MCP 2.3) records each round's input requests as untrusted observed input (`observe`, source `mcp:/input_required`). When the server provides a `request_state` and the call needed no human approval, it is decided and completed once, under its initial decision, when the terminal result arrives. A round continues the call its own asyncio task left waiting, as `mcp.Client.call_tool` drives the rounds, so concurrent calls never share a decision even when the server returns the same `request_state` to both; a task that makes another call on the session before the next round ends the waiting call, completed with the input request it returned. Rounds from a server that returns no `request_state`, or for a decision that required human approval (as the signer refuses to consume a used approval), are decided and completed as separate calls per round. `` is the name in the server's initialize result, with every character outside `A-Za-z0-9_.-` replaced by `_` (`unknown` when the server gave none). The policy's `tools` map should map `mcp:*/*` to the `mcp` class (the coding @@ -64,7 +64,7 @@ a lost connection) propagates unchanged. Both are recorded as `error`. ## Not covered -- MCP calls carry no call id, so every `call_tool` is a call of its own (attempt 0); multi-round input-required calls reuse their decision across rounds, while an independent retry is a new decision. A call whose later rounds never reach the session (the elicitation callback fails, or mcp's `input_required_max_rounds` runs out, both inside `mcp.Client`) stays decided and is never completed; at run close the signer records it as a signed `decided_not_completed` gap. +- MCP calls carry no call id, so every `call_tool` is a call of its own (attempt 0); multi-round input-required calls reuse their decision across rounds, while an independent retry is a new decision. A call whose later rounds never reach the session (the elicitation callback fails, mcp's `input_required_max_rounds` runs out, both inside `mcp.Client`, or the task driving it ends) stays decided and is never completed; at run close the signer records it as a signed `decided_not_completed` gap. Rounds you drive yourself from different tasks are decided as calls of their own. - There is no saved state: a call waiting for an approval does not survive the process. - Servers the agent reaches without this session (another client, a hosted MCP tool of a model provider) are not seen. - The server's name in `mcp:/` is the one the server gives. An mcp 1.x session doesn't keep it: diff --git a/docs/mcp-gateway.md b/docs/mcp-gateway.md index 9e765e81f..b5b277c7d 100644 --- a/docs/mcp-gateway.md +++ b/docs/mcp-gateway.md @@ -106,6 +106,9 @@ gateway, for the identity it decided the call for, alone; a run closed mid-call ([Kubernetes](deploy-kubernetes.md#custody)). - Tools, resources and prompts are listed up to 100 pages from each upstream; past that, the whole listing fails with an error naming the upstream. +- One tools/list, resources/list or prompts/list has 600 s for all the upstreams and their pages together, listed one + after another, so a slow upstream leaves less time for those after it; past that the listing fails with an error + naming the upstream it was waiting for. A tools/call, prompts/get or resources/read has 600 s of its own. - A resource is read through the upstream that listed it: list resources before reading one; a URI from a resource template isn't routed. - No server-initiated messages: sampling, elicitation, subscriptions and progress notifications from an upstream don't diff --git a/tests/test_contract_mcp.py b/tests/test_contract_mcp.py index f1c4bdf44..0d31a92a7 100644 --- a/tests/test_contract_mcp.py +++ b/tests/test_contract_mcp.py @@ -815,3 +815,136 @@ async def run_call(): self.assertEqual(out.result_type, "input_required") observed = [r for m, r in signer.requests if m == "observe"] self.assertEqual(len(observed), 0) + + def _state_reusing_server(self): + """A server that answers every first round with the same request_state (a stateless server, or one whose state + only encodes progress), and finishes on the next.""" + from mcp.server.lowlevel import Server + import mcp.types as t + + async def on_list_tools(ctx, params): + return t.ListToolsResult(tools=[t.Tool(name="ask", inputSchema={"type": "object"})]) + + async def on_call_tool(ctx, params): + if params.request_state is None: + return t.InputRequiredResult(request_state="S", input_requests={"q": t.ElicitRequest( + params=t.ElicitRequestFormParams(message="who?", + requested_schema={"type": "object", "properties": {}}))}) + await asyncio.sleep(0.05) # the other call's rounds run meanwhile + return t.CallToolResult(content=[t.TextContent(type="text", text="done")]) + return Server("srv", on_list_tools=on_list_tools, on_call_tool=on_call_tool) + + def _wired(self): + """instrument()'s MCP wiring on a signer whose decisions have ids of their own and send results whole.""" + from tracekit import autowire + ids = iter(range(1, 100)) + + class Signer(_Signer): + def __getattr__(self, method): + call = super().__getattr__(method) + if method == "decide": + return lambda req: dict(call(req), decision_id=f"dec-{next(ids)}", send_result=True) + return call + signer = Signer() + autowire._mcp(mock.Mock(client=signer, registered=self.RUN)) + return signer + + @staticmethod + async def _elicit(ctx, params): + import mcp.types as t + return t.ElicitResult(action="accept", content={}) + + def test_concurrent_calls_on_a_server_that_reuses_request_state_never_share_a_decision(self): + """#718: B's first round, kept under the same (tool, arguments, state) as A's, completed B "ok" at round 1, and + B's second round was decided a third time.""" + server, signer = self._state_reusing_server(), self._wired() + + async def both(): + async with MCPClient(server, elicitation_callback=self._elicit) as c: + return await asyncio.gather(c.call_tool("ask", {"k": "v"}), c.call_tool("ask", {"k": "v"})) + self.assertEqual([o.content[0].text for o in asyncio.run(both())], ["done", "done"]) + decided = {r["tool_call_id"]: r for m, r in signer.requests if m == "decide"} + self.assertEqual(len(decided), 2) + completes = [r for m, r in signer.requests if m == "complete"] + self.assertEqual(sorted(r["tool_call_id"] for r in completes), sorted(decided)) + for r in completes: + self.assertEqual((r["status"], r["result"]["content"][0]["text"]), ("ok", "done")) + self.assertEqual(r["decision_id"], f"dec-{list(decided).index(r['tool_call_id']) + 1}") + + def test_a_call_left_between_rounds_ends_when_its_task_makes_another(self): + """#718: call A stops after its first round; call B, same tool, arguments and state, ran its last round under A's + id and decision. Now A completes with the input request it got, and B under its own decision.""" + server, signer = self._state_reusing_server(), self._wired() + + async def abandoned_then_new(): + async with MCPClient(server, elicitation_callback=self._elicit) as c: + await c.session.call_tool("ask", {"k": "v"}, allow_input_required=True) # A: round 1 only + return await c.call_tool("ask", {"k": "v"}) # B, to its end + self.assertEqual(asyncio.run(abandoned_then_new()).content[0].text, "done") + a, b = [r["tool_call_id"] for m, r in signer.requests if m == "decide"] + completes = [(r["tool_call_id"], r["decision_id"], r["status"]) for m, r in signer.requests if m == "complete"] + self.assertEqual(completes, [(a, "dec-1", "ok"), (b, "dec-2", "ok")]) + first, last = [r["result"] for m, r in signer.requests if m == "complete"] + self.assertEqual((first["resultType"], last["content"][0]["text"]), ("input_required", "done")) + + +@unittest.skipUnless(HAVE_MCP and hasattr(mcp.types, "InputRequiredResult"), "mcp 2.3+ with InputRequiredResult required") +class BigLists(unittest.TestCase): + """#685: a tools/list over the signer's 1 MiB frame (300 tools with 40-field schemas, about 1.4 MB of JSON), on the + real signer, under instrument(). It was sent whole in `observe`, so the signer refused it (quota_exceeded) whatever + the fail mode, and a call_tool failed after the tool ran: mcp lists the tools again to check its result.""" + + def setUp(self): + from mcp.client.session import ClientSession + from tracekit import autowire + from tracekit.policy2.engine import Engine + from tracekit.signer.service import SignerService + for name in ("call_tool", "get_prompt", "list_tools", "list_resources", "list_prompts"): # autowire patches them + patcher = mock.patch.object(ClientSession, name, getattr(ClientSession, name)) + patcher.start() + self.addCleanup(patcher.stop) + self.service = SignerService(ac.tmpdir(self), policy=Engine({})) + self.addCleanup(self.service.close) + client = Client(ac._serve(self, self.service.handle_frame)) + self.addCleanup(client.close) + self.run = client.register_run({"agent": {"name": "big"}}) + autowire._mcp(mock.Mock(client=client, registered=self.run)) + + @staticmethod + def server(): + from mcp.server.lowlevel import Server + import mcp.types as t + props = {f"field_{i}": {"type": "string", "description": "a field " * 8} for i in range(40)} + + async def on_list_tools(ctx, params): + return t.ListToolsResult(tools=[t.Tool(name=f"tool_{i}", description="does a thing " * 10, + inputSchema={"type": "object", "properties": props}) + for i in range(300)]) + + async def on_call_tool(ctx, params): + return t.CallToolResult(content=[t.TextContent(type="text", text="ran " + params.name)]) + return Server("big", on_list_tools=on_list_tools, on_call_tool=on_call_tool) + + def events(self, typ): + return [r["event"]["data"] for r in self.service.log.storage.iter_run("default", self.run["run_id"]) + if r["event"]["type"] == typ] + + def test_a_list_over_half_a_frame_is_recorded_as_its_digest(self): + async def listed(): + async with MCPClient(self.server()) as c: + return await c.list_tools() + out = asyncio.run(listed()) + self.assertEqual(len(out.tools), 300) + self.assertGreater(len(out.model_dump_json()), 1 << 20) + [seen] = self.events("input.observed") + self.assertEqual((seen["source"], seen["trust"]), ("mcp:big/tools", "untrusted")) + self.assertLess(seen["output"]["size"], 100) # the digest, not the page + + def test_the_list_mcp_makes_inside_call_tool_is_not_recorded(self): + async def called(): + async with MCPClient(self.server()) as c: # no list first: mcp lists to check the result + return await c.call_tool("tool_7", {}) + out = asyncio.run(called()) + self.assertEqual((out.is_error, out.content[0].text), (False, "ran tool_7")) + self.assertEqual(self.events("input.observed"), []) + self.assertEqual([r["ok"] for r in self.events("tool.result")], [True]) diff --git a/tests/test_instrument.py b/tests/test_instrument.py index 7b2770eda..5474eb657 100644 --- a/tests/test_instrument.py +++ b/tests/test_instrument.py @@ -98,6 +98,19 @@ def test_last_open_explains_other_coding_agent_session(self): self.assertIn("it finishes when the session ends", codex_line) self.assertNotIn("`/exit` in Claude Code", codex_line) + def test_last_open_explains_an_agent_whose_hooks_get_no_session_end(self): + """OpenCode's and Kilo's plugin and goose's and Antigravity's hooks install no session-end event: their run + never finished "when the session ends", as `tracekit last` said.""" + agents = ("opencode", "kilo", "goose", "antigravity") + self.sh("-c", "import os; from tracekit.sdk.client import Client\n" + f"for a in {agents!r}: Client().run(a).decide('c1', 'Bash', {{'command': 'ls'}})\n" + "os._exit(0)") + stderr = self.sh("-m", "tracekit", "last", code=1).stderr + for agent in agents: + line = next(line for line in stderr.splitlines() if f"({agent})" in line) + self.assertIn(f"still open: Tracekit's {agent} hooks get no session end, so it finishes after the signer's " + "idle timeout", line) + class TestThreeSteps(DevSigner): def test_each_framework(self): diff --git a/tests/test_integration_langchain.py b/tests/test_integration_langchain.py index 2ab5537bc..f6790619b 100644 --- a/tests/test_integration_langchain.py +++ b/tests/test_integration_langchain.py @@ -1,8 +1,9 @@ """tracekit.integrations.langchain beyond the adapter contract (tests/test_contract_langchain.py): the result commitment, `ask` without a checkpointer, an interrupt raised inside a tool, an `ask` under each ToolNode error -handling, and the async gate's signer calls off the event loop.""" +handling, two asks in one ToolNode, and the async gate's signer calls off the event loop.""" import asyncio import json +import threading import time import types import unittest @@ -306,6 +307,50 @@ def test_the_tools_own_errors_are_handled_as_configured(self): self.assertIn("please fix your mistakes", tool_message(out).content) +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class ParallelAsks(unittest.TestCase): + """#687: two calls of one message that both ask, in one tracekit_tool_node. LangGraph hands the node's one resume + value to whichever call reaches interrupt() first, so the call a person approved ran only when it got there first + (7 of 16 under ainvoke, 4 of 16 under invoke). Each trial here makes the other call get there first.""" + + def trial(self, mode): + """(the call the pause showed and a person approved, what the resume answered with)""" + ac.RAN.clear() + signer = FakeSigner(ac._rule) + graph = tool_graph(tracekit_tool_node(TOOLS, signer, signer.register_run({"request_id": "reg", "agent": {"name": "test"}}))) + go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) + calls = [{"name": "pay", "args": {"to": "a", "cents": i}, "id": f"c{i}"} for i in (1, 2)] + shown = go({"messages": [HumanMessage(json.dumps(calls))]}, THREAD)["__interrupt__"][0].value["tracekit"] + signer.approval_decide({"request_id": "ok", "approval_id": shown["approval_id"], "decision": "approve"}) + other_paused, request = threading.Event(), signer.approval_request + + def pause(value): + try: + return interrupt(value) + finally: + other_paused.set() + + def approval_request(req): # the approved call reaches the gate's interrupt() only after the other one + if req["tool_call_id"] == shown["tool_call_id"]: + other_paused.wait(5) + return request(req) + signer.approval_request = approval_request + with mock.patch("tracekit.integrations.langchain.interrupt", pause): + out = go(Command(resume={"approval_id": shown["approval_id"]}), THREAD) + return shown["tool_call_id"], out + + def test_the_approved_call_runs_whichever_call_gets_the_resume_value(self): + for mode in ("sync", "async"): + outcomes = [] + for _ in range(16): + approved, out = self.trial(mode) + seen = {m.tool_call_id: m.content for m in out["messages"] if isinstance(m, ToolMessage)} + other = ({"c1", "c2"} - {approved}).pop() + outcomes.append(("__interrupt__" in out, list(ac.RAN), seen.get(approved) == f"paid {approved[1]}", + seen.get(other, "").startswith("Tool call blocked by policy: TK-APPROVAL-UNBOUND"))) + self.assertEqual(outcomes, [(False, ["pay"], True, True)] * 16, mode) + + class Jittery: """The client, with the pause a busy process may take between a call's client_seq and its send.""" diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index b223876cc..2dba9fe06 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -368,15 +368,27 @@ async def lists(c): ("mcp:up/prompts", "untrusted")]) def test_repeated_cursor_is_an_error(self): - """A cursor returned twice raises an error naming the upstream with advice.""" + """A cursor returned twice raises an error naming the upstream and what it did (it named the 100-page cap, + which the listing never reached).""" async def on_list_tools(ctx, params): return mcp_types.ListToolsResult(tools=[], next_cursor="same-forever") self._swap_upstream(LowServer("bad", on_list_tools=on_list_tools)) with self.assertRaises(Exception) as e: self.agent(lambda c: c.list_tools()) - self.assertIn("upstream 'up': tools/list returned a repeated cursor; " - "the gateway lists at most 100 pages per upstream; " - "give the upstream larger pages or fix its cursor", + self.assertIn("upstream 'up': tools/list page 2 returned a cursor an earlier page had returned, so the listing " + "would never end; fix the upstream's cursor", messages(e.exception)) + self.assertNotIn("100 pages", messages(e.exception)) + + def test_a_listing_past_its_shared_budget_names_the_upstream(self): + """One listing has UPSTREAM_TIMEOUT_S for every upstream and page; past it the agent read "upstream error: + TimeoutError: ; the gateway connects to an upstream that went away again on the next call".""" + async def on_list_tools(ctx, params): + await asyncio.sleep(1) + return mcp_types.ListToolsResult(tools=[]) + self._swap_upstream(LowServer("slow", on_list_tools=on_list_tools)) + with mock.patch.object(mcp_gateway, "UPSTREAM_TIMEOUT_S", 0.3), self.assertRaises(Exception) as e: + self.agent(lambda c: c.list_tools()) + self.assertIn("upstream 'up': tools/list timed out (one listing has 0.3 s for every upstream and page)", messages(e.exception)) def test_page_bound_error_forgets_page1_resources(self): diff --git a/tracekit/cli.py b/tracekit/cli.py index 70c0d6e4f..79dc16d5d 100644 --- a/tracekit/cli.py +++ b/tracekit/cli.py @@ -1373,6 +1373,9 @@ def _last(a): elif agent == "claude-code": why = ("still open: it finishes when the session ends (`/exit` in Claude Code) " "or after the signer's idle timeout; tracekit last --open verifies it as it stands") + elif agent in ("opencode", "kilo", "goose", "antigravity"): # their hooks get no session end + why = (f"still open: Tracekit's {agent} hooks get no session end, so it finishes after the signer's idle " + "timeout; tracekit last --open verifies it as it stands") elif agent in agent_hooks.AGENT_NAMES.values(): why = ("still open: it finishes when the session ends " "or after the signer's idle timeout; tracekit last --open verifies it as it stands") diff --git a/tracekit/integrations/langchain.py b/tracekit/integrations/langchain.py index 22b81c05a..2d5d29a02 100644 --- a/tracekit/integrations/langchain.py +++ b/tracekit/integrations/langchain.py @@ -7,7 +7,9 @@ fail_modes=run.get("fail_modes"))]) `deny` replaces the call with an error ToolMessage and the run goes on. `ask` pauses the run with LangGraph -`interrupt()`; resume it with `Command(resume={"approval_id": ...})`. Every call that is not denied runs only after the +`interrupt()`; resume it with `Command(resume={"approval_id": ...})`. A call whose approval was answered before the +resume doesn't pause again, so of two calls of one ToolNode that ask, the one a person approved runs on resume and the +other, still waiting, is refused. Every call that is not denied runs only after the signer's `approval_consume` agrees, so an approval is checked by the signer, never by the saved state; an approval with a T2 executor runs the arguments the consume returns. `ask` needs a checkpointer: without one the call is denied. @@ -116,9 +118,17 @@ def _gate(self, request): # gets the same approval back apr = self.signer.approval_request({"request_id": uuid.uuid4().hex, **self.run, "tool_call_id": call["id"], "attempt": attempt}) - resume = interrupt({"tracekit": {"approval_id": apr["approval_id"], "tool_call_id": call["id"], - "tool": call["name"], "rule_ids": d["rule_ids"]}}) - hint = resume.get("approval_id") if isinstance(resume, dict) else None + # answered already (the re-run on resume): no pause, the consume decides. LangGraph hands a node's one + # resume value to whichever of its calls pauses first, so an approved call that paused might not get it. + # lean: three or more asks in one ToolNode: one still waiting after a resume pauses the node again, and + # LangGraph's re-run then refuses a call that ran (once); give each call a task of its own (Send, as + # create_agent does) if graphs need that + if apr["state"] == "requested": + resume = interrupt({"tracekit": {"approval_id": apr["approval_id"], "tool_call_id": call["id"], + "tool": call["name"], "rule_ids": d["rule_ids"]}}) + hint = resume.get("approval_id") if isinstance(resume, dict) else None + else: + hint = apr["approval_id"] # every call, allowed or approved: the saved state can't vouch that no approval is needed c = self.signer.approval_consume({"request_id": uuid.uuid4().hex, **self.run, "tool_call_id": call["id"], "attempt": attempt, "tool": call["name"], "args_source": "parsed", diff --git a/tracekit/integrations/mcp.py b/tracekit/integrations/mcp.py index 8fa359002..5e1d7a5f0 100644 --- a/tracekit/integrations/mcp.py +++ b/tracekit/integrations/mcp.py @@ -13,19 +13,28 @@ up to `approval_wait_s` for a person to decide in the signer; with `approval_wait_s=0` the call is refused instead. Every call that is not denied is sent only after the signer's `approval_consume` agrees, with the arguments it returns for an approval with a T2 executor. Multi-round input-required calls (MCP 2.3) with a `request_state` under an allowed -decision are decided and completed once across rounds; rounds without `request_state`, or whose decision required human -approval, are decided as calls of their own. A result with `isError: true` completes as an error; an exception from the session -propagates unchanged and completes as an error. `list_tools`, `list_resources` and `list_prompts` are recorded as -untrusted observed input (`mcp:/tools`, `/resources`, `/prompts`) and returned once recorded. A list the -signer cannot reach is withheld unless the run fails open for this class, when it is returned with a warning; a -refusal is withheld either way. Only those calls made through the wrapped session are seen. +decision are decided and completed once across rounds: a round continues the call its own asyncio task left waiting (as +mcp's Client.call_tool drives them), so concurrent calls never share one, whatever `request_state` the server returns. +A task that makes any other call on the session first ends the waiting one, completed with the input request it +returned; a task that ends between rounds leaves it to the signer's decided_not_completed gap. Rounds without +`request_state`, or whose decision required human approval, are decided as calls of their own. A result with +`isError: true` completes as an error; an exception from the session propagates unchanged and completes as an error. +`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`mcp:/tools`, +`/resources`, `/prompts`; a page over half a signer frame as its digest) and returned once recorded. A list the signer +cannot reach is withheld unless the run fails open for this class, when it is returned with a warning; a refusal is +withheld either way. The list mcp itself asks for inside `call_tool` (to check a result against the tool's output +schema) never reaches the model and isn't recorded. Only those calls made through the wrapped session are seen. """ +import asyncio +import functools import re import uuid import warnings +import weakref from mcp.types import CallToolResult, TextContent +from tracekit.autowire import DECIDED_UPSTREAM from tracekit.format.canon import event_hash from tracekit.integrations.held import HeldCalls from tracekit.sdk.client import SignerUnavailable, fail_open, refusal, result_field @@ -46,8 +55,8 @@ def __init__(self, session, signer, run, approval_wait_s=300, server=None): `server`: the server's name, else the session's server_info.""" super().__init__(signer, run, approval_wait_s) self._session, self._server = session, server - # lean: abandoned multi-round calls stay in _pending for the session's life; at run close they become decided_not_completed - self._pending = {} + # the asyncio task -> the multi-round call it left waiting for its next round + self._pending = weakref.WeakKeyDictionary() def __getattr__(self, name): return getattr(self._session, name) @@ -81,13 +90,22 @@ async def _list(self, kind, *a, **kw): warnings.warn(_UNNAMED, stacklevel=4) # list_* -> _list -> warn: the caller's line, as _call's warning try: await self._rpc("observe", source=f"mcp:{server}/{kind}", trust="untrusted", - value=out.model_dump(mode="json", by_alias=True, exclude_none=True)) + value=result_field(True, out.model_dump(mode="json", by_alias=True, exclude_none=True))) except SignerUnavailable as e: # unreachable: the run's fail mode for this class, as _gate if not fail_open(self.fail_modes, self.CLASS): raise warnings.warn(f"tracekit: MCP {kind} list not recorded: {e}", stacklevel=2) return out + async def _send(self, method, *a, **kw): + """The session's `method`. mcp's call_tool lists the tools itself to check a result against the tool's output + schema: that list never reaches the model, so instrument()'s session lets it through unrecorded.""" + upstream = DECIDED_UPSTREAM.set(True) + try: + return await getattr(self._session, method)(*a, **kw) + finally: + DECIDED_UPSTREAM.reset(upstream) + async def _call(self, method, prefix, name, arguments, *a, **kw): raw, server = self._server_name() if not raw: @@ -95,10 +113,14 @@ async def _call(self, method, prefix, name, arguments, *a, **kw): tool = f"mcp:{server}/{prefix}{name}" args = arguments or {} caller_args_digest = event_hash({"tool": tool, "args": args}) - req_state = kw.get("request_state") - key = (tool, caller_args_digest, req_state) if req_state is not None else None - if key is not None and key in self._pending: - call_id, req, d, args = self._pending.pop(key) + task = asyncio.current_task() + held = self._pending.pop(task, None) + if held is not None and held[0] != (tool, caller_args_digest, kw.get("request_state")): + # this task moved on without the round: the call ended with the input request it returned + await self._complete(held[1], held[2], status="ok", result=held[5]) + held = None + if held is not None: + _, call_id, req, d, args, _ = held try: c = await self._rpc("approval_consume", tool_call_id=call_id, tool=tool, args_source="parsed", args=args) except (RPCError, SignerUnavailable) as e: @@ -121,7 +143,7 @@ async def _call(self, method, prefix, name, arguments, *a, **kw): return CallToolResult(content=[TextContent(type="text", text=BLOCKED + why)], isError=True) req = None if d is None else {"tool_call_id": call_id, "decision_id": d["decision_id"], "args_digest": event_hash({"tool": tool, "args": args})} - send = getattr(self._session, method) + send = functools.partial(self._send, method) if d is None: # the signer unreachable and the run fails open: the call runs unrecorded return await send(name, arguments, *a, **kw) try: @@ -135,7 +157,7 @@ async def _call(self, method, prefix, name, arguments, *a, **kw): if input_requests: try: await self._rpc("observe", source=f"mcp:{server}/input_required", trust="untrusted", - value=input_requests) + value=result_field(True, input_requests)) except SignerUnavailable as e: if not fail_open(self.fail_modes, self.CLASS): await self._complete(call_id, req, status="error", error=f"{type(e).__name__}: {e}"[:4096]) @@ -145,12 +167,11 @@ async def _call(self, method, prefix, name, arguments, *a, **kw): await self._complete(call_id, req, status="error", error=f"{type(e).__name__}: {e}"[:4096]) raise next_state = getattr(out, "request_state", None) - key = (tool, caller_args_digest, next_state) if next_state is not None else None - if key is not None and d.get("decision") == "allow" and key not in self._pending: - self._pending[key] = (call_id, req, d, args) + result = result_field(d.get("send_result"), dump) + if next_state is not None and d.get("decision") == "allow": + self._pending[task] = ((tool, caller_args_digest, next_state), call_id, req, d, args, result) else: - await self._complete(call_id, req, status="ok", - result=result_field(d.get("send_result"), dump)) + await self._complete(call_id, req, status="ok", result=result) return out failed = getattr(out, "is_error", getattr(out, "isError", False)) # mcp 2.x, mcp 1.x await self._complete(call_id, req, status="error" if failed else "ok", diff --git a/tracekit/mcp_gateway.py b/tracekit/mcp_gateway.py index d617d2e1c..55d318e06 100644 --- a/tracekit/mcp_gateway.py +++ b/tracekit/mcp_gateway.py @@ -189,8 +189,11 @@ def _answer(self, gw, key, method, params): for up in ups.clients: cursor, n, seen, all_items = None, 0, set(), [] while True: - result = ups.call(up, call, cursor=cursor, - timeout=deadline - time.monotonic()) + try: + result = ups.call(up, call, cursor=cursor, timeout=deadline - time.monotonic()) + except TimeoutError: + raise _Error(-32603, f"upstream {up!r}: {kind}/list timed out (one listing has " + f"{UPSTREAM_TIMEOUT_S:g} s for every upstream and page)") from None items = getattr(result, kind) if kind == "tools": # one named like a prompt would be decided as that prompt: never offered items = [t for t in items if not t.name.startswith(PROMPT)] @@ -200,9 +203,8 @@ def _answer(self, gw, key, method, params): if cursor is None: break if cursor in seen: - raise _Error(-32603, f"upstream {up!r}: {kind}/list returned a repeated cursor; " - f"the gateway lists at most {MAX_LIST_PAGES} pages per upstream; " - "give the upstream larger pages or fix its cursor") + raise _Error(-32603, f"upstream {up!r}: {kind}/list page {n} returned a cursor an earlier " + "page had returned, so the listing would never end; fix the upstream's cursor") seen.add(cursor) if n >= MAX_LIST_PAGES: raise _Error(-32603, f"upstream {up!r}: {kind}/list exceeded {MAX_LIST_PAGES} pages; " From cf3061ce2bfab20ac0e3b361908a2624a1edb11e Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:05:51 +0000 Subject: [PATCH 2/9] LangChain: a ToolNode decides every call of a message before any runs, so no call runs in a pass that then pauses ToolNode runs a message's calls in one node, which LangGraph runs again from its start on each resume. A call that ran in a pass that then paused ran again or read as refused: an allowed sibling of one ask ran twice, and with three or more asks a call that ran was refused on the next pass while an approved one never ran. A gated ToolNode (tracekit_tool_node, or one instrument() wired) now decides a pass's calls together first (TracekitMiddleware._pass) and pauses once, before any runs, while an approval is unanswered; the payload's `approvals` lists every waiting one. On resume each allowed or approved call runs once and each rejected or unanswered one is refused. The hook is ToolNode's class-level node function and acts only while the node's wrapper is the gate's, so tracekit why's replay copy (its own wrapper) runs no real tool. Replaces the lean note. --- docs/integrations/langchain.md | 15 +-- tests/test_integration_langchain.py | 132 +++++++++++++++----------- tracekit/autowire.py | 8 +- tracekit/integrations/langchain.py | 138 ++++++++++++++++++++++------ 4 files changed, 203 insertions(+), 90 deletions(-) diff --git a/docs/integrations/langchain.md b/docs/integrations/langchain.md index 4c2ea42be..ed8e9ee72 100644 --- a/docs/integrations/langchain.md +++ b/docs/integrations/langchain.md @@ -75,7 +75,8 @@ calls with `tracekit.instrument()` or the [LLM gateway](../gateway.md). The mess - **deny**: the tool does not run. The model gets an error ToolMessage, `Tool call blocked by policy: `, and the agent run goes on. - **ask**: the middleware opens an approval and pauses the run with LangGraph `interrupt()`. The interrupt payload is - `{"tracekit": {"approval_id", "tool_call_id", "tool", "rule_ids"}}`. A person approves or rejects in the signer; + `{"tracekit": {"approval_id", "tool_call_id", "tool", "rule_ids", "approvals"}}` (`approvals`: those four for each + call that waits, the first one included). A person approves or rejects in the signer; then the app resumes the thread: ```python @@ -88,11 +89,13 @@ calls with `tracekit.instrument()` or the [LLM gateway](../gateway.md). The mess value is only a hint; leaving it out does not skip the check. A call whose approval was already answered when the node re-runs doesn't pause again: the signer decides it at once. - A `tracekit_tool_node` runs every call of a model message in one node, and LangGraph gives that node one resume - value, to whichever call pauses first. When two calls ask, the one a person approved runs on resume and the other, - still waiting, is denied (`TK-APPROVAL-UNBOUND`, naming its approval): approve that one too, and the model's retry - runs it. With three or more, a call still waiting after a resume pauses the node again, and LangGraph's re-run then - denies a call that already ran once. `create_agent` gives each call a task of its own and has neither limit. + A `tracekit_tool_node` (or a `ToolNode` that `tracekit.instrument()` wired) runs every call of a model message in + one node, which LangGraph runs again from its start on each resume. So when a message has several calls, all of + them are decided before any runs, and while any approval is unanswered the run pauses once, before any call runs: + the payload names the first, and `approvals` lists every one, so a person can answer them all before the resume. + On resume each call that was allowed or approved runs once; one rejected or still unanswered is denied + (`TK-APPROVAL-REQUESTED` names the approval to grant, and the model's retry then runs). No call runs in a pass that + then pauses, so none runs twice or reads as denied after it ran. (`create_agent` gives each call a task of its own.) Exceptions raised by the tool are recorded and re-raised unchanged. diff --git a/tests/test_integration_langchain.py b/tests/test_integration_langchain.py index f6790619b..6dc8943c1 100644 --- a/tests/test_integration_langchain.py +++ b/tests/test_integration_langchain.py @@ -1,9 +1,9 @@ """tracekit.integrations.langchain beyond the adapter contract (tests/test_contract_langchain.py): the result commitment, `ask` without a checkpointer, an interrupt raised inside a tool, an `ask` under each ToolNode error -handling, two asks in one ToolNode, and the async gate's signer calls off the event loop.""" +handling, several asks in one ToolNode, and the async gate's signer calls off the event loop.""" import asyncio +import itertools import json -import threading import time import types import unittest @@ -42,7 +42,14 @@ def confirm(question: str) -> str: """Ask the user, pausing the run.""" return interrupt(question) + @tool("pay") + def counted_pay(to: str, cents: int) -> str: + """Send money.""" + PAID.append(cents) + return f"paid {cents}" + THREAD = {"configurable": {"thread_id": "t"}} +PAID = [] # the cents of each counted_pay that ran def prompt(name, **args): @@ -240,32 +247,37 @@ def tool_graph(node): "str, a type that isn't an exception": {"handle_tool_errors": str}} -@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") -class ErrorHandling(unittest.TestCase): - """An `ask` pauses the graph whatever the ToolNode's handle_tool_errors and inside with_fallbacks, wired by hand - or by instrument(); the tool's own errors are handled as configured.""" +class GatedNodes(unittest.TestCase): + """Gated ToolNodes, built by hand and as instrument() wires them.""" def setUp(self): - for name in ("__init__", "with_fallbacks"): # instrument() patches ToolNode for the process + for name in ("__init__", "_func", "_afunc", "with_fallbacks"): # instrument() patches ToolNode for the process patcher = mock.patch.object(ToolNode, name, getattr(ToolNode, name)) patcher.start() self.addCleanup(patcher.stop) self.wired = ToolNode.__init__ - def nodes(self, **kw): - """(how, a function building the gated node): by hand, and a plain ToolNode as instrument() wires it, each on a - signer of its own""" + def nodes(self, tools=None, **kw): + """(how, a function building the gated node of `tools`, else TOOLS): by hand, and a plain ToolNode as + instrument() wires it, each on a signer of its own""" + tools = tools or TOOLS for how in ("by hand", "instrument()"): ac.RAN.clear() ToolNode.__init__ = self.wired self.signer = FakeSigner(ac._rule) self.run = self.signer.register_run({"request_id": "reg", "agent": {"name": "test"}}) if how == "by hand": - yield how, lambda: tracekit_tool_node(TOOLS, self.signer, self.run, **kw) + yield how, lambda: tracekit_tool_node(tools, self.signer, self.run, **kw) else: autowire._langgraph(types.SimpleNamespace(client=self.signer, registered=self.run, **{ k: self.run[k] for k in ("run_id", "run_token")})) - yield how, lambda: ToolNode(TOOLS, **kw) + yield how, lambda: ToolNode(tools, **kw) + + +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class ErrorHandling(GatedNodes): + """An `ask` pauses the graph whatever the ToolNode's handle_tool_errors and inside with_fallbacks, wired by hand + or by instrument(); the tool's own errors are handled as configured.""" def pause_and_approve(self, graph, mode="sync"): run = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) @@ -308,47 +320,63 @@ def test_the_tools_own_errors_are_handled_as_configured(self): @unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") -class ParallelAsks(unittest.TestCase): - """#687: two calls of one message that both ask, in one tracekit_tool_node. LangGraph hands the node's one resume - value to whichever call reaches interrupt() first, so the call a person approved ran only when it got there first - (7 of 16 under ainvoke, 4 of 16 under invoke). Each trial here makes the other call get there first.""" - - def trial(self, mode): - """(the call the pause showed and a person approved, what the resume answered with)""" - ac.RAN.clear() - signer = FakeSigner(ac._rule) - graph = tool_graph(tracekit_tool_node(TOOLS, signer, signer.register_run({"request_id": "reg", "agent": {"name": "test"}}))) +class ManyAsks(GatedNodes): + """#687: a message whose calls ask, beside one allowed and one denied call, in one ToolNode. LangGraph gives the node + one resume value, by the order its calls reach interrupt(), and runs the node again from its start on each resume: + the call a person approved ran only when it got there first (7 of 16 under ainvoke, 4 of 16 under invoke), and with + three or more asks a call that ran was refused on the next pass, or the allowed call ran twice. Each ask is + approved, rejected or left waiting, answered in a different order each time, then the run resumes once.""" + + def outcomes(self, node, mode, answers, order): + """{tool call id: (times it ran before the resume, whether the pause listed its approval, times it ran in all, + what the model read: its result or the rule that refused it, the results the signer recorded, the rules of its + recorded refusal)}, and whether the resume paused again""" + graph = tool_graph(node) go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) - calls = [{"name": "pay", "args": {"to": "a", "cents": i}, "id": f"c{i}"} for i in (1, 2)] - shown = go({"messages": [HumanMessage(json.dumps(calls))]}, THREAD)["__interrupt__"][0].value["tracekit"] - signer.approval_decide({"request_id": "ok", "approval_id": shown["approval_id"], "decision": "approve"}) - other_paused, request = threading.Event(), signer.approval_request - - def pause(value): - try: - return interrupt(value) - finally: - other_paused.set() - - def approval_request(req): # the approved call reaches the gate's interrupt() only after the other one - if req["tool_call_id"] == shown["tool_call_id"]: - other_paused.wait(5) - return request(req) - signer.approval_request = approval_request - with mock.patch("tracekit.integrations.langchain.interrupt", pause): - out = go(Command(resume={"approval_id": shown["approval_id"]}), THREAD) - return shown["tool_call_id"], out - - def test_the_approved_call_runs_whichever_call_gets_the_resume_value(self): - for mode in ("sync", "async"): - outcomes = [] - for _ in range(16): - approved, out = self.trial(mode) - seen = {m.tool_call_id: m.content for m in out["messages"] if isinstance(m, ToolMessage)} - other = ({"c1", "c2"} - {approved}).pop() - outcomes.append(("__interrupt__" in out, list(ac.RAN), seen.get(approved) == f"paid {approved[1]}", - seen.get(other, "").startswith("Tool call blocked by policy: TK-APPROVAL-UNBOUND"))) - self.assertEqual(outcomes, [(False, ["pay"], True, True)] * 16, mode) + calls = [{"name": "pay", "args": {"to": "a", "cents": i}, "id": f"pay-{i}"} for i in range(len(answers))] + calls += [{"name": "echo", "args": {"text": "hi"}, "id": "echo"}, + {"name": "wipe", "args": {"path": "/"}, "id": "wipe"}] + + def ran(): + return {c["id"]: PAID.count(c["args"]["cents"]) if c["name"] == "pay" else ac.RAN.count(c["name"]) + for c in calls} + asked = go({"messages": [HumanMessage(json.dumps(calls))]}, THREAD)["__interrupt__"][0].value["tracekit"] + before, listed = ran(), {a["tool_call_id"] for a in asked.get("approvals", ())} + approvals = {r["tool_call_id"]: r["approval_id"] for r in self.recorded("approval.request")} + for i in order: + if answers[i] != "wait": + self.signer.approval_decide({"request_id": f"{i}", "decision": answers[i], + "approval_id": approvals[f"pay-{i}"]}) + out = go(Command(resume={"approval_id": asked["approval_id"]}), THREAD) + blocked = "Tool call blocked by policy: " + seen = {m.tool_call_id: m.content[len(blocked):].split(" ")[0] if m.content.startswith(blocked) else m.content + for m in out["messages"] if isinstance(m, ToolMessage)} + results = [r["tool_call_id"] for r in self.recorded("tool.result")] + refused = {r["tool_call_id"]: r["rule_ids"] for r in self.recorded("approval.refused")} + after = ran() + return {c["id"]: (before[c["id"]], c["id"] in listed, after[c["id"]], seen.get(c["id"]), + results.count(c["id"]), refused.get(c["id"])) for c in calls}, "__interrupt__" in out + + def recorded(self, typ): + return [e["data"] for e in self.signer.read({"run_id": self.run["run_id"], "run_token": self.run["run_token"], + "limit": 1000})["events"] if e["type"] == typ] + + def test_each_call_runs_once_or_is_refused_and_is_recorded_as_it_did(self): + want = {"approve": lambda i: (0, True, 1, f"paid {i}", 1, None), + "reject": lambda i: (0, True, 0, "TK-APPROVAL-REJECTED", 0, ["TK-APPROVAL-REJECTED"]), + "wait": lambda i: (0, True, 0, "TK-APPROVAL-REQUESTED", 0, ["TK-APPROVAL-REQUESTED"])} + cases = [*itertools.product(want, repeat=2), *itertools.product(want, repeat=3), + *itertools.product(("approve", "wait"), repeat=5)] + for k, answers in enumerate(cases): + orders = list(itertools.permutations(range(len(answers)))) + for mode in ("sync", "async"): + for how, node in self.nodes(tools=[TOOLS[0], counted_pay, TOOLS[2]]): + with self.subTest(answers=answers, mode=mode, wired=how): + PAID.clear() + self.assertEqual(self.outcomes(node(), mode, answers, orders[k % len(orders)]), + ({**{f"pay-{i}": want[a](i) for i, a in enumerate(answers)}, + "echo": (0, False, 1, "hi", 1, None), + "wipe": (0, False, 0, "R-WIPE", 0, None)}, False)) class Jittery: diff --git a/tracekit/autowire.py b/tracekit/autowire.py index 98e608257..60105458b 100644 --- a/tracekit/autowire.py +++ b/tracekit/autowire.py @@ -199,14 +199,15 @@ def gated(self): def _langgraph(run): from langgraph.prebuilt.tool_node import ToolNode - from .integrations.langchain import TracekitMiddleware, pausing, with_fallbacks + from .integrations.langchain import TracekitMiddleware, pass_afunc, pass_func, pausing, with_fallbacks gate = TracekitMiddleware(run.client, run.run_id, run.run_token, run.registered.get("fail_modes")) init = ToolNode.__init__ @functools.wraps(init) def gated(self, tools, *a, wrap_tool_call=None, awrap_tool_call=None, **kw): w, aw = wrap_tool_call, awrap_tool_call - if not isinstance(getattr(w, "__self__", None), TracekitMiddleware): # tracekit_tool_node gates already + wired = not isinstance(getattr(w, "__self__", None), TracekitMiddleware) # tracekit_tool_node gates already + if wired: wrap_tool_call = gate.wrap_tool_call if w is None else ( lambda req, h: w(req, lambda r: gate.wrap_tool_call(r, h))) # no async wrapper of its own: ToolNode runs the sync one, gate included, for async calls too @@ -215,7 +216,10 @@ def gated(self, tools, *a, wrap_tool_call=None, awrap_tool_call=None, **kw): if "handle_tool_errors" in kw: # an ask pauses whatever the node's error handling kw["handle_tool_errors"] = pausing(kw["handle_tool_errors"]) init(self, tools, *a, wrap_tool_call=wrap_tool_call, awrap_tool_call=awrap_tool_call, **kw) + if wired: + self._tracekit = gate, wrap_tool_call # the gate decides each pass's calls together (pass_func) ToolNode.__init__ = gated + ToolNode._func, ToolNode._afunc = pass_func, pass_afunc ToolNode.with_fallbacks = with_fallbacks # and through the fallbacks the node is wrapped in diff --git a/tracekit/integrations/langchain.py b/tracekit/integrations/langchain.py index 2d5d29a02..7efc0e1d3 100644 --- a/tracekit/integrations/langchain.py +++ b/tracekit/integrations/langchain.py @@ -8,8 +8,9 @@ `deny` replaces the call with an error ToolMessage and the run goes on. `ask` pauses the run with LangGraph `interrupt()`; resume it with `Command(resume={"approval_id": ...})`. A call whose approval was answered before the -resume doesn't pause again, so of two calls of one ToolNode that ask, the one a person approved runs on resume and the -other, still waiting, is refused. Every call that is not denied runs only after the +resume doesn't pause again. A ToolNode with several calls of one message decides them all before any runs and pauses +once while any approval is unanswered (the payload's `approvals` lists them); on resume each call allowed or approved +runs once and one still unanswered is refused. Every call that is not denied runs only after the signer's `approval_consume` agrees, so an approval is checked by the signer, never by the saved state; an approval with a T2 executor runs the arguments the consume returns. `ask` needs a checkpointer: without one the call is denied. @@ -34,6 +35,7 @@ recorded by `tracekit.instrument()` or the LLM gateway. """ import asyncio +import contextvars import copy import hashlib import itertools @@ -61,6 +63,9 @@ from tracekit.sdk.client import SignerUnavailable, fail_open, refusal, result_field from tracekit.signer.rpc_schema import RPCError +# the ToolNode pass this call runs in: what TracekitMiddleware._pass decided for each of its calls +_PASS = contextvars.ContextVar("tracekit_tool_node_pass", default=None) + class TracekitMiddleware(AgentMiddleware): def __init__(self, signer, run_id, run_token, fail_modes=None): @@ -97,42 +102,76 @@ async def awrap_model_call(self, request, handler): def _gate(self, request): """(the error ToolMessage that replaces a refused call or None when it may run, the decision or None)""" - call, hint = request.tool_call, None - attempt = self._attempts.get(call["id"], 0) + call = request.tool_call + planned = (_PASS.get() or {}).get((call["id"], self._attempts.get(call["id"], 0))) + if planned is not None: # decided with the other calls of its message (_pass) + blocked, d, hint = planned + else: + blocked, d, apr = self._ask(call, request.runtime.config) + hint = None + if apr is not None and apr["state"] == "requested": + resume = self._pause([(call, d, apr)]) + hint = resume.get("approval_id") if isinstance(resume, dict) else None + elif apr is not None: # answered already: the consume decides, with no pause + hint = apr["approval_id"] + if blocked is not None or d is None: # refused, or the signer unreachable and the run fails open + return blocked, d + return self._consume(call, d, hint) + + def _pass(self, calls, config): + """ToolNode runs every call of a model message in one node, which LangGraph runs again from its start on each + resume: a call that ran in a pass that then paused would run again, or read as refused. So a message's calls + are all decided before any runs, and the pass pauses once, before any runs, while an approval is unanswered. + On resume each call allowed or approved runs and one whose approval is still unanswered is refused, so no pass + that runs a call pauses. {(tool_call_id, attempt): (refusal or None, decision or None, approval id)} for _gate. + One call alone _gate decides by itself.""" + asked = [(call, *self._ask(call, config)) for call in calls] + waiting = [(call, d, apr) for call, _, d, apr in asked if apr is not None and apr["state"] == "requested"] + if waiting: + self._pause(waiting) # raises until the resume + return {(call["id"], self._attempts.get(call["id"], 0)): (blocked, d, apr and apr["approval_id"]) + for call, blocked, d, apr in asked} + + def _ask(self, call, config): + """(the error ToolMessage that replaces a refused call or None, the decision or None, the approval when the + decision asks: approval_request's answer)""" self.said.decided(call["id"]) # the call its recorded response left for now try: # LangChain hands over JSON-decoded args, never the model's raw string d = self._event("decide", call, tool=call["name"], args_source="parsed", args=call["args"]) except SignerUnavailable as e: # the run's fail mode for the tool's class open_ = fail_open(self.fail_modes, _tool_class(call["name"])) - return (None if open_ else self._refusal(call, f"signer unavailable: {e}")), None + return (None if open_ else self._refusal(call, f"signer unavailable: {e}")), None, None except RPCError as e: - return self._refusal(call, f"signer refused the call: {e}"), None + return self._refusal(call, f"signer refused the call: {e}"), None, None if d["decision"] == "deny": - return self._refusal(call, refusal(d) or "deny"), d + return self._refusal(call, refusal(d) or "deny"), d, None d["args_digest"] = event_hash({"tool": call["name"], "args": call["args"]}) - try: - if d["decision"] == "ask": - if request.runtime.config["configurable"].get(CONFIG_KEY_CHECKPOINTER) is None: - return self._refusal(call, "approval required, and the agent has no checkpointer to wait for it"), d - # the signer keeps one approval per call attempt, so the re-run on resume (even in another process) - # gets the same approval back - apr = self.signer.approval_request({"request_id": uuid.uuid4().hex, **self.run, - "tool_call_id": call["id"], "attempt": attempt}) - # answered already (the re-run on resume): no pause, the consume decides. LangGraph hands a node's one - # resume value to whichever of its calls pauses first, so an approved call that paused might not get it. - # lean: three or more asks in one ToolNode: one still waiting after a resume pauses the node again, and - # LangGraph's re-run then refuses a call that ran (once); give each call a task of its own (Send, as - # create_agent does) if graphs need that - if apr["state"] == "requested": - resume = interrupt({"tracekit": {"approval_id": apr["approval_id"], "tool_call_id": call["id"], - "tool": call["name"], "rule_ids": d["rule_ids"]}}) - hint = resume.get("approval_id") if isinstance(resume, dict) else None - else: - hint = apr["approval_id"] - # every call, allowed or approved: the saved state can't vouch that no approval is needed + if d["decision"] != "ask": + return None, d, None + if config["configurable"].get(CONFIG_KEY_CHECKPOINTER) is None: + return self._refusal(call, "approval required, and the agent has no checkpointer to wait for it"), d, None + try: # the signer keeps one approval per call attempt: the re-run on resume, in any process, gets it back + return None, d, self.signer.approval_request({"request_id": uuid.uuid4().hex, **self.run, + "tool_call_id": call["id"], + "attempt": self._attempts.get(call["id"], 0)}) + except (RPCError, SignerUnavailable) as e: + return self._refusal(call, str(e)), d, None + + @staticmethod + def _pause(waiting): + """interrupt() for the approvals of `waiting` [(call, decision, approval)], the first named at the top: the + resume value.""" + approvals = [{"approval_id": apr["approval_id"], "tool_call_id": call["id"], "tool": call["name"], + "rule_ids": d["rule_ids"]} for call, d, apr in waiting] + return interrupt({"tracekit": {**approvals[0], "approvals": approvals}}) + + def _consume(self, call, d, hint): + """approval_consume, the signer's last word before the call runs: (the refusal or None, the decision)""" + try: # every call, allowed or approved: the saved state can't vouch that no approval is needed c = self.signer.approval_consume({"request_id": uuid.uuid4().hex, **self.run, "tool_call_id": call["id"], - "attempt": attempt, "tool": call["name"], "args_source": "parsed", - "args": call["args"], **({"approval_id_hint": hint} if hint else {})}) + "attempt": self._attempts.get(call["id"], 0), "tool": call["name"], + "args_source": "parsed", "args": call["args"], + **({"approval_id_hint": hint} if hint else {})}) except (RPCError, SignerUnavailable) as e: # e.g. a hint that is not an id at all c = {"ok": False, "rule_ids": [], "reason": str(e)} if c["ok"]: @@ -306,8 +345,44 @@ def with_fallbacks(self, fallbacks, **kw): return _Fallbacks(runnable=self, fallbacks=fallbacks, **kw) +def _held(node, input): + """(the gate that holds ToolNode `node`, this pass's calls) when the gate decides them together (two or more: + TracekitMiddleware._pass), else (None, None). A node whose wrapper is no longer the gate's, as in `tracekit why`'s + replay copy, which serves recorded results instead, isn't held.""" + gate, wrap = getattr(node, "_tracekit", (None, None)) + if gate is None or node._wrap_tool_call is not wrap: + return None, None + calls = node._parse_input(input)[0] + return (gate, calls) if len(calls) > 1 else (None, None) + + +_TOOL_NODE_FUNC, _TOOL_NODE_AFUNC = ToolNode._func, ToolNode._afunc + + +def pass_func(self, input, config, runtime): + """ToolNode._func, the calls of a held node's pass decided together first (_held); each call's gate finds its + decision in _PASS.""" + gate, calls = _held(self, input) + token = _PASS.set(gate and gate._pass(calls, config)) + try: + return _TOOL_NODE_FUNC(self, input, config, runtime) + finally: + _PASS.reset(token) + + +async def pass_afunc(self, input, config, runtime): + """ToolNode._afunc, as pass_func; the signer calls in a worker thread.""" + gate, calls = _held(self, input) + token = _PASS.set(gate and await asyncio.to_thread(gate._pass, calls, config)) + try: + return await _TOOL_NODE_AFUNC(self, input, config, runtime) + finally: + _PASS.reset(token) + + class _GatedToolNode(ToolNode): with_fallbacks = with_fallbacks + _func, _afunc = pass_func, pass_afunc def tracekit_tool_node(tools, signer, run, **kw): @@ -316,7 +391,10 @@ def tracekit_tool_node(tools, signer, run, **kw): gate = TracekitMiddleware(signer, run["run_id"], run["run_token"], run.get("fail_modes")) if "handle_tool_errors" in kw: kw["handle_tool_errors"] = pausing(kw["handle_tool_errors"]) - return _GatedToolNode(tools, wrap_tool_call=gate.wrap_tool_call, awrap_tool_call=gate.awrap_tool_call, **kw) + wrap = gate.wrap_tool_call + node = _GatedToolNode(tools, wrap_tool_call=wrap, awrap_tool_call=gate.awrap_tool_call, **kw) + node._tracekit = gate, wrap # the gate holds the node while this is its wrapper (_held) + return node _SERDE = JsonPlusSerializer() From ce6c16227565590647a8b9ec417f9c854b514bc6 Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:06:40 +0000 Subject: [PATCH 3/9] tracekit last: a Cline run finishes when its task completes (#732 added Cline) --- tests/test_instrument.py | 10 ++++++---- tracekit/cli.py | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_instrument.py b/tests/test_instrument.py index 5474eb657..210a76ea8 100644 --- a/tests/test_instrument.py +++ b/tests/test_instrument.py @@ -100,16 +100,18 @@ def test_last_open_explains_other_coding_agent_session(self): def test_last_open_explains_an_agent_whose_hooks_get_no_session_end(self): """OpenCode's and Kilo's plugin and goose's and Antigravity's hooks install no session-end event: their run - never finished "when the session ends", as `tracekit last` said.""" + never finished "when the session ends", as `tracekit last` said. A Cline run is one task.""" agents = ("opencode", "kilo", "goose", "antigravity") self.sh("-c", "import os; from tracekit.sdk.client import Client\n" - f"for a in {agents!r}: Client().run(a).decide('c1', 'Bash', {{'command': 'ls'}})\n" + f"for a in {agents + ('cline',)!r}: Client().run(a).decide('c1', 'Bash', {{'command': 'ls'}})\n" "os._exit(0)") - stderr = self.sh("-m", "tracekit", "last", code=1).stderr + lines = self.sh("-m", "tracekit", "last", code=1).stderr.splitlines() for agent in agents: - line = next(line for line in stderr.splitlines() if f"({agent})" in line) + line = next(line for line in lines if f"({agent})" in line) self.assertIn(f"still open: Tracekit's {agent} hooks get no session end, so it finishes after the signer's " "idle timeout", line) + self.assertIn("still open: it finishes when the Cline task completes", + next(line for line in lines if "(cline)" in line)) class TestThreeSteps(DevSigner): diff --git a/tracekit/cli.py b/tracekit/cli.py index 79dc16d5d..cda0796a4 100644 --- a/tracekit/cli.py +++ b/tracekit/cli.py @@ -1376,6 +1376,9 @@ def _last(a): elif agent in ("opencode", "kilo", "goose", "antigravity"): # their hooks get no session end why = (f"still open: Tracekit's {agent} hooks get no session end, so it finishes after the signer's idle " "timeout; tracekit last --open verifies it as it stands") + elif agent == "cline": # a run is one task (agent_hooks: TaskComplete ends it) + why = ("still open: it finishes when the Cline task completes, or after the signer's idle timeout (a " + "cancelled task sends no hook); tracekit last --open verifies it as it stands") elif agent in agent_hooks.AGENT_NAMES.values(): why = ("still open: it finishes when the session ends " "or after the signer's idle timeout; tracekit last --open verifies it as it stands") From f40ac8e2933ba277698642b05142eb95a0bcdd7c Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:05:11 +0000 Subject: [PATCH 4/9] LangChain: a tool's own interrupt() gets the person's answer; an executor's call stays the executor's LangGraph pairs resume values with a node's interrupt() calls by order. Once an approval was answered the gate stopped pausing, so a tool that asked the person itself got the approval's resume value and ran without asking. The gate now pauses for an ask in every pass of its node, answered or not (a ToolNode pass once, when any call asks), so a tool's interrupt() always comes after it. A call whose tool paused the run after its consume goes on, consumed once, on resume (main refused it as TK-APPROVAL-CONSUMED); in a held ToolNode pass, calls that already ended are given back, not decided, asked, run or recorded again. Both are kept in memory; docs and limits say what a resume in another process does. _pass leaves a tool executor's calls to the executor. The MCP gateway's listing timeout also catches concurrent.futures.TimeoutError, a separate class before Python 3.11. --- docs/integrations/langchain.md | 27 +++-- docs/limits.md | 1 + tests/test_integration_langchain.py | 106 +++++++++++++++++- tests/test_mcp_gateway.py | 14 +++ tracekit/integrations/langchain.py | 160 ++++++++++++++++++++-------- tracekit/mcp_gateway.py | 2 +- 6 files changed, 253 insertions(+), 57 deletions(-) diff --git a/docs/integrations/langchain.md b/docs/integrations/langchain.md index ed8e9ee72..3ca7acb74 100644 --- a/docs/integrations/langchain.md +++ b/docs/integrations/langchain.md @@ -86,16 +86,27 @@ calls with `tracekit.instrument()` or the [LLM gateway](../gateway.md). The mess On resume LangGraph re-runs the tool node, so the middleware asks the signer again, this time naming the approval. The signer, not the saved state, decides: the call runs only if that approval was granted for this exact call and these exact args, and was not used before. Otherwise the call is denied as above. The `approval_id` in the resume - value is only a hint; leaving it out does not skip the check. A call whose approval was already answered when the - node re-runs doesn't pause again: the signer decides it at once. + value is only a hint; leaving it out does not skip the check. + + LangGraph pairs resume values with a node's `interrupt()` calls by their order, and runs the node again from its + start on each resume. So an `ask` pauses first in every pass of its node, answered or not, and a tool that calls + `interrupt()` itself (a confirmation) pauses after it and gets the person's own answer. A call whose tool paused the + run after its approval was consumed goes on when the run resumes, under that one consume. A `tracekit_tool_node` (or a `ToolNode` that `tracekit.instrument()` wired) runs every call of a model message in - one node, which LangGraph runs again from its start on each resume. So when a message has several calls, all of - them are decided before any runs, and while any approval is unanswered the run pauses once, before any call runs: - the payload names the first, and `approvals` lists every one, so a person can answer them all before the resume. - On resume each call that was allowed or approved runs once; one rejected or still unanswered is denied - (`TK-APPROVAL-REQUESTED` names the approval to grant, and the model's retry then runs). No call runs in a pass that - then pauses, so none runs twice or reads as denied after it ran. (`create_agent` gives each call a task of its own.) + one node. So when a message has several calls, all of them are decided before any runs, and when any asks, the run + pauses once, before any call runs: the payload names the first approval, and `approvals` lists every one, so a + person can answer them all before the resume. On resume each call that was allowed or approved runs once; one + rejected or still unanswered is denied (`TK-APPROVAL-REQUESTED` names the approval to grant; once it is granted, + the model's retry pauses on it once more and runs on the resume). When a tool of the message then pauses the run itself, the calls that already ended aren't decided, + asked, run or recorded again on the next resume: the gate gives back their result, refusal or exception. A call to a + tool executor's tool is the executor's to decide and isn't decided with the others. (`create_agent` gives each call + a task of its own.) + + The gate keeps those calls in memory. Resumed in another process after a tool's own pause (a durable checkpointer, + after a restart), they are decided again: one that asked is denied as already used (`TK-APPROVAL-CONSUMED`), even + if it ran before the pause, and one allowed runs again. The tool that paused still gets its answer + ([limits](../limits.md#approvals)). Exceptions raised by the tool are recorded and re-raised unchanged. diff --git a/docs/limits.md b/docs/limits.md index 781c9179b..9ade5a753 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -359,6 +359,7 @@ What Tracekit guarantees and doesn't, per deployment, on one page: [guarantees]( | An MCP call is named by the server's own name (`mcp:/`); an mcp 1.x session whose `initialize()` ran before `tracekit.instrument()`, or one wrapped by hand without `server=`, is recorded as `mcp:unknown/`, with a warning. | The name comes from the server's initialize result, which an mcp 1.x session doesn't keep. | Call `tracekit.instrument()` before any session initializes, or pass `TracekitSession(..., server="")`; write rules for server-named tools knowing the server picks its name. | [MCP](integrations/mcp.md#not-covered) | | An adapter without saved state (MCP, Browser Use, the Claude Agent SDK hook) holds a call while waiting; the wait doesn't survive the process, and a long wait ends in a denial. | There is no resumable state to carry the approval. | Answer within the adapter's wait; use a framework with saved state for long waits. | [MCP](integrations/mcp.md#not-covered), [Claude Agent SDK](integrations/claude-agent-sdk.md#not-covered) | | LangGraph: code that catches exceptions around a gated `ToolNode` (a `try`/`except Exception` in a node that invokes it, a middleware outside `TracekitMiddleware` that turns exceptions into messages) ends an `ask`'s pause: the model gets an error, the call doesn't run, and its approval expires. The node's own `handle_tool_errors` and `.with_fallbacks()` let the pause through. | LangGraph pauses by raising `GraphInterrupt` out of the node; code that catches it ends the pause before LangGraph sees it. | Re-raise `langgraph.errors.GraphBubbleUp` before any broad `except`, or put the error handling in the ToolNode (`handle_tool_errors`, `.with_fallbacks()`). | [LangGraph](quickstarts/langchain.md#approvals-wire-it-by-hand) | +| LangGraph: when a tool calls `interrupt()` itself, the gate keeps in memory which calls of its node already ran (or consumed their approval before the tool paused). Resumed in another process, those calls are decided again: one that asked is denied as already used (`TK-APPROVAL-CONSUMED`), even if it ran before the pause, and one allowed runs again. The tool that paused still gets its answer. | LangGraph runs the node again from its start on each resume and keeps nothing of a pass that paused but the resume values; the signer's record of what ran is a commitment, not the result to give back. | Resume a run paused by a tool's own `interrupt()` in the process that paused it. | [LangChain](integrations/langchain.md#decisions) | | A Browser Use action waits for an approval at most five sixths of Browser Use's action timeout (150 s of 180 s); then the approval is withdrawn and the action refused. | Browser Use cancels an action that outlives its timeout, approval wait included, with a misleading "dead CDP WebSocket" error. | Answer within the wait, or raise `BROWSER_USE_ACTION_TIMEOUT_S` (and the Agent's `step_timeout`) to give people longer. | [Browser Use](integrations/browser-use.md#decisions) | | OpenAI Agents' `LocalShellTool` and `ApplyPatchTool` can't wait for an approval: an `ask` for them is refused. | The SDK gives no approval hook for them. | Allow or deny them by policy. | [OpenAI Agents](integrations/openai-agents.md#not-covered) | | v1 approvals name the held call, not its exact arguments. | v1 approvals predate argument binding. | Use the v2 signer, which binds approvals to the exact arguments. | [laptop threat model](threat-model-laptop.md#open-problems) | diff --git a/tests/test_integration_langchain.py b/tests/test_integration_langchain.py index 6dc8943c1..827b27ad4 100644 --- a/tests/test_integration_langchain.py +++ b/tests/test_integration_langchain.py @@ -31,6 +31,7 @@ from test_contract_langchain import TOOLS, StubModel from tracekit import autowire from tracekit.format.canon import event_hash + from tracekit.integrations.executor_tools import EXECUTOR from tracekit.integrations.langchain import TracekitMiddleware, tracekit_tool_node except ImportError: # optional: the dev extra installs them HAVE_LC = False @@ -48,8 +49,15 @@ def counted_pay(to: str, cents: int) -> str: PAID.append(cents) return f"paid {cents}" + @tool("pay") + def confirmed_pay(to: str, cents: int) -> str: + """Send money once the person confirms: the tool's own pause.""" + answer = interrupt(f"pay {cents}?") + PAID.append((cents, answer)) + return f"paid {cents} after {answer}" + THREAD = {"configurable": {"thread_id": "t"}} -PAID = [] # the cents of each counted_pay that ran +PAID = [] # the cents of each counted_pay that ran; (cents, the answer it got) for confirmed_pay def prompt(name, **args): @@ -379,6 +387,102 @@ def test_each_call_runs_once_or_is_refused_and_is_recorded_as_it_did(self): "wipe": (0, False, 0, "R-WIPE", 0, None)}, False)) +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class ToolPauses(GatedNodes): + """A tool that pauses the run itself (interrupt()), beside or under an `ask`. LangGraph pairs resume values with a + node's interrupt() calls by their order, and runs the node again from its start on each resume. Once an approval + was answered the gate stopped pausing, so the tool got the approval's resume value as the person's answer and ran + without asking; before that, the tool asked, but its approved call was then refused as already used + (TK-APPROVAL-CONSUMED), and a call that ran before the tool's pause was asked for a second approval.""" + + def graphs(self, tools): + """(how, the graph): create_agent, and the gated ToolNodes (GatedNodes.nodes), each on a signer of its own""" + ToolNode.__init__ = self.wired + self.signer = FakeSigner(ac._rule) + self.run = self.signer.register_run({"request_id": "reg", "agent": {"name": "test"}}) + yield "create_agent", create_agent(StubModel(), tools, checkpointer=InMemorySaver(), middleware=[ + TracekitMiddleware(self.signer, self.run["run_id"], self.run["run_token"])]) + for how, node in self.nodes(tools=tools): + yield how, tool_graph(node()) + + def answered(self, graph, mode, calls, approve): + """(each pause in order: "tracekit" or the tool's question; what the model read of each call; the signer's + approval requests and results, by call), a person approving the calls in `approve` and answering each tool's + question "yes: "; every other approval left unanswered""" + go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) + out, pauses = go({"messages": [HumanMessage(json.dumps(calls))]}, THREAD), [] + while "__interrupt__" in out and len(pauses) < 10: + answers = {} + for i in out["__interrupt__"]: # create_agent's calls are tasks of their own: they pause together + shown = i.value.get("tracekit") if isinstance(i.value, dict) else None + pauses.append("tracekit" if shown else i.value) + for a in shown.get("approvals", [shown]) if shown else (): + if a["tool_call_id"] in approve and self.recorded("approval", a["approval_id"]) == []: + self.signer.approval_decide({"request_id": a["approval_id"], "decision": "approve", + "approval_id": a["approval_id"]}) + answers[i.id] = {"approval_id": shown["approval_id"]} if shown else f"yes: {i.value}" + out = go(Command(resume=answers), THREAD) + seen = {m.tool_call_id: m.content for m in out["messages"] if isinstance(m, ToolMessage)} + return pauses, seen, {t: sorted(r["tool_call_id"] for r in self.recorded(t)) for t in ("approval.request", + "tool.result")} + + def recorded(self, typ, approval_id=None): + return [e["data"] for e in self.signer.read({"run_id": self.run["run_id"], "run_token": self.run["run_token"], + "limit": 1000})["events"] + if e["type"] == typ and approval_id in (None, e["data"].get("approval_id"))] + + def test_a_tool_asked_under_an_approval_gets_the_persons_own_answer(self): + for mode in ("sync", "async"): + for how, graph in self.graphs([confirmed_pay]): + with self.subTest(mode=mode, built=how): + PAID.clear() + got = self.answered(graph, mode, [{"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay-1"}], + approve={"pay-1"}) + self.assertEqual(got, (["tracekit", "pay 1?"], {"pay-1": "paid 1 after yes: pay 1?"}, + {"approval.request": ["pay-1"], "tool.result": ["pay-1"]})) + self.assertEqual(PAID, [(1, "yes: pay 1?")]) # it ran once, with the person's answer + + def test_a_tool_that_pauses_beside_asks_gets_the_answer_and_no_call_runs_or_asks_twice(self): + calls = [{"name": "pay", "args": {"to": "a", "cents": 0}, "id": "pay-0"}, + {"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay-1"}, + {"name": "confirm", "args": {"question": "sure?"}, "id": "confirm"}, + {"name": "echo", "args": {"text": "hi"}, "id": "echo"}] + for mode in ("sync", "async"): + for how, graph in self.graphs([counted_pay, confirm, TOOLS[0]]): + with self.subTest(mode=mode, built=how): + PAID.clear() + ac.RAN.clear() + pauses, seen, recorded = self.answered(graph, mode, calls, approve={"pay-0"}) + self.assertEqual((set(pauses), pauses.count("sure?")), ({"tracekit", "sure?"}, 1)) + self.assertEqual({k: v.split(" (")[0] for k, v in seen.items()}, + {"pay-0": "paid 0", "pay-1": "Tool call blocked by policy: TK-APPROVAL-REQUESTED", + "confirm": "yes: sure?", "echo": "hi"}) + self.assertEqual(recorded, {"approval.request": ["pay-0", "pay-1"], + "tool.result": ["confirm", "echo", "pay-0"]}) + self.assertEqual((PAID, ac.RAN), ([0], ["echo"])) # each ran once + + def test_an_executors_tool_is_left_to_the_executor(self): + """_pass decided an executor's call with the gate's own decision and paused for an approval the executor never + consumes.""" + def send(to: str, cents: int) -> str: + """Send money, through the tool executor.""" + return f"sent {cents}" + executed = tool("pay")(send) + executed.metadata = {EXECUTOR: True} + for mode in ("sync", "async"): + for how, node in self.nodes(tools=[executed, TOOLS[0]]): + with self.subTest(mode=mode, wired=how): + graph = tool_graph(node()) + go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) + out = go({"messages": [HumanMessage(json.dumps([ + {"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay"}, + {"name": "echo", "args": {"text": "hi"}, "id": "echo"}]))]}, THREAD) + self.assertNotIn("__interrupt__", out) + self.assertEqual({m.tool_call_id: m.content for m in out["messages"] + if isinstance(m, ToolMessage)}, {"pay": "sent 1", "echo": "hi"}) + self.assertEqual([r["tool_call_id"] for r in self.recorded("policy.decision")], ["echo"]) + + class Jittery: """The client, with the pause a busy process may take between a call's client_seq and its send.""" diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index 2dba9fe06..776c65b29 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -391,6 +391,20 @@ async def on_list_tools(ctx, params): self.assertIn("upstream 'up': tools/list timed out (one listing has 0.3 s for every upstream and page)", messages(e.exception)) + def test_a_listing_past_its_budget_names_the_upstream_before_python_3_11(self): + """Before 3.11 the timeout of the future a listing waits on (concurrent.futures.TimeoutError) isn't the built-in + TimeoutError, so the listing's own error was missed.""" + class FutureTimeout(Exception): # concurrent.futures.TimeoutError on Python 3.10 + pass + + def call(up, method, *a, **kw): + raise FutureTimeout() + with mock.patch("concurrent.futures.TimeoutError", FutureTimeout), \ + mock.patch.object(self.srv.mcp["upstreams"], "call", call), self.assertRaises(Exception) as e: + self.agent(lambda c: c.list_tools()) + self.assertIn("upstream 'up': tools/list timed out (one listing has 600 s for every upstream and page)", + messages(e.exception)) + def test_page_bound_error_forgets_page1_resources(self): """Hitting the page cap on resources/list returns an error and remembers nothing.""" self._swap_upstream(paged_upstream()) diff --git a/tracekit/integrations/langchain.py b/tracekit/integrations/langchain.py index 7efc0e1d3..50f593575 100644 --- a/tracekit/integrations/langchain.py +++ b/tracekit/integrations/langchain.py @@ -7,10 +7,10 @@ fail_modes=run.get("fail_modes"))]) `deny` replaces the call with an error ToolMessage and the run goes on. `ask` pauses the run with LangGraph -`interrupt()`; resume it with `Command(resume={"approval_id": ...})`. A call whose approval was answered before the -resume doesn't pause again. A ToolNode with several calls of one message decides them all before any runs and pauses -once while any approval is unanswered (the payload's `approvals` lists them); on resume each call allowed or approved -runs once and one still unanswered is refused. Every call that is not denied runs only after the +`interrupt()`; resume it with `Command(resume={"approval_id": ...})`. The pause comes first in every pass of its node, +so a tool's own interrupt() gets the person's answer. A ToolNode with several calls of one message decides them all +before any runs and pauses once when any asks (the payload's `approvals` lists them); on resume each call allowed or +approved runs once and one still unanswered is refused. Every call that is not denied runs only after the signer's `approval_consume` agrees, so an approval is checked by the signer, never by the saved state; an approval with a T2 executor runs the arguments the consume returns. `ask` needs a checkpointer: without one the call is denied. @@ -78,6 +78,11 @@ def __init__(self, signer, run_id, run_token, fail_modes=None): self.stream, self._seq, self._lock = "langchain-" + uuid.uuid4().hex, itertools.count(), threading.Lock() # lean: one entry per executed tool call for the middleware's life; prune at run end if agents run for days self._attempts = {} # tool_call_id -> the attempt its next execution is (a retry runs the same call again) + # (node task, tool_call_id) -> ... while its node may run again (_gate). In memory: a run resumed in another + # process decides such a call again (docs/integrations/langchain.md) + # lean: a run never resumed from a tool's own pause keeps its entries for the middleware's life + self._paused = {} # -> (decision, pause entry or None): consumed, then its tool paused the run + self._ended = {} # -> (how it ended: a ToolMessage or the exception, pause entry or None), in a held pass self.said = said.Said(lambda **f: self._send("model_event", **f), lambda: signer.status({})) def _send(self, method, **kw): @@ -101,36 +106,67 @@ async def awrap_model_call(self, request, handler): return out def _gate(self, request): - """(the error ToolMessage that replaces a refused call or None when it may run, the decision or None)""" - call = request.tool_call - planned = (_PASS.get() or {}).get((call["id"], self._attempts.get(call["id"], 0))) + """(what replaces the call: a refusal, or how it ended in an earlier pass of its node, a ToolMessage or the + exception it raised; None when it may run, the decision or None, its pause entry when it asked) + + LangGraph runs a node again from its start on each resume and pairs resume values with interrupt() calls by + their order in the node. So a call that asks pauses here in every pass, answered or not, and a tool's own + interrupt() always comes after it and gets the answer meant for it. A call whose tool paused the run after its + consume goes on when the run resumes, consumed once (_paused).""" + call, key = request.tool_call, _call_key(request) + if key in self._ended: # its node runs again: a tool of the same message paused the run + return self._ended[key][0], None, self._ended[key][1] + if key in self._paused: + d, entry = self._paused[key] + if entry is not None and _PASS.get() is None: # _pass pauses for the calls of its node + self._pause([entry]) + return None, d, entry + planned = (_PASS.get() or {}).get(call["id"]) if planned is not None: # decided with the other calls of its message (_pass) - blocked, d, hint = planned + blocked, d, entry = planned + hint = entry and entry["approval_id"] else: blocked, d, apr = self._ask(call, request.runtime.config) - hint = None - if apr is not None and apr["state"] == "requested": - resume = self._pause([(call, d, apr)]) + entry, hint = apr and _entry(call, d, apr), None + if entry is not None: + resume = self._pause([entry]) hint = resume.get("approval_id") if isinstance(resume, dict) else None - elif apr is not None: # answered already: the consume decides, with no pause - hint = apr["approval_id"] if blocked is not None or d is None: # refused, or the signer unreachable and the run fails open - return blocked, d - return self._consume(call, d, hint) + return blocked, d, None + return *self._consume(call, d, hint), entry def _pass(self, calls, config): - """ToolNode runs every call of a model message in one node, which LangGraph runs again from its start on each - resume: a call that ran in a pass that then paused would run again, or read as refused. So a message's calls - are all decided before any runs, and the pass pauses once, before any runs, while an approval is unanswered. - On resume each call allowed or approved runs and one whose approval is still unanswered is refused, so no pass - that runs a call pauses. {(tool_call_id, attempt): (refusal or None, decision or None, approval id)} for _gate. - One call alone _gate decides by itself.""" - asked = [(call, *self._ask(call, config)) for call in calls] - waiting = [(call, d, apr) for call, _, d, apr in asked if apr is not None and apr["state"] == "requested"] - if waiting: - self._pause(waiting) # raises until the resume - return {(call["id"], self._attempts.get(call["id"], 0)): (blocked, d, apr and apr["approval_id"]) - for call, blocked, d, apr in asked} + """ToolNode runs every call of a model message in one node, and LangGraph runs it again from its start on each + resume. So a message's calls are decided before any runs, and the pass pauses once, before any runs, when any + of them asks (in every pass, as _gate); on resume each call allowed or approved runs and one unanswered is + refused. A call that ran in an earlier pass (a tool of the message paused the run) isn't decided again: _gate + gives back how it ended, or lets it go on. {tool_call_id: (refusal or None, decision or None, its pause entry + or None)} for _gate.""" + ns, plan, asked = config["configurable"].get("checkpoint_ns"), {}, [] + for call in calls: + earlier = self._ended.get((ns, call["id"])) or self._paused.get((ns, call["id"])) + if earlier is not None: + asked += [earlier[1]] if earlier[1] is not None else [] + continue + blocked, d, apr = self._ask(call, config) + plan[call["id"]] = blocked, d, apr and _entry(call, d, apr) + asked += [plan[call["id"]][2]] if apr is not None else [] + if asked: + self._pause(asked) # raises until the run resumes + return plan + + def _end(self, request, out, entry=None): + """How the call ended (`out`, its ToolMessage or exception), kept while its node may run again (a held pass, + _PASS) for _gate to give back; its pause entry when it asked.""" + key = _call_key(request) + self._paused.pop(key, None) + if _PASS.get() is not None: + self._ended[key] = out, entry + + def _forget(self, ns): + """The node task `ns` ended: none of its calls runs again.""" + for key in [k for k in self._ended if k[0] == ns]: + del self._ended[key] def _ask(self, call, config): """(the error ToolMessage that replaces a refused call or None, the decision or None, the approval when the @@ -158,12 +194,9 @@ def _ask(self, call, config): return self._refusal(call, str(e)), d, None @staticmethod - def _pause(waiting): - """interrupt() for the approvals of `waiting` [(call, decision, approval)], the first named at the top: the - resume value.""" - approvals = [{"approval_id": apr["approval_id"], "tool_call_id": call["id"], "tool": call["name"], - "rule_ids": d["rule_ids"]} for call, d, apr in waiting] - return interrupt({"tracekit": {**approvals[0], "approvals": approvals}}) + def _pause(entries): + """interrupt() for the approvals of `entries` (_entry), the first named at the top: the resume value.""" + return interrupt({"tracekit": {**entries[0], "approvals": entries}}) def _consume(self, call, d, hint): """approval_consume, the signer's last word before the call runs: (the refusal or None, the decision)""" @@ -206,52 +239,78 @@ def _complete(self, call, decision, out=None, exc=None): # lean: wrap_tool_call's signer calls block; awrap_tool_call runs them in a worker thread (issue #680) def wrap_tool_call(self, request, handler): - if _executor(request): # the executor decides it, under the model's id for the call + if _executor(request.tool): # the executor decides it, under the model's id for the call self.said.decided(request.tool_call["id"]) token = CALL_ID.set(request.tool_call["id"]) try: return handler(request) finally: CALL_ID.reset(token) - blocked, d = self._gate(request) - if blocked is not None: - return blocked + out, d, entry = self._gate(request) + if out is not None: + return self._ended_with(request, out, entry) if d is not None and "args" in d: request = request.override(tool_call={**request.tool_call, "args": d["args"]}) try: out = handler(request) except GraphBubbleUp: # interrupt() or a handoff inside the tool: control flow, not an outcome + if d is not None: # it goes on, consumed once, when the run resumes (_gate) + self._paused[_call_key(request)] = d, entry raise except Exception as e: self._complete(request.tool_call, d, exc=e) + self._end(request, e, entry) raise self._complete(request.tool_call, d, out) + self._end(request, out, entry) + return out + + def _ended_with(self, request, out, entry): + """The call's refusal, or how it ended in an earlier pass of its node: returned, or raised.""" + self._end(request, out, entry) + if isinstance(out, BaseException): + raise out return out async def awrap_tool_call(self, request, handler): - if _executor(request): + if _executor(request.tool): await asyncio.to_thread(self.said.decided, request.tool_call["id"]) token = CALL_ID.set(request.tool_call["id"]) try: return await handler(request) finally: CALL_ID.reset(token) - blocked, d = await asyncio.to_thread(self._gate, request) - if blocked is not None: - return blocked + out, d, entry = await asyncio.to_thread(self._gate, request) + if out is not None: + return self._ended_with(request, out, entry) if d is not None and "args" in d: request = request.override(tool_call={**request.tool_call, "args": d["args"]}) try: out = await handler(request) except GraphBubbleUp: # interrupt() or a handoff inside the tool: control flow, not an outcome + if d is not None: + self._paused[_call_key(request)] = d, entry raise except Exception as e: await asyncio.to_thread(self._complete, request.tool_call, d, exc=e) + self._end(request, e, entry) raise await asyncio.to_thread(self._complete, request.tool_call, d, out) + self._end(request, out, entry) return out +def _call_key(request): + """A call in its node's task, the same in each pass of the node (LangGraph runs it again on each resume).""" + return request.runtime.config["configurable"].get("checkpoint_ns"), request.tool_call["id"] + + +def _entry(call, d, apr): + """A call's approval as the pause shows it (TracekitMiddleware._pause).""" + return {"approval_id": apr["approval_id"], "tool_call_id": call["id"], "tool": call["name"], + "rule_ids": d["rule_ids"]} + + def _said(request, out): """(the items of what the model said, its model, its provider) in a model call's response: the calls of the agent's tools, which the gate or a tool executor decides under their ids (not a structured-output tool's), as @@ -281,9 +340,9 @@ def items(): return items(), meta.get("model_name"), meta.get("model_provider") -def _executor(request): - """Whether the call is to a tool executor's tool (executor_tools), which the executor decides and records (T2).""" - return bool((getattr(request.tool, "metadata", None) or {}).get(EXECUTOR)) +def _executor(tool): + """Whether `tool` is a tool executor's (executor_tools), whose calls the executor decides and records (T2).""" + return bool((getattr(tool, "metadata", None) or {}).get(EXECUTOR)) def pausing(flag): @@ -352,7 +411,8 @@ def _held(node, input): gate, wrap = getattr(node, "_tracekit", (None, None)) if gate is None or node._wrap_tool_call is not wrap: return None, None - calls = node._parse_input(input)[0] + # an executor's tool is the executor's to decide (wrap_tool_call passes it through) + calls = [c for c in node._parse_input(input)[0] if not _executor(node.tools_by_name.get(c["name"]))] return (gate, calls) if len(calls) > 1 else (None, None) @@ -365,9 +425,12 @@ def pass_func(self, input, config, runtime): gate, calls = _held(self, input) token = _PASS.set(gate and gate._pass(calls, config)) try: - return _TOOL_NODE_FUNC(self, input, config, runtime) + out = _TOOL_NODE_FUNC(self, input, config, runtime) finally: _PASS.reset(token) + if gate: # the node ended: none of its calls runs again + gate._forget(config["configurable"].get("checkpoint_ns")) + return out async def pass_afunc(self, input, config, runtime): @@ -375,9 +438,12 @@ async def pass_afunc(self, input, config, runtime): gate, calls = _held(self, input) token = _PASS.set(gate and await asyncio.to_thread(gate._pass, calls, config)) try: - return await _TOOL_NODE_AFUNC(self, input, config, runtime) + out = await _TOOL_NODE_AFUNC(self, input, config, runtime) finally: _PASS.reset(token) + if gate: + gate._forget(config["configurable"].get("checkpoint_ns")) + return out class _GatedToolNode(ToolNode): diff --git a/tracekit/mcp_gateway.py b/tracekit/mcp_gateway.py index 55d318e06..e4bc2efe1 100644 --- a/tracekit/mcp_gateway.py +++ b/tracekit/mcp_gateway.py @@ -191,7 +191,7 @@ def _answer(self, gw, key, method, params): while True: try: result = ups.call(up, call, cursor=cursor, timeout=deadline - time.monotonic()) - except TimeoutError: + except (TimeoutError, concurrent.futures.TimeoutError): # one class from Python 3.11, two before raise _Error(-32603, f"upstream {up!r}: {kind}/list timed out (one listing has " f"{UPSTREAM_TIMEOUT_S:g} s for every upstream and page)") from None items = getattr(result, kind) From 79c5ee3df3fe3b3098e5be299f15911739fbaf71 Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:55:33 +0000 Subject: [PATCH 5/9] LangChain: back to main's parallel-ask behaviour; the redesign moves to its own PR The multi-call ToolNode changes (#687) keep exposing new cases (RetryPolicy re-raising a kept exception, async sibling cancellation), so this PR keeps main's langchain.py, autowire.py, their tests and doc. The held-pass design moves to a draft PR for after 1.7.0. docs/limits.md says what main does with several asks in one ToolNode. The fork-replay leak found earlier came from this branch's own hook and isn't on main. --- docs/integrations/langchain.md | 23 +-- docs/limits.md | 2 +- tests/test_integration_langchain.py | 199 ++----------------------- tracekit/autowire.py | 8 +- tracekit/integrations/langchain.py | 220 +++++----------------------- 5 files changed, 48 insertions(+), 404 deletions(-) diff --git a/docs/integrations/langchain.md b/docs/integrations/langchain.md index 3ca7acb74..82cdea08a 100644 --- a/docs/integrations/langchain.md +++ b/docs/integrations/langchain.md @@ -75,8 +75,7 @@ calls with `tracekit.instrument()` or the [LLM gateway](../gateway.md). The mess - **deny**: the tool does not run. The model gets an error ToolMessage, `Tool call blocked by policy: `, and the agent run goes on. - **ask**: the middleware opens an approval and pauses the run with LangGraph `interrupt()`. The interrupt payload is - `{"tracekit": {"approval_id", "tool_call_id", "tool", "rule_ids", "approvals"}}` (`approvals`: those four for each - call that waits, the first one included). A person approves or rejects in the signer; + `{"tracekit": {"approval_id", "tool_call_id", "tool", "rule_ids"}}`. A person approves or rejects in the signer; then the app resumes the thread: ```python @@ -88,26 +87,6 @@ calls with `tracekit.instrument()` or the [LLM gateway](../gateway.md). The mess these exact args, and was not used before. Otherwise the call is denied as above. The `approval_id` in the resume value is only a hint; leaving it out does not skip the check. - LangGraph pairs resume values with a node's `interrupt()` calls by their order, and runs the node again from its - start on each resume. So an `ask` pauses first in every pass of its node, answered or not, and a tool that calls - `interrupt()` itself (a confirmation) pauses after it and gets the person's own answer. A call whose tool paused the - run after its approval was consumed goes on when the run resumes, under that one consume. - - A `tracekit_tool_node` (or a `ToolNode` that `tracekit.instrument()` wired) runs every call of a model message in - one node. So when a message has several calls, all of them are decided before any runs, and when any asks, the run - pauses once, before any call runs: the payload names the first approval, and `approvals` lists every one, so a - person can answer them all before the resume. On resume each call that was allowed or approved runs once; one - rejected or still unanswered is denied (`TK-APPROVAL-REQUESTED` names the approval to grant; once it is granted, - the model's retry pauses on it once more and runs on the resume). When a tool of the message then pauses the run itself, the calls that already ended aren't decided, - asked, run or recorded again on the next resume: the gate gives back their result, refusal or exception. A call to a - tool executor's tool is the executor's to decide and isn't decided with the others. (`create_agent` gives each call - a task of its own.) - - The gate keeps those calls in memory. Resumed in another process after a tool's own pause (a durable checkpointer, - after a restart), they are decided again: one that asked is denied as already used (`TK-APPROVAL-CONSUMED`), even - if it ran before the pause, and one allowed runs again. The tool that paused still gets its answer - ([limits](../limits.md#approvals)). - Exceptions raised by the tool are recorded and re-raised unchanged. ## Approvals need a checkpointer diff --git a/docs/limits.md b/docs/limits.md index 9ade5a753..b1b8a96c3 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -359,7 +359,7 @@ What Tracekit guarantees and doesn't, per deployment, on one page: [guarantees]( | An MCP call is named by the server's own name (`mcp:/`); an mcp 1.x session whose `initialize()` ran before `tracekit.instrument()`, or one wrapped by hand without `server=`, is recorded as `mcp:unknown/`, with a warning. | The name comes from the server's initialize result, which an mcp 1.x session doesn't keep. | Call `tracekit.instrument()` before any session initializes, or pass `TracekitSession(..., server="")`; write rules for server-named tools knowing the server picks its name. | [MCP](integrations/mcp.md#not-covered) | | An adapter without saved state (MCP, Browser Use, the Claude Agent SDK hook) holds a call while waiting; the wait doesn't survive the process, and a long wait ends in a denial. | There is no resumable state to carry the approval. | Answer within the adapter's wait; use a framework with saved state for long waits. | [MCP](integrations/mcp.md#not-covered), [Claude Agent SDK](integrations/claude-agent-sdk.md#not-covered) | | LangGraph: code that catches exceptions around a gated `ToolNode` (a `try`/`except Exception` in a node that invokes it, a middleware outside `TracekitMiddleware` that turns exceptions into messages) ends an `ask`'s pause: the model gets an error, the call doesn't run, and its approval expires. The node's own `handle_tool_errors` and `.with_fallbacks()` let the pause through. | LangGraph pauses by raising `GraphInterrupt` out of the node; code that catches it ends the pause before LangGraph sees it. | Re-raise `langgraph.errors.GraphBubbleUp` before any broad `except`, or put the error handling in the ToolNode (`handle_tool_errors`, `.with_fallbacks()`). | [LangGraph](quickstarts/langchain.md#approvals-wire-it-by-hand) | -| LangGraph: when a tool calls `interrupt()` itself, the gate keeps in memory which calls of its node already ran (or consumed their approval before the tool paused). Resumed in another process, those calls are decided again: one that asked is denied as already used (`TK-APPROVAL-CONSUMED`), even if it ran before the pause, and one allowed runs again. The tool that paused still gets its answer. | LangGraph runs the node again from its start on each resume and keeps nothing of a pass that paused but the resume values; the signer's record of what ran is a commitment, not the result to give back. | Resume a run paused by a tool's own `interrupt()` in the process that paused it. | [LangChain](integrations/langchain.md#decisions) | +| LangGraph: when one model message has several calls that ask, in one `ToolNode` (a `tracekit_tool_node`, or a `ToolNode` that `tracekit.instrument()` wired), each resume answers one pause and the run pauses again for the next. Under `ainvoke` a pause may show a call a person already approved. A call that ran on an earlier resume is gated again on the next: it gets a new approval request and is then denied (`TK-APPROVAL-UNBOUND`), though it ran once. An allowed call in the same message runs again on each resume. No asking call runs without an approval. | LangGraph runs the node again from its start on each resume and pairs resume values with the node's `interrupt()` calls by their order, which the node's parallel calls reach in no fixed order. | Answer each pause; read the run's `tool.result` records, not the messages the model got, for what ran; or build the agent with `create_agent`, which runs each call as a task of its own. | [LangChain](integrations/langchain.md#decisions) | | A Browser Use action waits for an approval at most five sixths of Browser Use's action timeout (150 s of 180 s); then the approval is withdrawn and the action refused. | Browser Use cancels an action that outlives its timeout, approval wait included, with a misleading "dead CDP WebSocket" error. | Answer within the wait, or raise `BROWSER_USE_ACTION_TIMEOUT_S` (and the Agent's `step_timeout`) to give people longer. | [Browser Use](integrations/browser-use.md#decisions) | | OpenAI Agents' `LocalShellTool` and `ApplyPatchTool` can't wait for an approval: an `ask` for them is refused. | The SDK gives no approval hook for them. | Allow or deny them by policy. | [OpenAI Agents](integrations/openai-agents.md#not-covered) | | v1 approvals name the held call, not its exact arguments. | v1 approvals predate argument binding. | Use the v2 signer, which binds approvals to the exact arguments. | [laptop threat model](threat-model-laptop.md#open-problems) | diff --git a/tests/test_integration_langchain.py b/tests/test_integration_langchain.py index 827b27ad4..2ab5537bc 100644 --- a/tests/test_integration_langchain.py +++ b/tests/test_integration_langchain.py @@ -1,8 +1,7 @@ """tracekit.integrations.langchain beyond the adapter contract (tests/test_contract_langchain.py): the result commitment, `ask` without a checkpointer, an interrupt raised inside a tool, an `ask` under each ToolNode error -handling, several asks in one ToolNode, and the async gate's signer calls off the event loop.""" +handling, and the async gate's signer calls off the event loop.""" import asyncio -import itertools import json import time import types @@ -31,7 +30,6 @@ from test_contract_langchain import TOOLS, StubModel from tracekit import autowire from tracekit.format.canon import event_hash - from tracekit.integrations.executor_tools import EXECUTOR from tracekit.integrations.langchain import TracekitMiddleware, tracekit_tool_node except ImportError: # optional: the dev extra installs them HAVE_LC = False @@ -43,21 +41,7 @@ def confirm(question: str) -> str: """Ask the user, pausing the run.""" return interrupt(question) - @tool("pay") - def counted_pay(to: str, cents: int) -> str: - """Send money.""" - PAID.append(cents) - return f"paid {cents}" - - @tool("pay") - def confirmed_pay(to: str, cents: int) -> str: - """Send money once the person confirms: the tool's own pause.""" - answer = interrupt(f"pay {cents}?") - PAID.append((cents, answer)) - return f"paid {cents} after {answer}" - THREAD = {"configurable": {"thread_id": "t"}} -PAID = [] # the cents of each counted_pay that ran; (cents, the answer it got) for confirmed_pay def prompt(name, **args): @@ -255,37 +239,32 @@ def tool_graph(node): "str, a type that isn't an exception": {"handle_tool_errors": str}} -class GatedNodes(unittest.TestCase): - """Gated ToolNodes, built by hand and as instrument() wires them.""" +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class ErrorHandling(unittest.TestCase): + """An `ask` pauses the graph whatever the ToolNode's handle_tool_errors and inside with_fallbacks, wired by hand + or by instrument(); the tool's own errors are handled as configured.""" def setUp(self): - for name in ("__init__", "_func", "_afunc", "with_fallbacks"): # instrument() patches ToolNode for the process + for name in ("__init__", "with_fallbacks"): # instrument() patches ToolNode for the process patcher = mock.patch.object(ToolNode, name, getattr(ToolNode, name)) patcher.start() self.addCleanup(patcher.stop) self.wired = ToolNode.__init__ - def nodes(self, tools=None, **kw): - """(how, a function building the gated node of `tools`, else TOOLS): by hand, and a plain ToolNode as - instrument() wires it, each on a signer of its own""" - tools = tools or TOOLS + def nodes(self, **kw): + """(how, a function building the gated node): by hand, and a plain ToolNode as instrument() wires it, each on a + signer of its own""" for how in ("by hand", "instrument()"): ac.RAN.clear() ToolNode.__init__ = self.wired self.signer = FakeSigner(ac._rule) self.run = self.signer.register_run({"request_id": "reg", "agent": {"name": "test"}}) if how == "by hand": - yield how, lambda: tracekit_tool_node(tools, self.signer, self.run, **kw) + yield how, lambda: tracekit_tool_node(TOOLS, self.signer, self.run, **kw) else: autowire._langgraph(types.SimpleNamespace(client=self.signer, registered=self.run, **{ k: self.run[k] for k in ("run_id", "run_token")})) - yield how, lambda: ToolNode(tools, **kw) - - -@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") -class ErrorHandling(GatedNodes): - """An `ask` pauses the graph whatever the ToolNode's handle_tool_errors and inside with_fallbacks, wired by hand - or by instrument(); the tool's own errors are handled as configured.""" + yield how, lambda: ToolNode(TOOLS, **kw) def pause_and_approve(self, graph, mode="sync"): run = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) @@ -327,162 +306,6 @@ def test_the_tools_own_errors_are_handled_as_configured(self): self.assertIn("please fix your mistakes", tool_message(out).content) -@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") -class ManyAsks(GatedNodes): - """#687: a message whose calls ask, beside one allowed and one denied call, in one ToolNode. LangGraph gives the node - one resume value, by the order its calls reach interrupt(), and runs the node again from its start on each resume: - the call a person approved ran only when it got there first (7 of 16 under ainvoke, 4 of 16 under invoke), and with - three or more asks a call that ran was refused on the next pass, or the allowed call ran twice. Each ask is - approved, rejected or left waiting, answered in a different order each time, then the run resumes once.""" - - def outcomes(self, node, mode, answers, order): - """{tool call id: (times it ran before the resume, whether the pause listed its approval, times it ran in all, - what the model read: its result or the rule that refused it, the results the signer recorded, the rules of its - recorded refusal)}, and whether the resume paused again""" - graph = tool_graph(node) - go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) - calls = [{"name": "pay", "args": {"to": "a", "cents": i}, "id": f"pay-{i}"} for i in range(len(answers))] - calls += [{"name": "echo", "args": {"text": "hi"}, "id": "echo"}, - {"name": "wipe", "args": {"path": "/"}, "id": "wipe"}] - - def ran(): - return {c["id"]: PAID.count(c["args"]["cents"]) if c["name"] == "pay" else ac.RAN.count(c["name"]) - for c in calls} - asked = go({"messages": [HumanMessage(json.dumps(calls))]}, THREAD)["__interrupt__"][0].value["tracekit"] - before, listed = ran(), {a["tool_call_id"] for a in asked.get("approvals", ())} - approvals = {r["tool_call_id"]: r["approval_id"] for r in self.recorded("approval.request")} - for i in order: - if answers[i] != "wait": - self.signer.approval_decide({"request_id": f"{i}", "decision": answers[i], - "approval_id": approvals[f"pay-{i}"]}) - out = go(Command(resume={"approval_id": asked["approval_id"]}), THREAD) - blocked = "Tool call blocked by policy: " - seen = {m.tool_call_id: m.content[len(blocked):].split(" ")[0] if m.content.startswith(blocked) else m.content - for m in out["messages"] if isinstance(m, ToolMessage)} - results = [r["tool_call_id"] for r in self.recorded("tool.result")] - refused = {r["tool_call_id"]: r["rule_ids"] for r in self.recorded("approval.refused")} - after = ran() - return {c["id"]: (before[c["id"]], c["id"] in listed, after[c["id"]], seen.get(c["id"]), - results.count(c["id"]), refused.get(c["id"])) for c in calls}, "__interrupt__" in out - - def recorded(self, typ): - return [e["data"] for e in self.signer.read({"run_id": self.run["run_id"], "run_token": self.run["run_token"], - "limit": 1000})["events"] if e["type"] == typ] - - def test_each_call_runs_once_or_is_refused_and_is_recorded_as_it_did(self): - want = {"approve": lambda i: (0, True, 1, f"paid {i}", 1, None), - "reject": lambda i: (0, True, 0, "TK-APPROVAL-REJECTED", 0, ["TK-APPROVAL-REJECTED"]), - "wait": lambda i: (0, True, 0, "TK-APPROVAL-REQUESTED", 0, ["TK-APPROVAL-REQUESTED"])} - cases = [*itertools.product(want, repeat=2), *itertools.product(want, repeat=3), - *itertools.product(("approve", "wait"), repeat=5)] - for k, answers in enumerate(cases): - orders = list(itertools.permutations(range(len(answers)))) - for mode in ("sync", "async"): - for how, node in self.nodes(tools=[TOOLS[0], counted_pay, TOOLS[2]]): - with self.subTest(answers=answers, mode=mode, wired=how): - PAID.clear() - self.assertEqual(self.outcomes(node(), mode, answers, orders[k % len(orders)]), - ({**{f"pay-{i}": want[a](i) for i, a in enumerate(answers)}, - "echo": (0, False, 1, "hi", 1, None), - "wipe": (0, False, 0, "R-WIPE", 0, None)}, False)) - - -@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") -class ToolPauses(GatedNodes): - """A tool that pauses the run itself (interrupt()), beside or under an `ask`. LangGraph pairs resume values with a - node's interrupt() calls by their order, and runs the node again from its start on each resume. Once an approval - was answered the gate stopped pausing, so the tool got the approval's resume value as the person's answer and ran - without asking; before that, the tool asked, but its approved call was then refused as already used - (TK-APPROVAL-CONSUMED), and a call that ran before the tool's pause was asked for a second approval.""" - - def graphs(self, tools): - """(how, the graph): create_agent, and the gated ToolNodes (GatedNodes.nodes), each on a signer of its own""" - ToolNode.__init__ = self.wired - self.signer = FakeSigner(ac._rule) - self.run = self.signer.register_run({"request_id": "reg", "agent": {"name": "test"}}) - yield "create_agent", create_agent(StubModel(), tools, checkpointer=InMemorySaver(), middleware=[ - TracekitMiddleware(self.signer, self.run["run_id"], self.run["run_token"])]) - for how, node in self.nodes(tools=tools): - yield how, tool_graph(node()) - - def answered(self, graph, mode, calls, approve): - """(each pause in order: "tracekit" or the tool's question; what the model read of each call; the signer's - approval requests and results, by call), a person approving the calls in `approve` and answering each tool's - question "yes: "; every other approval left unanswered""" - go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) - out, pauses = go({"messages": [HumanMessage(json.dumps(calls))]}, THREAD), [] - while "__interrupt__" in out and len(pauses) < 10: - answers = {} - for i in out["__interrupt__"]: # create_agent's calls are tasks of their own: they pause together - shown = i.value.get("tracekit") if isinstance(i.value, dict) else None - pauses.append("tracekit" if shown else i.value) - for a in shown.get("approvals", [shown]) if shown else (): - if a["tool_call_id"] in approve and self.recorded("approval", a["approval_id"]) == []: - self.signer.approval_decide({"request_id": a["approval_id"], "decision": "approve", - "approval_id": a["approval_id"]}) - answers[i.id] = {"approval_id": shown["approval_id"]} if shown else f"yes: {i.value}" - out = go(Command(resume=answers), THREAD) - seen = {m.tool_call_id: m.content for m in out["messages"] if isinstance(m, ToolMessage)} - return pauses, seen, {t: sorted(r["tool_call_id"] for r in self.recorded(t)) for t in ("approval.request", - "tool.result")} - - def recorded(self, typ, approval_id=None): - return [e["data"] for e in self.signer.read({"run_id": self.run["run_id"], "run_token": self.run["run_token"], - "limit": 1000})["events"] - if e["type"] == typ and approval_id in (None, e["data"].get("approval_id"))] - - def test_a_tool_asked_under_an_approval_gets_the_persons_own_answer(self): - for mode in ("sync", "async"): - for how, graph in self.graphs([confirmed_pay]): - with self.subTest(mode=mode, built=how): - PAID.clear() - got = self.answered(graph, mode, [{"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay-1"}], - approve={"pay-1"}) - self.assertEqual(got, (["tracekit", "pay 1?"], {"pay-1": "paid 1 after yes: pay 1?"}, - {"approval.request": ["pay-1"], "tool.result": ["pay-1"]})) - self.assertEqual(PAID, [(1, "yes: pay 1?")]) # it ran once, with the person's answer - - def test_a_tool_that_pauses_beside_asks_gets_the_answer_and_no_call_runs_or_asks_twice(self): - calls = [{"name": "pay", "args": {"to": "a", "cents": 0}, "id": "pay-0"}, - {"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay-1"}, - {"name": "confirm", "args": {"question": "sure?"}, "id": "confirm"}, - {"name": "echo", "args": {"text": "hi"}, "id": "echo"}] - for mode in ("sync", "async"): - for how, graph in self.graphs([counted_pay, confirm, TOOLS[0]]): - with self.subTest(mode=mode, built=how): - PAID.clear() - ac.RAN.clear() - pauses, seen, recorded = self.answered(graph, mode, calls, approve={"pay-0"}) - self.assertEqual((set(pauses), pauses.count("sure?")), ({"tracekit", "sure?"}, 1)) - self.assertEqual({k: v.split(" (")[0] for k, v in seen.items()}, - {"pay-0": "paid 0", "pay-1": "Tool call blocked by policy: TK-APPROVAL-REQUESTED", - "confirm": "yes: sure?", "echo": "hi"}) - self.assertEqual(recorded, {"approval.request": ["pay-0", "pay-1"], - "tool.result": ["confirm", "echo", "pay-0"]}) - self.assertEqual((PAID, ac.RAN), ([0], ["echo"])) # each ran once - - def test_an_executors_tool_is_left_to_the_executor(self): - """_pass decided an executor's call with the gate's own decision and paused for an approval the executor never - consumes.""" - def send(to: str, cents: int) -> str: - """Send money, through the tool executor.""" - return f"sent {cents}" - executed = tool("pay")(send) - executed.metadata = {EXECUTOR: True} - for mode in ("sync", "async"): - for how, node in self.nodes(tools=[executed, TOOLS[0]]): - with self.subTest(mode=mode, wired=how): - graph = tool_graph(node()) - go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) - out = go({"messages": [HumanMessage(json.dumps([ - {"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay"}, - {"name": "echo", "args": {"text": "hi"}, "id": "echo"}]))]}, THREAD) - self.assertNotIn("__interrupt__", out) - self.assertEqual({m.tool_call_id: m.content for m in out["messages"] - if isinstance(m, ToolMessage)}, {"pay": "sent 1", "echo": "hi"}) - self.assertEqual([r["tool_call_id"] for r in self.recorded("policy.decision")], ["echo"]) - - class Jittery: """The client, with the pause a busy process may take between a call's client_seq and its send.""" diff --git a/tracekit/autowire.py b/tracekit/autowire.py index 60105458b..98e608257 100644 --- a/tracekit/autowire.py +++ b/tracekit/autowire.py @@ -199,15 +199,14 @@ def gated(self): def _langgraph(run): from langgraph.prebuilt.tool_node import ToolNode - from .integrations.langchain import TracekitMiddleware, pass_afunc, pass_func, pausing, with_fallbacks + from .integrations.langchain import TracekitMiddleware, pausing, with_fallbacks gate = TracekitMiddleware(run.client, run.run_id, run.run_token, run.registered.get("fail_modes")) init = ToolNode.__init__ @functools.wraps(init) def gated(self, tools, *a, wrap_tool_call=None, awrap_tool_call=None, **kw): w, aw = wrap_tool_call, awrap_tool_call - wired = not isinstance(getattr(w, "__self__", None), TracekitMiddleware) # tracekit_tool_node gates already - if wired: + if not isinstance(getattr(w, "__self__", None), TracekitMiddleware): # tracekit_tool_node gates already wrap_tool_call = gate.wrap_tool_call if w is None else ( lambda req, h: w(req, lambda r: gate.wrap_tool_call(r, h))) # no async wrapper of its own: ToolNode runs the sync one, gate included, for async calls too @@ -216,10 +215,7 @@ def gated(self, tools, *a, wrap_tool_call=None, awrap_tool_call=None, **kw): if "handle_tool_errors" in kw: # an ask pauses whatever the node's error handling kw["handle_tool_errors"] = pausing(kw["handle_tool_errors"]) init(self, tools, *a, wrap_tool_call=wrap_tool_call, awrap_tool_call=awrap_tool_call, **kw) - if wired: - self._tracekit = gate, wrap_tool_call # the gate decides each pass's calls together (pass_func) ToolNode.__init__ = gated - ToolNode._func, ToolNode._afunc = pass_func, pass_afunc ToolNode.with_fallbacks = with_fallbacks # and through the fallbacks the node is wrapped in diff --git a/tracekit/integrations/langchain.py b/tracekit/integrations/langchain.py index 50f593575..22b81c05a 100644 --- a/tracekit/integrations/langchain.py +++ b/tracekit/integrations/langchain.py @@ -7,10 +7,7 @@ fail_modes=run.get("fail_modes"))]) `deny` replaces the call with an error ToolMessage and the run goes on. `ask` pauses the run with LangGraph -`interrupt()`; resume it with `Command(resume={"approval_id": ...})`. The pause comes first in every pass of its node, -so a tool's own interrupt() gets the person's answer. A ToolNode with several calls of one message decides them all -before any runs and pauses once when any asks (the payload's `approvals` lists them); on resume each call allowed or -approved runs once and one still unanswered is refused. Every call that is not denied runs only after the +`interrupt()`; resume it with `Command(resume={"approval_id": ...})`. Every call that is not denied runs only after the signer's `approval_consume` agrees, so an approval is checked by the signer, never by the saved state; an approval with a T2 executor runs the arguments the consume returns. `ask` needs a checkpointer: without one the call is denied. @@ -35,7 +32,6 @@ recorded by `tracekit.instrument()` or the LLM gateway. """ import asyncio -import contextvars import copy import hashlib import itertools @@ -63,9 +59,6 @@ from tracekit.sdk.client import SignerUnavailable, fail_open, refusal, result_field from tracekit.signer.rpc_schema import RPCError -# the ToolNode pass this call runs in: what TracekitMiddleware._pass decided for each of its calls -_PASS = contextvars.ContextVar("tracekit_tool_node_pass", default=None) - class TracekitMiddleware(AgentMiddleware): def __init__(self, signer, run_id, run_token, fail_modes=None): @@ -78,11 +71,6 @@ def __init__(self, signer, run_id, run_token, fail_modes=None): self.stream, self._seq, self._lock = "langchain-" + uuid.uuid4().hex, itertools.count(), threading.Lock() # lean: one entry per executed tool call for the middleware's life; prune at run end if agents run for days self._attempts = {} # tool_call_id -> the attempt its next execution is (a retry runs the same call again) - # (node task, tool_call_id) -> ... while its node may run again (_gate). In memory: a run resumed in another - # process decides such a call again (docs/integrations/langchain.md) - # lean: a run never resumed from a tool's own pause keeps its entries for the middleware's life - self._paused = {} # -> (decision, pause entry or None): consumed, then its tool paused the run - self._ended = {} # -> (how it ended: a ToolMessage or the exception, pause entry or None), in a held pass self.said = said.Said(lambda **f: self._send("model_event", **f), lambda: signer.status({})) def _send(self, method, **kw): @@ -106,105 +94,35 @@ async def awrap_model_call(self, request, handler): return out def _gate(self, request): - """(what replaces the call: a refusal, or how it ended in an earlier pass of its node, a ToolMessage or the - exception it raised; None when it may run, the decision or None, its pause entry when it asked) - - LangGraph runs a node again from its start on each resume and pairs resume values with interrupt() calls by - their order in the node. So a call that asks pauses here in every pass, answered or not, and a tool's own - interrupt() always comes after it and gets the answer meant for it. A call whose tool paused the run after its - consume goes on when the run resumes, consumed once (_paused).""" - call, key = request.tool_call, _call_key(request) - if key in self._ended: # its node runs again: a tool of the same message paused the run - return self._ended[key][0], None, self._ended[key][1] - if key in self._paused: - d, entry = self._paused[key] - if entry is not None and _PASS.get() is None: # _pass pauses for the calls of its node - self._pause([entry]) - return None, d, entry - planned = (_PASS.get() or {}).get(call["id"]) - if planned is not None: # decided with the other calls of its message (_pass) - blocked, d, entry = planned - hint = entry and entry["approval_id"] - else: - blocked, d, apr = self._ask(call, request.runtime.config) - entry, hint = apr and _entry(call, d, apr), None - if entry is not None: - resume = self._pause([entry]) - hint = resume.get("approval_id") if isinstance(resume, dict) else None - if blocked is not None or d is None: # refused, or the signer unreachable and the run fails open - return blocked, d, None - return *self._consume(call, d, hint), entry - - def _pass(self, calls, config): - """ToolNode runs every call of a model message in one node, and LangGraph runs it again from its start on each - resume. So a message's calls are decided before any runs, and the pass pauses once, before any runs, when any - of them asks (in every pass, as _gate); on resume each call allowed or approved runs and one unanswered is - refused. A call that ran in an earlier pass (a tool of the message paused the run) isn't decided again: _gate - gives back how it ended, or lets it go on. {tool_call_id: (refusal or None, decision or None, its pause entry - or None)} for _gate.""" - ns, plan, asked = config["configurable"].get("checkpoint_ns"), {}, [] - for call in calls: - earlier = self._ended.get((ns, call["id"])) or self._paused.get((ns, call["id"])) - if earlier is not None: - asked += [earlier[1]] if earlier[1] is not None else [] - continue - blocked, d, apr = self._ask(call, config) - plan[call["id"]] = blocked, d, apr and _entry(call, d, apr) - asked += [plan[call["id"]][2]] if apr is not None else [] - if asked: - self._pause(asked) # raises until the run resumes - return plan - - def _end(self, request, out, entry=None): - """How the call ended (`out`, its ToolMessage or exception), kept while its node may run again (a held pass, - _PASS) for _gate to give back; its pause entry when it asked.""" - key = _call_key(request) - self._paused.pop(key, None) - if _PASS.get() is not None: - self._ended[key] = out, entry - - def _forget(self, ns): - """The node task `ns` ended: none of its calls runs again.""" - for key in [k for k in self._ended if k[0] == ns]: - del self._ended[key] - - def _ask(self, call, config): - """(the error ToolMessage that replaces a refused call or None, the decision or None, the approval when the - decision asks: approval_request's answer)""" + """(the error ToolMessage that replaces a refused call or None when it may run, the decision or None)""" + call, hint = request.tool_call, None + attempt = self._attempts.get(call["id"], 0) self.said.decided(call["id"]) # the call its recorded response left for now try: # LangChain hands over JSON-decoded args, never the model's raw string d = self._event("decide", call, tool=call["name"], args_source="parsed", args=call["args"]) except SignerUnavailable as e: # the run's fail mode for the tool's class open_ = fail_open(self.fail_modes, _tool_class(call["name"])) - return (None if open_ else self._refusal(call, f"signer unavailable: {e}")), None, None + return (None if open_ else self._refusal(call, f"signer unavailable: {e}")), None except RPCError as e: - return self._refusal(call, f"signer refused the call: {e}"), None, None + return self._refusal(call, f"signer refused the call: {e}"), None if d["decision"] == "deny": - return self._refusal(call, refusal(d) or "deny"), d, None + return self._refusal(call, refusal(d) or "deny"), d d["args_digest"] = event_hash({"tool": call["name"], "args": call["args"]}) - if d["decision"] != "ask": - return None, d, None - if config["configurable"].get(CONFIG_KEY_CHECKPOINTER) is None: - return self._refusal(call, "approval required, and the agent has no checkpointer to wait for it"), d, None - try: # the signer keeps one approval per call attempt: the re-run on resume, in any process, gets it back - return None, d, self.signer.approval_request({"request_id": uuid.uuid4().hex, **self.run, - "tool_call_id": call["id"], - "attempt": self._attempts.get(call["id"], 0)}) - except (RPCError, SignerUnavailable) as e: - return self._refusal(call, str(e)), d, None - - @staticmethod - def _pause(entries): - """interrupt() for the approvals of `entries` (_entry), the first named at the top: the resume value.""" - return interrupt({"tracekit": {**entries[0], "approvals": entries}}) - - def _consume(self, call, d, hint): - """approval_consume, the signer's last word before the call runs: (the refusal or None, the decision)""" - try: # every call, allowed or approved: the saved state can't vouch that no approval is needed + try: + if d["decision"] == "ask": + if request.runtime.config["configurable"].get(CONFIG_KEY_CHECKPOINTER) is None: + return self._refusal(call, "approval required, and the agent has no checkpointer to wait for it"), d + # the signer keeps one approval per call attempt, so the re-run on resume (even in another process) + # gets the same approval back + apr = self.signer.approval_request({"request_id": uuid.uuid4().hex, **self.run, + "tool_call_id": call["id"], "attempt": attempt}) + resume = interrupt({"tracekit": {"approval_id": apr["approval_id"], "tool_call_id": call["id"], + "tool": call["name"], "rule_ids": d["rule_ids"]}}) + hint = resume.get("approval_id") if isinstance(resume, dict) else None + # every call, allowed or approved: the saved state can't vouch that no approval is needed c = self.signer.approval_consume({"request_id": uuid.uuid4().hex, **self.run, "tool_call_id": call["id"], - "attempt": self._attempts.get(call["id"], 0), "tool": call["name"], - "args_source": "parsed", "args": call["args"], - **({"approval_id_hint": hint} if hint else {})}) + "attempt": attempt, "tool": call["name"], "args_source": "parsed", + "args": call["args"], **({"approval_id_hint": hint} if hint else {})}) except (RPCError, SignerUnavailable) as e: # e.g. a hint that is not an id at all c = {"ok": False, "rule_ids": [], "reason": str(e)} if c["ok"]: @@ -239,78 +157,52 @@ def _complete(self, call, decision, out=None, exc=None): # lean: wrap_tool_call's signer calls block; awrap_tool_call runs them in a worker thread (issue #680) def wrap_tool_call(self, request, handler): - if _executor(request.tool): # the executor decides it, under the model's id for the call + if _executor(request): # the executor decides it, under the model's id for the call self.said.decided(request.tool_call["id"]) token = CALL_ID.set(request.tool_call["id"]) try: return handler(request) finally: CALL_ID.reset(token) - out, d, entry = self._gate(request) - if out is not None: - return self._ended_with(request, out, entry) + blocked, d = self._gate(request) + if blocked is not None: + return blocked if d is not None and "args" in d: request = request.override(tool_call={**request.tool_call, "args": d["args"]}) try: out = handler(request) except GraphBubbleUp: # interrupt() or a handoff inside the tool: control flow, not an outcome - if d is not None: # it goes on, consumed once, when the run resumes (_gate) - self._paused[_call_key(request)] = d, entry raise except Exception as e: self._complete(request.tool_call, d, exc=e) - self._end(request, e, entry) raise self._complete(request.tool_call, d, out) - self._end(request, out, entry) - return out - - def _ended_with(self, request, out, entry): - """The call's refusal, or how it ended in an earlier pass of its node: returned, or raised.""" - self._end(request, out, entry) - if isinstance(out, BaseException): - raise out return out async def awrap_tool_call(self, request, handler): - if _executor(request.tool): + if _executor(request): await asyncio.to_thread(self.said.decided, request.tool_call["id"]) token = CALL_ID.set(request.tool_call["id"]) try: return await handler(request) finally: CALL_ID.reset(token) - out, d, entry = await asyncio.to_thread(self._gate, request) - if out is not None: - return self._ended_with(request, out, entry) + blocked, d = await asyncio.to_thread(self._gate, request) + if blocked is not None: + return blocked if d is not None and "args" in d: request = request.override(tool_call={**request.tool_call, "args": d["args"]}) try: out = await handler(request) except GraphBubbleUp: # interrupt() or a handoff inside the tool: control flow, not an outcome - if d is not None: - self._paused[_call_key(request)] = d, entry raise except Exception as e: await asyncio.to_thread(self._complete, request.tool_call, d, exc=e) - self._end(request, e, entry) raise await asyncio.to_thread(self._complete, request.tool_call, d, out) - self._end(request, out, entry) return out -def _call_key(request): - """A call in its node's task, the same in each pass of the node (LangGraph runs it again on each resume).""" - return request.runtime.config["configurable"].get("checkpoint_ns"), request.tool_call["id"] - - -def _entry(call, d, apr): - """A call's approval as the pause shows it (TracekitMiddleware._pause).""" - return {"approval_id": apr["approval_id"], "tool_call_id": call["id"], "tool": call["name"], - "rule_ids": d["rule_ids"]} - - def _said(request, out): """(the items of what the model said, its model, its provider) in a model call's response: the calls of the agent's tools, which the gate or a tool executor decides under their ids (not a structured-output tool's), as @@ -340,9 +232,9 @@ def items(): return items(), meta.get("model_name"), meta.get("model_provider") -def _executor(tool): - """Whether `tool` is a tool executor's (executor_tools), whose calls the executor decides and records (T2).""" - return bool((getattr(tool, "metadata", None) or {}).get(EXECUTOR)) +def _executor(request): + """Whether the call is to a tool executor's tool (executor_tools), which the executor decides and records (T2).""" + return bool((getattr(request.tool, "metadata", None) or {}).get(EXECUTOR)) def pausing(flag): @@ -404,51 +296,8 @@ def with_fallbacks(self, fallbacks, **kw): return _Fallbacks(runnable=self, fallbacks=fallbacks, **kw) -def _held(node, input): - """(the gate that holds ToolNode `node`, this pass's calls) when the gate decides them together (two or more: - TracekitMiddleware._pass), else (None, None). A node whose wrapper is no longer the gate's, as in `tracekit why`'s - replay copy, which serves recorded results instead, isn't held.""" - gate, wrap = getattr(node, "_tracekit", (None, None)) - if gate is None or node._wrap_tool_call is not wrap: - return None, None - # an executor's tool is the executor's to decide (wrap_tool_call passes it through) - calls = [c for c in node._parse_input(input)[0] if not _executor(node.tools_by_name.get(c["name"]))] - return (gate, calls) if len(calls) > 1 else (None, None) - - -_TOOL_NODE_FUNC, _TOOL_NODE_AFUNC = ToolNode._func, ToolNode._afunc - - -def pass_func(self, input, config, runtime): - """ToolNode._func, the calls of a held node's pass decided together first (_held); each call's gate finds its - decision in _PASS.""" - gate, calls = _held(self, input) - token = _PASS.set(gate and gate._pass(calls, config)) - try: - out = _TOOL_NODE_FUNC(self, input, config, runtime) - finally: - _PASS.reset(token) - if gate: # the node ended: none of its calls runs again - gate._forget(config["configurable"].get("checkpoint_ns")) - return out - - -async def pass_afunc(self, input, config, runtime): - """ToolNode._afunc, as pass_func; the signer calls in a worker thread.""" - gate, calls = _held(self, input) - token = _PASS.set(gate and await asyncio.to_thread(gate._pass, calls, config)) - try: - out = await _TOOL_NODE_AFUNC(self, input, config, runtime) - finally: - _PASS.reset(token) - if gate: - gate._forget(config["configurable"].get("checkpoint_ns")) - return out - - class _GatedToolNode(ToolNode): with_fallbacks = with_fallbacks - _func, _afunc = pass_func, pass_afunc def tracekit_tool_node(tools, signer, run, **kw): @@ -457,10 +306,7 @@ def tracekit_tool_node(tools, signer, run, **kw): gate = TracekitMiddleware(signer, run["run_id"], run["run_token"], run.get("fail_modes")) if "handle_tool_errors" in kw: kw["handle_tool_errors"] = pausing(kw["handle_tool_errors"]) - wrap = gate.wrap_tool_call - node = _GatedToolNode(tools, wrap_tool_call=wrap, awrap_tool_call=gate.awrap_tool_call, **kw) - node._tracekit = gate, wrap # the gate holds the node while this is its wrapper (_held) - return node + return _GatedToolNode(tools, wrap_tool_call=gate.wrap_tool_call, awrap_tool_call=gate.awrap_tool_call, **kw) _SERDE = JsonPlusSerializer() From 9a9579d1bb9b4bb005d29ae42d0fa7aac21a0615 Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:27:04 +0000 Subject: [PATCH 6/9] Limits: under ainvoke, a repeated approval can leave another approved call denied and unrun --- docs/limits.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/limits.md b/docs/limits.md index b1b8a96c3..0291bb31d 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -359,7 +359,7 @@ What Tracekit guarantees and doesn't, per deployment, on one page: [guarantees]( | An MCP call is named by the server's own name (`mcp:/`); an mcp 1.x session whose `initialize()` ran before `tracekit.instrument()`, or one wrapped by hand without `server=`, is recorded as `mcp:unknown/`, with a warning. | The name comes from the server's initialize result, which an mcp 1.x session doesn't keep. | Call `tracekit.instrument()` before any session initializes, or pass `TracekitSession(..., server="")`; write rules for server-named tools knowing the server picks its name. | [MCP](integrations/mcp.md#not-covered) | | An adapter without saved state (MCP, Browser Use, the Claude Agent SDK hook) holds a call while waiting; the wait doesn't survive the process, and a long wait ends in a denial. | There is no resumable state to carry the approval. | Answer within the adapter's wait; use a framework with saved state for long waits. | [MCP](integrations/mcp.md#not-covered), [Claude Agent SDK](integrations/claude-agent-sdk.md#not-covered) | | LangGraph: code that catches exceptions around a gated `ToolNode` (a `try`/`except Exception` in a node that invokes it, a middleware outside `TracekitMiddleware` that turns exceptions into messages) ends an `ask`'s pause: the model gets an error, the call doesn't run, and its approval expires. The node's own `handle_tool_errors` and `.with_fallbacks()` let the pause through. | LangGraph pauses by raising `GraphInterrupt` out of the node; code that catches it ends the pause before LangGraph sees it. | Re-raise `langgraph.errors.GraphBubbleUp` before any broad `except`, or put the error handling in the ToolNode (`handle_tool_errors`, `.with_fallbacks()`). | [LangGraph](quickstarts/langchain.md#approvals-wire-it-by-hand) | -| LangGraph: when one model message has several calls that ask, in one `ToolNode` (a `tracekit_tool_node`, or a `ToolNode` that `tracekit.instrument()` wired), each resume answers one pause and the run pauses again for the next. Under `ainvoke` a pause may show a call a person already approved. A call that ran on an earlier resume is gated again on the next: it gets a new approval request and is then denied (`TK-APPROVAL-UNBOUND`), though it ran once. An allowed call in the same message runs again on each resume. No asking call runs without an approval. | LangGraph runs the node again from its start on each resume and pairs resume values with the node's `interrupt()` calls by their order, which the node's parallel calls reach in no fixed order. | Answer each pause; read the run's `tool.result` records, not the messages the model got, for what ran; or build the agent with `create_agent`, which runs each call as a task of its own. | [LangChain](integrations/langchain.md#decisions) | +| LangGraph: when one model message has several calls that ask, in one `ToolNode` (a `tracekit_tool_node`, or a `ToolNode` that `tracekit.instrument()` wired), each resume answers one pause and the run pauses again for the next. Under `ainvoke` a pause may show a call a person already approved; when it does, another approved call's approval is never shown, and that call is denied (`TK-APPROVAL-UNBOUND`) and never runs, however each pause is answered. A call that ran on an earlier resume is gated again on the next: it gets a new approval request and is then denied (`TK-APPROVAL-UNBOUND`), though it ran once. An allowed call in the same message runs again on each resume. No asking call runs without an approval. | LangGraph runs the node again from its start on each resume and pairs resume values with the node's `interrupt()` calls by their order, which the node's parallel calls reach in no fixed order. | Answer each pause; read the run's `tool.result` records, not the messages the model got, for what ran; or build the agent with `create_agent`, which runs each call as a task of its own. | [LangChain](integrations/langchain.md#decisions) | | A Browser Use action waits for an approval at most five sixths of Browser Use's action timeout (150 s of 180 s); then the approval is withdrawn and the action refused. | Browser Use cancels an action that outlives its timeout, approval wait included, with a misleading "dead CDP WebSocket" error. | Answer within the wait, or raise `BROWSER_USE_ACTION_TIMEOUT_S` (and the Agent's `step_timeout`) to give people longer. | [Browser Use](integrations/browser-use.md#decisions) | | OpenAI Agents' `LocalShellTool` and `ApplyPatchTool` can't wait for an approval: an `ask` for them is refused. | The SDK gives no approval hook for them. | Allow or deny them by policy. | [OpenAI Agents](integrations/openai-agents.md#not-covered) | | v1 approvals name the held call, not its exact arguments. | v1 approvals predate argument binding. | Use the v2 signer, which binds approvals to the exact arguments. | [laptop threat model](threat-model-laptop.md#open-problems) | From bb36f02f69b8dc37f9686d207975d5b8bdde0b70 Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:34:33 +0530 Subject: [PATCH 7/9] Review fixes: MCP rounds left waiting stay gaps, listing timeouts, LangGraph row - MCP: a multi-round call its task leaves (the elicitation callback fails, the rounds run out, the caller moves on) is never completed later; it was completed "ok" with its input request when the task made another call, though its caller had got an exception. It ends as the signer's decided_not_completed gap at run close, as the docs say. - MCP gateway: only the listing's own time running out (Upstreams._run's wait, now UpstreamTimeout) is reported as the listing's budget; a TimeoutError the upstream call or the reconnect raises keeps the upstream error and its advice. - Tests: a round's input requests over half a frame are recorded as a digest. - Limits: the LangGraph parallel-asks row says what happens under invoke and ainvoke alike, with what-to-dos checked against the code; a row for MCP rounds sent from different tasks. Signed-off-by: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> --- docs/integrations/mcp.md | 4 +- docs/limits.md | 3 +- tests/test_contract_mcp.py | 71 +++++++++++++++++++++++++++--------- tests/test_mcp_gateway.py | 35 +++++++++++------- tracekit/integrations/mcp.py | 24 ++++++------ tracekit/mcp_gateway.py | 11 +++++- 6 files changed, 100 insertions(+), 48 deletions(-) diff --git a/docs/integrations/mcp.md b/docs/integrations/mcp.md index 998f47f0e..3bf85c7b1 100644 --- a/docs/integrations/mcp.md +++ b/docs/integrations/mcp.md @@ -39,7 +39,7 @@ Every other attribute is the wrapped session's. Only calls made through the wrap | right before it is sent | `approval_consume` | the same call and arguments, and the approval id when the call waited for one | | after the server answers | `complete` | `status` `ok`, or `error` for a result with `isError: true`; a `sha256:` commitment to the result (not the result), or the exception type and message | -`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`observe`, source `mcp:/tools`, `/resources` or `/prompts`): the model reads the descriptions the server wrote, and the page the caller asked for is what is recorded (a page over half the signer's 1 MiB frame as its `sha256:` digest, as the [MCP gateway](../mcp-gateway.md) records one). The list mcp's `call_tool` makes itself, to check a result against the tool's output schema, never reaches the model and isn't recorded. A page the signer cannot record is withheld, unless the run fails open for `mcp`, when an unreachable signer returns it with a warning; a refusal is withheld either way. A tool call or prompt that returns an `InputRequiredResult` (MCP 2.3) records each round's input requests as untrusted observed input (`observe`, source `mcp:/input_required`). When the server provides a `request_state` and the call needed no human approval, it is decided and completed once, under its initial decision, when the terminal result arrives. A round continues the call its own asyncio task left waiting, as `mcp.Client.call_tool` drives the rounds, so concurrent calls never share a decision even when the server returns the same `request_state` to both; a task that makes another call on the session before the next round ends the waiting call, completed with the input request it returned. Rounds from a server that returns no `request_state`, or for a decision that required human approval (as the signer refuses to consume a used approval), are decided and completed as separate calls per round. +`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`observe`, source `mcp:/tools`, `/resources` or `/prompts`): the model reads the descriptions the server wrote, and the page the caller asked for is what is recorded (a page over half the signer's 1 MiB frame as its `sha256:` digest, as the [MCP gateway](../mcp-gateway.md) records one). The list mcp's `call_tool` makes itself, to check a result against the tool's output schema, never reaches the model and isn't recorded. A page the signer cannot record is withheld, unless the run fails open for `mcp`, when an unreachable signer returns it with a warning; a refusal is withheld either way. A tool call or prompt that returns an `InputRequiredResult` (MCP 2.3) records each round's input requests as untrusted observed input (`observe`, source `mcp:/input_required`). When the server provides a `request_state` and the call needed no human approval, it is decided and completed once, under its initial decision, when the terminal result arrives. A round continues the call its own asyncio task left waiting, as `mcp.Client.call_tool` drives the rounds, so concurrent calls never share a decision even when the server returns the same `request_state` to both; a waiting call whose task makes another call on the session first is never completed ([not covered](#not-covered)). Rounds from a server that returns no `request_state`, or for a decision that required human approval (as the signer refuses to consume a used approval), are decided and completed as separate calls per round. `` is the name in the server's initialize result, with every character outside `A-Za-z0-9_.-` replaced by `_` (`unknown` when the server gave none). The policy's `tools` map should map `mcp:*/*` to the `mcp` class (the coding @@ -64,7 +64,7 @@ a lost connection) propagates unchanged. Both are recorded as `error`. ## Not covered -- MCP calls carry no call id, so every `call_tool` is a call of its own (attempt 0); multi-round input-required calls reuse their decision across rounds, while an independent retry is a new decision. A call whose later rounds never reach the session (the elicitation callback fails, mcp's `input_required_max_rounds` runs out, both inside `mcp.Client`, or the task driving it ends) stays decided and is never completed; at run close the signer records it as a signed `decided_not_completed` gap. Rounds you drive yourself from different tasks are decided as calls of their own. +- MCP calls carry no call id, so every `call_tool` is a call of its own (attempt 0); multi-round input-required calls reuse their decision across rounds, while an independent retry is a new decision. A call whose later rounds never reach the session (the elicitation callback fails, mcp's `input_required_max_rounds` runs out, both inside `mcp.Client`, or the task driving it ends or makes another call on the session first) stays decided and is never completed; at run close the signer records it as a signed `decided_not_completed` gap. Rounds you drive yourself from different tasks are decided as calls of their own, so each round of an allowed call before its last is such a gap ([limits](../limits.md#approvals)). - There is no saved state: a call waiting for an approval does not survive the process. - Servers the agent reaches without this session (another client, a hosted MCP tool of a model provider) are not seen. - The server's name in `mcp:/` is the one the server gives. An mcp 1.x session doesn't keep it: diff --git a/docs/limits.md b/docs/limits.md index 0291bb31d..2fcd2ca71 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -358,8 +358,9 @@ What Tracekit guarantees and doesn't, per deployment, on one page: [guarantees]( | **(1.5.1)** A retry uses an earlier call's T2 approval (`executor: t2`) only when its consume names it (`approval_id_hint`): without the hint it is refused `TK-APPROVAL-REQUIRED`. | The signer reads its copy of the approved arguments, which a T2 consume returns, before the writer runs, for the approval it is told. | Pass the `approval_id` that `approval_request` returned, as the shipped hooks do. | [approve, then retry](approvals.md#approve-then-retry) | | An MCP call is named by the server's own name (`mcp:/`); an mcp 1.x session whose `initialize()` ran before `tracekit.instrument()`, or one wrapped by hand without `server=`, is recorded as `mcp:unknown/`, with a warning. | The name comes from the server's initialize result, which an mcp 1.x session doesn't keep. | Call `tracekit.instrument()` before any session initializes, or pass `TracekitSession(..., server="")`; write rules for server-named tools knowing the server picks its name. | [MCP](integrations/mcp.md#not-covered) | | An adapter without saved state (MCP, Browser Use, the Claude Agent SDK hook) holds a call while waiting; the wait doesn't survive the process, and a long wait ends in a denial. | There is no resumable state to carry the approval. | Answer within the adapter's wait; use a framework with saved state for long waits. | [MCP](integrations/mcp.md#not-covered), [Claude Agent SDK](integrations/claude-agent-sdk.md#not-covered) | +| MCP: an input-required call (MCP 2.3) whose rounds are sent from different asyncio tasks (a web app that keeps the `request_state` and sends the next round in its next HTTP request) is decided again in each round, and each round of an allowed call before its last stays decided and never completed: a signed `decided_not_completed` gap at run close, one per round, counted toward the run's 100 such gaps (past them, one gap counts the rest). | A round continues only the call its own task left waiting, so two calls never share a decision, whatever `request_state` a server returns to both; a round sent from another task can't be told from a new call that reuses the state. | Drive all of a call's rounds in one task, as `mcp.Client.call_tool` does; read such a gap next to the later round with the same tool and arguments. | [MCP](integrations/mcp.md#not-covered) | | LangGraph: code that catches exceptions around a gated `ToolNode` (a `try`/`except Exception` in a node that invokes it, a middleware outside `TracekitMiddleware` that turns exceptions into messages) ends an `ask`'s pause: the model gets an error, the call doesn't run, and its approval expires. The node's own `handle_tool_errors` and `.with_fallbacks()` let the pause through. | LangGraph pauses by raising `GraphInterrupt` out of the node; code that catches it ends the pause before LangGraph sees it. | Re-raise `langgraph.errors.GraphBubbleUp` before any broad `except`, or put the error handling in the ToolNode (`handle_tool_errors`, `.with_fallbacks()`). | [LangGraph](quickstarts/langchain.md#approvals-wire-it-by-hand) | -| LangGraph: when one model message has several calls that ask, in one `ToolNode` (a `tracekit_tool_node`, or a `ToolNode` that `tracekit.instrument()` wired), each resume answers one pause and the run pauses again for the next. Under `ainvoke` a pause may show a call a person already approved; when it does, another approved call's approval is never shown, and that call is denied (`TK-APPROVAL-UNBOUND`) and never runs, however each pause is answered. A call that ran on an earlier resume is gated again on the next: it gets a new approval request and is then denied (`TK-APPROVAL-UNBOUND`), though it ran once. An allowed call in the same message runs again on each resume. No asking call runs without an approval. | LangGraph runs the node again from its start on each resume and pairs resume values with the node's `interrupt()` calls by their order, which the node's parallel calls reach in no fixed order. | Answer each pause; read the run's `tool.result` records, not the messages the model got, for what ran; or build the agent with `create_agent`, which runs each call as a task of its own. | [LangChain](integrations/langchain.md#decisions) | +| LangGraph: when one model message has more than one call that asks and one `ToolNode` runs them all, as when `tools_condition` routes the message to it (a `tracekit_tool_node`, or a `ToolNode` that `tracekit.instrument()` wired), a resume can reach a call other than the one its pause showed, under `invoke` and `ainvoke` alike. Any of those calls may then be denied (`TK-APPROVAL-UNBOUND`) and never run, even after its own pause was approved. A pause may repeat one already approved, or show a call that already ran under a new approval, which runs it again once approved. A call that ran is gated again on each later resume, so the model may get a denial for a call that ran. An allowed call in the same message may run again on each resume. No asking call runs without an approval a person gave for it. | LangGraph runs the node again from its start on each resume and pairs resume values with the node's `interrupt()` calls by the order they are reached, which the node's parallel calls (threads under `invoke`, tasks under `ainvoke`) reach in no fixed order. | Give each call a task of its own: build the agent with `create_agent`, or route each call to the node with its own `Send("tools", [call])`, and resume every pending pause by its interrupt id (`Command(resume={i.id: {"approval_id": i.value["tracekit"]["approval_id"]} for i in out["__interrupt__"]})`). Or have the model make one call per message (`bind_tools(..., parallel_tool_calls=False)` on a chat model that takes it). Read the run's `tool.result` records, not the messages the model got, for what ran. | [LangChain](integrations/langchain.md#decisions) | | A Browser Use action waits for an approval at most five sixths of Browser Use's action timeout (150 s of 180 s); then the approval is withdrawn and the action refused. | Browser Use cancels an action that outlives its timeout, approval wait included, with a misleading "dead CDP WebSocket" error. | Answer within the wait, or raise `BROWSER_USE_ACTION_TIMEOUT_S` (and the Agent's `step_timeout`) to give people longer. | [Browser Use](integrations/browser-use.md#decisions) | | OpenAI Agents' `LocalShellTool` and `ApplyPatchTool` can't wait for an approval: an `ask` for them is refused. | The SDK gives no approval hook for them. | Allow or deny them by policy. | [OpenAI Agents](integrations/openai-agents.md#not-covered) | | v1 approvals name the held call, not its exact arguments. | v1 approvals predate argument binding. | Use the v2 signer, which binds approvals to the exact arguments. | [laptop threat model](threat-model-laptop.md#open-problems) | diff --git a/tests/test_contract_mcp.py b/tests/test_contract_mcp.py index 0d31a92a7..1acdce56b 100644 --- a/tests/test_contract_mcp.py +++ b/tests/test_contract_mcp.py @@ -725,7 +725,8 @@ async def run_calls(): self.assertEqual(d1, 1) self.assertEqual(d2, 2) - def test_server_that_reuses_request_state_completes_both_calls(self): + def test_server_that_reuses_request_state_decides_both_calls(self): + """The first call, left after its first round, is never completed; the second completes.""" from mcp.server.lowlevel import Server import mcp.types as t from tracekit import autowire @@ -765,7 +766,7 @@ async def run_test(): decides = [r for m, r in signer.requests if m == "decide"] completes = [r for m, r in signer.requests if m == "complete"] self.assertEqual(len(decides), 2) - self.assertEqual(len(completes), 2) + self.assertEqual([(r["tool_call_id"], r["status"]) for r in completes], [(decides[1]["tool_call_id"], "ok")]) def test_round_whose_observe_fails_warns_when_fail_open(self): from tracekit import autowire @@ -871,21 +872,35 @@ async def both(): self.assertEqual((r["status"], r["result"]["content"][0]["text"]), ("ok", "done")) self.assertEqual(r["decision_id"], f"dec-{list(decided).index(r['tool_call_id']) + 1}") - def test_a_call_left_between_rounds_ends_when_its_task_makes_another(self): - """#718: call A stops after its first round; call B, same tool, arguments and state, ran its last round under A's - id and decision. Now A completes with the input request it got, and B under its own decision.""" - server, signer = self._state_reusing_server(), self._wired() - - async def abandoned_then_new(): - async with MCPClient(server, elicitation_callback=self._elicit) as c: - await c.session.call_tool("ask", {"k": "v"}, allow_input_required=True) # A: round 1 only - return await c.call_tool("ask", {"k": "v"}) # B, to its end - self.assertEqual(asyncio.run(abandoned_then_new()).content[0].text, "done") - a, b = [r["tool_call_id"] for m, r in signer.requests if m == "decide"] - completes = [(r["tool_call_id"], r["decision_id"], r["status"]) for m, r in signer.requests if m == "complete"] - self.assertEqual(completes, [(a, "dec-1", "ok"), (b, "dec-2", "ok")]) - first, last = [r["result"] for m, r in signer.requests if m == "complete"] - self.assertEqual((first["resultType"], last["content"][0]["text"]), ("input_required", "done")) + def test_a_call_left_between_rounds_is_never_completed(self): + """#718: call A is left after its first round, by its caller or by mcp.Client when the elicitation callback + raises; call B, same tool, arguments and state, ran its last round under A's id and decision. Then A was + completed "ok" with its input request when B began, though its caller got an exception. A stays decided and + is never completed (the signer's decided_not_completed gap at run close); B completes under its own decision.""" + import mcp.types as t + for elicitation_fails in (False, True): + with self.subTest(elicitation_fails=elicitation_fails): + self._restore() # _wired() wraps ClientSession's methods again + server, signer = self._state_reusing_server(), self._wired() + fails = [RuntimeError("the person closed the dialog")] if elicitation_fails else [] + + async def elicit(ctx, params): + if fails: + raise fails.pop() + return t.ElicitResult(action="accept", content={}) + + async def left_then_new(): + async with MCPClient(server, elicitation_callback=elicit) as c: + if elicitation_fails: + with self.assertRaises(Exception): # an ExceptionGroup holding the callback's error + await c.call_tool("ask", {"k": "v"}) # A + else: + await c.session.call_tool("ask", {"k": "v"}, allow_input_required=True) # A: round 1 + return await c.call_tool("ask", {"k": "v"}) # B, to its end + self.assertEqual(asyncio.run(left_then_new()).content[0].text, "done") + _, b = [r["tool_call_id"] for m, r in signer.requests if m == "decide"] + self.assertEqual([(r["tool_call_id"], r["decision_id"], r["status"], r["result"]["content"][0]["text"]) + for m, r in signer.requests if m == "complete"], [(b, "dec-2", "ok", "done")]) @unittest.skipUnless(HAVE_MCP and hasattr(mcp.types, "InputRequiredResult"), "mcp 2.3+ with InputRequiredResult required") @@ -948,3 +963,25 @@ async def called(): self.assertEqual((out.is_error, out.content[0].text), (False, "ran tool_7")) self.assertEqual(self.events("input.observed"), []) self.assertEqual([r["ok"] for r in self.events("tool.result")], [True]) + + def test_input_requests_over_half_a_frame_are_recorded_as_their_digest(self): + """A round's input requests are recorded as a list is: sent whole, the signer refused requests over its frame + (quota_exceeded) and the call failed.""" + from mcp.server.lowlevel import Server + import mcp.types as t + + async def on_list_tools(ctx, params): + return t.ListToolsResult(tools=[t.Tool(name="ask", inputSchema={"type": "object"})]) + + async def on_call_tool(ctx, params): + return t.InputRequiredResult(request_state="S", input_requests={"q": t.ElicitRequest( + params=t.ElicitRequestFormParams(message="who? " * 300_000, # 1.5 MB + requested_schema={"type": "object", "properties": {}}))}) + + async def asked(): + async with MCPClient(Server("big", on_list_tools=on_list_tools, on_call_tool=on_call_tool)) as c: + return await c.session.call_tool("ask", {}, allow_input_required=True) + self.assertEqual(asyncio.run(asked()).result_type, "input_required") + [seen] = self.events("input.observed") + self.assertEqual((seen["source"], seen["trust"]), ("mcp:big/input_required", "untrusted")) + self.assertLess(seen["output"]["size"], 100) # the digest, not the requests diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index 776c65b29..54eae3c00 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -1,6 +1,8 @@ """The MCP gateway (tracekit/mcp_gateway.py): an MCP client connects over Streamable HTTP; the gateway proxies an in-process upstream MCP server, deciding and recording through an in-process v2 signer as source gateway (tier T2).""" import asyncio +import concurrent.futures +import contextlib import http.client import json import os @@ -391,19 +393,26 @@ async def on_list_tools(ctx, params): self.assertIn("upstream 'up': tools/list timed out (one listing has 0.3 s for every upstream and page)", messages(e.exception)) - def test_a_listing_past_its_budget_names_the_upstream_before_python_3_11(self): - """Before 3.11 the timeout of the future a listing waits on (concurrent.futures.TimeoutError) isn't the built-in - TimeoutError, so the listing's own error was missed.""" - class FutureTimeout(Exception): # concurrent.futures.TimeoutError on Python 3.10 - pass - - def call(up, method, *a, **kw): - raise FutureTimeout() - with mock.patch("concurrent.futures.TimeoutError", FutureTimeout), \ - mock.patch.object(self.srv.mcp["upstreams"], "call", call), self.assertRaises(Exception) as e: - self.agent(lambda c: c.list_tools()) - self.assertIn("upstream 'up': tools/list timed out (one listing has 600 s for every upstream and page)", - messages(e.exception)) + def test_a_timeout_the_upstream_call_raises_is_not_the_listings(self): + """Only the listing's own time running out names its budget. A TimeoutError raised inside the upstream call (a + socket's, anyio's fail_after) or by the reconnect's wait read "tools/list timed out (one listing has 600 s for + every upstream and page)"; it keeps the upstream error and its advice.""" + ups = self.srv.mcp["upstreams"] + advice = "; the gateway connects to an upstream that went away again on the next call" + + def listing_error(*patches): + with contextlib.ExitStack() as stack, self.assertRaises(Exception) as e: + for p in patches: + stack.enter_context(p) + self.agent(lambda c: c.list_tools()) + self.assertNotIn("one listing has", messages(e.exception)) + return messages(e.exception) + self.assertIn("upstream error: TimeoutError: read timed out" + advice, listing_error( + mock.patch.object(ups.clients["up"], "list_tools", new_callable=mock.AsyncMock, + side_effect=TimeoutError("read timed out")))) + self.assertIn("upstream error: TimeoutError: " + advice, listing_error( # the reconnect's wait for a session + mock.patch.object(mcp_gateway, "_closed", return_value=True), + mock.patch.object(ups, "_open", side_effect=concurrent.futures.TimeoutError()))) def test_page_bound_error_forgets_page1_resources(self): """Hitting the page cap on resources/list returns an error and remembers nothing.""" diff --git a/tracekit/integrations/mcp.py b/tracekit/integrations/mcp.py index 5e1d7a5f0..6d8f38d45 100644 --- a/tracekit/integrations/mcp.py +++ b/tracekit/integrations/mcp.py @@ -15,10 +15,10 @@ for an approval with a T2 executor. Multi-round input-required calls (MCP 2.3) with a `request_state` under an allowed decision are decided and completed once across rounds: a round continues the call its own asyncio task left waiting (as mcp's Client.call_tool drives them), so concurrent calls never share one, whatever `request_state` the server returns. -A task that makes any other call on the session first ends the waiting one, completed with the input request it -returned; a task that ends between rounds leaves it to the signer's decided_not_completed gap. Rounds without -`request_state`, or whose decision required human approval, are decided as calls of their own. A result with -`isError: true` completes as an error; an exception from the session propagates unchanged and completes as an error. +A waiting call whose task makes any other call on the session first, or ends, is never completed: the signer's +decided_not_completed gap at run close. Rounds without `request_state`, or whose decision required human approval, are +decided as calls of their own. A result with `isError: true` completes as an error; an exception from the session +propagates unchanged and completes as an error. `list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`mcp:/tools`, `/resources`, `/prompts`; a page over half a signer frame as its digest) and returned once recorded. A list the signer cannot reach is withheld unless the run fails open for this class, when it is returned with a warning; a refusal is @@ -114,13 +114,11 @@ async def _call(self, method, prefix, name, arguments, *a, **kw): args = arguments or {} caller_args_digest = event_hash({"tool": tool, "args": args}) task = asyncio.current_task() + # a call this task left waiting and doesn't continue now is never completed: its caller may have seen an + # exception (mcp's elicitation callback, its round cap), so it ends as the signer's decided_not_completed gap held = self._pending.pop(task, None) - if held is not None and held[0] != (tool, caller_args_digest, kw.get("request_state")): - # this task moved on without the round: the call ended with the input request it returned - await self._complete(held[1], held[2], status="ok", result=held[5]) - held = None - if held is not None: - _, call_id, req, d, args, _ = held + if held is not None and held[0] == (tool, caller_args_digest, kw.get("request_state")): + _, call_id, req, d, args = held try: c = await self._rpc("approval_consume", tool_call_id=call_id, tool=tool, args_source="parsed", args=args) except (RPCError, SignerUnavailable) as e: @@ -167,11 +165,11 @@ async def _call(self, method, prefix, name, arguments, *a, **kw): await self._complete(call_id, req, status="error", error=f"{type(e).__name__}: {e}"[:4096]) raise next_state = getattr(out, "request_state", None) - result = result_field(d.get("send_result"), dump) if next_state is not None and d.get("decision") == "allow": - self._pending[task] = ((tool, caller_args_digest, next_state), call_id, req, d, args, result) + self._pending[task] = ((tool, caller_args_digest, next_state), call_id, req, d, args) else: - await self._complete(call_id, req, status="ok", result=result) + await self._complete(call_id, req, status="ok", + result=result_field(d.get("send_result"), dump)) return out failed = getattr(out, "is_error", getattr(out, "isError", False)) # mcp 2.x, mcp 1.x await self._complete(call_id, req, status="error" if failed else "ok", diff --git a/tracekit/mcp_gateway.py b/tracekit/mcp_gateway.py index e4bc2efe1..ac66122dd 100644 --- a/tracekit/mcp_gateway.py +++ b/tracekit/mcp_gateway.py @@ -72,6 +72,11 @@ "resources/list": ("list_resources", "resources")} +class UpstreamTimeout(TimeoutError): + """An upstream call that didn't answer within the time Upstreams waits for it, unlike a TimeoutError the call + raised itself (a socket's, anyio's fail_after, a reconnect's).""" + + class Upstreams: """One session per upstream MCP server, held open on an event loop of its own; `call(name, method, ...)` runs one of the mcp Client's methods on it from any thread. A session that ended (a stdio upstream that exited, a remote @@ -109,8 +114,10 @@ def _run(self, coro, timeout=UPSTREAM_TIMEOUT_S): try: return fut.result(timeout) except concurrent.futures.TimeoutError: + if fut.done(): # a TimeoutError the call raised (from Python 3.11 the same class) + raise fut.cancel() # else the call keeps running (and holding its buffers) after its answer was given up - raise + raise UpstreamTimeout(f"no answer in {timeout:g} s") from None def call(self, name, method, *a, timeout=UPSTREAM_TIMEOUT_S, **kw): with self._locks[name]: @@ -191,7 +198,7 @@ def _answer(self, gw, key, method, params): while True: try: result = ups.call(up, call, cursor=cursor, timeout=deadline - time.monotonic()) - except (TimeoutError, concurrent.futures.TimeoutError): # one class from Python 3.11, two before + except UpstreamTimeout: # the listing's time ran out; any other TimeoutError is the upstream's raise _Error(-32603, f"upstream {up!r}: {kind}/list timed out (one listing has " f"{UPSTREAM_TIMEOUT_S:g} s for every upstream and page)") from None items = getattr(result, kind) From 4a9f4c94e76204cf45f93f4b13afa1ff3ab38e80 Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:16:12 +0000 Subject: [PATCH 8/9] MCP gateway: a call that ends as its wait runs out gives its own answer; a reconnect with no session names the upstream; MCP docs say which decisions wait and which calls end a waiting one Signed-off-by: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> --- docs/integrations/mcp.md | 2 +- docs/mcp-gateway.md | 4 ++- tests/test_mcp_gateway.py | 55 ++++++++++++++++++++++++++++++++++++ tracekit/integrations/mcp.py | 6 ++-- tracekit/mcp_gateway.py | 12 ++++++-- 5 files changed, 71 insertions(+), 8 deletions(-) diff --git a/docs/integrations/mcp.md b/docs/integrations/mcp.md index 3bf85c7b1..6b595ad2d 100644 --- a/docs/integrations/mcp.md +++ b/docs/integrations/mcp.md @@ -39,7 +39,7 @@ Every other attribute is the wrapped session's. Only calls made through the wrap | right before it is sent | `approval_consume` | the same call and arguments, and the approval id when the call waited for one | | after the server answers | `complete` | `status` `ok`, or `error` for a result with `isError: true`; a `sha256:` commitment to the result (not the result), or the exception type and message | -`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`observe`, source `mcp:/tools`, `/resources` or `/prompts`): the model reads the descriptions the server wrote, and the page the caller asked for is what is recorded (a page over half the signer's 1 MiB frame as its `sha256:` digest, as the [MCP gateway](../mcp-gateway.md) records one). The list mcp's `call_tool` makes itself, to check a result against the tool's output schema, never reaches the model and isn't recorded. A page the signer cannot record is withheld, unless the run fails open for `mcp`, when an unreachable signer returns it with a warning; a refusal is withheld either way. A tool call or prompt that returns an `InputRequiredResult` (MCP 2.3) records each round's input requests as untrusted observed input (`observe`, source `mcp:/input_required`). When the server provides a `request_state` and the call needed no human approval, it is decided and completed once, under its initial decision, when the terminal result arrives. A round continues the call its own asyncio task left waiting, as `mcp.Client.call_tool` drives the rounds, so concurrent calls never share a decision even when the server returns the same `request_state` to both; a waiting call whose task makes another call on the session first is never completed ([not covered](#not-covered)). Rounds from a server that returns no `request_state`, or for a decision that required human approval (as the signer refuses to consume a used approval), are decided and completed as separate calls per round. +`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`observe`, source `mcp:/tools`, `/resources` or `/prompts`): the model reads the descriptions the server wrote, and the page the caller asked for is what is recorded (a page over half the signer's 1 MiB frame as its `sha256:` digest, as the [MCP gateway](../mcp-gateway.md) records one). The list mcp's `call_tool` makes itself, to check a result against the tool's output schema, never reaches the model and isn't recorded. A page the signer cannot record is withheld, unless the run fails open for `mcp`, when an unreachable signer returns it with a warning; a refusal is withheld either way. A tool call or prompt that returns an `InputRequiredResult` (MCP 2.3) records each round's input requests as untrusted observed input (`observe`, source `mcp:/input_required`). When the server provides a `request_state` and the decision is `allow`, it is decided and completed once, under its initial decision, when the terminal result arrives. A round continues the call its own asyncio task left waiting, as `mcp.Client.call_tool` drives the rounds, so concurrent calls never share a decision even when the server returns the same `request_state` to both; a waiting call whose task calls a tool or gets a prompt on the session first is never completed ([not covered](#not-covered)). Rounds from a server that returns no `request_state`, or under a decision that isn't `allow` (a flag, or one that required human approval, as the signer refuses to consume a used approval), are decided and completed as separate calls per round. `` is the name in the server's initialize result, with every character outside `A-Za-z0-9_.-` replaced by `_` (`unknown` when the server gave none). The policy's `tools` map should map `mcp:*/*` to the `mcp` class (the coding diff --git a/docs/mcp-gateway.md b/docs/mcp-gateway.md index b5b277c7d..14f12f8a2 100644 --- a/docs/mcp-gateway.md +++ b/docs/mcp-gateway.md @@ -108,7 +108,9 @@ gateway, for the identity it decided the call for, alone; a run closed mid-call the whole listing fails with an error naming the upstream. - One tools/list, resources/list or prompts/list has 600 s for all the upstreams and their pages together, listed one after another, so a slow upstream leaves less time for those after it; past that the listing fails with an error - naming the upstream it was waiting for. A tools/call, prompts/get or resources/read has 600 s of its own. + naming the upstream it was waiting for. Opening a new session to an upstream that went away has 600 s of its own, + outside the listing's, and fails naming the upstream. A tools/call, prompts/get or resources/read has 600 s of its + own. - A resource is read through the upstream that listed it: list resources before reading one; a URI from a resource template isn't routed. - No server-initiated messages: sampling, elicitation, subscriptions and progress notifications from an upstream don't diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index 54eae3c00..b32d57b5c 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -414,6 +414,61 @@ def listing_error(*patches): mock.patch.object(mcp_gateway, "_closed", return_value=True), mock.patch.object(ups, "_open", side_effect=concurrent.futures.TimeoutError()))) + def test_a_call_that_ends_as_its_wait_runs_out_gives_its_own_answer(self): + """An upstream answering between the wait running out and the check gives that answer, not the wait's empty + TimeoutError; a TimeoutError the call raised itself is raised as it was.""" + ups = self.srv.mcp["upstreams"] + + class Late(concurrent.futures.Future): # the wait runs out, and the answer is there at once + def result(self, timeout=None): + if timeout is not None: + raise concurrent.futures.TimeoutError() + return super().result() + + def run(outcome): + def submit(coro, loop): + coro.close() + fut = Late() + if isinstance(outcome, BaseException): + fut.set_exception(outcome) + else: + fut.set_result(outcome) + return fut + + async def nothing(): + pass + with mock.patch.object(mcp_gateway.asyncio, "run_coroutine_threadsafe", submit): + return ups._run(nothing(), timeout=1) + self.assertEqual("listed", run("listed")) + with self.assertRaisesRegex(TimeoutError, "read timed out"): + run(TimeoutError("read timed out")) + + def test_a_reconnect_that_gets_no_session_names_the_upstream(self): + """The reconnect's wait for a session names the upstream, and gives up the attempt so the next call opens a + new one.""" + ups = self.srv.mcp["upstreams"] + + class Never: # an upstream whose session never opens + def __init__(self, target): + pass + + async def __aenter__(self): + await asyncio.Event().wait() + + async def __aexit__(self, *exc): + return False + held = ups._held["up"] + try: + with mock.patch("mcp.Client", Never), mock.patch.object(mcp_gateway, "UPSTREAM_TIMEOUT_S", 0.2), \ + self.assertRaisesRegex(TimeoutError, r"^upstream 'up' gave no session in 0.2 s$"): + ups._open("up") + attempt = ups._held["up"] + with contextlib.suppress(concurrent.futures.CancelledError, concurrent.futures.TimeoutError): + attempt.result(5) + self.assertTrue(attempt.cancelled()) + finally: + ups._held["up"] = held # the open session this test left alone + def test_page_bound_error_forgets_page1_resources(self): """Hitting the page cap on resources/list returns an error and remembers nothing.""" self._swap_upstream(paged_upstream()) diff --git a/tracekit/integrations/mcp.py b/tracekit/integrations/mcp.py index 6d8f38d45..e27d4ddf4 100644 --- a/tracekit/integrations/mcp.py +++ b/tracekit/integrations/mcp.py @@ -15,9 +15,9 @@ for an approval with a T2 executor. Multi-round input-required calls (MCP 2.3) with a `request_state` under an allowed decision are decided and completed once across rounds: a round continues the call its own asyncio task left waiting (as mcp's Client.call_tool drives them), so concurrent calls never share one, whatever `request_state` the server returns. -A waiting call whose task makes any other call on the session first, or ends, is never completed: the signer's -decided_not_completed gap at run close. Rounds without `request_state`, or whose decision required human approval, are -decided as calls of their own. A result with `isError: true` completes as an error; an exception from the session +A waiting call whose task calls a tool or gets a prompt on the session first, or ends, is never completed: the +signer's decided_not_completed gap at run close. Rounds without `request_state`, or whose decision isn't `allow` (a +flag, or an approval a person gave), are decided as calls of their own. A result with `isError: true` completes as an error; an exception from the session propagates unchanged and completes as an error. `list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`mcp:/tools`, `/resources`, `/prompts`; a page over half a signer frame as its digest) and returned once recorded. A list the signer diff --git a/tracekit/mcp_gateway.py b/tracekit/mcp_gateway.py index ac66122dd..1e6197b46 100644 --- a/tracekit/mcp_gateway.py +++ b/tracekit/mcp_gateway.py @@ -107,15 +107,21 @@ async def hold(ready): raise ready = concurrent.futures.Future() self._held[name] = asyncio.run_coroutine_threadsafe(hold(ready), self.loop) - ready.result(UPSTREAM_TIMEOUT_S) + try: + ready.result(UPSTREAM_TIMEOUT_S) + except concurrent.futures.TimeoutError: + if ready.done(): # the connection raised a TimeoutError of its own (from Python 3.11 the same class) + raise + self._held[name].cancel() # the next call opens a new session + raise TimeoutError(f"upstream {name!r} gave no session in {UPSTREAM_TIMEOUT_S:g} s") from None def _run(self, coro, timeout=UPSTREAM_TIMEOUT_S): fut = asyncio.run_coroutine_threadsafe(coro, self.loop) try: return fut.result(timeout) except concurrent.futures.TimeoutError: - if fut.done(): # a TimeoutError the call raised (from Python 3.11 the same class) - raise + if fut.done(): # the call raised a TimeoutError of its own (from 3.11 the same class), or just ended + return fut.result() fut.cancel() # else the call keeps running (and holding its buffers) after its answer was given up raise UpstreamTimeout(f"no answer in {timeout:g} s") from None From 4a171bbea99ef4d87e8b11034fe682c8c108c641 Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:23:40 +0000 Subject: [PATCH 9/9] MCP gateway: a call to an upstream that can't be opened again says nothing was sent; a call that ends as its wait is cancelled gives its answer; docs on the reconnect's time and which decisions wait Signed-off-by: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> --- docs/integrations/mcp.md | 2 +- docs/mcp-gateway.md | 6 +++--- tests/test_mcp_gateway.py | 35 +++++++++++++++++++++++++++++++---- tracekit/integrations/mcp.py | 2 +- tracekit/mcp_gateway.py | 28 +++++++++++++++++++--------- 5 files changed, 55 insertions(+), 18 deletions(-) diff --git a/docs/integrations/mcp.md b/docs/integrations/mcp.md index 6b595ad2d..291e87486 100644 --- a/docs/integrations/mcp.md +++ b/docs/integrations/mcp.md @@ -39,7 +39,7 @@ Every other attribute is the wrapped session's. Only calls made through the wrap | right before it is sent | `approval_consume` | the same call and arguments, and the approval id when the call waited for one | | after the server answers | `complete` | `status` `ok`, or `error` for a result with `isError: true`; a `sha256:` commitment to the result (not the result), or the exception type and message | -`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`observe`, source `mcp:/tools`, `/resources` or `/prompts`): the model reads the descriptions the server wrote, and the page the caller asked for is what is recorded (a page over half the signer's 1 MiB frame as its `sha256:` digest, as the [MCP gateway](../mcp-gateway.md) records one). The list mcp's `call_tool` makes itself, to check a result against the tool's output schema, never reaches the model and isn't recorded. A page the signer cannot record is withheld, unless the run fails open for `mcp`, when an unreachable signer returns it with a warning; a refusal is withheld either way. A tool call or prompt that returns an `InputRequiredResult` (MCP 2.3) records each round's input requests as untrusted observed input (`observe`, source `mcp:/input_required`). When the server provides a `request_state` and the decision is `allow`, it is decided and completed once, under its initial decision, when the terminal result arrives. A round continues the call its own asyncio task left waiting, as `mcp.Client.call_tool` drives the rounds, so concurrent calls never share a decision even when the server returns the same `request_state` to both; a waiting call whose task calls a tool or gets a prompt on the session first is never completed ([not covered](#not-covered)). Rounds from a server that returns no `request_state`, or under a decision that isn't `allow` (a flag, or one that required human approval, as the signer refuses to consume a used approval), are decided and completed as separate calls per round. +`list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`observe`, source `mcp:/tools`, `/resources` or `/prompts`): the model reads the descriptions the server wrote, and the page the caller asked for is what is recorded (a page over half the signer's 1 MiB frame as its `sha256:` digest, as the [MCP gateway](../mcp-gateway.md) records one). The list mcp's `call_tool` makes itself, to check a result against the tool's output schema, never reaches the model and isn't recorded. A page the signer cannot record is withheld, unless the run fails open for `mcp`, when an unreachable signer returns it with a warning; a refusal is withheld either way. A tool call or prompt that returns an `InputRequiredResult` (MCP 2.3) records each round's input requests as untrusted observed input (`observe`, source `mcp:/input_required`). When the server provides a `request_state` and the decision is `allow`, it is decided and completed once, under its initial decision, when the terminal result arrives. A round continues the call its own asyncio task left waiting, as `mcp.Client.call_tool` drives the rounds, so concurrent calls never share a decision even when the server returns the same `request_state` to both; a waiting call whose task calls a tool or gets a prompt on the session first is never completed ([not covered](#not-covered)). Rounds from a server that returns no `request_state`, or under a decision that isn't `allow` (a flag, or an ask, whose approval the signer consumes only once), are decided and completed as separate calls per round. `` is the name in the server's initialize result, with every character outside `A-Za-z0-9_.-` replaced by `_` (`unknown` when the server gave none). The policy's `tools` map should map `mcp:*/*` to the `mcp` class (the coding diff --git a/docs/mcp-gateway.md b/docs/mcp-gateway.md index 14f12f8a2..bd40bc738 100644 --- a/docs/mcp-gateway.md +++ b/docs/mcp-gateway.md @@ -89,7 +89,7 @@ answers each request with one JSON response and keeps no session; it serves `ini untrusted observed input (`mcp:/tools`, `/resources`, `/prompts`): the model reads the descriptions the upstream wrote. A tool named `prompt:...` is neither listed nor called: it would be decided as a prompt. - An upstream that fails during a tools/call or prompts/get answers an error saying the call may have run, and the - call completes as an error. + call completes as an error. One that went away and can't be opened again answers that nothing was sent to it. The upstreams' credentials never reach the agent or the signer: the agent's own headers are never forwarded, and each `env` and `headers` value is cut from the decoded string values of everything the gateway answers or records (never @@ -109,8 +109,8 @@ gateway, for the identity it decided the call for, alone; a run closed mid-call - One tools/list, resources/list or prompts/list has 600 s for all the upstreams and their pages together, listed one after another, so a slow upstream leaves less time for those after it; past that the listing fails with an error naming the upstream it was waiting for. Opening a new session to an upstream that went away has 600 s of its own, - outside the listing's, and fails naming the upstream. A tools/call, prompts/get or resources/read has 600 s of its - own. + not capped by what is left of the listing's (the time it takes still counts against the listing), and fails naming + the upstream. A tools/call, prompts/get or resources/read has 600 s of its own. - A resource is read through the upstream that listed it: list resources before reading one; a URI from a resource template isn't routed. - No server-initiated messages: sampling, elicitation, subscriptions and progress notifications from an upstream don't diff --git a/tests/test_mcp_gateway.py b/tests/test_mcp_gateway.py index b32d57b5c..1afce26a4 100644 --- a/tests/test_mcp_gateway.py +++ b/tests/test_mcp_gateway.py @@ -410,9 +410,27 @@ def listing_error(*patches): self.assertIn("upstream error: TimeoutError: read timed out" + advice, listing_error( mock.patch.object(ups.clients["up"], "list_tools", new_callable=mock.AsyncMock, side_effect=TimeoutError("read timed out")))) - self.assertIn("upstream error: TimeoutError: " + advice, listing_error( # the reconnect's wait for a session - mock.patch.object(mcp_gateway, "_closed", return_value=True), - mock.patch.object(ups, "_open", side_effect=concurrent.futures.TimeoutError()))) + self.assertIn("upstream error: upstream 'up' gave no session in 600 s; nothing was sent to it", + listing_error( # the reconnect's wait for a session + mock.patch.object(mcp_gateway, "_closed", return_value=True), + mock.patch.object(ups, "_open", side_effect=mcp_gateway.Unreachable( + "upstream 'up' gave no session in 600 s")))) + + def test_a_call_to_an_upstream_that_cant_be_opened_was_not_sent(self): + """A tools/call whose upstream went away and can't be opened again says nothing was sent, not that it may + have run, and completes as an error.""" + ups = self.srv.mcp["upstreams"] + with mock.patch.object(mcp_gateway, "_closed", return_value=True), \ + mock.patch.object(ups, "_open", side_effect=mcp_gateway.Unreachable( + "upstream 'up' couldn't be opened: ConnectionRefusedError: refused")), \ + self.assertRaises(Exception) as e: + self.agent(lambda c: c.call_tool("up__echo", {"text": "hi"})) + self.assertIn("upstream error: upstream 'up' couldn't be opened: ConnectionRefusedError: refused; nothing was " + "sent to it", messages(e.exception)) + self.assertNotIn("may have run", messages(e.exception)) + self.assertEqual(RAN, []) + [r] = [e for e in self.events() if e["type"] == "tool.result"] + self.assertFalse(r["data"]["ok"]) def test_a_call_that_ends_as_its_wait_runs_out_gives_its_own_answer(self): """An upstream answering between the wait running out and the check gives that answer, not the wait's empty @@ -443,6 +461,15 @@ async def nothing(): with self.assertRaisesRegex(TimeoutError, "read timed out"): run(TimeoutError("read timed out")) + class Ending(Late): # not done when checked, and already finishing when cancelled + def done(self): + return False + + def cancel(self): + return False + Late = Ending + self.assertEqual("listed", run("listed")) + def test_a_reconnect_that_gets_no_session_names_the_upstream(self): """The reconnect's wait for a session names the upstream, and gives up the attempt so the next call opens a new one.""" @@ -460,7 +487,7 @@ async def __aexit__(self, *exc): held = ups._held["up"] try: with mock.patch("mcp.Client", Never), mock.patch.object(mcp_gateway, "UPSTREAM_TIMEOUT_S", 0.2), \ - self.assertRaisesRegex(TimeoutError, r"^upstream 'up' gave no session in 0.2 s$"): + self.assertRaisesRegex(mcp_gateway.Unreachable, r"^upstream 'up' gave no session in 0.2 s$"): ups._open("up") attempt = ups._held["up"] with contextlib.suppress(concurrent.futures.CancelledError, concurrent.futures.TimeoutError): diff --git a/tracekit/integrations/mcp.py b/tracekit/integrations/mcp.py index e27d4ddf4..018fccfd1 100644 --- a/tracekit/integrations/mcp.py +++ b/tracekit/integrations/mcp.py @@ -17,7 +17,7 @@ mcp's Client.call_tool drives them), so concurrent calls never share one, whatever `request_state` the server returns. A waiting call whose task calls a tool or gets a prompt on the session first, or ends, is never completed: the signer's decided_not_completed gap at run close. Rounds without `request_state`, or whose decision isn't `allow` (a -flag, or an approval a person gave), are decided as calls of their own. A result with `isError: true` completes as an error; an exception from the session +flag or an ask), are decided as calls of their own. A result with `isError: true` completes as an error; an exception from the session propagates unchanged and completes as an error. `list_tools`, `list_resources` and `list_prompts` are recorded as untrusted observed input (`mcp:/tools`, `/resources`, `/prompts`; a page over half a signer frame as its digest) and returned once recorded. A list the signer diff --git a/tracekit/mcp_gateway.py b/tracekit/mcp_gateway.py index 1e6197b46..790973dfc 100644 --- a/tracekit/mcp_gateway.py +++ b/tracekit/mcp_gateway.py @@ -74,7 +74,11 @@ class UpstreamTimeout(TimeoutError): """An upstream call that didn't answer within the time Upstreams waits for it, unlike a TimeoutError the call - raised itself (a socket's, anyio's fail_after, a reconnect's).""" + raised itself (a socket's, anyio's fail_after).""" + + +class Unreachable(ConnectionError): + """An upstream whose session couldn't be opened (again): nothing was sent to it.""" class Upstreams: @@ -109,20 +113,24 @@ async def hold(ready): self._held[name] = asyncio.run_coroutine_threadsafe(hold(ready), self.loop) try: ready.result(UPSTREAM_TIMEOUT_S) - except concurrent.futures.TimeoutError: - if ready.done(): # the connection raised a TimeoutError of its own (from Python 3.11 the same class) - raise - self._held[name].cancel() # the next call opens a new session - raise TimeoutError(f"upstream {name!r} gave no session in {UPSTREAM_TIMEOUT_S:g} s") from None + except Exception: + if not ready.done(): # the wait ran out: give up this attempt, the next call opens a new session + self._held[name].cancel() + raise Unreachable(f"upstream {name!r} gave no session in {UPSTREAM_TIMEOUT_S:g} s") from None + e = ready.exception() + if e is None: # it opened just as the wait ran out + return + raise Unreachable(f"upstream {name!r} couldn't be opened: {type(e).__name__}: {e}") from e def _run(self, coro, timeout=UPSTREAM_TIMEOUT_S): fut = asyncio.run_coroutine_threadsafe(coro, self.loop) try: return fut.result(timeout) except concurrent.futures.TimeoutError: - if fut.done(): # the call raised a TimeoutError of its own (from 3.11 the same class), or just ended + # the call raised a TimeoutError of its own (from 3.11 the same class), or ended as the wait ran out; + # else cancel it, so it doesn't keep running (and holding its buffers) after its answer was given up + if fut.done() or not fut.cancel(): return fut.result() - fut.cancel() # else the call keeps running (and holding its buffers) after its answer was given up raise UpstreamTimeout(f"no answer in {timeout:g} s") from None def call(self, name, method, *a, timeout=UPSTREAM_TIMEOUT_S, **kw): @@ -263,7 +271,7 @@ def _call(self, gw, key, method, up, name, args): return why[1] try: out = gw["cut"](_dump(gw["upstreams"].call(up, method, name, args))) - except Exception as e: # it reached the upstream: the call may have run + except Exception as e: # it reached the upstream, and may have run, unless the upstream couldn't be opened why = _upstream_error(e, ran=True) gw["custody"].complete(key, tcid, d, tool, args, "error", error=gw["cut"](why)[:4096]) raise _Error(-32603, why) from None @@ -290,6 +298,8 @@ class _Error(Exception): def _upstream_error(e, ran=False): + if isinstance(e, Unreachable): + return f"upstream error: {e}; nothing was sent to it, and the gateway tries to open it again on the next call" return (f"upstream error: {type(e).__name__}: {e}" + ("; the call may have run: check before making it again" if ran else "") + "; the gateway connects to an upstream that went away again on the next call")