From d9799e9abc38ca146bceba8f6e530fc080d21ef6 Mon Sep 17 00:00:00 2001 From: LoopGlitch26 <53336715+LoopGlitch26@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:58:37 +0000 Subject: [PATCH] Design inputs for the LangChain held ToolNode pass (#687): probes and tests, no code The design is the draft PR's description. These files are the cases it must hold: the 1.7.0 reviews' probes and the tests from #753 at f40ac8e293. They become tests in tests/ when the design is built. --- .../probes/allowed_sibling_runs.py | 77 +++ .../probes/approve_then_retry.py | 73 +++ .../probes/fresh_gate_after_tool_pause.py | 83 +++ .../probes/many_asks_one_at_a_time.py | 69 +++ .../probes/parallel_ask_ainvoke.py | 85 +++ .../probes/parallel_ask_invoke.py | 85 +++ .../probes/probe_async_race.py | 64 ++ .../probes/probe_cross_process.py | 116 ++++ .../probes/probe_executor_and_mixed.py | 74 +++ .../probes/probe_executor_ask.py | 25 + .../langchain-held-pass/probes/probe_retry.py | 51 ++ .../probes/probe_tool_interrupt.py | 55 ++ .../probes/probe_tool_interrupt2.py | 59 ++ .../probes/tool_interrupt_eight_shapes.py | 109 ++++ ...est_integration_langchain_at_f40ac8e293.py | 577 ++++++++++++++++++ 15 files changed, 1602 insertions(+) create mode 100644 design/langchain-held-pass/probes/allowed_sibling_runs.py create mode 100644 design/langchain-held-pass/probes/approve_then_retry.py create mode 100644 design/langchain-held-pass/probes/fresh_gate_after_tool_pause.py create mode 100644 design/langchain-held-pass/probes/many_asks_one_at_a_time.py create mode 100644 design/langchain-held-pass/probes/parallel_ask_ainvoke.py create mode 100644 design/langchain-held-pass/probes/parallel_ask_invoke.py create mode 100644 design/langchain-held-pass/probes/probe_async_race.py create mode 100644 design/langchain-held-pass/probes/probe_cross_process.py create mode 100644 design/langchain-held-pass/probes/probe_executor_and_mixed.py create mode 100644 design/langchain-held-pass/probes/probe_executor_ask.py create mode 100644 design/langchain-held-pass/probes/probe_retry.py create mode 100644 design/langchain-held-pass/probes/probe_tool_interrupt.py create mode 100644 design/langchain-held-pass/probes/probe_tool_interrupt2.py create mode 100644 design/langchain-held-pass/probes/tool_interrupt_eight_shapes.py create mode 100644 design/langchain-held-pass/test_integration_langchain_at_f40ac8e293.py diff --git a/design/langchain-held-pass/probes/allowed_sibling_runs.py b/design/langchain-held-pass/probes/allowed_sibling_runs.py new file mode 100644 index 000000000..cbed23c60 --- /dev/null +++ b/design/langchain-held-pass/probes/allowed_sibling_runs.py @@ -0,0 +1,77 @@ +"""One message: an allowed echo and N asking pays in one tracekit_tool_node. Approve every pay as each pause shows it, +resuming after each; print how often each tool ran and what the model read.""" +import asyncio +import json +import sys +import warnings + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import Command + +from tracekit.integrations.langchain import tracekit_tool_node +from tracekit.testing import FakeSigner + +warnings.simplefilter("ignore") +RAN = [] +N = int(sys.argv[1]) if len(sys.argv) > 1 else 1 + + +@tool +def pay(to: str) -> str: + """Pay someone.""" + RAN.append(to) + return f"paid {to}" + + +@tool +def echo(text: str) -> str: + """Echo.""" + RAN.append("echo") + return text + + +class Model(BaseChatModel): + def _generate(self, messages, stop=None, run_manager=None, **kw): + if isinstance(messages[-1], HumanMessage): + calls = [{"name": "echo", "args": {"text": "hi"}, "id": "e"}] + calls += [{"name": "pay", "args": {"to": f"p{i}"}, "id": f"c{i}"} for i in range(N)] + msg = AIMessage("", tool_calls=calls) + else: + msg = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=msg)]) + + def bind_tools(self, tools, **kw): + return self + + @property + def _llm_type(self): + return "stub" + + +signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"]) if tool == "pay" else ("allow", [])) +run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}}) +g = StateGraph(MessagesState) +g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]}) +g.add_node("tools", tracekit_tool_node([pay, echo], signer, run)) +g.add_edge(START, "model") +g.add_conditional_edges("model", tools_condition) +g.add_edge("tools", "model") +a = g.compile(checkpointer=InMemorySaver()) +cfg = {"configurable": {"thread_id": "t"}} +out = a.invoke({"messages": [HumanMessage("go")]}, cfg) +step = 0 +while "__interrupt__" in out and step < 10: + shown = out["__interrupt__"][0].value["tracekit"] + for x in shown.get("approvals", [shown]): + signer.approval_decide({"request_id": f"d{step}{x['approval_id']}", "approval_id": x["approval_id"], + "decision": "approve"}) + out = a.invoke(Command(resume={"approval_id": shown["approval_id"]}), cfg) + step += 1 +print("resumes:", step, "| ran:", sorted(RAN), "|", + {m.tool_call_id: m.content[:60] for m in out["messages"] if isinstance(m, ToolMessage)}) diff --git a/design/langchain-held-pass/probes/approve_then_retry.py b/design/langchain-held-pass/probes/approve_then_retry.py new file mode 100644 index 000000000..bf9422e12 --- /dev/null +++ b/design/langchain-held-pass/probes/approve_then_retry.py @@ -0,0 +1,73 @@ +"""Two asks in one ToolNode: approve the shown one, resume; then approve the refused one's approval and send the model's +retry of it (a new call id, same tool and arguments). Does the retry run without a pause?""" +import json +import sys +import warnings + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import Command + +from tracekit.integrations.langchain import tracekit_tool_node +from tracekit.testing import FakeSigner + +warnings.simplefilter("ignore") +RAN = [] + + +@tool +def pay(to: str) -> str: + """Pay someone.""" + RAN.append(to) + return f"paid {to}" + + +class Model(BaseChatModel): + def _generate(self, messages, stop=None, run_manager=None, **kw): + if isinstance(messages[-1], HumanMessage): + msg = AIMessage("", tool_calls=json.loads(messages[-1].content)) + else: + msg = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=msg)]) + + def bind_tools(self, tools, **kw): + return self + + @property + def _llm_type(self): + return "stub" + + +signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"])) +run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}}) +g = StateGraph(MessagesState) +g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]}) +g.add_node("tools", tracekit_tool_node([pay], signer, run)) +g.add_edge(START, "model") +g.add_conditional_edges("model", tools_condition) +g.add_edge("tools", "model") +a = g.compile(checkpointer=InMemorySaver()) +cfg = {"configurable": {"thread_id": "t"}} +calls = [{"name": "pay", "args": {"to": "alice"}, "id": "c1"}, {"name": "pay", "args": {"to": "bob"}, "id": "c2"}] +out = a.invoke({"messages": [HumanMessage(json.dumps(calls))]}, cfg) +shown = out["__interrupt__"][0].value["tracekit"] +signer.approval_decide({"request_id": "d1", "approval_id": shown["approval_id"], "decision": "approve"}) +out = a.invoke(Command(resume={"approval_id": shown["approval_id"]}), cfg) +refused = [m for m in out["messages"] if isinstance(m, ToolMessage) and "blocked" in m.content][0] +print("after resume:", RAN, "|", refused.tool_call_id, refused.content) +other_aid = refused.content.split("(approval ")[1].split(")")[0] +signer.approval_decide({"request_id": "d2", "approval_id": other_aid, "decision": "approve"}) +other = next(c for c in calls if c["id"] == refused.tool_call_id) +out = a.invoke({"messages": [HumanMessage(json.dumps([dict(other, id="c3")]))]}, cfg) +print("retry:", RAN, "| paused:", "__interrupt__" in out, "|", + [m.content for m in out["messages"] if isinstance(m, ToolMessage) and m.tool_call_id == "c3"]) +if "__interrupt__" in out: + again = out["__interrupt__"][0].value["tracekit"] + out = a.invoke(Command(resume={"approval_id": again["approval_id"]}), cfg) + print("after resuming the retry (its approval", again["approval_id"], "):", RAN, + [m.content for m in out["messages"] if isinstance(m, ToolMessage) and m.tool_call_id == "c3"]) diff --git a/design/langchain-held-pass/probes/fresh_gate_after_tool_pause.py b/design/langchain-held-pass/probes/fresh_gate_after_tool_pause.py new file mode 100644 index 000000000..46943479d --- /dev/null +++ b/design/langchain-held-pass/probes/fresh_gate_after_tool_pause.py @@ -0,0 +1,83 @@ +"""After a tool's own pause, resume with a fresh gate (as a new process would, same checkpointer and signer): what the +model reads and what ran, for one asking call whose tool pauses, and for [pay-0 approved, confirm pausing, echo].""" +import json +import warnings + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import Command, interrupt + +from tracekit.integrations.langchain import tracekit_tool_node +from tracekit.testing import FakeSigner + +warnings.simplefilter("ignore") +RAN = [] + + +@tool +def pay(to: str, ask: bool = False) -> str: + """Pay.""" + answer = interrupt(f"pay {to}?") if ask else "-" + RAN.append(to) + return f"paid {to} ({answer})" + + +@tool +def confirm(question: str) -> str: + """Ask.""" + return interrupt(question) + + +@tool +def echo(text: str) -> str: + """Echo.""" + RAN.append("echo") + return text + + +class Model(BaseChatModel): + def _generate(self, messages, stop=None, run_manager=None, **kw): + msg = AIMessage("", tool_calls=json.loads(messages[-1].content)) if isinstance(messages[-1], HumanMessage) \ + else AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=msg)]) + + def bind_tools(self, tools, **kw): + return self + + @property + def _llm_type(self): + return "stub" + + +def graph(signer, run, saver): + g = StateGraph(MessagesState) + g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]}) + g.add_node("tools", tracekit_tool_node([pay, confirm, echo], signer, run)) + g.add_edge(START, "model") + g.add_conditional_edges("model", tools_condition) + g.add_edge("tools", "model") + return g.compile(checkpointer=saver) + + +for name, calls in (("one asking call, its tool pauses", [{"name": "pay", "args": {"to": "p0", "ask": True}, "id": "p0"}]), + ("pay-0 approved, confirm pauses, echo", [{"name": "pay", "args": {"to": "p0"}, "id": "p0"}, + {"name": "confirm", "args": {"question": "sure?"}, "id": "c"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e"}])): + RAN.clear() + signer = FakeSigner(lambda t, a: ("ask", ["R-PAY"]) if t == "pay" else ("allow", [])) + run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}}) + saver, cfg = InMemorySaver(), {"configurable": {"thread_id": "t"}} + out = graph(signer, run, saver).invoke({"messages": [HumanMessage(json.dumps(calls))]}, cfg) + shown = out["__interrupt__"][0].value["tracekit"] + signer.approval_decide({"request_id": "d", "approval_id": shown["approval_id"], "decision": "approve"}) + out = graph(signer, run, saver).invoke(Command(resume={"approval_id": shown["approval_id"]}), cfg) + asked = out["__interrupt__"][0].value + before = list(RAN) + out = graph(signer, run, saver).invoke(Command(resume="yes"), cfg) # a new gate: the memory of a new process + print(f"{name}: tool asked {asked!r}; ran before {before}, after {RAN};", + {m.tool_call_id: m.content[:60] for m in out["messages"] if isinstance(m, ToolMessage)}) diff --git a/design/langchain-held-pass/probes/many_asks_one_at_a_time.py b/design/langchain-held-pass/probes/many_asks_one_at_a_time.py new file mode 100644 index 000000000..414eeb662 --- /dev/null +++ b/design/langchain-held-pass/probes/many_asks_one_at_a_time.py @@ -0,0 +1,69 @@ +"""Three `ask` calls in one ToolNode message: approve the shown one, resume, and see what runs; then approve the next +shown one and resume again. Prints what ran and the ToolMessages after each resume.""" +import asyncio +import sys +import warnings + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import Command + +from tracekit.integrations.langchain import tracekit_tool_node +from tracekit.testing import FakeSigner + +warnings.simplefilter("ignore") +RAN = [] +N = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + + +@tool +def pay(to: str) -> str: + """Pay someone.""" + RAN.append(to) + return f"paid {to}" + + +class Model(BaseChatModel): + def _generate(self, messages, stop=None, run_manager=None, **kw): + if isinstance(messages[-1], HumanMessage): + msg = AIMessage("", tool_calls=[{"name": "pay", "args": {"to": f"p{i}"}, "id": f"c{i}"} for i in range(N)]) + else: + msg = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=msg)]) + + def bind_tools(self, tools, **kw): + return self + + @property + def _llm_type(self): + return "stub" + + +signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"])) +run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}}) +g = StateGraph(MessagesState) +g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]}) +g.add_node("tools", tracekit_tool_node([pay], signer, run)) +g.add_edge(START, "model") +g.add_conditional_edges("model", tools_condition) +g.add_edge("tools", "model") +a = g.compile(checkpointer=InMemorySaver()) +cfg = {"configurable": {"thread_id": "t"}} +mode = sys.argv[2] if len(sys.argv) > 2 else "sync" +inv = a.invoke if mode == "sync" else (lambda v, c: asyncio.run(a.ainvoke(v, c))) +out = inv({"messages": [HumanMessage("go")]}, cfg) +for step in range(N + 1): + if "__interrupt__" not in out: + break + shown = out["__interrupt__"][0].value["tracekit"] + signer.approval_decide({"request_id": f"d{step}", "approval_id": shown["approval_id"], "decision": "approve"}) + out = inv(Command(resume={"approval_id": shown["approval_id"]}), cfg) + msgs = {m.tool_call_id: m.content[:90] for m in out.get("messages", []) if isinstance(m, ToolMessage)} + print(f"resume {step}: approved {shown['tool_call_id']}; ran so far {RAN}; paused again {'__interrupt__' in out}") + for k, v in msgs.items(): + print(" ", k, v) diff --git a/design/langchain-held-pass/probes/parallel_ask_ainvoke.py b/design/langchain-held-pass/probes/parallel_ask_ainvoke.py new file mode 100644 index 000000000..3a0dc933a --- /dev/null +++ b/design/langchain-held-pass/probes/parallel_ask_ainvoke.py @@ -0,0 +1,85 @@ +"""Probe #687: tracekit_tool_node (one ToolNode task runs every call of a message) with two calls that both ask, under +ainvoke. LangGraph numbers interrupts in the order the gates reach interrupt(); the gate now runs in worker threads. +Resume with the approval id the first pause showed, after a person approves it, and see which call gets it.""" +import asyncio +import json +import random +import sys +import time +import warnings + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import Command + +from tracekit.integrations.langchain import tracekit_tool_node +from tracekit.testing import FakeSigner + +warnings.simplefilter("ignore") +RAN = [] + + +@tool +def pay(to: str) -> str: + """Pay someone.""" + RAN.append(to) + return f"paid {to}" + + +class Model(BaseChatModel): + def _generate(self, messages, stop=None, run_manager=None, **kw): + if isinstance(messages[-1], HumanMessage): + msg = AIMessage("", tool_calls=[{"name": "pay", "args": {"to": "alice"}, "id": "c1"}, + {"name": "pay", "args": {"to": "bob"}, "id": "c2"}]) + else: + msg = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=msg)]) + + def bind_tools(self, tools, **kw): + return self + + @property + def _llm_type(self): + return "stub" + + +def trial(jitter): + RAN.clear() + signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"])) + run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}}) + decide = signer.approval_request + signer.approval_request = lambda req: (time.sleep(random.random() * jitter), decide(req))[1] + g = StateGraph(MessagesState) + g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]}) + g.add_node("tools", tracekit_tool_node([pay], signer, run)) + g.add_edge(START, "model") + g.add_conditional_edges("model", tools_condition) + g.add_edge("tools", "model") + a = g.compile(checkpointer=InMemorySaver()) + cfg = {"configurable": {"thread_id": "t"}} + out = asyncio.run(a.ainvoke({"messages": [HumanMessage("go")]}, cfg)) + shown = out["__interrupt__"][0].value["tracekit"] + signer.approval_decide({"request_id": "d1", + "approval_id": shown["approval_id"], "decision": "approve"}) + out = asyncio.run(a.ainvoke(Command(resume={"approval_id": shown["approval_id"]}), cfg)) + msgs = {m.tool_call_id: m.content[:60] for m in out.get("messages", []) if isinstance(m, ToolMessage)} + return shown["tool_call_id"], list(RAN), msgs, "__interrupt__" in out + + +jitter = float(sys.argv[1]) if len(sys.argv) > 1 else 0.05 +results = {} +for i in range(int(sys.argv[2]) if len(sys.argv) > 2 else 12): + try: + shown, ran, msgs, paused = trial(jitter) + key = f"pause showed {shown}; ran {ran}; paused again {paused}; refused: " + \ + ", ".join(f"{k}" for k, v in msgs.items() if "blocked" in v) + except Exception as e: + key = f"raised {type(e).__name__}: {str(e)[:120]}" + results[key] = results.get(key, 0) + 1 +for k, v in results.items(): + print(v, "x", k) diff --git a/design/langchain-held-pass/probes/parallel_ask_invoke.py b/design/langchain-held-pass/probes/parallel_ask_invoke.py new file mode 100644 index 000000000..e80b39057 --- /dev/null +++ b/design/langchain-held-pass/probes/parallel_ask_invoke.py @@ -0,0 +1,85 @@ +"""Probe #687: tracekit_tool_node (one ToolNode task runs every call of a message) with two calls that both ask, under +ainvoke. LangGraph numbers interrupts in the order the gates reach interrupt(); the gate now runs in worker threads. +Resume with the approval id the first pause showed, after a person approves it, and see which call gets it.""" +import asyncio +import json +import random +import sys +import time +import warnings + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import Command + +from tracekit.integrations.langchain import tracekit_tool_node +from tracekit.testing import FakeSigner + +warnings.simplefilter("ignore") +RAN = [] + + +@tool +def pay(to: str) -> str: + """Pay someone.""" + RAN.append(to) + return f"paid {to}" + + +class Model(BaseChatModel): + def _generate(self, messages, stop=None, run_manager=None, **kw): + if isinstance(messages[-1], HumanMessage): + msg = AIMessage("", tool_calls=[{"name": "pay", "args": {"to": "alice"}, "id": "c1"}, + {"name": "pay", "args": {"to": "bob"}, "id": "c2"}]) + else: + msg = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=msg)]) + + def bind_tools(self, tools, **kw): + return self + + @property + def _llm_type(self): + return "stub" + + +def trial(jitter): + RAN.clear() + signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"])) + run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}}) + decide = signer.approval_request + signer.approval_request = lambda req: (time.sleep(random.random() * jitter), decide(req))[1] + g = StateGraph(MessagesState) + g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]}) + g.add_node("tools", tracekit_tool_node([pay], signer, run)) + g.add_edge(START, "model") + g.add_conditional_edges("model", tools_condition) + g.add_edge("tools", "model") + a = g.compile(checkpointer=InMemorySaver()) + cfg = {"configurable": {"thread_id": "t"}} + out = a.invoke({"messages": [HumanMessage("go")]}, cfg) + shown = out["__interrupt__"][0].value["tracekit"] + signer.approval_decide({"request_id": "d1", + "approval_id": shown["approval_id"], "decision": "approve"}) + out = a.invoke(Command(resume={"approval_id": shown["approval_id"]}), cfg) + msgs = {m.tool_call_id: m.content[:60] for m in out.get("messages", []) if isinstance(m, ToolMessage)} + return shown["tool_call_id"], list(RAN), msgs, "__interrupt__" in out + + +jitter = float(sys.argv[1]) if len(sys.argv) > 1 else 0.05 +results = {} +for i in range(int(sys.argv[2]) if len(sys.argv) > 2 else 12): + try: + shown, ran, msgs, paused = trial(jitter) + key = f"pause showed {shown}; ran {ran}; paused again {paused}; refused: " + \ + ", ".join(f"{k}" for k, v in msgs.items() if "blocked" in v) + except Exception as e: + key = f"raised {type(e).__name__}: {str(e)[:120]}" + results[key] = results.get(key, 0) + 1 +for k, v in results.items(): + print(v, "x", k) diff --git a/design/langchain-held-pass/probes/probe_async_race.py b/design/langchain-held-pass/probes/probe_async_race.py new file mode 100644 index 000000000..aaad5259f --- /dev/null +++ b/design/langchain-held-pass/probes/probe_async_race.py @@ -0,0 +1,64 @@ +"""Async held pass where a tool's own interrupt() comes back before a sibling call finishes: is the sibling run once, +recorded once, and not asked twice? (sync for comparison)""" +import json, sys, time, asyncio +sys.path.insert(0, sys.argv[1] + "/tests") +from langchain_core.messages import HumanMessage, ToolMessage +from langchain_core.tools import tool +from langgraph.types import Command, interrupt +from test_integration_langchain import tool_graph +from tracekit.testing import FakeSigner +from tracekit.integrations.langchain import tracekit_tool_node +import adapter_contract as ac + +RAN = [] +@tool +def slow_echo(text: str) -> str: + """Echo, slowly (a sync tool).""" + time.sleep(0.3) + RAN.append("slow_echo") + return text + +@tool("pay") +def slow_pay(to: str, cents: int) -> str: + """Send money, slowly.""" + time.sleep(0.3) + RAN.append(f"pay{cents}") + return f"paid {cents}" + +@tool +def confirm(question: str) -> str: + """Ask the person.""" + a = interrupt(question) + RAN.append(f"confirm:{a}") + return a + +T = {"configurable": {"thread_id": "t"}} +for label, tools, calls, approve in [ + ("allowed slow sibling", [slow_echo, confirm], + [{"name": "slow_echo", "args": {"text": "hi"}, "id": "e"}, {"name": "confirm", "args": {"question": "sure?"}, "id": "q"}], set()), + ("approved slow sibling", [slow_pay, confirm], + [{"name": "pay", "args": {"to": "a", "cents": 0}, "id": "p0"}, {"name": "confirm", "args": {"question": "sure?"}, "id": "q"}], {"p0"})]: + for mode in ("sync", "async"): + RAN.clear() + s = FakeSigner(ac._rule) + r = s.register_run({"request_id": "reg", "agent": {"name": "p"}}) + run = {"run_id": r["run_id"], "run_token": r["run_token"]} + g = tool_graph(tracekit_tool_node(tools, s, r)) + inv = g.invoke if mode == "sync" else (lambda v, c: asyncio.run(g.ainvoke(v, c))) + out, pauses = inv({"messages": [HumanMessage(json.dumps(calls))]}, T), [] + while "__interrupt__" in out and len(pauses) < 6: + v = out["__interrupt__"][0].value + if isinstance(v, dict) and "tracekit" in v: + pauses.append("tk") + for a in v["tracekit"]["approvals"]: + if a["tool_call_id"] in approve and s._approvals[a["approval_id"]]["state"] == "requested": + s.approval_decide({"request_id": a["approval_id"], "decision": "approve", "approval_id": a["approval_id"]}) + out = inv(Command(resume={"approval_id": v["tracekit"]["approval_id"]}), T) + else: + pauses.append(v) + out = inv(Command(resume="yes"), T) + ev = s.read({**run, "limit": 1000})["events"] + print(f"{label:22} {mode:5} pauses={pauses} ran={RAN} " + f"read={ {m.tool_call_id: m.content[:40] for m in out['messages'] if isinstance(m, ToolMessage)} } " + f"results={[e['data']['tool_call_id'] for e in ev if e['type'] == 'tool.result']} " + f"approval requests={[e['data']['tool_call_id'] for e in ev if e['type'] == 'approval.request']}") diff --git a/design/langchain-held-pass/probes/probe_cross_process.py b/design/langchain-held-pass/probes/probe_cross_process.py new file mode 100644 index 000000000..a4e4a1194 --- /dev/null +++ b/design/langchain-held-pass/probes/probe_cross_process.py @@ -0,0 +1,116 @@ +"""Resume in "another process" (a new gate and graph on the same checkpointer and the real signer) after a tool's own +pause. Does every call that runs have a decision, does none run twice under one approval, and what is recorded?""" +import json, os, sys, tempfile, time, asyncio +sys.path.insert(0, sys.argv[1] + "/tests") +from langchain_core.messages import HumanMessage, ToolMessage +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import Command, interrupt +from test_contract_langchain import StubModel +from test_signer_service import ME, OTHER, PAY_ASKS +from tracekit.signer import service as svc +from tracekit.integrations.langchain import tracekit_tool_node + +RAN = [] +@tool("pay") +def pay(to: str, cents: int) -> str: + """Send money.""" + RAN.append(f"pay{cents}") + return f"paid {cents}" + +@tool("pay") +def confirmed_pay(to: str, cents: int) -> str: + """Send money after the person confirms (its own pause).""" + a = interrupt(f"pay {cents}?") + RAN.append(f"pay{cents}:{a}") + return f"paid {cents} after {a}" + +@tool +def echo(text: str) -> str: + """Echo.""" + RAN.append("echo") + return text + +@tool +def confirm(question: str) -> str: + """Ask the person.""" + a = interrupt(question) + RAN.append(f"confirm:{a}") + return a + +class Agent: + def __init__(self, s, who): + self.s, self.who = s, who + def __getattr__(self, m): + return lambda req: self.s.call(self.who, m, req) + +def graph(tools, signer, run, saver): + g = StateGraph(MessagesState) + g.add_node("model", lambda st: {"messages": [StubModel().invoke(st["messages"])]}) + g.add_node("tools", tracekit_tool_node(tools, signer, run)) + g.add_edge(START, "model"); g.add_conditional_edges("model", tools_condition); g.add_edge("tools", "model") + return g.compile(checkpointer=saver) + +def scenario(name, tools, calls, mode, approve, switch_after): + """switch_after: the number of resumes made in process 1 before process 2 takes over""" + RAN.clear() + d = tempfile.mkdtemp(dir="/tmp") + s = svc.SignerService(os.path.join(d, "data"), policy=PAY_ASKS, grace_s=0, fsck_every_s=0) + try: + signer = Agent(s, ME) + run = signer.register_run({"request_id": "reg", "agent": {"name": "p"}}) + saver, T = InMemorySaver(), {"configurable": {"thread_id": "t"}} + g = graph(tools, signer, run, saver) + inv = lambda g, v: g.invoke(v, T) if mode == "sync" else asyncio.run(g.ainvoke(v, T)) + out, pauses, n = inv(g, {"messages": [HumanMessage(json.dumps(calls))]}), [], 0 + while "__interrupt__" in out and n < 6: + if n == switch_after: + g = graph(tools, signer, run, saver) # another process: a new gate, the same saved state + pauses.append("|P2|") + v = out["__interrupt__"][0].value + if isinstance(v, dict) and "tracekit" in v: + pauses.append("tk") + for a in v["tracekit"]["approvals"]: + if a["tool_call_id"] in approve: + try: + s.call(OTHER, "approval_decide", {"request_id": a["approval_id"] + str(n), + "approval_id": a["approval_id"], "decision": "approve"}) + except Exception as e: + pass # answered already + out = inv(g, Command(resume={"approval_id": v["tracekit"]["approval_id"]})) + else: + pauses.append(v) + out = inv(g, Command(resume="yes")) + n += 1 + read = {m.tool_call_id: m.content.split(" (")[0][:48] for m in out["messages"] if isinstance(m, ToolMessage)} + signer.close_run({"request_id": "close", "run_id": run["run_id"], "run_token": run["run_token"]}) + s.sweep(time.monotonic() + 5) + ev = [r["event"] for r in s.log.storage.iter_run("default", run["run_id"])] + pick = lambda t, f: [e["data"].get(f) or e.get(f) for e in ev if e["type"] == t] + results = [(e.get("tool_call_id") or e["data"].get("tool_use_id"), e["data"].get("ok"), e.get("attempt")) + for e in ev if e["type"] == "tool.result"] + print(f"{name} [{mode}] pauses={pauses}\n ran={RAN}\n model read={read}\n results={results}" + f"\n decisions={[(e.get('tool_call_id'), e['data'].get('decision'), e.get('attempt')) for e in ev if e['type'] == 'policy.decision']}" + f"\n approval.request={len([e for e in ev if e['type'] == 'approval.request'])} " + f"consumed={len([e for e in ev if e['type'] == 'approval.consumed'])} " + f"refused={[(e.get('tool_call_id'), e['data'].get('rule_ids')) for e in ev if e['type'] == 'approval.refused']}" + f"\n gaps={[(e['data'].get('kind'), e['data'].get('tool_use_id')) for e in ev if e['type'] == 'capture.gap']}") + finally: + s.close() + +for mode in ("sync", "async"): + scenario("X1 single ask+own pause, P2 answers the tool", [confirmed_pay], + [{"name": "pay", "args": {"to": "a", "cents": 1}, "id": "p1"}], mode, {"p1"}, 1) + scenario("X2 held [pay0 ask, echo, confirm], P2 answers confirm", [pay, echo, confirm], + [{"name": "pay", "args": {"to": "a", "cents": 0}, "id": "p0"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e"}, + {"name": "confirm", "args": {"question": "sure?"}, "id": "q"}], mode, {"p0"}, 1) + scenario("X3 held [confirmed_pay ask, echo], P2 answers the tool", [confirmed_pay, echo], + [{"name": "pay", "args": {"to": "a", "cents": 2}, "id": "p2"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e"}], mode, {"p2"}, 1) + scenario("X2-control all in P1", [pay, echo, confirm], + [{"name": "pay", "args": {"to": "a", "cents": 0}, "id": "p0"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e"}, + {"name": "confirm", "args": {"question": "sure?"}, "id": "q"}], mode, {"p0"}, 99) diff --git a/design/langchain-held-pass/probes/probe_executor_and_mixed.py b/design/langchain-held-pass/probes/probe_executor_and_mixed.py new file mode 100644 index 000000000..15bd23041 --- /dev/null +++ b/design/langchain-held-pass/probes/probe_executor_and_mixed.py @@ -0,0 +1,74 @@ +"""Probe 1: an executor tool beside another call in one ToolNode message: does the middleware decide it (it must not; +the executor decides it)? Probe 2: [pay0 ask, pay1 ask, confirm (its own interrupt)]: approve pay0, leave pay1, resume +twice; how often does pay0 run, and what does the model read?""" +import json, sys, asyncio +sys.path.insert(0, sys.argv[1] + "/tests") +from langchain_core.messages import HumanMessage, ToolMessage +from langchain_core.tools import tool +from langgraph.types import Command, interrupt +from test_contract_langchain import TOOLS +from test_integration_langchain import tool_graph +from tracekit.testing import FakeSigner +from tracekit.integrations.langchain import tracekit_tool_node +from tracekit.integrations.executor_tools import EXECUTOR, CALL_ID +import adapter_contract as ac + +T = {"configurable": {"thread_id": "t"}} +def recorded(s, run, typ): + return [e["data"] for e in s.read({**run, "limit": 1000})["events"] if e["type"] == typ] + +EX = [] +@tool +def refund(cents: int) -> str: + """An executor action (the executor decides it).""" + EX.append(CALL_ID.get()) + return "refunded" +refund.metadata = {EXECUTOR: True} + +for mode in ("sync", "async"): + s = FakeSigner(ac._rule) + r = s.register_run({"request_id": "reg", "agent": {"name": "p"}}) + run = {"run_id": r["run_id"], "run_token": r["run_token"]} + g = tool_graph(tracekit_tool_node([*TOOLS, refund], s, r)) + inv = g.invoke if mode == "sync" else (lambda v, c: asyncio.run(g.ainvoke(v, c))) + out = inv({"messages": [HumanMessage(json.dumps([{"name": "refund", "args": {"cents": 1}, "id": "x1"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e1"}]))]}, T) + print(f"executor {mode}: middleware decisions for={[d['tool_call_id'] for d in recorded(s, run, 'policy.decision')]}" + f" results for={[d['tool_call_id'] for d in recorded(s, run, 'tool.result')]} executor ran={EX}") + EX.clear() + +PAID = [] +@tool +def pay(to: str, cents: int) -> str: + """Send money.""" + PAID.append(cents) + return f"paid {cents}" + +@tool +def confirm(question: str) -> str: + """Ask the user.""" + return interrupt(question) + +for mode in ("sync", "async"): + PAID.clear() + s = FakeSigner(ac._rule) + r = s.register_run({"request_id": "reg", "agent": {"name": "p"}}) + run = {"run_id": r["run_id"], "run_token": r["run_token"]} + g = tool_graph(tracekit_tool_node([*TOOLS, pay, confirm], s, r)) + inv = g.invoke if mode == "sync" else (lambda v, c: asyncio.run(g.ainvoke(v, c))) + calls = [{"name": "pay", "args": {"to": "a", "cents": 0}, "id": "p0"}, + {"name": "pay", "args": {"to": "a", "cents": 1}, "id": "p1"}, + {"name": "confirm", "args": {"question": "sure?"}, "id": "q"}] + out = inv({"messages": [HumanMessage(json.dumps(calls))]}, T) + aprs = {a["tool_call_id"]: a["approval_id"] for a in recorded(s, run, "approval.request")} + s.approval_decide({"request_id": "d", "decision": "approve", "approval_id": aprs["p0"]}) + out = inv(Command(resume={"approval_id": aprs["p0"]}), T) + steps = [("resume1", out.get("__interrupt__"), list(PAID))] + if out.get("__interrupt__"): + out = inv(Command(resume="yes"), T) + steps.append(("resume2", out.get("__interrupt__"), list(PAID))) + print(f"mixed {mode}: " + "; ".join(f"{n}: paused={bool(i)} paid={p}" for n, i, p in steps)) + print(f" model read={ {m.tool_call_id: m.content[:60] for m in out['messages'] if isinstance(m, ToolMessage)} }") + print(f" recorded results={[(d['tool_call_id'], d['status']) for d in recorded(s, run, 'tool.result')]} " + f"refused={[d['tool_call_id'] for d in recorded(s, run, 'approval.refused')]} " + f"approval requests={[(a['tool_call_id'], a.get('attempt')) for a in recorded(s, run, 'approval.request')]}") diff --git a/design/langchain-held-pass/probes/probe_executor_ask.py b/design/langchain-held-pass/probes/probe_executor_ask.py new file mode 100644 index 000000000..55b25a2b2 --- /dev/null +++ b/design/langchain-held-pass/probes/probe_executor_ask.py @@ -0,0 +1,25 @@ +"""An executor tool whose policy asks, beside echo in one ToolNode message: does the middleware pause for it?""" +import json, sys +sys.path.insert(0, sys.argv[1] + "/tests") +from langchain_core.messages import HumanMessage +from langchain_core.tools import tool +from test_contract_langchain import TOOLS +from test_integration_langchain import tool_graph +from tracekit.testing import FakeSigner +from tracekit.integrations.langchain import tracekit_tool_node +from tracekit.integrations.executor_tools import EXECUTOR + +@tool +def refund(cents: int) -> str: + """An executor action.""" + return "refunded" +refund.metadata = {EXECUTOR: True} +s = FakeSigner(lambda t, a: ("ask", ["R-REFUND"]) if t == "refund" else ("allow", [])) +r = s.register_run({"request_id": "reg", "agent": {"name": "p"}}) +g = tool_graph(tracekit_tool_node([*TOOLS, refund], s, r)) +out = g.invoke({"messages": [HumanMessage(json.dumps([{"name": "refund", "args": {"cents": 1}, "id": "x1"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e1"}]))]}, + {"configurable": {"thread_id": "t"}}) +ev = s.read({"run_id": r["run_id"], "run_token": r["run_token"], "limit": 100})["events"] +print("paused:", out.get("__interrupt__", [None])[0] and out["__interrupt__"][0].value, + "| middleware approval requests:", [e["data"]["tool_call_id"] for e in ev if e["type"] == "approval.request"]) diff --git a/design/langchain-held-pass/probes/probe_retry.py b/design/langchain-held-pass/probes/probe_retry.py new file mode 100644 index 000000000..62e7e70ea --- /dev/null +++ b/design/langchain-held-pass/probes/probe_retry.py @@ -0,0 +1,51 @@ +"""A held ToolNode under a LangGraph RetryPolicy: a flaky tool fails once beside echo. Does the retry run the flaky tool +again?""" +import json, sys, asyncio +sys.path.insert(0, sys.argv[1] + "/tests") +from langchain_core.messages import HumanMessage, ToolMessage +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import RetryPolicy +from test_contract_langchain import StubModel +from tracekit.testing import FakeSigner +from tracekit.integrations.langchain import tracekit_tool_node +import adapter_contract as ac + +RAN, FAILS = [], [] +@tool +def flaky(x: int) -> str: + """Fails the first time.""" + RAN.append("flaky") + if not FAILS: + FAILS.append(1) + raise ConnectionError("transient") + return "ok" + +@tool +def echo(text: str) -> str: + """Echo.""" + RAN.append("echo") + return text + +for mode in ("sync", "async"): + RAN.clear(); FAILS.clear() + s = FakeSigner(ac._rule) + r = s.register_run({"request_id": "reg", "agent": {"name": "p"}}) + g = StateGraph(MessagesState) + g.add_node("model", lambda st: {"messages": [StubModel().invoke(st["messages"])]}) + g.add_node("tools", tracekit_tool_node([flaky, echo], s, r), retry_policy=RetryPolicy(max_attempts=3, initial_interval=0.01, + retry_on=ConnectionError)) + g.add_edge(START, "model"); g.add_conditional_edges("model", tools_condition); g.add_edge("tools", "model") + g = g.compile(checkpointer=InMemorySaver()) + msg = {"messages": [HumanMessage(json.dumps([{"name": "flaky", "args": {"x": 1}, "id": "f"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e"}]))]} + T = {"configurable": {"thread_id": "t"}} + try: + out = g.invoke(msg, T) if mode == "sync" else asyncio.run(g.ainvoke(msg, T)) + res = {m.tool_call_id: m.content for m in out["messages"] if isinstance(m, ToolMessage)} + except Exception as e: + res = f"raised {type(e).__name__}: {e}" + ev = s.read({"run_id": r["run_id"], "run_token": r["run_token"], "limit": 100})["events"] + print(f"{mode:5} ran={RAN} outcome={res} results={[(e['data']['tool_call_id'], e['data']['status']) for e in ev if e['type'] == 'tool.result']}") diff --git a/design/langchain-held-pass/probes/probe_tool_interrupt.py b/design/langchain-held-pass/probes/probe_tool_interrupt.py new file mode 100644 index 000000000..9eb6a923f --- /dev/null +++ b/design/langchain-held-pass/probes/probe_tool_interrupt.py @@ -0,0 +1,55 @@ +"""Probe: a tool that calls interrupt() itself (a HITL confirm) under a Tracekit `ask`. Does the tool's own question +reach the person, or does it get the approval's resume value?""" +import json, sys, asyncio +sys.path.insert(0, sys.argv[1] + "/tests") +from langchain.agents import create_agent +from langchain_core.messages import HumanMessage, ToolMessage +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.types import Command, interrupt +from test_contract_langchain import StubModel, TOOLS +from test_integration_langchain import tool_graph +from tracekit.testing import FakeSigner +from tracekit.integrations.langchain import TracekitMiddleware, tracekit_tool_node + +ANSWERS = [] + +@tool +def transfer(cents: int) -> str: + """Move money after the user confirms.""" + answer = interrupt(f"confirm transfer of {cents}?") + ANSWERS.append(answer) + return "moved" if answer else "cancelled" + +T = {"configurable": {"thread_id": "t"}} + +def recorded(s, run, typ): + return [e["data"] for e in s.read({**run, "limit": 1000})["events"] if e["type"] == typ] + +def go(kind, calls, mode): + ANSWERS.clear() + s = FakeSigner(lambda tool, args: ("ask", ["R-T"]) if tool == "transfer" else ("allow", [])) + r = s.register_run({"request_id": "reg", "agent": {"name": "p"}}) + run = {"run_id": r["run_id"], "run_token": r["run_token"]} + if kind == "create_agent": + g = create_agent(StubModel(), [*TOOLS, transfer], checkpointer=InMemorySaver(), + middleware=[TracekitMiddleware(s, run["run_id"], run["run_token"])]) + else: + g = tool_graph(tracekit_tool_node([*TOOLS, transfer], s, r)) + inv = g.invoke if mode == "sync" else (lambda v, c: asyncio.run(g.ainvoke(v, c))) + out = inv({"messages": [HumanMessage(json.dumps(calls))]}, T) + first = out["__interrupt__"][0].value + [apr] = [a for a in recorded(s, run, "approval.request")] + s.approval_decide({"request_id": "d", "decision": "approve", "approval_id": apr["approval_id"]}) + out = inv(Command(resume={"approval_id": apr["approval_id"]}), T) + second = out.get("__interrupt__") + print(f"{kind:13} {mode:5} calls={[c['name'] for c in calls]}: first pause={first}") + print(f" after approve+resume: paused again={bool(second)} {second[0].value if second else ''}; " + f"tool got answers={ANSWERS}; tool msgs={[m.content for m in out['messages'] if isinstance(m, ToolMessage)]}") + +for kind in ("create_agent", "tool_node"): + for mode in ("sync", "async"): + go(kind, [{"name": "transfer", "args": {"cents": 5}, "id": "c1"}], mode) +for mode in ("sync", "async"): + go("tool_node", [{"name": "transfer", "args": {"cents": 5}, "id": "c1"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e1"}], mode) diff --git a/design/langchain-held-pass/probes/probe_tool_interrupt2.py b/design/langchain-held-pass/probes/probe_tool_interrupt2.py new file mode 100644 index 000000000..d29abac7e --- /dev/null +++ b/design/langchain-held-pass/probes/probe_tool_interrupt2.py @@ -0,0 +1,59 @@ +"""Six variants, driven to the end: approve, resume, answer the tool's own question; what did the tool get, how many +times did its body pass the interrupt, what was recorded?""" +import json, sys, asyncio +sys.path.insert(0, sys.argv[1] + "/tests") +from langchain.agents import create_agent +from langchain_core.messages import HumanMessage, ToolMessage +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.types import Command, interrupt +from test_contract_langchain import StubModel, TOOLS +from test_integration_langchain import tool_graph +from tracekit.testing import FakeSigner +from tracekit.integrations.langchain import TracekitMiddleware, tracekit_tool_node + +ANSWERS = [] +@tool +def transfer(cents: int) -> str: + """Move money after the user confirms.""" + answer = interrupt(f"confirm transfer of {cents}?") + ANSWERS.append(answer) + return f"moved after {answer}" + +T = {"configurable": {"thread_id": "t"}} +def rec(s, run, typ): + return [e["data"] for e in s.read({**run, "limit": 1000})["events"] if e["type"] == typ] + +def go(kind, calls, mode): + ANSWERS.clear() + s = FakeSigner(lambda tool, args: ("ask", ["R-T"]) if tool == "transfer" else ("allow", [])) + r = s.register_run({"request_id": "reg", "agent": {"name": "p"}}) + run = {"run_id": r["run_id"], "run_token": r["run_token"]} + g = (create_agent(StubModel(), [*TOOLS, transfer], checkpointer=InMemorySaver(), + middleware=[TracekitMiddleware(s, run["run_id"], run["run_token"])]) + if kind == "create_agent" else tool_graph(tracekit_tool_node([*TOOLS, transfer], s, r))) + inv = g.invoke if mode == "sync" else (lambda v, c: asyncio.run(g.ainvoke(v, c))) + out = inv({"messages": [HumanMessage(json.dumps(calls))]}, T) + pauses = [] + while "__interrupt__" in out and len(pauses) < 6: + v = out["__interrupt__"][0].value + if isinstance(v, dict) and "tracekit" in v: + pauses.append("tracekit") + aid = v["tracekit"]["approval_id"] + s.approval_decide({"request_id": aid, "decision": "approve", "approval_id": aid}) + out = inv(Command(resume={"approval_id": aid}), T) + else: + pauses.append(v) + out = inv(Command(resume="PERSON-SAID-YES"), T) + print(f"{kind:12} {mode:5} {[c['name'] for c in calls]}: pauses={pauses} tool got={ANSWERS} " + f"read={[m.content for m in out['messages'] if isinstance(m, ToolMessage)]} " + f"results={[(d['tool_call_id'], d['status']) for d in rec(s, run, 'tool.result')]} " + f"refused={[d['tool_call_id'] for d in rec(s, run, 'approval.refused')]} " + f"approvals={len(rec(s, run, 'approval.request'))} decisions={len(rec(s, run, 'policy.decision'))}") + +for kind in ("create_agent", "tool_node"): + for mode in ("sync", "async"): + go(kind, [{"name": "transfer", "args": {"cents": 5}, "id": "c1"}], mode) +for mode in ("sync", "async"): + go("tool_node", [{"name": "transfer", "args": {"cents": 5}, "id": "c1"}, + {"name": "echo", "args": {"text": "hi"}, "id": "e1"}], mode) diff --git a/design/langchain-held-pass/probes/tool_interrupt_eight_shapes.py b/design/langchain-held-pass/probes/tool_interrupt_eight_shapes.py new file mode 100644 index 000000000..33205cb3e --- /dev/null +++ b/design/langchain-held-pass/probes/tool_interrupt_eight_shapes.py @@ -0,0 +1,109 @@ +"""A tool that asks the person itself (interrupt("confirm?")) under a Tracekit ask, plus siblings. For each builder +(create_agent, tracekit_tool_node), shape (one call, several) and mode (invoke, ainvoke): approve what Tracekit shows, +answer the tool's own question with "yes-", and print what the tool got, what ran, what the model read, and the +signer's records.""" +import asyncio +import json +import sys +import warnings + +from langchain.agents import create_agent +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool +from langgraph.checkpoint.memory import InMemorySaver +from langgraph.graph import START, MessagesState, StateGraph +from langgraph.prebuilt import tools_condition +from langgraph.types import Command, interrupt + +from tracekit.integrations.langchain import TracekitMiddleware, tracekit_tool_node +from tracekit.testing import FakeSigner + +warnings.simplefilter("ignore") +RAN, GOT = [], [] + + +@tool +def pay(to: str) -> str: + """Pay, after the person confirms.""" + answer = interrupt(f"confirm paying {to}?") + GOT.append((to, answer)) + RAN.append(to) + return f"paid {to} ({answer})" + + +@tool +def echo(text: str) -> str: + """Echo.""" + RAN.append("echo") + return text + + +class Model(BaseChatModel): + def _generate(self, messages, stop=None, run_manager=None, **kw): + if isinstance(messages[-1], HumanMessage): + msg = AIMessage("", tool_calls=json.loads(messages[-1].content)) + else: + msg = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=msg)]) + + def bind_tools(self, tools, **kw): + return self + + @property + def _llm_type(self): + return "stub" + + +def trial(builder, shape, mode): + RAN.clear(), GOT.clear() + signer = FakeSigner(lambda t, a: ("ask", ["R-PAY"]) if t == "pay" else ("allow", [])) + run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}}) + if builder == "create_agent": + a = create_agent(Model(), [pay, echo], checkpointer=InMemorySaver(), + middleware=[TracekitMiddleware(signer, run["run_id"], run["run_token"])]) + else: + g = StateGraph(MessagesState) + g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]}) + g.add_node("tools", tracekit_tool_node([pay, echo], signer, run)) + g.add_edge(START, "model") + g.add_conditional_edges("model", tools_condition) + g.add_edge("tools", "model") + a = g.compile(checkpointer=InMemorySaver()) + cfg = {"configurable": {"thread_id": "t"}} + go = a.invoke if mode == "sync" else (lambda v, c: asyncio.run(a.ainvoke(v, c))) + calls = [{"name": "pay", "args": {"to": "p0"}, "id": "c0"}] + if shape == "several": + calls += [{"name": "pay", "args": {"to": "p1"}, "id": "c1"}, {"name": "echo", "args": {"text": "hi"}, "id": "e"}] + out = go({"messages": [HumanMessage(json.dumps(calls))]}, cfg) + steps, n = [], 0 + while "__interrupt__" in out and n < 8: + n += 1 + answers = {} + for i in out["__interrupt__"]: # parallel tasks (create_agent's Send) pause together: one answer each + v = i.value + if isinstance(v, dict) and "tracekit" in v: + first = v["tracekit"] + signer.approval_decide({"request_id": f"d{n}{first['approval_id']}", "approval_id": first["approval_id"], + "decision": "approve"}) + steps.append(f"approved {first['tool_call_id']}") + answers[i.id] = {"approval_id": first["approval_id"]} + else: + steps.append(f"tool asked {v!r}") + answers[i.id] = f"yes-{n}" + out = go(Command(resume=answers), cfg) + ev = signer.read({"run_id": run["run_id"], "run_token": run["run_token"], "limit": 1000})["events"] + count = lambda t: sorted(e["data"].get("tool_call_id") or e["data"].get("approval_id") for e in ev if e["type"] == t) + msgs = {m.tool_call_id: m.content[:48] for m in out.get("messages", []) if isinstance(m, ToolMessage)} + print(f"{builder:13} {shape:7} {mode:5} | {' -> '.join(steps)} | got {GOT} | ran {RAN} | {msgs} | " + f"approval.request {len(count('approval.request'))} results {count('tool.result')}") + + +for builder in ("create_agent", "tool_node"): + for shape in ("one", "several"): + for mode in ("sync", "async"): + try: + trial(builder, shape, mode) + except Exception as e: + print(builder, shape, mode, "raised", type(e).__name__, str(e)[:150]) diff --git a/design/langchain-held-pass/test_integration_langchain_at_f40ac8e293.py b/design/langchain-held-pass/test_integration_langchain_at_f40ac8e293.py new file mode 100644 index 000000000..827b27ad4 --- /dev/null +++ b/design/langchain-held-pass/test_integration_langchain_at_f40ac8e293.py @@ -0,0 +1,577 @@ +"""tracekit.integrations.langchain beyond the adapter contract (tests/test_contract_langchain.py): the result +commitment, `ask` without a checkpointer, an interrupt raised inside a tool, an `ask` under each ToolNode error +handling, several asks in one ToolNode, and the async gate's signer calls off the event loop.""" +import asyncio +import itertools +import json +import time +import types +import unittest +from unittest import mock + +import adapter_contract as ac +import test_said as ts +from tracekit.sdk.client import Client, SignerUnavailable +from tracekit.testing import FakeSigner + +try: + from langchain.agents import create_agent + from langchain.agents.middleware import wrap_tool_call + from langchain.agents.structured_output import ToolStrategy + from langchain_core.language_models import BaseChatModel + from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + from langchain_core.outputs import ChatGeneration, ChatResult + from langchain_core.runnables import RunnableLambda + from langchain_core.tools import tool + from langgraph.checkpoint.memory import InMemorySaver + from langgraph.graph import START, MessagesState, StateGraph + from langgraph.prebuilt import ToolNode, tools_condition + from langgraph.types import Command, interrupt + + from test_contract_langchain import TOOLS, StubModel + from tracekit import autowire + from tracekit.format.canon import event_hash + from tracekit.integrations.executor_tools import EXECUTOR + from tracekit.integrations.langchain import TracekitMiddleware, tracekit_tool_node +except ImportError: # optional: the dev extra installs them + HAVE_LC = False +else: + HAVE_LC = True + + @tool + def confirm(question: str) -> str: + """Ask the user, pausing the run.""" + return interrupt(question) + + @tool("pay") + def counted_pay(to: str, cents: int) -> str: + """Send money.""" + PAID.append(cents) + return f"paid {cents}" + + @tool("pay") + def confirmed_pay(to: str, cents: int) -> str: + """Send money once the person confirms: the tool's own pause.""" + answer = interrupt(f"pay {cents}?") + PAID.append((cents, answer)) + return f"paid {cents} after {answer}" + +THREAD = {"configurable": {"thread_id": "t"}} +PAID = [] # the cents of each counted_pay that ran; (cents, the answer it got) for confirmed_pay + + +def prompt(name, **args): + return {"messages": [HumanMessage(json.dumps({"name": name, "args": args, "id": "call-1"}))]} + + +def tool_message(out): + return next(m for m in out["messages"] if isinstance(m, ToolMessage)) + + +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class Middleware(unittest.TestCase): + def setUp(self): + self.signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"]) if tool == "pay" else ("allow", [])) + r = self.signer.register_run({"request_id": "reg", "agent": {"name": "test"}}) + self.run = {"run_id": r["run_id"], "run_token": r["run_token"]} + + def agent(self, checkpointer=None): + return create_agent(StubModel(), [*TOOLS, confirm], checkpointer=checkpointer, + middleware=[TracekitMiddleware(self.signer, self.run["run_id"], self.run["run_token"])]) + + def recorded(self, typ): + return [e["data"] for e in self.signer.read({**self.run, "limit": 1000})["events"] if e["type"] == typ] + + def test_result_is_recorded_as_a_commitment(self): + sent = [] + complete = self.signer.complete + self.signer.complete = lambda req: sent.append(req) or complete(req) + self.agent().invoke(prompt("echo", text="hi")) + self.assertEqual([r["result"] for r in sent], [event_hash("hi")]) + + def test_ask_without_a_checkpointer_is_denied(self): + out = self.agent().invoke(prompt("pay", to="a", cents=1)) + self.assertIn("no checkpointer", tool_message(out).content) + self.assertEqual(self.recorded("approval.request"), []) + + def test_interrupt_inside_a_tool_is_not_recorded_as_an_outcome(self): + a = self.agent(InMemorySaver()) + a.invoke(prompt("confirm", question="sure?"), THREAD) + self.assertEqual(self.recorded("tool.result"), []) + out = a.invoke(Command(resume="yes"), THREAD) + self.assertEqual(tool_message(out).content, "yes") + self.assertEqual([r["status"] for r in self.recorded("tool.result")], ["ok"]) + + def test_args_changed_after_the_decision_complete_as_error(self): + @wrap_tool_call + def rewrite(request, handler): + request.tool_call["args"]["text"] = "other" + return handler(request) + a = create_agent(StubModel(), TOOLS, middleware=[ + TracekitMiddleware(self.signer, self.run["run_id"], self.run["run_token"]), rewrite]) + a.invoke(prompt("echo", text="hi")) + [r] = self.recorded("tool.result") + self.assertEqual(r["status"], "error") + + def test_a_blocking_signer_call_does_not_stall_the_event_loop(self): # issue #680 + """The async gate's signer calls run in a worker thread: while `decide` sleeps 0.3s, other tasks on the + loop (here a 10ms tick) keep running - no inter-tick gap approaches the sleep.""" + decide = self.signer.decide + self.signer.decide = lambda req: time.sleep(0.3) or decide(req) + + async def one_run_then_tick(): + a_task = asyncio.create_task(self.agent().ainvoke(prompt("echo", text="hi"))) + gaps, last = [], time.monotonic() + while not a_task.done(): + await asyncio.sleep(0.01) + now = time.monotonic() + gaps.append(now - last) + last = now + out = await a_task + self.assertEqual(tool_message(out).content, "hi") + return max(gaps) + slowest = asyncio.run(one_run_then_tick()) + self.assertEqual([d["tool_call_id"] for d in self.recorded("policy.decision")], ["call-1"]) + self.assertEqual([r["status"] for r in self.recorded("tool.result")], ["ok"]) + self.assertLess(slowest, 0.2, f"the event loop stalled {slowest:.3f}s for the signer") + + +if HAVE_LC: + class ThinkingModel(BaseChatModel): + """Thinks (a signed summary, and an open model's reasoning_content), says what it will do and calls echo; then + answers, in the agent's structured output when it has one.""" + + def _generate(self, messages, stop=None, run_manager=None, **kw): + if isinstance(messages[-1], HumanMessage): + msg = AIMessage([{"type": "thinking", "thinking": "the user wants an echo", "signature": "sig-1"}, + {"type": "text", "text": "Echoing."}], + additional_kwargs={"reasoning_content": "raw chain of thought"}, + tool_calls=[{"id": "call-1", "name": "echo", "args": {"text": "hi"}}], + response_metadata={"model_name": "m-1", "model_provider": "anthropic"}) + elif self.structured: + msg = AIMessage("", tool_calls=[{"id": "call-2", "name": "Echo", "args": {"text": "hi"}}]) + else: + msg = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=msg)]) + + structured: bool = False + + def bind_tools(self, tools, **kw): + return self + + @property + def _llm_type(self): + return "stub" + + +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class Said(unittest.TestCase): + """create_agent's model responses are recorded: reasoning and text before each call, and each call, under its id, + once it reaches the gate; the response itself only with the vault; a failure to record changes nothing for the + agent.""" + + def setUp(self): + self.signer = FakeSigner() + r = self.signer.register_run({"request_id": "reg", "agent": {"name": "test"}}) + self.run = {"run_id": r["run_id"], "run_token": r["run_token"]} + self.sent = [] + model_event = self.signer.model_event + self.signer.model_event = lambda req: self.sent.append(req) or model_event(req) + + def invoke(self, **kw): + a = create_agent(ThinkingModel(structured="response_format" in kw), TOOLS, middleware=[ + TracekitMiddleware(self.signer, self.run["run_id"], self.run["run_token"])], **kw) + return a.invoke({"messages": [HumanMessage("echo hi")]}) + + def test_the_reasoning_before_a_call_is_linked_to_it(self): + status = self.signer.status + self.signer.status = lambda req: {**status(req), **ts.VAULT} + self.invoke() + first, call, last = self.sent + self.assertNotIn("tool_uses", first) + self.assertEqual(([t["id"] for t in call["tool_uses"]], call["exchange_id"]), (["call-1"], first["exchange_id"])) + decided = [e["data"]["tool_call_id"] for e in self.signer.read({**self.run, "limit": 100})["events"] + if e["type"] == "policy.decision"] + self.assertEqual(decided, ["call-1"]) + self.assertEqual((first["model"], first["provider"]), ("m-1", "anthropic")) + self.assertEqual(ts.segments(first), [("reasoning", "full", "raw chain of thought", []), + ("reasoning", "summary", "the user wants an echo", []), + ("text", "full", "Echoing.", [])]) + self.assertEqual(first["body"]["output"][1]["encrypted_content"], "sig-1") + self.assertEqual(ts.segments(last), [("text", "full", "done", [])]) + + def test_without_the_vault_only_the_commitment_and_no_structured_output_call(self): + out = self.invoke(response_format=ToolStrategy(dict(title="Echo", type="object", + properties={"text": {"type": "string"}}))) + self.assertEqual(out["structured_response"], {"text": "hi"}) + self.assertEqual(len(self.sent), 2) # the response and its echo call: the structured answer said nothing else + self.assertTrue(all("body" not in r and r["content_digest"] for r in self.sent)) + self.assertEqual([t["name"] for r in self.sent for t in r.get("tool_uses", ())], ["echo"]) # not Echo + + def test_a_call_another_middleware_stops_before_the_gate_is_not_recorded(self): + """A person's reject, a call limit: the call never ran and no hook saw it; recorded, the signer would read it + as one that ran with no decision (reconcile.hook_missing).""" + from langchain.agents.middleware import HumanInTheLoopMiddleware, ToolCallLimitMiddleware + mw = lambda: TracekitMiddleware(self.signer, self.run["run_id"], self.run["run_token"]) + a = create_agent(StubModel(), TOOLS, checkpointer=InMemorySaver(), + middleware=[HumanInTheLoopMiddleware({"echo": True}), mw()]) + a.invoke(prompt("echo", text="hi"), THREAD) + a.invoke(Command(resume={"decisions": [{"type": "reject", "message": "no"}]}), THREAD) + calls = [{"name": "echo", "args": {"text": str(i)}, "id": f"c{i}"} for i in range(3)] + create_agent(StubModel(), TOOLS, middleware=[ToolCallLimitMiddleware(run_limit=1), mw()]).invoke( + {"messages": [HumanMessage(json.dumps(calls))]}) + self.assertEqual([t["id"] for r in self.sent for t in r.get("tool_uses", ())], ["c0"]) # the one that ran + decided = [e["data"]["tool_call_id"] for e in self.signer.read({**self.run, "limit": 100})["events"] + if e["type"] == "policy.decision"] + self.assertEqual(decided, ["c0"]) + + def test_a_failure_to_record_changes_nothing_for_the_agent(self): + self.signer.model_event = mock.Mock(side_effect=SignerUnavailable("down")) + with self.assertWarnsRegex(UserWarning, "what the model said was not recorded"): + out = self.invoke() + self.assertEqual(out["messages"][-1].content, "done") + + +def handle_tool_error(state): + """The fallback of LangGraph's customer-support tutorial: `ToolNode(tools).with_fallbacks([RunnableLambda( + handle_tool_error)], exception_key="error")`.""" + return {"messages": [ToolMessage(f"Error: {state.get('error')!r}\n please fix your mistakes.", tool_call_id=c["id"]) + for c in state["messages"][-1].tool_calls]} + + +def tool_graph(node): + g = StateGraph(MessagesState) + g.add_node("model", lambda state: {"messages": [StubModel().invoke(state["messages"])]}) + g.add_node("tools", node) + g.add_edge(START, "model") + g.add_conditional_edges("model", tools_condition) + g.add_edge("tools", "model") + return g.compile(checkpointer=InMemorySaver()) + + +ERROR_HANDLING = {"default": {}, "True": {"handle_tool_errors": True}, "a string": {"handle_tool_errors": "failed"}, + "a callable": {"handle_tool_errors": lambda e: f"Error: {e!r}"}, + "a tuple": {"handle_tool_errors": (ValueError,)}, "a type": {"handle_tool_errors": ValueError}, + "str, a type that isn't an exception": {"handle_tool_errors": str}} + + +class GatedNodes(unittest.TestCase): + """Gated ToolNodes, built by hand and as instrument() wires them.""" + + def setUp(self): + for name in ("__init__", "_func", "_afunc", "with_fallbacks"): # instrument() patches ToolNode for the process + patcher = mock.patch.object(ToolNode, name, getattr(ToolNode, name)) + patcher.start() + self.addCleanup(patcher.stop) + self.wired = ToolNode.__init__ + + def nodes(self, tools=None, **kw): + """(how, a function building the gated node of `tools`, else TOOLS): by hand, and a plain ToolNode as + instrument() wires it, each on a signer of its own""" + tools = tools or TOOLS + for how in ("by hand", "instrument()"): + ac.RAN.clear() + ToolNode.__init__ = self.wired + self.signer = FakeSigner(ac._rule) + self.run = self.signer.register_run({"request_id": "reg", "agent": {"name": "test"}}) + if how == "by hand": + yield how, lambda: tracekit_tool_node(tools, self.signer, self.run, **kw) + else: + autowire._langgraph(types.SimpleNamespace(client=self.signer, registered=self.run, **{ + k: self.run[k] for k in ("run_id", "run_token")})) + yield how, lambda: ToolNode(tools, **kw) + + +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class ErrorHandling(GatedNodes): + """An `ask` pauses the graph whatever the ToolNode's handle_tool_errors and inside with_fallbacks, wired by hand + or by instrument(); the tool's own errors are handled as configured.""" + + def pause_and_approve(self, graph, mode="sync"): + run = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) + out = run(prompt("pay", to="a", cents=1), THREAD) + self.assertIn("__interrupt__", out, [m.content for m in out["messages"]]) + self.assertEqual(ac.RAN, []) + aid = out["__interrupt__"][0].value["tracekit"]["approval_id"] + self.signer.approval_decide({"request_id": "ok-" + aid, "approval_id": aid, "decision": "approve"}) + out = run(Command(resume={"approval_id": aid}), THREAD) + self.assertEqual((tool_message(out).content, ac.RAN), ("paid 1", ["pay"])) + + def test_an_ask_pauses_under_every_handle_tool_errors(self): + for setting, kw in ERROR_HANDLING.items(): + for mode in ("sync", "async"): + for how, node in self.nodes(**kw): + with self.subTest(setting=setting, wired=how, mode=mode): + self.pause_and_approve(tool_graph(node()), mode) + + def test_an_ask_pauses_through_with_fallbacks(self): # the tutorial's create_tool_node_with_fallback + for mode in ("sync", "async"): + for how, node in self.nodes(): + with self.subTest(wired=how, mode=mode): + fallback = node().with_fallbacks([RunnableLambda(handle_tool_error)], exception_key="error") + self.pause_and_approve(tool_graph(fallback), mode) + + def test_the_tools_own_errors_are_handled_as_configured(self): + for how, node in self.nodes(handle_tool_errors=True): + with self.subTest(wired=how): + out = tool_graph(node()).invoke(prompt("fail", why="boom"), THREAD) + self.assertIn("ValueError('boom')", tool_message(out).content) + for how, node in self.nodes(handle_tool_errors=str): # a callable that is a type: ToolNode calls it + with self.subTest(wired=how, handler=str): + self.assertEqual(tool_message(tool_graph(node()).invoke(prompt("fail", why="boom"), THREAD)).content, + "boom") + for how, node in self.nodes(): + with self.subTest(wired=how, fallbacks=True): + fallback = node().with_fallbacks([RunnableLambda(handle_tool_error)], exception_key="error") + out = tool_graph(fallback).invoke(prompt("fail", why="boom"), THREAD) + self.assertIn("please fix your mistakes", tool_message(out).content) + + +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class ManyAsks(GatedNodes): + """#687: a message whose calls ask, beside one allowed and one denied call, in one ToolNode. LangGraph gives the node + one resume value, by the order its calls reach interrupt(), and runs the node again from its start on each resume: + the call a person approved ran only when it got there first (7 of 16 under ainvoke, 4 of 16 under invoke), and with + three or more asks a call that ran was refused on the next pass, or the allowed call ran twice. Each ask is + approved, rejected or left waiting, answered in a different order each time, then the run resumes once.""" + + def outcomes(self, node, mode, answers, order): + """{tool call id: (times it ran before the resume, whether the pause listed its approval, times it ran in all, + what the model read: its result or the rule that refused it, the results the signer recorded, the rules of its + recorded refusal)}, and whether the resume paused again""" + graph = tool_graph(node) + go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) + calls = [{"name": "pay", "args": {"to": "a", "cents": i}, "id": f"pay-{i}"} for i in range(len(answers))] + calls += [{"name": "echo", "args": {"text": "hi"}, "id": "echo"}, + {"name": "wipe", "args": {"path": "/"}, "id": "wipe"}] + + def ran(): + return {c["id"]: PAID.count(c["args"]["cents"]) if c["name"] == "pay" else ac.RAN.count(c["name"]) + for c in calls} + asked = go({"messages": [HumanMessage(json.dumps(calls))]}, THREAD)["__interrupt__"][0].value["tracekit"] + before, listed = ran(), {a["tool_call_id"] for a in asked.get("approvals", ())} + approvals = {r["tool_call_id"]: r["approval_id"] for r in self.recorded("approval.request")} + for i in order: + if answers[i] != "wait": + self.signer.approval_decide({"request_id": f"{i}", "decision": answers[i], + "approval_id": approvals[f"pay-{i}"]}) + out = go(Command(resume={"approval_id": asked["approval_id"]}), THREAD) + blocked = "Tool call blocked by policy: " + seen = {m.tool_call_id: m.content[len(blocked):].split(" ")[0] if m.content.startswith(blocked) else m.content + for m in out["messages"] if isinstance(m, ToolMessage)} + results = [r["tool_call_id"] for r in self.recorded("tool.result")] + refused = {r["tool_call_id"]: r["rule_ids"] for r in self.recorded("approval.refused")} + after = ran() + return {c["id"]: (before[c["id"]], c["id"] in listed, after[c["id"]], seen.get(c["id"]), + results.count(c["id"]), refused.get(c["id"])) for c in calls}, "__interrupt__" in out + + def recorded(self, typ): + return [e["data"] for e in self.signer.read({"run_id": self.run["run_id"], "run_token": self.run["run_token"], + "limit": 1000})["events"] if e["type"] == typ] + + def test_each_call_runs_once_or_is_refused_and_is_recorded_as_it_did(self): + want = {"approve": lambda i: (0, True, 1, f"paid {i}", 1, None), + "reject": lambda i: (0, True, 0, "TK-APPROVAL-REJECTED", 0, ["TK-APPROVAL-REJECTED"]), + "wait": lambda i: (0, True, 0, "TK-APPROVAL-REQUESTED", 0, ["TK-APPROVAL-REQUESTED"])} + cases = [*itertools.product(want, repeat=2), *itertools.product(want, repeat=3), + *itertools.product(("approve", "wait"), repeat=5)] + for k, answers in enumerate(cases): + orders = list(itertools.permutations(range(len(answers)))) + for mode in ("sync", "async"): + for how, node in self.nodes(tools=[TOOLS[0], counted_pay, TOOLS[2]]): + with self.subTest(answers=answers, mode=mode, wired=how): + PAID.clear() + self.assertEqual(self.outcomes(node(), mode, answers, orders[k % len(orders)]), + ({**{f"pay-{i}": want[a](i) for i, a in enumerate(answers)}, + "echo": (0, False, 1, "hi", 1, None), + "wipe": (0, False, 0, "R-WIPE", 0, None)}, False)) + + +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class ToolPauses(GatedNodes): + """A tool that pauses the run itself (interrupt()), beside or under an `ask`. LangGraph pairs resume values with a + node's interrupt() calls by their order, and runs the node again from its start on each resume. Once an approval + was answered the gate stopped pausing, so the tool got the approval's resume value as the person's answer and ran + without asking; before that, the tool asked, but its approved call was then refused as already used + (TK-APPROVAL-CONSUMED), and a call that ran before the tool's pause was asked for a second approval.""" + + def graphs(self, tools): + """(how, the graph): create_agent, and the gated ToolNodes (GatedNodes.nodes), each on a signer of its own""" + ToolNode.__init__ = self.wired + self.signer = FakeSigner(ac._rule) + self.run = self.signer.register_run({"request_id": "reg", "agent": {"name": "test"}}) + yield "create_agent", create_agent(StubModel(), tools, checkpointer=InMemorySaver(), middleware=[ + TracekitMiddleware(self.signer, self.run["run_id"], self.run["run_token"])]) + for how, node in self.nodes(tools=tools): + yield how, tool_graph(node()) + + def answered(self, graph, mode, calls, approve): + """(each pause in order: "tracekit" or the tool's question; what the model read of each call; the signer's + approval requests and results, by call), a person approving the calls in `approve` and answering each tool's + question "yes: "; every other approval left unanswered""" + go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) + out, pauses = go({"messages": [HumanMessage(json.dumps(calls))]}, THREAD), [] + while "__interrupt__" in out and len(pauses) < 10: + answers = {} + for i in out["__interrupt__"]: # create_agent's calls are tasks of their own: they pause together + shown = i.value.get("tracekit") if isinstance(i.value, dict) else None + pauses.append("tracekit" if shown else i.value) + for a in shown.get("approvals", [shown]) if shown else (): + if a["tool_call_id"] in approve and self.recorded("approval", a["approval_id"]) == []: + self.signer.approval_decide({"request_id": a["approval_id"], "decision": "approve", + "approval_id": a["approval_id"]}) + answers[i.id] = {"approval_id": shown["approval_id"]} if shown else f"yes: {i.value}" + out = go(Command(resume=answers), THREAD) + seen = {m.tool_call_id: m.content for m in out["messages"] if isinstance(m, ToolMessage)} + return pauses, seen, {t: sorted(r["tool_call_id"] for r in self.recorded(t)) for t in ("approval.request", + "tool.result")} + + def recorded(self, typ, approval_id=None): + return [e["data"] for e in self.signer.read({"run_id": self.run["run_id"], "run_token": self.run["run_token"], + "limit": 1000})["events"] + if e["type"] == typ and approval_id in (None, e["data"].get("approval_id"))] + + def test_a_tool_asked_under_an_approval_gets_the_persons_own_answer(self): + for mode in ("sync", "async"): + for how, graph in self.graphs([confirmed_pay]): + with self.subTest(mode=mode, built=how): + PAID.clear() + got = self.answered(graph, mode, [{"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay-1"}], + approve={"pay-1"}) + self.assertEqual(got, (["tracekit", "pay 1?"], {"pay-1": "paid 1 after yes: pay 1?"}, + {"approval.request": ["pay-1"], "tool.result": ["pay-1"]})) + self.assertEqual(PAID, [(1, "yes: pay 1?")]) # it ran once, with the person's answer + + def test_a_tool_that_pauses_beside_asks_gets_the_answer_and_no_call_runs_or_asks_twice(self): + calls = [{"name": "pay", "args": {"to": "a", "cents": 0}, "id": "pay-0"}, + {"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay-1"}, + {"name": "confirm", "args": {"question": "sure?"}, "id": "confirm"}, + {"name": "echo", "args": {"text": "hi"}, "id": "echo"}] + for mode in ("sync", "async"): + for how, graph in self.graphs([counted_pay, confirm, TOOLS[0]]): + with self.subTest(mode=mode, built=how): + PAID.clear() + ac.RAN.clear() + pauses, seen, recorded = self.answered(graph, mode, calls, approve={"pay-0"}) + self.assertEqual((set(pauses), pauses.count("sure?")), ({"tracekit", "sure?"}, 1)) + self.assertEqual({k: v.split(" (")[0] for k, v in seen.items()}, + {"pay-0": "paid 0", "pay-1": "Tool call blocked by policy: TK-APPROVAL-REQUESTED", + "confirm": "yes: sure?", "echo": "hi"}) + self.assertEqual(recorded, {"approval.request": ["pay-0", "pay-1"], + "tool.result": ["confirm", "echo", "pay-0"]}) + self.assertEqual((PAID, ac.RAN), ([0], ["echo"])) # each ran once + + def test_an_executors_tool_is_left_to_the_executor(self): + """_pass decided an executor's call with the gate's own decision and paused for an approval the executor never + consumes.""" + def send(to: str, cents: int) -> str: + """Send money, through the tool executor.""" + return f"sent {cents}" + executed = tool("pay")(send) + executed.metadata = {EXECUTOR: True} + for mode in ("sync", "async"): + for how, node in self.nodes(tools=[executed, TOOLS[0]]): + with self.subTest(mode=mode, wired=how): + graph = tool_graph(node()) + go = graph.invoke if mode == "sync" else lambda v, c: asyncio.run(graph.ainvoke(v, c)) + out = go({"messages": [HumanMessage(json.dumps([ + {"name": "pay", "args": {"to": "a", "cents": 1}, "id": "pay"}, + {"name": "echo", "args": {"text": "hi"}, "id": "echo"}]))]}, THREAD) + self.assertNotIn("__interrupt__", out) + self.assertEqual({m.tool_call_id: m.content for m in out["messages"] + if isinstance(m, ToolMessage)}, {"pay": "sent 1", "echo": "hi"}) + self.assertEqual([r["tool_call_id"] for r in self.recorded("policy.decision")], ["echo"]) + + +class Jittery: + """The client, with the pause a busy process may take between a call's client_seq and its send.""" + + def __init__(self, client): + self.client = client + + def __getattr__(self, name): + def call(req): + if "tool_call_id" in req: # a model event, or the vault's status, names no call + time.sleep(int(req["tool_call_id"][1:]) % 3 / 1000) + return getattr(self.client, name)(req) + return call + + +class Down: + """A signer that cannot be reached.""" + + def __getattr__(self, name): + def call(req): + raise SignerUnavailable("no signer answering") + return call + + +@unittest.skipUnless(HAVE_LC, "langchain>=1 / langgraph-checkpoint-sqlite not installed") +class Failures(ac.OnReal, unittest.TestCase): + def setUp(self): + ac.RAN.clear() + self.client = Client(self.serve_signer()) + self.addCleanup(self.client.close) + r = self.client.register_run({"agent": {"name": "test"}}) + self.run = {"run_id": r["run_id"], "run_token": r["run_token"]} + + def agent(self, signer, **kw): + return create_agent(StubModel(), TOOLS, middleware=[TracekitMiddleware(signer, **self.run, **kw)]) + + def test_parallel_sync_calls_reach_the_signer_in_order(self): + calls = [{"name": "echo", "args": {"text": str(i)}, "id": f"c{i}"} for i in range(48)] + for _ in range(2): # within the signer's event burst + out = self.agent(Jittery(self.client)).invoke({"messages": [HumanMessage(json.dumps(calls))]}) + self.assertEqual(out["messages"][-1].content, "done") + types = [e["type"] for e in self.events(self.run)] + self.assertEqual((types.count("policy.decision"), types.count("tool.result")), (96, 96)) + self.assertNotIn("capture.gap", types) + + def test_signer_unreachable_follows_the_fail_mode(self): + out = self.agent(Down(), fail_modes={"default": "closed"}).invoke(prompt("echo", text="hi")) + self.assertIn("signer unavailable", tool_message(out).content) + self.assertEqual(ac.RAN, []) + out = self.agent(Down(), fail_modes={"default": "open"}).invoke(prompt("echo", text="hi")) + self.assertEqual((tool_message(out).content, ac.RAN), ("hi", ["echo"])) + + def test_signer_unreachable_follows_the_fail_mode_of_the_tools_class(self): + @tool + def terminal(commands: str) -> str: # LangChain's ShellTool name: class shell in the default policy + """Run commands.""" + ac.RAN.append("terminal") + return "ran" + agent = create_agent(StubModel(), [*TOOLS, terminal], middleware=[TracekitMiddleware( + Down(), **self.run, fail_modes={"default": "closed", "shell": "open"})]) + self.assertEqual(tool_message(agent.invoke(prompt("terminal", commands="ls"))).content, "ran") + self.assertIn("signer unavailable", tool_message(agent.invoke(prompt("echo", text="hi"))).content) + self.assertEqual(ac.RAN, ["terminal"]) + + def test_an_outage_reads_the_default_policy_once(self): + """The fail mode's class comes from the shipped default policy, compiled once, not on every refused call.""" + from tracekit.integrations.claude_code import _tool_class + from tracekit.signer import service + _tool_class.cache_clear() + with mock.patch.object(service, "load_policy", wraps=service.load_policy) as load: + agent = self.agent(Down(), fail_modes={"default": "closed"}) + for _ in range(3): + self.assertIn("signer unavailable", tool_message(agent.invoke(prompt("echo", text="hi"))).content) + self.assertEqual(load.call_count, 1) + + def test_closed_run_refuses_the_call(self): + self.client.close_run(dict(self.run)) + out = self.agent(self.client).invoke(prompt("echo", text="hi")) + self.assertIn("run_closed", tool_message(out).content) + self.assertEqual(ac.RAN, []) + + def test_a_failed_complete_leaves_the_result_unchanged(self): + signer = mock.Mock(wraps=self.client) + signer.complete.side_effect = SignerUnavailable("gone") + with self.assertWarns(UserWarning): + out = self.agent(signer).invoke(prompt("echo", text="hi")) + self.assertEqual((tool_message(out).content, out["messages"][-1].content), ("hi", "done")) + with self.assertWarns(UserWarning), self.assertRaises(ValueError): + self.agent(signer).invoke(prompt("fail", why="boom")) + + +if __name__ == "__main__": + unittest.main()