Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions design/langchain-held-pass/probes/allowed_sibling_runs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
"""One message: an allowed echo and N asking pays in one tracekit_tool_node. Approve every pay as each pause shows it,
resuming after each; print how often each tool ran and what the model read."""
import asyncio
import json
import sys
import warnings

from langchain_core.language_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.tools import tool
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph import START, MessagesState, StateGraph
from langgraph.prebuilt import tools_condition
from langgraph.types import Command

from tracekit.integrations.langchain import tracekit_tool_node
from tracekit.testing import FakeSigner

warnings.simplefilter("ignore")
RAN = []
N = int(sys.argv[1]) if len(sys.argv) > 1 else 1


@tool
def pay(to: str) -> str:
"""Pay someone."""
RAN.append(to)
return f"paid {to}"


@tool
def echo(text: str) -> str:
"""Echo."""
RAN.append("echo")
return text


class Model(BaseChatModel):
def _generate(self, messages, stop=None, run_manager=None, **kw):
if isinstance(messages[-1], HumanMessage):
calls = [{"name": "echo", "args": {"text": "hi"}, "id": "e"}]
calls += [{"name": "pay", "args": {"to": f"p{i}"}, "id": f"c{i}"} for i in range(N)]
msg = AIMessage("", tool_calls=calls)
else:
msg = AIMessage("done")
return ChatResult(generations=[ChatGeneration(message=msg)])

def bind_tools(self, tools, **kw):
return self

@property
def _llm_type(self):
return "stub"


signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"]) if tool == "pay" else ("allow", []))
run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}})
g = StateGraph(MessagesState)
g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]})
g.add_node("tools", tracekit_tool_node([pay, echo], signer, run))
g.add_edge(START, "model")
g.add_conditional_edges("model", tools_condition)
g.add_edge("tools", "model")
a = g.compile(checkpointer=InMemorySaver())
cfg = {"configurable": {"thread_id": "t"}}
out = a.invoke({"messages": [HumanMessage("go")]}, cfg)
step = 0
while "__interrupt__" in out and step < 10:
shown = out["__interrupt__"][0].value["tracekit"]
for x in shown.get("approvals", [shown]):
signer.approval_decide({"request_id": f"d{step}{x['approval_id']}", "approval_id": x["approval_id"],
"decision": "approve"})
out = a.invoke(Command(resume={"approval_id": shown["approval_id"]}), cfg)
step += 1
print("resumes:", step, "| ran:", sorted(RAN), "|",
{m.tool_call_id: m.content[:60] for m in out["messages"] if isinstance(m, ToolMessage)})
73 changes: 73 additions & 0 deletions design/langchain-held-pass/probes/approve_then_retry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""Two asks in one ToolNode: approve the shown one, resume; then approve the refused one's approval and send the model's
retry of it (a new call id, same tool and arguments). Does the retry run without a pause?"""
import json
import sys
import warnings

from langchain_core.language_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.tools import tool
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph import START, MessagesState, StateGraph
from langgraph.prebuilt import tools_condition
from langgraph.types import Command

from tracekit.integrations.langchain import tracekit_tool_node
from tracekit.testing import FakeSigner

warnings.simplefilter("ignore")
RAN = []


@tool
def pay(to: str) -> str:
"""Pay someone."""
RAN.append(to)
return f"paid {to}"


class Model(BaseChatModel):
def _generate(self, messages, stop=None, run_manager=None, **kw):
if isinstance(messages[-1], HumanMessage):
msg = AIMessage("", tool_calls=json.loads(messages[-1].content))
else:
msg = AIMessage("done")
return ChatResult(generations=[ChatGeneration(message=msg)])

def bind_tools(self, tools, **kw):
return self

@property
def _llm_type(self):
return "stub"


signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"]))
run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}})
g = StateGraph(MessagesState)
g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]})
g.add_node("tools", tracekit_tool_node([pay], signer, run))
g.add_edge(START, "model")
g.add_conditional_edges("model", tools_condition)
g.add_edge("tools", "model")
a = g.compile(checkpointer=InMemorySaver())
cfg = {"configurable": {"thread_id": "t"}}
calls = [{"name": "pay", "args": {"to": "alice"}, "id": "c1"}, {"name": "pay", "args": {"to": "bob"}, "id": "c2"}]
out = a.invoke({"messages": [HumanMessage(json.dumps(calls))]}, cfg)
shown = out["__interrupt__"][0].value["tracekit"]
signer.approval_decide({"request_id": "d1", "approval_id": shown["approval_id"], "decision": "approve"})
out = a.invoke(Command(resume={"approval_id": shown["approval_id"]}), cfg)
refused = [m for m in out["messages"] if isinstance(m, ToolMessage) and "blocked" in m.content][0]
print("after resume:", RAN, "|", refused.tool_call_id, refused.content)
other_aid = refused.content.split("(approval ")[1].split(")")[0]
signer.approval_decide({"request_id": "d2", "approval_id": other_aid, "decision": "approve"})
other = next(c for c in calls if c["id"] == refused.tool_call_id)
out = a.invoke({"messages": [HumanMessage(json.dumps([dict(other, id="c3")]))]}, cfg)
print("retry:", RAN, "| paused:", "__interrupt__" in out, "|",
[m.content for m in out["messages"] if isinstance(m, ToolMessage) and m.tool_call_id == "c3"])
if "__interrupt__" in out:
again = out["__interrupt__"][0].value["tracekit"]
out = a.invoke(Command(resume={"approval_id": again["approval_id"]}), cfg)
print("after resuming the retry (its approval", again["approval_id"], "):", RAN,
[m.content for m in out["messages"] if isinstance(m, ToolMessage) and m.tool_call_id == "c3"])
83 changes: 83 additions & 0 deletions design/langchain-held-pass/probes/fresh_gate_after_tool_pause.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""After a tool's own pause, resume with a fresh gate (as a new process would, same checkpointer and signer): what the
model reads and what ran, for one asking call whose tool pauses, and for [pay-0 approved, confirm pausing, echo]."""
import json
import warnings

from langchain_core.language_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.tools import tool
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph import START, MessagesState, StateGraph
from langgraph.prebuilt import tools_condition
from langgraph.types import Command, interrupt

from tracekit.integrations.langchain import tracekit_tool_node
from tracekit.testing import FakeSigner

warnings.simplefilter("ignore")
RAN = []


@tool
def pay(to: str, ask: bool = False) -> str:
"""Pay."""
answer = interrupt(f"pay {to}?") if ask else "-"
RAN.append(to)
return f"paid {to} ({answer})"


@tool
def confirm(question: str) -> str:
"""Ask."""
return interrupt(question)


@tool
def echo(text: str) -> str:
"""Echo."""
RAN.append("echo")
return text


class Model(BaseChatModel):
def _generate(self, messages, stop=None, run_manager=None, **kw):
msg = AIMessage("", tool_calls=json.loads(messages[-1].content)) if isinstance(messages[-1], HumanMessage) \
else AIMessage("done")
return ChatResult(generations=[ChatGeneration(message=msg)])

def bind_tools(self, tools, **kw):
return self

@property
def _llm_type(self):
return "stub"


def graph(signer, run, saver):
g = StateGraph(MessagesState)
g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]})
g.add_node("tools", tracekit_tool_node([pay, confirm, echo], signer, run))
g.add_edge(START, "model")
g.add_conditional_edges("model", tools_condition)
g.add_edge("tools", "model")
return g.compile(checkpointer=saver)


for name, calls in (("one asking call, its tool pauses", [{"name": "pay", "args": {"to": "p0", "ask": True}, "id": "p0"}]),
("pay-0 approved, confirm pauses, echo", [{"name": "pay", "args": {"to": "p0"}, "id": "p0"},
{"name": "confirm", "args": {"question": "sure?"}, "id": "c"},
{"name": "echo", "args": {"text": "hi"}, "id": "e"}])):
RAN.clear()
signer = FakeSigner(lambda t, a: ("ask", ["R-PAY"]) if t == "pay" else ("allow", []))
run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}})
saver, cfg = InMemorySaver(), {"configurable": {"thread_id": "t"}}
out = graph(signer, run, saver).invoke({"messages": [HumanMessage(json.dumps(calls))]}, cfg)
shown = out["__interrupt__"][0].value["tracekit"]
signer.approval_decide({"request_id": "d", "approval_id": shown["approval_id"], "decision": "approve"})
out = graph(signer, run, saver).invoke(Command(resume={"approval_id": shown["approval_id"]}), cfg)
asked = out["__interrupt__"][0].value
before = list(RAN)
out = graph(signer, run, saver).invoke(Command(resume="yes"), cfg) # a new gate: the memory of a new process
print(f"{name}: tool asked {asked!r}; ran before {before}, after {RAN};",
{m.tool_call_id: m.content[:60] for m in out["messages"] if isinstance(m, ToolMessage)})
69 changes: 69 additions & 0 deletions design/langchain-held-pass/probes/many_asks_one_at_a_time.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""Three `ask` calls in one ToolNode message: approve the shown one, resume, and see what runs; then approve the next
shown one and resume again. Prints what ran and the ToolMessages after each resume."""
import asyncio
import sys
import warnings

from langchain_core.language_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.tools import tool
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph import START, MessagesState, StateGraph
from langgraph.prebuilt import tools_condition
from langgraph.types import Command

from tracekit.integrations.langchain import tracekit_tool_node
from tracekit.testing import FakeSigner

warnings.simplefilter("ignore")
RAN = []
N = int(sys.argv[1]) if len(sys.argv) > 1 else 3


@tool
def pay(to: str) -> str:
"""Pay someone."""
RAN.append(to)
return f"paid {to}"


class Model(BaseChatModel):
def _generate(self, messages, stop=None, run_manager=None, **kw):
if isinstance(messages[-1], HumanMessage):
msg = AIMessage("", tool_calls=[{"name": "pay", "args": {"to": f"p{i}"}, "id": f"c{i}"} for i in range(N)])
else:
msg = AIMessage("done")
return ChatResult(generations=[ChatGeneration(message=msg)])

def bind_tools(self, tools, **kw):
return self

@property
def _llm_type(self):
return "stub"


signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"]))
run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}})
g = StateGraph(MessagesState)
g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]})
g.add_node("tools", tracekit_tool_node([pay], signer, run))
g.add_edge(START, "model")
g.add_conditional_edges("model", tools_condition)
g.add_edge("tools", "model")
a = g.compile(checkpointer=InMemorySaver())
cfg = {"configurable": {"thread_id": "t"}}
mode = sys.argv[2] if len(sys.argv) > 2 else "sync"
inv = a.invoke if mode == "sync" else (lambda v, c: asyncio.run(a.ainvoke(v, c)))
out = inv({"messages": [HumanMessage("go")]}, cfg)
for step in range(N + 1):
if "__interrupt__" not in out:
break
shown = out["__interrupt__"][0].value["tracekit"]
signer.approval_decide({"request_id": f"d{step}", "approval_id": shown["approval_id"], "decision": "approve"})
out = inv(Command(resume={"approval_id": shown["approval_id"]}), cfg)
msgs = {m.tool_call_id: m.content[:90] for m in out.get("messages", []) if isinstance(m, ToolMessage)}
print(f"resume {step}: approved {shown['tool_call_id']}; ran so far {RAN}; paused again {'__interrupt__' in out}")
for k, v in msgs.items():
print(" ", k, v)
85 changes: 85 additions & 0 deletions design/langchain-held-pass/probes/parallel_ask_ainvoke.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
"""Probe #687: tracekit_tool_node (one ToolNode task runs every call of a message) with two calls that both ask, under
ainvoke. LangGraph numbers interrupts in the order the gates reach interrupt(); the gate now runs in worker threads.
Resume with the approval id the first pause showed, after a person approves it, and see which call gets it."""
import asyncio
import json
import random
import sys
import time
import warnings

from langchain_core.language_models import BaseChatModel
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.tools import tool
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph import START, MessagesState, StateGraph
from langgraph.prebuilt import tools_condition
from langgraph.types import Command

from tracekit.integrations.langchain import tracekit_tool_node
from tracekit.testing import FakeSigner

warnings.simplefilter("ignore")
RAN = []


@tool
def pay(to: str) -> str:
"""Pay someone."""
RAN.append(to)
return f"paid {to}"


class Model(BaseChatModel):
def _generate(self, messages, stop=None, run_manager=None, **kw):
if isinstance(messages[-1], HumanMessage):
msg = AIMessage("", tool_calls=[{"name": "pay", "args": {"to": "alice"}, "id": "c1"},
{"name": "pay", "args": {"to": "bob"}, "id": "c2"}])
else:
msg = AIMessage("done")
return ChatResult(generations=[ChatGeneration(message=msg)])

def bind_tools(self, tools, **kw):
return self

@property
def _llm_type(self):
return "stub"


def trial(jitter):
RAN.clear()
signer = FakeSigner(lambda tool, args: ("ask", ["R-PAY"]))
run = signer.register_run({"request_id": "reg", "agent": {"name": "t"}})
decide = signer.approval_request
signer.approval_request = lambda req: (time.sleep(random.random() * jitter), decide(req))[1]
g = StateGraph(MessagesState)
g.add_node("model", lambda s: {"messages": [Model().invoke(s["messages"])]})
g.add_node("tools", tracekit_tool_node([pay], signer, run))
g.add_edge(START, "model")
g.add_conditional_edges("model", tools_condition)
g.add_edge("tools", "model")
a = g.compile(checkpointer=InMemorySaver())
cfg = {"configurable": {"thread_id": "t"}}
out = asyncio.run(a.ainvoke({"messages": [HumanMessage("go")]}, cfg))
shown = out["__interrupt__"][0].value["tracekit"]
signer.approval_decide({"request_id": "d1",
"approval_id": shown["approval_id"], "decision": "approve"})
out = asyncio.run(a.ainvoke(Command(resume={"approval_id": shown["approval_id"]}), cfg))
msgs = {m.tool_call_id: m.content[:60] for m in out.get("messages", []) if isinstance(m, ToolMessage)}
return shown["tool_call_id"], list(RAN), msgs, "__interrupt__" in out


jitter = float(sys.argv[1]) if len(sys.argv) > 1 else 0.05
results = {}
for i in range(int(sys.argv[2]) if len(sys.argv) > 2 else 12):
try:
shown, ran, msgs, paused = trial(jitter)
key = f"pause showed {shown}; ran {ran}; paused again {paused}; refused: " + \
", ".join(f"{k}" for k, v in msgs.items() if "blocked" in v)
except Exception as e:
key = f"raised {type(e).__name__}: {str(e)[:120]}"
results[key] = results.get(key, 0) + 1
for k, v in results.items():
print(v, "x", k)
Loading
Loading