Skip to content

Commit b0a2b59

Browse files
pcbeingused333claudemarandaneto
authored
feat(ai): record the Anthropic web search count in the LangChain callback handler (#1097)
* feat(ai): record the Anthropic web search count in the LangChain callback handler Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * chore: update public API snapshot for ModelUsage.web_search_count Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: regenerate LangChain ModelUsage API snapshot * fix(ai): read the LangChain web search count from llm_output usage and omit zero Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: Manoel Aranda Neto <marandaneto@gmail.com>
1 parent 7833e95 commit b0a2b59

4 files changed

Lines changed: 93 additions & 1 deletion

File tree

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
pypi/posthog: patch
3+
---
4+
5+
The LangChain callback handler now records `$ai_web_search_count` for Anthropic responses that report `server_tool_use.web_search_requests`, as the OpenAI, Anthropic and Gemini wrappers and `posthog-js` already do.

‎posthog/ai/langchain/callbacks.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -724,6 +724,8 @@ def _capture_generation(
724724
)
725725
if usage.reasoning_tokens is not None:
726726
event_properties["$ai_reasoning_tokens"] = usage.reasoning_tokens
727+
if usage.web_search_count is not None and usage.web_search_count > 0:
728+
event_properties["$ai_web_search_count"] = usage.web_search_count
727729

728730
# Generation results
729731
generation_result = output.generations[-1]
@@ -856,6 +858,23 @@ class ModelUsage:
856858
reasoning_tokens: Optional[int]
857859
cache_write_5m_tokens: Optional[int] = None
858860
cache_write_1h_tokens: Optional[int] = None
861+
web_search_count: Optional[int] = None
862+
863+
864+
def _extract_web_search_requests(usage: Union[BaseModel, dict]) -> Optional[int]:
865+
if isinstance(usage, BaseModel):
866+
usage = usage.__dict__
867+
server_tool_use = usage.get("server_tool_use")
868+
web_search_requests = (
869+
server_tool_use.get("web_search_requests")
870+
if isinstance(server_tool_use, dict)
871+
else getattr(server_tool_use, "web_search_requests", None)
872+
)
873+
if isinstance(web_search_requests, int) and not isinstance(
874+
web_search_requests, bool
875+
):
876+
return web_search_requests
877+
return None
859878

860879

861880
def _parse_usage_model(
@@ -924,6 +943,11 @@ def _parse_usage_model(
924943
):
925944
parsed_usage["reasoning"] = usage["output_token_details"].get("reasoning")
926945

946+
# Anthropic reports the exact number of web searches of a response
947+
web_search_requests = _extract_web_search_requests(usage)
948+
if web_search_requests is not None:
949+
parsed_usage["web_search"] = web_search_requests
950+
927951
field_mapping = {
928952
"input": "input_tokens",
929953
"output": "output_tokens",
@@ -938,6 +962,7 @@ def _parse_usage_model(
938962
},
939963
cache_write_5m_tokens=parsed_usage.get("cache_write_5m"),
940964
cache_write_1h_tokens=parsed_usage.get("cache_write_1h"),
965+
web_search_count=parsed_usage.get("web_search"),
941966
)
942967
# For Anthropic providers, LangChain reports input_tokens as the sum of all input tokens.
943968
# Our cost calculation expects them to be separate for Anthropic, so we subtract cache tokens.
@@ -975,12 +1000,19 @@ def _parse_usage(
9751000
reasoning_tokens=None,
9761001
)
9771002

1003+
# The generations loop below can replace the llm_output usage, which is the
1004+
# only place langchain-anthropic reports server_tool_use, so read it here.
1005+
web_search_count: Optional[int] = None
1006+
9781007
if response.llm_output is not None:
9791008
for key in llm_usage_keys:
9801009
if response.llm_output.get(key):
9811010
llm_usage = _parse_usage_model(
9821011
response.llm_output[key], provider, model
9831012
)
1013+
web_search_count = _extract_web_search_requests(
1014+
response.llm_output[key]
1015+
)
9841016
break
9851017

9861018
if hasattr(response, "generations"):
@@ -1026,6 +1058,9 @@ def _parse_usage(
10261058
llm_usage = _parse_usage_model(chunk_usage, provider, model)
10271059
break
10281060

1061+
if llm_usage.web_search_count is None:
1062+
llm_usage.web_search_count = web_search_count
1063+
10291064
return llm_usage
10301065

10311066

‎posthog/test/ai/langchain/test_callbacks.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2965,6 +2965,57 @@ def test_stop_reason_resolution(
29652965
assert props.get("$ai_stop_reason") == expected
29662966

29672967

2968+
@pytest.mark.parametrize(
2969+
"server_tool_use,expected",
2970+
[
2971+
# Anthropic reports the exact number of searches
2972+
({"web_search_requests": 2}, 2),
2973+
# no search is not worth an event property
2974+
({"web_search_requests": 0}, None),
2975+
# the provider reports nothing
2976+
(None, None),
2977+
],
2978+
)
2979+
def test_web_search_count(mock_client, server_tool_use, expected):
2980+
from langchain_core.outputs import ChatGeneration, LLMResult
2981+
2982+
cb = CallbackHandler(mock_client)
2983+
run_id = uuid.uuid4()
2984+
cb._set_llm_metadata(
2985+
serialized={},
2986+
run_id=run_id,
2987+
messages=[{"role": "user", "content": "test"}],
2988+
metadata={"ls_provider": "anthropic", "ls_model_name": "claude-sonnet"},
2989+
)
2990+
usage = {"input_tokens": 10, "output_tokens": 5}
2991+
if server_tool_use is not None:
2992+
usage["server_tool_use"] = server_tool_use
2993+
# langchain-anthropic always sets usage_metadata on the message, which the
2994+
# generations loop prefers over llm_output["usage"]
2995+
response = LLMResult(
2996+
generations=[
2997+
[
2998+
ChatGeneration(
2999+
message=AIMessage(
3000+
content="r",
3001+
usage_metadata={
3002+
"input_tokens": 10,
3003+
"output_tokens": 5,
3004+
"total_tokens": 15,
3005+
},
3006+
)
3007+
)
3008+
]
3009+
],
3010+
llm_output={"usage": usage},
3011+
)
3012+
3013+
cb._pop_run_and_capture_generation(run_id, None, response)
3014+
3015+
props = mock_client.capture.call_args.kwargs["properties"]
3016+
assert props.get("$ai_web_search_count") == expected
3017+
3018+
29683019
class TestGetHttpStatus:
29693020
def test_uses_the_status_code_of_an_http_error(self):
29703021
from posthog.ai.langchain.callbacks import _get_http_status

‎references/public_api_snapshot.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,7 @@ attribute posthog.ai.langchain.callbacks.ModelUsage.cache_write_tokens: Optional
596596
attribute posthog.ai.langchain.callbacks.ModelUsage.input_tokens: Optional[int]
597597
attribute posthog.ai.langchain.callbacks.ModelUsage.output_tokens: Optional[int]
598598
attribute posthog.ai.langchain.callbacks.ModelUsage.reasoning_tokens: Optional[int]
599+
attribute posthog.ai.langchain.callbacks.ModelUsage.web_search_count: Optional[int] = None
599600
attribute posthog.ai.langchain.callbacks.RunMetadata = Union[SpanMetadata, GenerationMetadata]
600601
attribute posthog.ai.langchain.callbacks.RunMetadataStorage = Dict[UUID, RunMetadata]
601602
attribute posthog.ai.langchain.callbacks.SpanMetadata.end_time: Optional[float]
@@ -1234,7 +1235,7 @@ class posthog.ai.gemini.gemini_converter.GeminiMessage
12341235
class posthog.ai.gemini.gemini_converter.GeminiPart
12351236
class posthog.ai.langchain.callbacks.CallbackHandler(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[Dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[Dict[str, Any]] = None)
12361237
class posthog.ai.langchain.callbacks.GenerationMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], provider: Optional[str] = None, model: Optional[str] = None, model_params: Optional[Dict[str, Any]] = None, base_url: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, posthog_properties: Optional[Dict[str, Any]] = None)
1237-
class posthog.ai.langchain.callbacks.ModelUsage(input_tokens: Optional[int], output_tokens: Optional[int], cache_write_tokens: Optional[int], cache_read_tokens: Optional[int], reasoning_tokens: Optional[int], cache_write_5m_tokens: Optional[int] = None, cache_write_1h_tokens: Optional[int] = None)
1238+
class posthog.ai.langchain.callbacks.ModelUsage(input_tokens: Optional[int], output_tokens: Optional[int], cache_write_tokens: Optional[int], cache_read_tokens: Optional[int], reasoning_tokens: Optional[int], cache_write_5m_tokens: Optional[int] = None, cache_write_1h_tokens: Optional[int] = None, web_search_count: Optional[int] = None)
12381239
class posthog.ai.langchain.callbacks.SpanMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any])
12391240
class posthog.ai.langchain.middleware.PostHogMiddleware(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[dict[str, Any]] = None)
12401241
class posthog.ai.openai.openai.OpenAI(posthog_client: Optional[PostHogClient] = None, **kwargs)

0 commit comments

Comments
 (0)