From e8a22d8a1e0cf2af12f9fb4d683f5b3108b525ae Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Thu, 24 Sep 2026 19:21:18 -0300 Subject: [PATCH 01/11] feat(growth): turn suggestion backend Classify each completed PostHog AI turn and publish an end-of-turn suggestion frame. The task runs on a new posthog_ai Celery queue, a short Redis lock drops duplicate turn reports before they are queued, and the offer ledger in the task state is the only stored outcome of a card: a reload reads it from the new state endpoint. Co-Authored-By: Claude Opus 5.5 (1M context) --- bin/celery-queues.env | 2 +- posthog/celery_queues.py | 1 + products/posthog_ai/backend/api/__init__.py | 3 +- .../backend/api/turn_suggestions.py | 86 ++ products/posthog_ai/backend/receivers.py | 8 +- products/posthog_ai/backend/routes.py | 8 +- products/posthog_ai/backend/tasks.py | 11 + .../backend/tests/test_turn_suggestions.py | 1043 +++++++++++++++++ .../tests/test_turn_suggestions_api.py | 122 ++ .../backend/turn_suggestions/__init__.py | 0 .../backend/turn_suggestions/classifier.py | 161 +++ .../backend/turn_suggestions/dispatch.py | 58 + .../backend/turn_suggestions/drafter.py | 187 +++ .../backend/turn_suggestions/judgment.py | 339 ++++++ .../backend/turn_suggestions/offer_ledger.py | 221 ++++ .../backend/turn_suggestions/service.py | 260 ++++ .../backend/turn_suggestions/transcript.py | 401 +++++++ .../backend/turn_suggestions/verdict.py | 139 +++ .../frontend/generated/api.schemas.ts | 49 + products/posthog_ai/frontend/generated/api.ts | 54 + .../posthog_ai/frontend/generated/api.zod.ts | 19 + products/posthog_ai/mcp/tools.yaml | 6 + services/mcp/src/api/generated.ts | 49 + tach.toml | 1 + 24 files changed, 3224 insertions(+), 4 deletions(-) create mode 100644 products/posthog_ai/backend/api/turn_suggestions.py create mode 100644 products/posthog_ai/backend/tasks.py create mode 100644 products/posthog_ai/backend/tests/test_turn_suggestions.py create mode 100644 products/posthog_ai/backend/tests/test_turn_suggestions_api.py create mode 100644 products/posthog_ai/backend/turn_suggestions/__init__.py create mode 100644 products/posthog_ai/backend/turn_suggestions/classifier.py create mode 100644 products/posthog_ai/backend/turn_suggestions/dispatch.py create mode 100644 products/posthog_ai/backend/turn_suggestions/drafter.py create mode 100644 products/posthog_ai/backend/turn_suggestions/judgment.py create mode 100644 products/posthog_ai/backend/turn_suggestions/offer_ledger.py create mode 100644 products/posthog_ai/backend/turn_suggestions/service.py create mode 100644 products/posthog_ai/backend/turn_suggestions/transcript.py create mode 100644 products/posthog_ai/backend/turn_suggestions/verdict.py diff --git a/bin/celery-queues.env b/bin/celery-queues.env index 5cd934b61789..fb53a511a82b 100644 --- a/bin/celery-queues.env +++ b/bin/celery-queues.env @@ -2,4 +2,4 @@ # Important: Add new queues to make Celery consume tasks from them. # NOTE: Keep in sync with posthog/tasks/utils.py -CELERY_WORKER_QUEUES=ai_research_privacy,celery,stats,email,analytics_queries,analytics_limited,long_running,exports,subscription_delivery,usage_reports,integrations,feature_flags,feature_flags_long_running +CELERY_WORKER_QUEUES=ai_research_privacy,celery,stats,email,analytics_queries,analytics_limited,long_running,exports,subscription_delivery,usage_reports,integrations,feature_flags,feature_flags_long_running,posthog_ai diff --git a/posthog/celery_queues.py b/posthog/celery_queues.py index a961189c9b94..cc8926d8254d 100644 --- a/posthog/celery_queues.py +++ b/posthog/celery_queues.py @@ -42,3 +42,4 @@ class CeleryQueue(Enum): INTEGRATIONS = "integrations" FEATURE_FLAGS = "feature_flags" FEATURE_FLAGS_LONG_RUNNING = "feature_flags_long_running" + POSTHOG_AI = "posthog_ai" diff --git a/products/posthog_ai/backend/api/__init__.py b/products/posthog_ai/backend/api/__init__.py index b96e41966634..0f6e42d9a69c 100644 --- a/products/posthog_ai/backend/api/__init__.py +++ b/products/posthog_ai/backend/api/__init__.py @@ -1,3 +1,4 @@ from .mcp_tools import MCPToolsViewSet +from .turn_suggestions import TurnSuggestionsViewSet -__all__ = ["MCPToolsViewSet"] +__all__ = ["MCPToolsViewSet", "TurnSuggestionsViewSet"] diff --git a/products/posthog_ai/backend/api/turn_suggestions.py b/products/posthog_ai/backend/api/turn_suggestions.py new file mode 100644 index 000000000000..2ba4c9a75a09 --- /dev/null +++ b/products/posthog_ai/backend/api/turn_suggestions.py @@ -0,0 +1,86 @@ +from drf_spectacular.utils import OpenApiResponse +from rest_framework import serializers +from rest_framework.decorators import action +from rest_framework.exceptions import NotFound +from rest_framework.response import Response +from rest_framework.viewsets import GenericViewSet + +from posthog.api.mixins import ValidatedRequest, validated_request +from posthog.api.routing import TeamAndOrgViewSetMixin + +from products.posthog_ai.backend.turn_suggestions.offer_ledger import TurnSuggestionResolution +from products.posthog_ai.backend.turn_suggestions.service import resolve_turn_suggestion, turn_suggestion_state +from products.tasks.backend.facade.api import task_visible + + +class ResolveTurnSuggestionSerializer(serializers.Serializer): + task_id = serializers.UUIDField( + help_text="ID of the PostHog AI conversation (task) the suggestion card belongs to." + ) + turn_index = serializers.IntegerField( + min_value=0, help_text="Zero-based index of the conversation turn the suggestion card was shown under." + ) + resolution = serializers.ChoiceField( + choices=TurnSuggestionResolution.choices, + help_text="What the user did with the card: `dismissed` mutes suggestions for the rest of the conversation, `accepted` means the offered scout, notebook, alert or subscription was created.", + ) + + +class ResolveTurnSuggestionResponseSerializer(serializers.Serializer): + recorded = serializers.BooleanField( + help_text="Whether a suggestion card existed for that turn and this call recorded its outcome. A card keeps the first outcome recorded for it." + ) + + +class TurnSuggestionStateQuerySerializer(serializers.Serializer): + task_id = serializers.UUIDField( + help_text="ID of the PostHog AI conversation (task) to read suggestion outcomes for." + ) + + +class TurnSuggestionStateSerializer(serializers.Serializer): + muted = serializers.BooleanField( + help_text="Whether the user dismissed a suggestion card in this conversation, which stops further cards." + ) + resolved_turns = serializers.ListField( + child=serializers.IntegerField(min_value=0), + help_text="Zero-based indexes of the turns whose suggestion card the user already dismissed or accepted.", + ) + + +class TurnSuggestionsViewSet(TeamAndOrgViewSetMixin, GenericViewSet): + scope_object = "task" + + @validated_request( + query_serializer=TurnSuggestionStateQuerySerializer, + responses={200: OpenApiResponse(response=TurnSuggestionStateSerializer)}, + summary="Read which PostHog AI turn suggestion cards the user already resolved", + ) + @action(detail=False, methods=["get"], required_scopes=["task:read"]) + def state(self, request: ValidatedRequest, *args, **kwargs) -> Response: + task_id = str(request.validated_query_data["task_id"]) + if not task_visible(task_id, self.team_id, request.user.id): + raise NotFound() + state = turn_suggestion_state(task_id, self.team_id) + return Response( + TurnSuggestionStateSerializer({"muted": state.muted, "resolved_turns": list(state.resolved_turns)}).data + ) + + @validated_request( + request_serializer=ResolveTurnSuggestionSerializer, + responses={200: OpenApiResponse(response=ResolveTurnSuggestionResponseSerializer)}, + summary="Record what the user did with a PostHog AI turn suggestion card", + ) + @action(detail=False, methods=["post"], required_scopes=["task:write"]) + def resolve(self, request: ValidatedRequest, *args, **kwargs) -> Response: + task_id = str(request.validated_data["task_id"]) + # A dismissal mutes the conversation for its creator, so a teammate who can only read it must not resolve cards. + if not task_visible(task_id, self.team_id, request.user.id, for_control=True): + raise NotFound() + recorded = resolve_turn_suggestion( + task_id, + self.team_id, + turn_index=request.validated_data["turn_index"], + resolution=TurnSuggestionResolution(request.validated_data["resolution"]), + ) + return Response(ResolveTurnSuggestionResponseSerializer({"recorded": recorded}).data) diff --git a/products/posthog_ai/backend/receivers.py b/products/posthog_ai/backend/receivers.py index be2498a9e948..0a55516d22d7 100644 --- a/products/posthog_ai/backend/receivers.py +++ b/products/posthog_ai/backend/receivers.py @@ -1,6 +1,6 @@ """Keep a copied chat and its task in step once the task runs for real. -Both hooks connect through the tasks facade so no model class crosses the product boundary. +Every hook connects through the tasks facade so no model class crosses the product boundary. - Before a run starts: the copy runs a few seconds behind the chat, so a run started in that window would resume without the last turn. The guard copies what is still missing and refuses the run @@ -10,6 +10,7 @@ path too, so a chat can never fork back into LangGraph after it has been continued as a task. - When tasks are read: a user without the sandbox runtime does not see the copies of their chats. The copy is written for everyone so the switch is instant later, but until then their chats are chats. +- After a turn completes: the end-of-turn suggestion is queued (``turn_suggestions/dispatch.py``). """ import logging @@ -22,9 +23,11 @@ from posthog.models import Team, User from products.posthog_ai.backend.models.assistant import Conversation +from products.posthog_ai.backend.turn_suggestions.dispatch import enqueue_turn_suggestion_on_turn_completed from products.tasks.backend.facade.task_run_signals import ( TaskOriginProduct, connect_task_run_post_save, + connect_task_run_turn_completed, register_task_read_exclusion, register_task_run_start_guard, ) @@ -51,6 +54,9 @@ def connect() -> None: connect_task_run_post_save( mark_conversation_sandbox_on_first_run, dispatch_uid="posthog_ai_conversation_follows_task_run" ) + connect_task_run_turn_completed( + enqueue_turn_suggestion_on_turn_completed, dispatch_uid="posthog_ai_turn_suggestion_on_turn_completed" + ) def copied_chats_hidden_without_sandbox_mode(team_id: int, user_id: int | None) -> Iterable[UUID]: diff --git a/products/posthog_ai/backend/routes.py b/products/posthog_ai/backend/routes.py index 22252ed457b3..18689d06504c 100644 --- a/products/posthog_ai/backend/routes.py +++ b/products/posthog_ai/backend/routes.py @@ -1,6 +1,6 @@ from posthog.api.routing import RouterRegistry -from products.posthog_ai.backend.api import MCPToolsViewSet +from products.posthog_ai.backend.api import MCPToolsViewSet, TurnSuggestionsViewSet from products.posthog_ai.backend.presentation.terminal_ai import TerminalAIViewSet @@ -12,3 +12,9 @@ def register_routes(routers: RouterRegistry) -> None: "project_mcp_tools", ["team_id"], ) + routers.projects.register( + r"turn_suggestions", + TurnSuggestionsViewSet, + "project_turn_suggestions", + ["team_id"], + ) diff --git a/products/posthog_ai/backend/tasks.py b/products/posthog_ai/backend/tasks.py new file mode 100644 index 000000000000..1c1a4079ffbc --- /dev/null +++ b/products/posthog_ai/backend/tasks.py @@ -0,0 +1,11 @@ +from celery import shared_task + +from posthog.celery_queues import CeleryQueue + +from products.posthog_ai.backend.turn_suggestions.service import generate_turn_suggestion + + +# No retries: a suggestion that misses its turn is worth less than a stale one arriving mid-conversation. +@shared_task(ignore_result=True, queue=CeleryQueue.POSTHOG_AI.value, soft_time_limit=90, time_limit=120) +def generate_turn_suggestion_task(*, run_id: str, team_id: int) -> None: + generate_turn_suggestion(run_id, team_id) diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions.py b/products/posthog_ai/backend/tests/test_turn_suggestions.py new file mode 100644 index 000000000000..46152649d9d6 --- /dev/null +++ b/products/posthog_ai/backend/tests/test_turn_suggestions.py @@ -0,0 +1,1043 @@ +import json +from dataclasses import replace +from datetime import date +from typing import Any + +from posthog.test.base import BaseTest +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase + +import requests +from parameterized import parameterized + +from posthog.celery_queues import CeleryQueue +from posthog.egress.typesafe import ( + Answer, + ChoiceAnswer, + ChoiceQuestion, + NoulAnswer, + SystemOneResult, + TypeSafeRequestFailed, +) +from posthog.llm.gateway_client import GatewayNotConfiguredError + +from products.posthog_ai.backend.tasks import generate_turn_suggestion_task +from products.posthog_ai.backend.turn_suggestions.classifier import CardCopy, card_copy, classify_turn, pick_offer +from products.posthog_ai.backend.turn_suggestions.dispatch import TURN_SETTLE_SECONDS, enqueue_turn_suggestion +from products.posthog_ai.backend.turn_suggestions.drafter import DRAFT_MODEL, draft_scout, render_turn_prompt +from products.posthog_ai.backend.turn_suggestions.judgment import ( + TurnJudgment, + build_judge_questions, + build_judge_state, + judge_turn, +) +from products.posthog_ai.backend.turn_suggestions.offer_ledger import ( + STATE_KEY, + OfferRecord, + OfferStatus, + TurnSuggestionResolution, + claim_turn, + read_ledger, + resolve_offer, +) +from products.posthog_ai.backend.turn_suggestions.service import ( + MAX_TRANSCRIPT_LOG_BYTES, + TURN_SUGGESTION_METHOD, + TurnSuggestionOutcome, + generate_turn_suggestion, +) +from products.posthog_ai.backend.turn_suggestions.transcript import ( + ErrorIssueRef, + SavedInsightRef, + build_turn_transcript, + redact_values, +) +from products.posthog_ai.backend.turn_suggestions.verdict import ( + AlertDirection, + AlertDraft, + Draft, + ErrorAlertDraft, + IncidentOutline, + NotebookDraft, + NotebookTemplate, + OfferKind, + ScoutCadence, + ScoutDraft, + ScoutMode, + SubscriptionDraft, + TurnIntent, + TurnVerdict, +) +from products.tasks.backend.facade.api import TaskClientProvenance +from products.tasks.backend.facade.contracts import StreamNotificationDelivery +from products.tasks.backend.facade.task_run_signals import task_run_turn_completed +from products.tasks.backend.models import Task, TaskRun + +SERVICE = "products.posthog_ai.backend.turn_suggestions.service" + + +def _offer(turn_index: int, *, run_id: str = "run-1") -> dict: + return {"turn_index": turn_index, "run_id": run_id, "kind": "scout", "status": "offered"} + + +ALL_OFFERS = frozenset(OfferKind) - {OfferKind.NONE} + + +def _notification(method: str, params: dict) -> dict: + return {"type": "notification", "notification": {"method": method, "params": params}} + + +def _user_message(text: str) -> dict: + return _notification("_posthog/user_message", {"content": text}) + + +def _wire_user_message(text: str) -> dict: + return _notification( + "session/update", + {"update": {"sessionUpdate": "user_message", "content": {"type": "text", "text": text}}}, + ) + + +def _agent_text(text: str) -> dict: + return _notification( + "session/update", + {"update": {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": text}}}, + ) + + +def _agent_final(text: str, message_id: str | None = "m1") -> dict: + update: dict = {"sessionUpdate": "agent_message", "content": {"type": "text", "text": text}} + if message_id is not None: + update["messageId"] = message_id + return _notification("session/update", {"update": update}) + + +def _builtin_tool_call(tool_call_id: str, tool_name: str, raw_input: dict) -> dict: + return _notification( + "session/update", + { + "update": { + "sessionUpdate": "tool_call", + "toolCallId": tool_call_id, + "title": tool_name, + "status": "completed", + "rawInput": raw_input, + "_meta": {"claudeCode": {"toolName": tool_name}}, + } + }, + ) + + +def _exec_tool_call(tool_call_id: str, command: str, status: str = "in_progress", output: dict | None = None) -> dict: + update: dict = { + "sessionUpdate": "tool_call", + "toolCallId": tool_call_id, + "title": "exec", + "serverName": "posthog", + "toolName": "exec", + "status": status, + "rawInput": {"command": command}, + "_meta": {"claudeCode": {"toolName": "mcp__posthog__exec"}}, + } + if output is not None: + update["rawOutput"] = output + return _notification("session/update", {"update": update}) + + +def _tool_call_update( + tool_call_id: str, status: str, raw_input: dict | None = None, output: dict | None = None +) -> dict: + update: dict = {"sessionUpdate": "tool_call_update", "toolCallId": tool_call_id, "status": status} + if raw_input is not None: + update["rawInput"] = raw_input + if output is not None: + update["rawOutput"] = output + return _notification("session/update", {"update": update}) + + +def _metric_turn() -> list[dict]: + return [ + _user_message( + "\n- dashboard: 12\n\nHow many signups did we get this week?" + ), + _exec_tool_call( + "t1", 'call --json query-trends {"series":[{"event":"signed_up"}],"dateRange":{"date_from":"-7d"}}' + ), + _tool_call_update("t1", "completed"), + _agent_text("You had 412 signups this week, up 8% on last week."), + _notification("_posthog/turn_complete", {"stopReason": "end_turn"}), + ] + + +SAVED_INSIGHT = SavedInsightRef(short_id="abc123", insight_id=42, name="Signups", query_kind="TrendsQuery") +ERROR_ISSUE = ErrorIssueRef(issue_id="0199c0de-1111-7000-8000-0000000000aa", name="Checkout error") + + +def _saved_insight_turn(query_kind: str = "TrendsQuery") -> list[dict]: + return [ + _user_message("How many signups did we get this week? Save it."), + _exec_tool_call( + "t1", + 'call insight-create {"name":"Signups"}', + "completed", + {"id": 42, "short_id": "abc123", "name": "Signups", "query": {"kind": query_kind, "series": []}}, + ), + _agent_text("Saved. You had 412 signups this week."), + ] + + +def _error_turn() -> list[dict]: + return [ + _user_message("Why did checkout break on Tuesday?"), + _exec_tool_call( + "t1", + "call query-error-tracking-issues-list {}", + "completed", + {"results": [{"id": ERROR_ISSUE.issue_id, "name": "Checkout error"}], "hasMore": False}, + ), + _agent_text("A checkout error started at 14:10."), + ] + + +_DRAFTS: dict[OfferKind, Draft] = { + OfferKind.SCOUT: ScoutDraft( + mode=ScoutMode.REPORT, + display_name="Weekly signups", + description="Counts signed_up events for the last 7 days.", + body="# Weekly signups\n\nQuery `signed_up` for the last 7 days...", + cadence=ScoutCadence.WEEKLY, + ), + OfferKind.NOTEBOOK: NotebookDraft( + title="Why signups dropped on Tuesday", + summary="A checkout error cut Tuesday's signups by a third.", + incident=IncidentOutline(timeline="- 14:10 release", cause="Checkout error.", fix="Rolled back."), + ), + OfferKind.ALERT: AlertDraft(insight=SAVED_INSIGHT, direction=AlertDirection.DECREASE, change_percent=20), + OfferKind.SUBSCRIPTION: SubscriptionDraft(insight=SAVED_INSIGHT, cadence=ScoutCadence.WEEKLY), + OfferKind.ERROR_ALERT: ErrorAlertDraft(issue=ERROR_ISSUE), +} + + +def _verdict(offer: OfferKind = OfferKind.SCOUT, intent: TurnIntent = TurnIntent.METRIC_STATE) -> TurnVerdict: + return TurnVerdict( + intent=intent, + show_probability=0.92, + picked=offer, + offer_probabilities={offer.value: 0.8}, + draft=_DRAFTS.get(offer), + ) + + +class TestBuildTurnTranscript(SimpleTestCase): + def test_resolves_exec_wrapped_tools_and_strips_context_blocks(self): + transcript = build_turn_transcript(_metric_turn()) + + assert transcript.human_messages == ("How many signups did we get this week?",) + assert transcript.assistant_text == "You had 412 signups this week, up 8% on last week." + assert [(call.name, call.status) for call in transcript.tool_calls] == [("query-trends", "completed")] + assert '"series"' in transcript.tool_calls[0].args_preview + + @parameterized.expand( + [ + ("discovery_verb", "search signups", None), + ("info_verb", "info insight-create", None), + ("flag_before_subtool", "call --confirm --no-skills insight-create {}", "insight-create"), + ("unknown_verb", "frobnicate x", None), + ] + ) + def test_exec_command_grammar(self, _name: str, command: str, expected_name: str | None): + transcript = build_turn_transcript([_user_message("q"), _exec_tool_call("t1", command)]) + + names = [call.name for call in transcript.tool_calls] + assert names == ([] if expected_name is None else [expected_name]) + + @parameterized.expand([("no_command_yet", {}), ("partial_call", {"command": "call"})]) + def test_exec_command_arriving_in_a_later_update_is_resolved(self, _name: str, first_input: dict): + entries = [ + _user_message("q"), + _notification( + "session/update", + { + "update": { + "sessionUpdate": "tool_call", + "toolCallId": "t1", + "serverName": "posthog", + "toolName": "exec", + "status": "pending", + "rawInput": first_input, + } + }, + ), + _tool_call_update("t1", "completed", {"command": 'call execute-sql {"query":"SELECT 1"}'}), + ] + + transcript = build_turn_transcript(entries) + + assert [(call.name, call.status) for call in transcript.tool_calls] == [("execute-sql", "completed")] + + @parameterized.expand( + [ + ("ids_on_both", "m1", "m1"), + ("id_only_on_chunks", "m1", None), + ("id_only_on_final", None, "m1"), + ("no_ids", None, None), + ] + ) + def test_a_closing_agent_message_replaces_its_streamed_chunks( + self, _name: str, chunk_message_id: str | None, final_message_id: str | None + ): + def chunk(text: str) -> dict: + update: dict = {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": text}} + if chunk_message_id is not None: + update["messageId"] = chunk_message_id + return _notification("session/update", {"update": update}) + + entries = [ + _user_message("q"), + chunk("You had 412 "), + chunk("signups."), + _agent_final("You had 412 signups.", final_message_id), + ] + + transcript = build_turn_transcript(entries) + + assert transcript.assistant_text == "You had 412 signups." + + def test_builtin_tools_keep_their_name_and_a_preview_of_their_input(self): + entries = [_user_message("q"), _builtin_tool_call("t1", "Bash", {"command": "ls -la", "timeout": 5})] + + transcript = build_turn_transcript(entries) + + assert [(call.name, call.args_preview) for call in transcript.tool_calls] == [("Bash", "ls -la")] + + def test_earlier_turns_are_kept_as_context_and_only_the_last_turn_is_folded(self): + entries = [ + *_metric_turn(), + _user_message("Why did it drop on Tuesday?"), + _exec_tool_call("t2", "call query-session-recordings-list {}", status="completed"), + _agent_text("Tuesday's drop lines up with a checkout error."), + ] + + transcript = build_turn_transcript(entries) + + assert transcript.last_human_message == "Why did it drop on Tuesday?" + assert [call.name for call in transcript.tool_calls] == ["query-session-recordings-list"] + assert transcript.assistant_text == "Tuesday's drop lines up with a checkout error." + assert [(turn.question, turn.tool_names, turn.answer_excerpt) for turn in transcript.earlier_turns] == [ + ( + "How many signups did we get this week?", + ("query-trends",), + "You had 412 signups this week, up 8% on last week.", + ) + ] + + def test_saved_insights_and_error_issues_come_from_the_tool_payloads(self): + assert build_turn_transcript(_saved_insight_turn()).saved_insights == (SAVED_INSIGHT,) + assert build_turn_transcript(_error_turn()).error_issues == (ERROR_ISSUE,) + assert build_turn_transcript(_metric_turn()).saved_insights == () + + @parameterized.expand( + [ + ( + "wire_user_messages_start_turns", + [ + _wire_user_message("How many users today?"), + _agent_text("120."), + _wire_user_message("Why did checkout break?"), + _agent_text("A deploy."), + ], + ("How many users today?", "Why did checkout break?"), + "A deploy.", + ), + ( + "resume_prompt_is_not_a_turn", + [ + _wire_user_message("You are resuming a previous conversation. Here is what happened..."), + _wire_user_message("How many users today?"), + _agent_text("120."), + ], + ("How many users today?",), + "120.", + ), + ( + "a_turn_in_both_wire_forms_counts_once", + [ + _user_message("How many users today?"), + _wire_user_message("How many users today?"), + _agent_text("120."), + ], + ("How many users today?",), + "120.", + ), + ( + "hidden_content_blocks_are_not_a_turn", + [ + _notification( + "_posthog/user_message", + {"content": [{"type": "text", "text": "Context", "_meta": {"ui": {"hidden": True}}}]}, + ), + _user_message("How many users today?"), + _agent_text("120."), + ], + ("How many users today?",), + "120.", + ), + ( + "session_prompt_is_not_a_turn", + [ + _notification("session/prompt", {"prompt": [{"type": "text", "text": "How many users today?"}]}), + _agent_text("120."), + ], + (), + "120.", + ), + ] + ) + def test_counts_human_messages_the_way_the_thread_renders_them( + self, _name: str, entries: list[dict], expected_messages: tuple[str, ...], expected_answer: str + ): + transcript = build_turn_transcript(entries) + + assert transcript.human_messages == expected_messages + assert transcript.assistant_text == expected_answer + + def test_prompt_carries_the_instruction_context_and_resolved_tools(self): + entries = [*_saved_insight_turn(), _user_message("Break that down by country"), _agent_text("Mostly the US.")] + + prompt = render_turn_prompt( + build_turn_transcript(entries), today=date(2026, 9, 16), instruction="Layout: incident." + ) + + assert "Layout: incident." in prompt + assert "- Q: How many signups did we get this week? Save it." in prompt + assert "\nBreak that down by country\n" in prompt + assert "posthog_untrusted_context" not in prompt + + @parameterized.expand( + [ + ( + "counts_and_percents", + "You had 412 signups, up 8% on last week.", + "You had signups, up % on last week.", + ), + ( + "money_dates_and_times", + "Revenue was $1,234.50 on 2026-09-16 at 14:10.", + "Revenue was on at .", + ), + ("identifiers_keep_their_digits", "p95 latency in v2 fell -12%.", "p95 latency in v2 fell -%."), + ( + "contact_details_and_ids", + "Ask ada@example.com about 0199c0de-1111-7000-8000-0000000000aa at https://example.com/x", + "Ask about at ", + ), + ] + ) + def test_redact_values_masks_what_an_answer_reports(self, _name: str, text: str, expected: str): + assert redact_values(text) == expected + + +def _judgment(**overrides: Any) -> TurnJudgment: + judgment = TurnJudgment( + show_probability=0.9, + intent=TurnIntent.METRIC_STATE, + offer=OfferKind.SCOUT, + offer_probabilities={"scout": 0.7, "notebook": 0.3}, + scout_mode=ScoutMode.REPORT, + cadence=ScoutCadence.WEEKLY, + notebook_template=NotebookTemplate.CONVERSATION, + alert_direction=AlertDirection.DECREASE, + alert_change_percent=20, + insight=SAVED_INSIGHT, + error_issue=ERROR_ISSUE, + issue_resolved_probability=0.9, + ) + return replace(judgment, **overrides) + + +def _answers(nouls: dict[str, float], choices: dict[str, str]) -> SystemOneResult: + answers: dict[str, Answer] = {key: NoulAnswer(probability=value) for key, value in nouls.items()} + answers.update( + {key: ChoiceAnswer(choice=value, probabilities={value: 1.0}, confidence=1.0) for key, value in choices.items()} + ) + return SystemOneResult(model="jev-1.13.0", answers=answers, input_tokens=300) + + +FUNNEL_INSIGHT = replace(SAVED_INSIGHT, query_kind="FunnelsQuery") +JUDGMENT = "products.posthog_ai.backend.turn_suggestions.judgment" +CLASSIFIER = "products.posthog_ai.backend.turn_suggestions.classifier" + + +class TestJudgeTurn(SimpleTestCase): + @parameterized.expand( + [("saved_insight", _saved_insight_turn(), "abc123"), ("error_issue", _error_turn(), ERROR_ISSUE.issue_id)] + ) + def test_jev_sees_the_question_and_a_masked_answer_but_no_outputs_or_ids( + self, _name: str, entries: list[dict], hidden_id: str + ): + transcript = build_turn_transcript(entries) + + state = build_judge_state(transcript) + questions = build_judge_questions(transcript, ALL_OFFERS) + + wire = json.dumps({"state": state, "questions": {key: q.to_json() for key, q in questions.items()}}) + assert hidden_id not in wire + latest = state["latest_turn"] + assert isinstance(latest, dict) + assert latest["question"] == transcript.last_human_message + assert "" in str(latest["answer"]) and not any(char.isdigit() for char in str(latest["answer"])) + # A "none" option would let the offer answer veto `show_offer` and cap the offer rate. + offer = questions["offer"] + assert isinstance(offer, ChoiceQuestion) and set(offer.criteria) == ALL_OFFERS + + def test_a_turn_with_one_possible_offer_skips_the_offer_question(self): + transcript = build_turn_transcript(_metric_turn()) + only_scout = frozenset({OfferKind.SCOUT}) + answers = _answers({"show_offer": 0.8}, {"intent": "metric_state", "scout_mode": "report", "cadence": "weekly"}) + + with patch(f"{JUDGMENT}.system_one", return_value=answers): + judgment = judge_turn(transcript, available=only_scout) + + assert "offer" not in build_judge_questions(transcript, only_scout) + assert judgment is not None and judgment.offer == OfferKind.SCOUT + + def test_option_keys_map_back_to_the_refs_they_stand_for(self): + answers = _answers( + {"show_offer": 0.8, "issue_resolved": 0.1}, + { + "intent": "metric_state", + "offer": "alert", + "scout_mode": "watch", + "cadence": "daily", + "notebook_template": "conversation", + "alert_direction": "increase", + "alert_change": "large", + "insight": "insight_1", + }, + ) + + with patch(f"{JUDGMENT}.system_one", return_value=answers): + judgment = judge_turn(build_turn_transcript(_saved_insight_turn()), available=ALL_OFFERS) + + assert judgment is not None + assert judgment.offer == OfferKind.ALERT + assert judgment.insight == SAVED_INSIGHT + assert judgment.alert_direction == AlertDirection.INCREASE + assert judgment.alert_change_percent == 50 + assert judgment.cadence == ScoutCadence.DAILY + assert judgment.error_issue is None + + @parameterized.expand( + [("typesafe_error", TypeSafeRequestFailed("HTTP 500")), ("network_error", requests.ConnectionError())] + ) + def test_a_failed_request_returns_none(self, _name: str, error: Exception): + with patch(f"{JUDGMENT}.system_one", side_effect=error): + assert judge_turn(build_turn_transcript(_metric_turn()), available=ALL_OFFERS) is None + + +class TestPickOffer(SimpleTestCase): + @parameterized.expand( + [ + ("offers_what_jev_picked", {}, ALL_OFFERS, OfferKind.SCOUT), + ("below_the_show_threshold", {"show_probability": 0.3}, ALL_OFFERS, OfferKind.NONE), + ("pick_the_project_cannot_act_on", {}, frozenset({OfferKind.NOTEBOOK}), OfferKind.NONE), + ("alert_without_an_insight", {"offer": OfferKind.ALERT, "insight": None}, ALL_OFFERS, OfferKind.NONE), + ("alert_on_a_funnel", {"offer": OfferKind.ALERT, "insight": FUNNEL_INSIGHT}, ALL_OFFERS, OfferKind.NONE), + ( + "subscription_on_a_funnel", + {"offer": OfferKind.SUBSCRIPTION, "insight": FUNNEL_INSIGHT}, + ALL_OFFERS, + OfferKind.SUBSCRIPTION, + ), + ( + "error_alert_on_an_active_issue", + {"offer": OfferKind.ERROR_ALERT, "issue_resolved_probability": 0.2}, + ALL_OFFERS, + OfferKind.NONE, + ), + ("error_alert_on_a_resolved_issue", {"offer": OfferKind.ERROR_ALERT}, ALL_OFFERS, OfferKind.ERROR_ALERT), + ] + ) + def test_policy(self, _name: str, overrides: dict, available: frozenset[OfferKind], expected: OfferKind): + assert pick_offer(_judgment(**overrides), available) == expected + + +class TestClassifyTurn(SimpleTestCase): + def _classify(self, judgment: TurnJudgment | None, scout_draft: ScoutDraft | None = None): + with ( + patch(f"{CLASSIFIER}.judge_turn", return_value=judgment), + patch(f"{CLASSIFIER}.draft_scout", return_value=scout_draft) as draft_scout_mock, + patch(f"{CLASSIFIER}.draft_notebook") as draft_notebook_mock, + ): + verdict = classify_turn( + build_turn_transcript(_saved_insight_turn()), team_id=1, today=date(2026, 9, 16), available=ALL_OFFERS + ) + return verdict, draft_scout_mock, draft_notebook_mock + + def test_an_alert_is_built_from_the_judgment_without_a_language_model(self): + verdict, draft_scout_mock, draft_notebook_mock = self._classify( + _judgment(offer=OfferKind.ALERT, alert_direction=AlertDirection.INCREASE, alert_change_percent=50) + ) + + draft = AlertDraft(insight=SAVED_INSIGHT, direction=AlertDirection.INCREASE, change_percent=50) + assert verdict is not None and verdict.draft == draft + assert card_copy(draft) == CardCopy( + title="Get an alert when this rises", + description="The alert checks once a day, compares with the day before, and posts to Slack.", + ) + draft_scout_mock.assert_not_called() + draft_notebook_mock.assert_not_called() + + def test_a_scout_is_drafted_with_the_judged_mode_and_cadence(self): + draft = ScoutDraft( + mode=ScoutMode.WATCH, + display_name="Signups watch", + description="Watches signups.", + body="# Signups", + cadence=ScoutCadence.DAILY, + ) + + verdict, draft_scout_mock, _ = self._classify( + _judgment(scout_mode=ScoutMode.WATCH, cadence=ScoutCadence.DAILY), scout_draft=draft + ) + + assert verdict is not None and verdict.draft == draft + assert draft_scout_mock.call_args.kwargs["mode"] == ScoutMode.WATCH + assert draft_scout_mock.call_args.kwargs["cadence"] == ScoutCadence.DAILY + assert card_copy(draft).title == "Get a Slack message when this changes" + + def test_a_failed_draft_keeps_the_pick_but_offers_nothing(self): + verdict, _, _ = self._classify(_judgment(), scout_draft=None) + + assert verdict is not None + assert verdict.picked == OfferKind.SCOUT + assert verdict.draft is None + + def test_a_failed_judgment_returns_none(self): + verdict, draft_scout_mock, _ = self._classify(None) + + assert verdict is None + draft_scout_mock.assert_not_called() + + +def _gateway_reply(payload: dict) -> MagicMock: + client = MagicMock() + client.with_options.return_value = client + client.chat.completions.create.return_value = MagicMock( + choices=[MagicMock(message=MagicMock(content=json.dumps(payload)))] + ) + return client + + +class TestDraftScout(SimpleTestCase): + def _draft(self, content: str) -> tuple[ScoutDraft | None, MagicMock]: + client = _gateway_reply({}) + client.chat.completions.create.return_value = MagicMock(choices=[MagicMock(message=MagicMock(content=content))]) + with patch("products.posthog_ai.backend.turn_suggestions.drafter.get_llm_client", return_value=client): + draft = draft_scout( + build_turn_transcript(_metric_turn()), + team_id=1, + today=date(2026, 9, 16), + mode=ScoutMode.WATCH, + cadence=ScoutCadence.DAILY, + ) + return draft, client + + def test_the_reply_becomes_a_draft_with_the_mode_and_cadence_it_was_given(self): + draft, client = self._draft( + json.dumps({"display_name": " Signups watch ", "description": "Watches signups.", "prompt": "# Signups"}) + ) + + assert draft == ScoutDraft( + mode=ScoutMode.WATCH, + display_name="Signups watch", + description="Watches signups.", + body="# Signups", + cadence=ScoutCadence.DAILY, + ) + request = client.chat.completions.create.call_args.kwargs + assert request["model"] == DRAFT_MODEL + assert request["response_format"]["json_schema"]["strict"] is True + assert "Scout mode: watch." in request["messages"][1]["content"] + + @parameterized.expand( + [ + ("prose", "I cannot tell."), + ("invalid_shape", json.dumps({"display_name": "x"})), + ("empty_prompt", json.dumps({"display_name": "x", "description": "y", "prompt": " "})), + ] + ) + def test_unusable_replies_return_none(self, _name: str, content: str): + draft, _ = self._draft(content) + + assert draft is None + + def test_an_unconfigured_gateway_returns_none(self): + with patch( + "products.posthog_ai.backend.turn_suggestions.drafter.get_llm_client", + side_effect=GatewayNotConfiguredError("LLM_GATEWAY_URL and an API key must be configured"), + ): + draft = draft_scout( + build_turn_transcript(_metric_turn()), + team_id=1, + today=date(2026, 9, 16), + mode=ScoutMode.WATCH, + cadence=ScoutCadence.DAILY, + ) + + assert draft is None + + +class TestEnqueueTurnSuggestion(BaseTest): + def _run(self, origin: str) -> TaskRun: + task = Task.objects.create( + team=self.team, title="t", description="d", origin_product=origin, created_by=self.user + ) + return task.create_run(mode="interactive") + + @parameterized.expand( + [ + ("posthog_ai", Task.OriginProduct.POSTHOG_AI, True), + ("other_product", Task.OriginProduct.USER_CREATED, False), + ] + ) + def test_only_posthog_ai_runs_are_classified(self, _name: str, origin: str, expected: bool): + task_run = self._run(origin) + + with patch("products.posthog_ai.backend.tasks.generate_turn_suggestion_task.apply_async") as apply_async: + assert enqueue_turn_suggestion(task_run) is expected + + assert apply_async.called is expected + if expected: + assert apply_async.call_args.kwargs == { + "kwargs": {"run_id": str(task_run.id), "team_id": self.team.id}, + "countdown": TURN_SETTLE_SECONDS, + } + + def test_a_turn_reported_twice_is_queued_once(self): + task_run = self._run(Task.OriginProduct.POSTHOG_AI) + + with patch("products.posthog_ai.backend.tasks.generate_turn_suggestion_task.apply_async") as apply_async: + task_run_turn_completed.send_robust(sender=TaskRun, task_run=task_run) + task_run_turn_completed.send_robust(sender=TaskRun, task_run=task_run) + + apply_async.assert_called_once() + + def test_the_task_runs_on_the_posthog_ai_queue(self): + assert generate_turn_suggestion_task.queue == CeleryQueue.POSTHOG_AI.value + + +class TestGenerateTurnSuggestion(BaseTest): + def setUp(self): + super().setUp() + task = Task.objects.create( + team=self.team, + title="t", + description="d", + origin_product=Task.OriginProduct.POSTHOG_AI, + created_by=self.user, + ) + self.task_run = task.create_run(mode="interactive") + + patchers = { + "history": patch(f"{SERVICE}.read_task_run_history", return_value=_metric_turn()), + "publish": patch( + f"{SERVICE}.publish_task_run_stream_notification", + return_value=StreamNotificationDelivery(live=True, persisted=True), + ), + "classify": patch(f"{SERVICE}.classify_turn", return_value=_verdict()), + "scouts": patch(f"{SERVICE}.scout_creation_available", return_value=True), + "flag": patch(f"{SERVICE}.feature_enabled_or_false", return_value=True), + "judge": patch(f"{SERVICE}.judge_configured", return_value=True), + "capture": patch(f"{SERVICE}.ph_scoped_capture"), + } + self.mocks = {name: patcher.start() for name, patcher in patchers.items()} + for patcher in patchers.values(): + self.addCleanup(patcher.stop) + + def _generate(self) -> TurnSuggestionOutcome: + return generate_turn_suggestion(str(self.task_run.id), self.team.id) + + def _published_params(self) -> dict: + return self.mocks["publish"].call_args.args[4] + + def _available(self) -> frozenset[OfferKind]: + return self.mocks["classify"].call_args.kwargs["available"] + + def test_publishes_a_scout_suggestion_for_a_metric_turn(self): + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="emitted", reason="scout") + run_id, task_id, team_id, method, params = self.mocks["publish"].call_args.args + assert (run_id, task_id, team_id, method) == ( + self.task_run.id, + self.task_run.task_id, + self.team.id, + TURN_SUGGESTION_METHOD, + ) + assert params["turnIndex"] == 0 + assert params["kind"] == "scout" + assert params["scout"] == { + "mode": "report", + "displayName": "Weekly signups", + "description": "Counts signed_up events for the last 7 days.", + "body": "# Weekly signups\n\nQuery `signed_up` for the last 7 days...", + "cadence": "weekly", + } + assert self._available() == frozenset({OfferKind.NOTEBOOK, OfferKind.SCOUT}) + + @parameterized.expand( + [ + ( + "incident_notebook", + OfferKind.NOTEBOOK, + "notebook", + { + "title": "Why signups dropped on Tuesday", + "summary": "A checkout error cut Tuesday's signups by a third.", + "incident": {"timeline": "- 14:10 release", "cause": "Checkout error.", "fix": "Rolled back."}, + }, + ), + ( + "alert", + OfferKind.ALERT, + "alert", + { + "insightShortId": "abc123", + "insightId": 42, + "insightName": "Signups", + "direction": "decrease", + "changePercent": 20, + }, + ), + ( + "subscription", + OfferKind.SUBSCRIPTION, + "subscription", + { + "insightShortId": "abc123", + "insightId": 42, + "insightName": "Signups", + "cadence": "weekly", + }, + ), + ( + "error_alert", + OfferKind.ERROR_ALERT, + "errorAlert", + {"issueId": ERROR_ISSUE.issue_id, "issueName": "Checkout error"}, + ), + ] + ) + def test_each_offer_kind_publishes_its_own_frame(self, _name: str, offer: OfferKind, key: str, expected: dict): + self.mocks["history"].return_value = _error_turn() if offer == OfferKind.ERROR_ALERT else _saved_insight_turn() + self.mocks["classify"].return_value = _verdict(offer=offer) + + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="emitted", reason=offer.value) + params = self._published_params() + assert params["kind"] == offer.value + assert params[key] == expected + + @parameterized.expand( + [ + ("trends_insight", "TrendsQuery", {OfferKind.NOTEBOOK, OfferKind.SUBSCRIPTION, OfferKind.ALERT}), + ("funnel_insight", "FunnelsQuery", {OfferKind.NOTEBOOK, OfferKind.SUBSCRIPTION}), + ] + ) + def test_offers_follow_what_the_turn_and_project_make_possible(self, _name: str, query_kind: str, expected: set): + self.mocks["history"].return_value = _saved_insight_turn(query_kind) + self.mocks["scouts"].return_value = False + + self._generate() + + assert self._available() == frozenset(expected) + + def test_a_turn_with_nothing_to_offer_never_reaches_the_classifier(self): + self.mocks["scouts"].return_value = False + self.mocks["history"].return_value = [_user_message("Why did signups drop?"), _agent_text("Hard to say.")] + + outcome = self._generate() + + assert outcome.reason == "no_offers_available" + self.mocks["classify"].assert_not_called() + + def test_verdict_without_an_offer_is_recorded_but_not_published(self): + self.mocks["classify"].return_value = _verdict(offer=OfferKind.NONE, intent=TurnIntent.KNOWLEDGE) + + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="skipped", reason="no_offer:knowledge") + self.mocks["publish"].assert_not_called() + capture = self.mocks["capture"].return_value.__enter__.return_value + assert capture.call_args.kwargs["properties"]["emitted"] is False + assert capture.call_args.kwargs["properties"]["offer"] is None + + def test_a_pick_whose_draft_failed_is_a_failure_and_not_published(self): + self.mocks["classify"].return_value = replace(_verdict(), draft=None) + + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="failed", reason="draft_failed") + self.mocks["publish"].assert_not_called() + capture = self.mocks["capture"].return_value.__enter__.return_value + assert capture.call_args.kwargs["properties"]["picked"] == "scout" + assert capture.call_args.kwargs["properties"]["draft_model"] == DRAFT_MODEL + + def test_follow_up_turns_are_classified_with_their_own_turn_index(self): + self.mocks["history"].return_value = [ + *_metric_turn(), + _user_message("Break that down by country"), + _exec_tool_call( + "t2", 'call query-trends {"breakdownFilter":{"breakdown":"$geoip_country_code"}}', "completed" + ), + _agent_text("Most signups came from the US."), + ] + + outcome = self._generate() + + assert outcome.status == "emitted" + assert self._published_params()["turnIndex"] == 1 + transcript = self.mocks["classify"].call_args.args[0] + assert transcript.last_human_message == "Break that down by country" + assert [turn.question for turn in transcript.earlier_turns] == ["How many signups did we get this week?"] + assert read_ledger(self.task_run.task_id, self.team.id).offers == ( + OfferRecord(turn_index=1, run_id=str(self.task_run.id), kind="scout", status=OfferStatus.OFFERED), + ) + + @parameterized.expand( + [ + ("next_turn_claims", "superseded"), + ("next_turn_offers_nothing", "superseded"), + ("earlier_card_dismissed", "dismissed"), + ] + ) + def test_a_card_that_lost_its_turn_while_drafting_is_not_published(self, interruption: str, reason: str): + self.mocks["history"].return_value = [ + *_metric_turn(), + *( + entry + for question in ("By country?", "By plan?") + for entry in (_user_message(question), _agent_text("A.")) + ), + ] + earlier = _offer(0, run_id=str(self.task_run.id)) + Task.objects.filter(id=self.task_run.task_id).update(state={STATE_KEY: {"offers": [earlier]}}) + + def classify_while_interrupted(*_args: Any, **_kwargs: Any) -> TurnVerdict: + if interruption == "next_turn_claims": + assert claim_turn(self.task_run.task_id, self.team.id, 3) is None + elif interruption == "next_turn_offers_nothing": + self.mocks["history"].return_value = [ + *self.mocks["history"].return_value, + _user_message("Thanks"), + _agent_text("You're welcome."), + ] + self.mocks["scouts"].return_value = False + assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="no_offers_available") + else: + resolution = TurnSuggestionResolution.DISMISSED + assert resolve_offer(self.task_run.task_id, self.team.id, turn_index=0, resolution=resolution) + return _verdict() + + self.mocks["classify"].side_effect = classify_while_interrupted + + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="skipped", reason=reason) + self.mocks["publish"].assert_not_called() + assert [offer.turn_index for offer in read_ledger(self.task_run.task_id, self.team.id).offers] == [0] + + @parameterized.expand( + [ + ("muted", {"offers": [{**_offer(0), "status": "dismissed"}]}, "dismissed"), + ("budget_spent", {"offers": [_offer(0), _offer(1)]}, "offer_budget_spent"), + ("card_on_the_previous_turn", {"offers": [_offer(0)]}, "follows_an_offer"), + ( + "malformed_entry_next_to_a_card", + {"offers": [{"turn_index": 0, "status": "unknown"}, {"turn_index": 0}, _offer(0)]}, + "follows_an_offer", + ), + ("turn_already_claimed", {"offers": [], "last_classified_turn": 1}, "already_classified"), + ] + ) + def test_the_offer_ledger_holds_back_a_card(self, _name: str, ledger: dict, reason: str): + Task.objects.filter(id=self.task_run.task_id).update(state={STATE_KEY: ledger}) + updated_at = Task.objects.get(id=self.task_run.task_id).updated_at + self.mocks["history"].return_value = [*_metric_turn(), _user_message("And by country?"), _agent_text("US.")] + + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="skipped", reason=reason) + self.mocks["classify"].assert_not_called() + assert Task.objects.get(id=self.task_run.task_id).updated_at == updated_at + + @parameterized.expand( + [ + ("slack", {"origin_product": Task.OriginProduct.SLACK}, "not_posthog_ai_conversation"), + ("desktop", {"client_provenance": TaskClientProvenance.POSTHOG_DESKTOP}, "not_started_in_web"), + ("other_client", {"client_provenance": "posthog_cli"}, None), + ] + ) + def test_only_conversations_started_in_the_web_app_get_a_card(self, _name: str, fields: dict, reason: str | None): + Task.objects.filter(id=self.task_run.task_id).update(**fields) + + outcome = self._generate() + + if reason is None: + assert outcome.status == "emitted" + else: + assert outcome == TurnSuggestionOutcome(status="skipped", reason=reason) + self.mocks["classify"].assert_not_called() + + @parameterized.expand( + [ + ("flag", "flag_off"), + ("judge", "judge_not_configured"), + ("ai_data_processing", "ai_data_processing_not_approved"), + ] + ) + def test_a_closed_gate_skips_before_classifying(self, gate: str, reason: str): + if gate == "ai_data_processing": + self.organization.is_ai_data_processing_approved = False + self.organization.save() + else: + self.mocks[gate].return_value = False + + outcome = self._generate() + + assert outcome.reason == reason + self.mocks["classify"].assert_not_called() + + def test_a_conversation_over_the_history_cap_gets_no_card(self): + self.mocks["history"].return_value = None + + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="skipped", reason="log_too_large") + assert self.mocks["history"].call_args.kwargs == {"max_bytes": MAX_TRANSCRIPT_LOG_BYTES} + self.mocks["classify"].assert_not_called() + + def test_classifier_failure_is_recorded_and_not_published(self): + self.mocks["classify"].return_value = None + + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="failed", reason="classifier_failed") + self.mocks["publish"].assert_not_called() + + def test_a_run_from_another_team_is_ignored(self): + outcome = generate_turn_suggestion(str(self.task_run.id), self.team.id + 1) + + assert outcome.reason == "run_missing" + self.mocks["classify"].assert_not_called() + + def test_second_report_of_the_same_turn_is_deduplicated(self): + first = self._generate() + second = self._generate() + + assert first.status == "emitted" + assert second == TurnSuggestionOutcome(status="skipped", reason="already_classified") + self.mocks["classify"].assert_called_once() diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions_api.py b/products/posthog_ai/backend/tests/test_turn_suggestions_api.py new file mode 100644 index 000000000000..e838f94f6910 --- /dev/null +++ b/products/posthog_ai/backend/tests/test_turn_suggestions_api.py @@ -0,0 +1,122 @@ +from posthog.test.base import APIBaseTest +from unittest.mock import patch + +from parameterized import parameterized + +from posthog.models import User +from posthog.models.scoping import team_scope + +from products.posthog_ai.backend.turn_suggestions.offer_ledger import STATE_KEY, OfferRecord, OfferStatus, read_ledger +from products.tasks.backend.facade.contracts import StreamNotificationDelivery +from products.tasks.backend.models import Channel, Task + +SERVICE = "products.posthog_ai.backend.turn_suggestions.service" + + +def _offer(turn_index: int, *, run_id: str) -> dict: + return OfferRecord(turn_index=turn_index, run_id=run_id, kind="scout", status=OfferStatus.OFFERED).to_json() + + +class TestResolveTurnSuggestion(APIBaseTest): + def setUp(self): + super().setUp() + self.task = Task.objects.create( + team=self.team, + title="t", + description="d", + origin_product=Task.OriginProduct.POSTHOG_AI, + created_by=self.user, + ) + self.task_run = self.task.create_run(mode="interactive") + Task.objects.filter(id=self.task.id).update( + state={STATE_KEY: {"offers": [_offer(0, run_id=str(self.task_run.id))]}} + ) + + def _resolve(self, task_id: str, turn_index: int, resolution: str, *, live: bool = True): + delivery = StreamNotificationDelivery(live=live, persisted=False) + with patch(f"{SERVICE}.publish_task_run_stream_notification", return_value=delivery) as publish: + response = self.client.post( + f"/api/projects/{self.team.id}/turn_suggestions/resolve/", + {"task_id": task_id, "turn_index": turn_index, "resolution": resolution}, + format="json", + ) + return response, publish + + def _state(self, task_id: str): + return self.client.get(f"/api/projects/{self.team.id}/turn_suggestions/state/", {"task_id": task_id}) + + @parameterized.expand( + [ + ("other_tabs_reached", True, False), + ("no_tab_listening", False, False), + ("warmed_successor_run", True, True), + ] + ) + def test_a_dismissal_mutes_the_conversation_and_a_reload_reads_it( + self, _name: str, live: bool, with_successor: bool + ): + run_ids = {str(self.task_run.id)} + if with_successor: + run_ids.add(str(self.task.create_run(mode="interactive").id)) + + response, publish = self._resolve(str(self.task.id), 0, "dismissed", live=live) + + assert response.status_code == 200 + assert response.json() == {"recorded": True} + assert self._state(str(self.task.id)).json() == {"muted": True, "resolved_turns": [0]} + assert {call.args[0] for call in publish.call_args_list} == run_ids + for call in publish.call_args_list: + assert call.args[1:] == ( + str(self.task.id), + self.team.id, + "_posthog/turn_suggestion_resolved", + {"turnIndex": 0, "outcome": "dismissed"}, + ) + assert call.kwargs == {"persist": False} + + def test_a_card_keeps_its_first_outcome(self): + self._resolve(str(self.task.id), 0, "accepted") + + response, publish = self._resolve(str(self.task.id), 0, "dismissed") + + assert response.json() == {"recorded": False} + publish.assert_not_called() + ledger = read_ledger(self.task.id, self.team.id) + assert ledger.offers[0].status == OfferStatus.ACCEPTED + assert ledger.muted is False + + @parameterized.expand( + [ + ("turn_without_a_card", "own_task", 3, 200), + ("task_of_another_team", "other_team", 0, 404), + ("teammate_task_readable_in_a_shared_channel", "read_only", 0, 404), + ] + ) + def test_nothing_is_recorded_for_a_card_that_does_not_exist( + self, _name: str, task_access: str, turn_index: int, status_code: int + ): + task_id = str(self.task.id) + if task_access == "other_team": + other = self.create_team_with_organization(organization=self.organization) + task_id = str( + Task.objects.create( + team=other, + title="t", + description="d", + origin_product=Task.OriginProduct.POSTHOG_AI, + created_by=self.user, + ).id + ) + elif task_access == "read_only": + creator = User.objects.create_and_join(self.organization, "creator@example.com", "password") + with team_scope(self.team.id): + shared = Channel.objects.create(team=self.team, name="general", created_by=creator) + Task.objects.filter(id=self.task.id).update(channel=shared, created_by=creator) + + response, publish = self._resolve(task_id, turn_index, "accepted") + + assert response.status_code == status_code + publish.assert_not_called() + if task_access == "other_team": + assert self._state(task_id).status_code == 404 + assert read_ledger(self.task.id, self.team.id).offers[0].status == OfferStatus.OFFERED diff --git a/products/posthog_ai/backend/turn_suggestions/__init__.py b/products/posthog_ai/backend/turn_suggestions/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/products/posthog_ai/backend/turn_suggestions/classifier.py b/products/posthog_ai/backend/turn_suggestions/classifier.py new file mode 100644 index 000000000000..218a6237827e --- /dev/null +++ b/products/posthog_ai/backend/turn_suggestions/classifier.py @@ -0,0 +1,161 @@ +"""Decides which follow-up, if any, a finished PostHog AI turn gets, and builds the offer. + +Two steps. Jev judges the turn in one fast request (see ``judgment.py``), and the policy below turns +those probabilities into at most one offer. Only a scout or a notebook then needs generated text, +which a language model writes in a second call (see ``drafter.py``). Alerts, subscriptions and error +alerts are built from the judgment alone, so those turns never wait on a language model. +""" + +from datetime import date + +import structlog + +from posthog.dataclasses import frozen + +from products.posthog_ai.backend.turn_suggestions.drafter import draft_notebook, draft_scout +from products.posthog_ai.backend.turn_suggestions.judgment import TurnJudgment, judge_turn +from products.posthog_ai.backend.turn_suggestions.transcript import TurnTranscript +from products.posthog_ai.backend.turn_suggestions.verdict import ( + AlertDirection, + AlertDraft, + Draft, + ErrorAlertDraft, + NotebookDraft, + OfferKind, + ScoutCadence, + ScoutDraft, + ScoutMode, + SubscriptionDraft, + TurnVerdict, +) + +logger = structlog.get_logger(__name__) + +# Starting points to tune from the `posthog ai turn suggestion classified` events, which record the +# probabilities behind every decision. +SHOW_THRESHOLD = 0.5 +ISSUE_RESOLVED_THRESHOLD = 0.5 + + +@frozen +class CardCopy: + title: str + description: str + + +def pick_offer( + judgment: TurnJudgment, available: frozenset[OfferKind], *, show_threshold: float = SHOW_THRESHOLD +) -> OfferKind: + """The offer the judgment supports, or ``NONE`` when it does not clear the bar for showing one.""" + offer = judgment.offer + if judgment.show_probability < show_threshold or offer not in available: + return OfferKind.NONE + match offer: + case OfferKind.ALERT if judgment.insight is None or not judgment.insight.alertable: + return OfferKind.NONE + case OfferKind.SUBSCRIPTION if judgment.insight is None: + return OfferKind.NONE + # The alert fires on a reopen, so it only means something for an issue that is resolved. + case OfferKind.ERROR_ALERT if ( + judgment.error_issue is None or judgment.issue_resolved_probability < ISSUE_RESOLVED_THRESHOLD + ): + return OfferKind.NONE + return offer + + +def build_draft( + offer: OfferKind, judgment: TurnJudgment, transcript: TurnTranscript, *, team_id: int, today: date +) -> Draft | None: + match offer: + case OfferKind.SCOUT: + return draft_scout( + transcript, team_id=team_id, today=today, mode=judgment.scout_mode, cadence=judgment.cadence + ) + case OfferKind.NOTEBOOK: + return draft_notebook(transcript, team_id=team_id, today=today, template=judgment.notebook_template) + case OfferKind.ALERT if judgment.insight is not None: + return AlertDraft( + insight=judgment.insight, + direction=judgment.alert_direction, + change_percent=judgment.alert_change_percent, + ) + case OfferKind.SUBSCRIPTION if judgment.insight is not None: + return SubscriptionDraft(insight=judgment.insight, cadence=judgment.cadence) + case OfferKind.ERROR_ALERT if judgment.error_issue is not None: + return ErrorAlertDraft(issue=judgment.error_issue) + case _: + return None + + +def _every(cadence: ScoutCadence) -> str: + return "every day" if cadence == ScoutCadence.DAILY else "every week" + + +def card_copy(draft: Draft) -> CardCopy: + match draft: + case ScoutDraft(mode=ScoutMode.WATCH, cadence=cadence): + return CardCopy( + title="Get a Slack message when this changes", + description=f"A scout checks this {_every(cadence)}. It posts only when the number crosses a threshold.", + ) + case ScoutDraft(mode=ScoutMode.INVESTIGATE, cadence=cadence): + return CardCopy( + title="Rerun this investigation when the metric drops", + description=f"A scout checks the metric {_every(cadence)}. When it drops, the scout repeats these steps and posts what it finds.", + ) + case ScoutDraft(mode=ScoutMode.DIGEST, cadence=cadence): + return CardCopy( + title="Get these metrics in one Slack post", + description=f"A scout posts the metrics from this conversation to Slack {_every(cadence)}.", + ) + case ScoutDraft(cadence=cadence): + return CardCopy( + title=f"Get this in Slack {_every(cadence)}", + description=f"A scout runs this analysis again {_every(cadence)} and posts the results to Slack.", + ) + case NotebookDraft(incident=None): + return CardCopy( + title="Save this conversation as a notebook", + description="The notebook keeps the messages, and each query becomes a cell you can run again.", + ) + case NotebookDraft(): + return CardCopy( + title="Save this as an incident report", + description="The notebook starts with the timeline, cause and fix, followed by the conversation and its queries.", + ) + case AlertDraft(direction=direction): + verb = "drops" if direction == AlertDirection.DECREASE else "rises" + return CardCopy( + title=f"Get an alert when this {verb}", + description="The alert checks once a day, compares with the day before, and posts to Slack.", + ) + case SubscriptionDraft(cadence=cadence): + schedule = "every day at 9:00" if cadence == ScoutCadence.DAILY else "every Monday at 9:00" + return CardCopy( + title=f"Send this chart to Slack {_every(cadence)}", + description=f"The chart posts to a Slack channel {schedule}.", + ) + case ErrorAlertDraft(): + return CardCopy( + title="Get an alert if this error comes back", + description="Slack gets a message if this issue reopens after it was resolved.", + ) + + +def classify_turn( + transcript: TurnTranscript, *, team_id: int, today: date, available: frozenset[OfferKind] +) -> TurnVerdict | None: + """``None`` means the judgment failed. A verdict whose draft is ``None`` offers nothing, either + because the policy picked nothing or because drafting the picked offer failed.""" + judgment = judge_turn(transcript, available=available) + if judgment is None: + return None + picked = pick_offer(judgment, available) + draft = build_draft(picked, judgment, transcript, team_id=team_id, today=today) + return TurnVerdict( + intent=judgment.intent, + show_probability=judgment.show_probability, + picked=picked, + offer_probabilities=judgment.offer_probabilities, + draft=draft, + ) diff --git a/products/posthog_ai/backend/turn_suggestions/dispatch.py b/products/posthog_ai/backend/turn_suggestions/dispatch.py new file mode 100644 index 000000000000..ec2a92ae1229 --- /dev/null +++ b/products/posthog_ai/backend/turn_suggestions/dispatch.py @@ -0,0 +1,58 @@ +from typing import TYPE_CHECKING, Any + +import structlog + +from posthog.redis import get_client + +from products.tasks.backend.facade.task_run_signals import TaskOriginProduct + +if TYPE_CHECKING: + from products.tasks.backend.models import TaskRun + +logger = structlog.get_logger(__name__) + +# The proxy callback, the event ingest and the sandbox relay can each report the same turn, within +# moments of each other. A person cannot finish two turns this close together. +ENQUEUE_DEDUP_SECONDS = 5 +# The proxy callback reports over its own channel and can beat the ingest writes of the turn's last +# frames, so the read waits for them instead of judging a turn whose answer has not landed. +TURN_SETTLE_SECONDS = 2 + + +def _enqueue_dedup_key(run_id: str) -> str: + return f"turn_suggestion:{run_id}" + + +def _first_report_of_turn(run_id: str) -> bool: + try: + return bool(get_client().set(_enqueue_dedup_key(run_id), 1, nx=True, ex=ENQUEUE_DEDUP_SECONDS)) + except Exception: + # The offer ledger still refuses a second claim of the same turn, so a duplicate only costs a read. + logger.warning("posthog_ai_turn_suggestion_dedup_failed", run_id=run_id, exc_info=True) + return True + + +def enqueue_turn_suggestion(task_run: "TaskRun") -> bool: + """Queue the end-of-turn suggestion for a PostHog AI run. Never raises: the turn completion that + calls this must not fail because a nudge could not be scheduled.""" + try: + if task_run.origin_product != TaskOriginProduct.POSTHOG_AI: + return False + run_id = str(task_run.id) + if not _first_report_of_turn(run_id): + return False + from products.posthog_ai.backend.tasks import ( + generate_turn_suggestion_task, # noqa: PLC0415 — keeps the judge and drafter clients off the Django startup path + ) + + generate_turn_suggestion_task.apply_async( + kwargs={"run_id": run_id, "team_id": task_run.team_id}, countdown=TURN_SETTLE_SECONDS + ) + except Exception: + logger.warning("posthog_ai_turn_suggestion_enqueue_failed", run_id=str(task_run.id), exc_info=True) + return False + return True + + +def enqueue_turn_suggestion_on_turn_completed(sender: type, task_run: "TaskRun", **kwargs: Any) -> None: + enqueue_turn_suggestion(task_run) diff --git a/products/posthog_ai/backend/turn_suggestions/drafter.py b/products/posthog_ai/backend/turn_suggestions/drafter.py new file mode 100644 index 000000000000..9f735fa84f6d --- /dev/null +++ b/products/posthog_ai/backend/turn_suggestions/drafter.py @@ -0,0 +1,187 @@ +"""Writes the text a scout or notebook offer needs, once the judgment has picked one. + +Jev decides whether and what to offer, but a scout prompt or an incident outline is generated text, +which a System One model does not write. This call runs only on the turns where the policy already +picked a scout or a notebook, with the mode or template fixed, so the language model writes and +never decides. +""" + +from datetime import date +from typing import Any, TypeVar + +import structlog +from openai.types.shared_params import ResponseFormatJSONSchema +from pydantic import BaseModel, ConfigDict, ValidationError + +from posthog.llm.gateway_client import get_llm_client +from posthog.llm.semantic_enrichment import extract_json_object + +from products.posthog_ai.backend.turn_suggestions.transcript import TurnTranscript, truncate_text +from products.posthog_ai.backend.turn_suggestions.verdict import ( + IncidentOutline, + NotebookDraft, + NotebookTemplate, + ScoutCadence, + ScoutDraft, + ScoutMode, +) + +logger = structlog.get_logger(__name__) + +DRAFT_MODEL = "gpt-5.6-luna" + +# A reasoning model spends the same token budget on its thinking and on the reply, and the reply +# carries a complete scout prompt, so the ceiling sits well above the JSON alone. +DRAFT_MAX_TOKENS = 4096 + +# The card waits for this call, so a slow one is dropped rather than retried into the user's next message. +DRAFT_TIMEOUT_SECONDS = 30.0 +DRAFT_MAX_RETRIES = 0 +EARLIER_QUESTION_LIMIT = 300 + +_ReplyT = TypeVar("_ReplyT", bound=BaseModel) + + +class _ScoutReply(BaseModel): + model_config = ConfigDict(extra="forbid") + + display_name: str + description: str + prompt: str + + +class _NotebookReply(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str + summary: str + incident_timeline: str + incident_cause: str + incident_fix: str + + +_SCOUT_MODE_GUIDES = { + ScoutMode.REPORT: "Rerun the analysis and post the numbers and what moved on every run.", + ScoutMode.WATCH: "Rerun the analysis and post only when the number crosses a bound you state in the prompt. Stay silent otherwise.", + ScoutMode.INVESTIGATE: "Check the metric, and when it dips again, rerun the investigation steps from the conversation (queries, recordings, comparisons) and post the findings.", + ScoutMode.DIGEST: "Cover every metric the conversation asked about, including the earlier turns, in one post.", +} + +SCOUT_SYSTEM_PROMPT = """You write the configuration of a scout from one PostHog AI conversation. A scout is a scheduled agent with the same PostHog tools the assistant used. It posts a short report to Slack and the inbox. The mode and cadence are already chosen; write the text. + +prompt is the complete markdown the scout runs on every run. It must stand alone: name the exact events, properties, filters, breakdowns and cohorts, restate the analysis with a relative window that matches the cadence, tell the scout to compare with the previous period, and say what to post and when to stay silent. Tell it to say plainly when the project has no matching data instead of guessing. Do not mention the user or this conversation. When the latest question refines an earlier one, the scout runs the refined version. + +display_name is at most 60 characters in sentence case. description is one sentence, at most 200 characters. Reply with the JSON object only.""" + +NOTEBOOK_SYSTEM_PROMPT = """You write the title and summary of a notebook that saves one PostHog AI conversation with its queries as live cells. The layout is already chosen; write the text. + +title is at most 80 characters in sentence case. summary is one or two sentences, at most 300 characters, saying what the investigation found. + +For the incident layout, fill incident_timeline (markdown bullets, one per event with its time), incident_cause (one or two sentences) and incident_fix (what fixed it or what to do next). For the conversation layout, leave those three empty. Reply with the JSON object only.""" + + +def render_turn_prompt(transcript: TurnTranscript, *, today: date, instruction: str) -> str: + sections = [f"Today is {today.isoformat()}.", instruction] + if transcript.earlier_turns: + lines = [] + for turn in transcript.earlier_turns: + tools = ", ".join(turn.tool_names) if turn.tool_names else "no tools" + lines.append( + f"- Q: {truncate_text(turn.question, EARLIER_QUESTION_LIMIT)}\n" + f" tools: {tools}\n" + f" A: {turn.answer_excerpt or '(empty)'}" + ) + sections.append("\n" + "\n".join(lines) + "\n") + tool_lines = [ + f"- {tool_call.name} [{tool_call.status}]" + (f": {tool_call.args_preview}" if tool_call.args_preview else "") + for tool_call in transcript.tool_calls + ] + sections.append(f"\n{transcript.last_human_message}\n") + sections.append("\n" + ("\n".join(tool_lines) if tool_lines else "(no tool calls)") + "\n") + sections.append(f"\n{transcript.assistant_text or '(empty)'}\n") + return "\n\n".join(sections) + + +def _response_format(name: str, reply_type: type[BaseModel]) -> ResponseFormatJSONSchema: + # Strict mode pins the reply to the schema, so a reasoning model cannot answer with its reasoning. + return { + "type": "json_schema", + "json_schema": {"name": name, "strict": True, "schema": reply_type.model_json_schema()}, + } + + +def _complete( + *, team_id: int, system_prompt: str, user_prompt: str, schema_name: str, reply_type: type[_ReplyT] +) -> _ReplyT | None: + try: + # Inside the guard: an instance without a configured gateway raises here, and a failed draft offers nothing. + client = get_llm_client("posthog_ai", team_id=team_id).with_options( + timeout=DRAFT_TIMEOUT_SECONDS, + max_retries=DRAFT_MAX_RETRIES, + ) + response = client.chat.completions.create( + model=DRAFT_MODEL, + messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}], + max_tokens=DRAFT_MAX_TOKENS, + response_format=_response_format(schema_name, reply_type), + ) + except Exception: + logger.exception("posthog_ai_turn_suggestion_draft_failed", team_id=team_id) + return None + + content = response.choices[0].message.content if response.choices else None + parsed: dict[str, Any] | None = extract_json_object(content) if content else None + if parsed is None: + logger.warning("posthog_ai_turn_suggestion_draft_unparseable", team_id=team_id) + return None + try: + return reply_type.model_validate(parsed) + except ValidationError: + logger.warning("posthog_ai_turn_suggestion_draft_invalid", team_id=team_id) + return None + + +def draft_scout( + transcript: TurnTranscript, *, team_id: int, today: date, mode: ScoutMode, cadence: ScoutCadence +) -> ScoutDraft | None: + instruction = f"Scout mode: {mode.value}. {_SCOUT_MODE_GUIDES[mode]} Cadence: {cadence.value}." + reply = _complete( + team_id=team_id, + system_prompt=SCOUT_SYSTEM_PROMPT, + user_prompt=render_turn_prompt(transcript, today=today, instruction=instruction), + schema_name="posthog_ai_scout_draft", + reply_type=_ScoutReply, + ) + if reply is None or not reply.prompt.strip() or not reply.display_name.strip(): + return None + return ScoutDraft( + mode=mode, + display_name=reply.display_name.strip()[:60], + description=reply.description.strip()[:200], + body=reply.prompt.strip(), + cadence=cadence, + ) + + +def draft_notebook( + transcript: TurnTranscript, *, team_id: int, today: date, template: NotebookTemplate +) -> NotebookDraft | None: + reply = _complete( + team_id=team_id, + system_prompt=NOTEBOOK_SYSTEM_PROMPT, + user_prompt=render_turn_prompt(transcript, today=today, instruction=f"Layout: {template.value}."), + schema_name="posthog_ai_notebook_draft", + reply_type=_NotebookReply, + ) + if reply is None or not reply.title.strip(): + return None + incident = ( + IncidentOutline( + timeline=reply.incident_timeline.strip(), + cause=reply.incident_cause.strip(), + fix=reply.incident_fix.strip(), + ) + if template == NotebookTemplate.INCIDENT and reply.incident_cause.strip() + else None + ) + return NotebookDraft(title=reply.title.strip()[:80], summary=reply.summary.strip()[:300], incident=incident) diff --git a/products/posthog_ai/backend/turn_suggestions/judgment.py b/products/posthog_ai/backend/turn_suggestions/judgment.py new file mode 100644 index 000000000000..ab0a57d37186 --- /dev/null +++ b/products/posthog_ai/backend/turn_suggestions/judgment.py @@ -0,0 +1,339 @@ +"""Asks Jev, TypeSafe's System One model, whether a finished PostHog AI turn deserves a follow-up and which. + +Jev answers typed questions instead of writing text, so one request carries every judgment the +policy might need: whether to show anything now, which offer, and the speculative parameters of each +offer (scout mode, cadence, alert direction, which saved insight). The policy in ``classifier.py`` +reads only the answers that apply to the offer it picks. + +The state Jev reads holds what the user asked, which tools ran, and the answer with its values +masked. Tool outputs never leave PostHog, and saved insights and error issues reach Jev as numbered +options, so their ids stay here too. +""" + +from collections.abc import Mapping, Sequence + +from django.conf import settings + +import structlog + +from posthog.dataclasses import frozen +from posthog.egress.limiter.policies import Priority +from posthog.egress.typesafe import ( + ChoiceAnswer, + ChoiceQuestion, + JsonValue, + NoulAnswer, + NoulQuestion, + Question, + SystemOneResult, + TypeSafeEgressBudgetExhausted, + TypeSafeNotConfigured, + TypeSafeRequestFailed, + system_one, +) + +from products.posthog_ai.backend.turn_suggestions.transcript import ( + ErrorIssueRef, + SavedInsightRef, + TurnTranscript, + redact_values, +) +from products.posthog_ai.backend.turn_suggestions.verdict import ( + AlertDirection, + NotebookTemplate, + OfferKind, + ScoutCadence, + ScoutMode, + TurnIntent, +) + +logger = structlog.get_logger(__name__) + +# Pinned rather than `jev-latest`, because the thresholds in classifier.py are tuned against this +# version's probabilities and an alias moves on each release. +JUDGE_MODEL = "jev-1.13.0" +JUDGE_SOURCE = "posthog_ai_turn_suggestions" +JUDGE_TIMEOUT: tuple[float, float] = (3.0, 10.0) + +_NO_MATCH = "none" + +# Option keys map to the percent the alert card preconfigures. +_ALERT_CHANGE_PERCENT = {"small": 10, "moderate": 20, "large": 50} + +_SHOW_OFFER = NoulQuestion( + instructions={ + "question": "Should PostHog AI show a follow-up offer under the answer in `latest_turn`, at this point in the conversation?", + "context": ( + "PostHog AI is the analytics agent inside PostHog. A follow-up offer is a small card under its answer " + "that turns the answer into something lasting: a scheduled report in Slack, an alert, a chart " + "subscription, or a saved notebook. An offer at the wrong moment interrupts the user, so show one " + "only when it clearly helps." + ), + "show_when": [ + "The turn ran PostHog tools that completed, and the answer gives a real result.", + "The user digs into something specific: a segment, filter, flow or feature, often compared with an earlier period, a review of what changed on a dashboard, a worry about a number, or a request to keep track of it.", + "The turn investigated a problem and found its cause.", + "The question is about one area of the product, such as a flow, page or feature, which the user will likely check again. This counts for more than a question about the whole product.", + "The answer is complete and does not wait for the user to reply.", + ], + "hold_back_when": [ + "The turn failed, a tool errored, or the answer apologizes or says it found no data.", + "The turn ran no PostHog tools, for example advice, brainstorming or planning.", + "The answer asks the user a question or offers options to pick from.", + "The question satisfies a one-time curiosity: a breakdown such as top pages, referrers, browsers, countries or the device split, a list of users or accounts, or a search for recordings, even when the answer points out something in them. Numbers in the answer do not make it recurring.", + "The question is a basic count of the whole product with no filters and no digging, such as how many events, people, users or daily active users there are. A count for one flow or feature, such as how many people finished onboarding, is not a basic count.", + "The question is a generic overview of the whole product, such as the most common error or the busiest page, and the user gives no sign they will ask again.", + "The turn found nothing worth keeping, and the user gives no sign they will want the answer again.", + "The question is a small clarification of an earlier answer, or small talk.", + "The turn explained documentation or a concept, or answered a how-to question.", + "The turn fixed or wrote a query the user asked for help with, such as a SQL error.", + "The turn created or changed something, such as a feature flag, survey or dashboard, and nothing about it needs watching.", + "The user asked to rename, reformat or fix something that already exists, such as a dashboard name or a chart axis, and the answer only confirms the change.", + ], + }, + criteria_true="An offer clearly helps the user at this point in the conversation.", + criteria_false="An offer now would interrupt the user or would not help.", +) + +_INTENT = ChoiceQuestion( + instructions="What does the question in `latest_turn` ask for?", + criteria={ + TurnIntent.METRIC_STATE: "What a metric or breakdown is right now or over a relative window, such as the last 7 days or week over week.", + TurnIntent.DIAGNOSTIC: "Why something happened, or an investigation of a specific incident, spike or drop.", + TurnIntent.ACTION: "To create or change something, such as a feature flag, survey, dashboard, cohort or experiment.", + TurnIntent.KNOWLEDGE: "Documentation, a how-to, or an explanation of a concept.", + TurnIntent.OTHER: "Anything else, including small talk.", + }, +) + +_OFFER_CRITERIA: dict[OfferKind, JsonValue] = { + OfferKind.SCOUT: { + "what": "A scheduled agent that reruns this analysis with the same PostHog tools and posts a short report to Slack.", + "fits": "The user asks how a metric is doing now, today, this week, or over a recent window, and will ask again next period. A metric the user digs into with specific filters, segments or a flow and will follow over time, a concern about a number, a check on one area of the product for a period (such as errors or drop-off in onboarding this week) that the user will want every period, a review of what changed on a dashboard or metric this period, several metrics from this conversation, or an investigation worth repeating when the metric dips.", + "not_for": "A question about why something happened, once the answer explains it. A basic count with no filters, such as how many events, people or daily active users. A generic overview of the whole product, such as the most common errors this week. An investigation whose cause was a one-time event, such as a release, an email or a migration. A turn about performance. A notebook keeps those last two better.", + }, + OfferKind.ALERT: { + "what": "A threshold alert on a saved insight from `saved_insights`, sent to Slack when the number moves.", + "fits": "The user wants to know when this number changes and a saved insight tracks it. Prefer it over a scheduled agent in that case.", + "not_for": "A turn that built insights or a dashboard, unless the user also asked to hear when a number moves.", + }, + OfferKind.SUBSCRIPTION: { + "what": "The chart of a saved insight from `saved_insights`, sent to Slack on a schedule.", + "fits": "The chart itself answers the question, and no analysis is needed on each run.", + }, + OfferKind.NOTEBOOK: { + "what": "The investigation saved as a notebook, with its queries as cells the user can rerun.", + "fits": "The user asks why something happened, and the answer names a cause. A diagnostic turn that found something worth keeping or sharing with the team, even when it will not happen again. Rank it first for any turn about performance, such as p95 or p99 latency, load times, web vitals or durations in milliseconds, since the notebook keeps the numbers and the queries behind them.", + "not_for": "A turn that built a new insight, funnel or dashboard, which is already saved. A check of a metric over a window, such as a rate by hour or by day, that looked for no cause. A scheduled agent fits that better.", + }, + OfferKind.ERROR_ALERT: { + "what": "A Slack message when an error tracking issue from `error_issues` comes back.", + "fits": "The user investigated an error, and the answer says it is fixed or resolved.", + }, +} + +_SCOUT_MODE = ChoiceQuestion( + instructions="If PostHog AI offered a scheduled agent that reruns the analysis in `latest_turn`, which kind of run would fit?", + criteria={ + ScoutMode.REPORT: "Posts the current numbers and what changed on every run. Fits a question about the current state of a metric.", + ScoutMode.WATCH: "Posts only when the number crosses a threshold. Fits a question that carries a concern, such as whether something is down.", + ScoutMode.INVESTIGATE: "Checks the metric and repeats the investigation steps when it dips. Fits a diagnostic turn whose steps form a runbook.", + ScoutMode.DIGEST: "Covers several metrics in one post. Fits a conversation that asked about several metrics across `earlier_turns` and `latest_turn`.", + }, +) + +_CADENCE = ChoiceQuestion( + instructions="If this analysis ran on a schedule, how often would a new result be useful?", + criteria={ + ScoutCadence.DAILY: "The metric moves day to day, or the question looks at today, yesterday or a daily window.", + ScoutCadence.WEEKLY: "The question looks at a week, a month, or a week-over-week change.", + }, +) + +_NOTEBOOK_TEMPLATE = ChoiceQuestion( + instructions="If the conversation were saved as a notebook, which layout fits it?", + criteria={ + NotebookTemplate.CONVERSATION: "The conversation as it is. Fits an exploration without one clear cause.", + NotebookTemplate.INCIDENT: "An incident writeup with a timeline, a cause and a fix. Fits an investigation that found a cause and when it started.", + }, +) + +_ALERT_DIRECTION = ChoiceQuestion( + instructions="If the user got an alert on the metric in `latest_turn`, which movement would they want to hear about?", + criteria={ + AlertDirection.DECREASE: "A drop, such as fewer sign-ups or less revenue.", + AlertDirection.INCREASE: "A rise, such as more errors, churn or latency.", + }, +) + +_ALERT_CHANGE = ChoiceQuestion( + instructions="How large a change from one period to the next is worth a message for the metric in `latest_turn`?", + criteria={ + "small": "About 10%. The metric is steady and small moves matter, such as revenue or a conversion rate.", + "moderate": "About 20%. A typical product usage metric.", + "large": "About 50%. The metric is noisy or low volume, so only large swings matter.", + }, +) + +_ISSUE_RESOLVED = NoulQuestion( + instructions="Does the answer in `latest_turn` say that the error the user investigated is fixed or resolved, or that its fix has shipped?", + criteria_true="The answer says the error is fixed, resolved, or that the fix shipped.", + criteria_false="The error is still active, or the answer does not say.", +) + + +@frozen +class TurnJudgment: + show_probability: float + intent: TurnIntent + offer: OfferKind + offer_probabilities: Mapping[str, float] + scout_mode: ScoutMode + cadence: ScoutCadence + notebook_template: NotebookTemplate + alert_direction: AlertDirection + alert_change_percent: int + insight: SavedInsightRef | None + error_issue: ErrorIssueRef | None + issue_resolved_probability: float + + +def judge_configured() -> bool: + return bool(settings.TYPESAFE_API_KEY) + + +def _numbered[T](prefix: str, refs: Sequence[T]) -> dict[str, T]: + return {f"{prefix}_{index}": ref for index, ref in enumerate(refs, start=1)} + + +def _insight_options(transcript: TurnTranscript) -> dict[str, SavedInsightRef]: + return _numbered("insight", transcript.saved_insights) + + +def _issue_options(transcript: TurnTranscript) -> dict[str, ErrorIssueRef]: + return _numbered("issue", transcript.error_issues) + + +def _insight_label(ref: SavedInsightRef) -> str: + return f"{ref.name or 'Untitled insight'} ({ref.query_kind or 'unknown chart type'})" + + +def _issue_label(ref: ErrorIssueRef) -> str: + return redact_values(ref.name) or "Unnamed issue" + + +def build_judge_state(transcript: TurnTranscript) -> dict[str, JsonValue]: + """The state Jev reads. The questions stay as the user wrote them; answers have their values masked.""" + state: dict[str, JsonValue] = { + "latest_turn": { + "question": transcript.last_human_message, + "tool_calls": [{"tool": call.name, "status": call.status} for call in transcript.tool_calls], + "answer": redact_values(transcript.assistant_text) or "(empty)", + } + } + if transcript.earlier_turns: + state["earlier_turns"] = [ + { + "question": turn.question, + "tools": list(turn.tool_names), + "answer": redact_values(turn.answer_excerpt) or "(empty)", + } + for turn in transcript.earlier_turns + ] + if transcript.saved_insights: + state["saved_insights"] = {key: _insight_label(ref) for key, ref in _insight_options(transcript).items()} + if transcript.error_issues: + state["error_issues"] = {key: _issue_label(ref) for key, ref in _issue_options(transcript).items()} + return state + + +def build_judge_questions(transcript: TurnTranscript, available: frozenset[OfferKind]) -> dict[str, Question]: + questions: dict[str, Question] = {"show_offer": _SHOW_OFFER, "intent": _INTENT} + # `show_offer` alone decides whether a card shows, so this question lists only the offers the turn + # can make and has no "none" option. With one offer there is nothing to pick, and a System One + # server can reject a choice with a single option. + if len(available) > 1: + questions["offer"] = ChoiceQuestion( + instructions="If PostHog AI showed a follow-up under the answer in `latest_turn`, which one would help the user most? Pick the lightest one that covers what they need next.", + criteria={kind: _OFFER_CRITERIA[kind] for kind in OfferKind if kind in available}, + ) + if OfferKind.SCOUT in available: + questions["scout_mode"] = _SCOUT_MODE + if available & {OfferKind.SCOUT, OfferKind.SUBSCRIPTION}: + questions["cadence"] = _CADENCE + if OfferKind.NOTEBOOK in available: + questions["notebook_template"] = _NOTEBOOK_TEMPLATE + if OfferKind.ALERT in available: + questions["alert_direction"] = _ALERT_DIRECTION + questions["alert_change"] = _ALERT_CHANGE + if transcript.saved_insights: + insight_criteria: dict[str, JsonValue] = dict.fromkeys(_insight_options(transcript)) + insight_criteria[_NO_MATCH] = "No listed insight tracks that metric." + questions["insight"] = ChoiceQuestion( + instructions="Which saved insight in `saved_insights` tracks the metric the user asked about in `latest_turn`?", + criteria=insight_criteria, + ) + if transcript.error_issues: + issue_criteria: dict[str, JsonValue] = dict.fromkeys(_issue_options(transcript)) + issue_criteria[_NO_MATCH] = "None of the listed issues is the error the user cares about." + questions["error_issue"] = ChoiceQuestion( + instructions="Which error tracking issue in `error_issues` is the error the user cares about in `latest_turn`?", + criteria=issue_criteria, + ) + questions["issue_resolved"] = _ISSUE_RESOLVED + return questions + + +def judge_turn(transcript: TurnTranscript, *, available: frozenset[OfferKind]) -> TurnJudgment | None: + """One Jev request. ``None`` means the call failed, was shed, or the instance has no key. + ``available`` must hold at least one offer, because the offer question needs an option.""" + try: + result = system_one( + state=build_judge_state(transcript), + questions=build_judge_questions(transcript, available), + source=JUDGE_SOURCE, + model=JUDGE_MODEL, + priority=Priority.BATCH, + timeout=JUDGE_TIMEOUT, + ) + except (TypeSafeNotConfigured, TypeSafeEgressBudgetExhausted) as error: + logger.info("posthog_ai_turn_suggestion_judge_skipped", reason=type(error).__name__) + return None + except (TypeSafeRequestFailed, OSError): + logger.warning("posthog_ai_turn_suggestion_judge_failed", exc_info=True) + return None + return read_judgment(result, transcript, available) + + +def read_judgment(result: SystemOneResult, transcript: TurnTranscript, available: frozenset[OfferKind]) -> TurnJudgment: + answers = result.answers + choices = {question_id: answer for question_id, answer in answers.items() if isinstance(answer, ChoiceAnswer)} + nouls = {question_id: answer for question_id, answer in answers.items() if isinstance(answer, NoulAnswer)} + + offer_answer = choices.get("offer") + if offer_answer is not None: + offer, offer_probabilities = OfferKind(offer_answer.choice), dict(offer_answer.probabilities) + else: + # Without an offer question the turn has exactly one offer. See `build_judge_questions`. + offer = next(iter(available)) + offer_probabilities = {offer: 1.0} + + def picked(question_id: str, default: str) -> str: + # Speculative questions are only asked when their offer is available, so an absent answer + # falls back to a default the policy never reads. + return choices[question_id].choice if question_id in choices else default + + return TurnJudgment( + show_probability=nouls["show_offer"].probability, + intent=TurnIntent(choices["intent"].choice), + offer=offer, + offer_probabilities=offer_probabilities, + scout_mode=ScoutMode(picked("scout_mode", ScoutMode.REPORT)), + cadence=ScoutCadence(picked("cadence", ScoutCadence.WEEKLY)), + notebook_template=NotebookTemplate(picked("notebook_template", NotebookTemplate.CONVERSATION)), + alert_direction=AlertDirection(picked("alert_direction", AlertDirection.DECREASE)), + alert_change_percent=_ALERT_CHANGE_PERCENT[picked("alert_change", "moderate")], + insight=_insight_options(transcript).get(picked("insight", _NO_MATCH)), + error_issue=_issue_options(transcript).get(picked("error_issue", _NO_MATCH)), + issue_resolved_probability=nouls["issue_resolved"].probability if "issue_resolved" in nouls else 0.0, + ) diff --git a/products/posthog_ai/backend/turn_suggestions/offer_ledger.py b/products/posthog_ai/backend/turn_suggestions/offer_ledger.py new file mode 100644 index 000000000000..32a1517a3cd7 --- /dev/null +++ b/products/posthog_ai/backend/turn_suggestions/offer_ledger.py @@ -0,0 +1,221 @@ +"""The record of the turn suggestions one conversation received, kept in the task's state. + +It lives in Postgres rather than Redis because it decides whether a conversation gets another card +and whether a dismissed or accepted card stays hidden after a reload. An evicted Redis key would +quietly bring both back. +""" + +from collections.abc import Callable +from dataclasses import replace +from enum import StrEnum +from typing import Any, Literal +from uuid import UUID + +from django.db import models + +from posthog.dataclasses import frozen + +from products.tasks.backend.facade.api import read_task_state_entry, update_task_state_entry + +STATE_KEY = "turn_suggestions" + +# A second offer exists for the case where the first one was superseded by the next message. +MAX_OFFERS_PER_CONVERSATION = 2 + + +class OfferStatus(models.TextChoices): + OFFERED = "offered", "Offered" + DISMISSED = "dismissed", "Dismissed" + ACCEPTED = "accepted", "Accepted" + + +class TurnSuggestionResolution(models.TextChoices): + DISMISSED = "dismissed", "Dismissed" + ACCEPTED = "accepted", "Accepted" + + +class ClaimRefusal(StrEnum): + MUTED = "dismissed" + BUDGET_SPENT = "offer_budget_spent" + FOLLOWS_AN_OFFER = "follows_an_offer" + ALREADY_CLASSIFIED = "already_classified" + SUPERSEDED = "superseded" + TASK_MISSING = "task_missing" + + +@frozen +class OfferRecord: + turn_index: int + run_id: str + kind: str + status: OfferStatus + + def to_json(self) -> dict[str, Any]: + return {"turn_index": self.turn_index, "run_id": self.run_id, "kind": self.kind, "status": self.status.value} + + @classmethod + def from_json(cls, raw: Any) -> "OfferRecord | None": + if not isinstance(raw, dict) or not isinstance(raw.get("turn_index"), int): + return None + run_id, kind, status = raw.get("run_id"), raw.get("kind"), raw.get("status") + if not isinstance(run_id, str) or not isinstance(kind, str) or status not in OfferStatus.values: + return None + return cls(turn_index=raw["turn_index"], run_id=run_id, kind=kind, status=OfferStatus(status)) + + +@frozen +class OfferLedger: + offers: tuple[OfferRecord, ...] = () + last_classified_turn: int = -1 + + @property + def muted(self) -> bool: + """A dismissed card mutes the rest of the conversation.""" + return any(offer.status == OfferStatus.DISMISSED for offer in self.offers) + + @property + def resolved_turns(self) -> tuple[int, ...]: + return tuple(offer.turn_index for offer in self.offers if offer.status != OfferStatus.OFFERED) + + def offer_at(self, turn_index: int) -> OfferRecord | None: + return next((offer for offer in self.offers if offer.turn_index == turn_index), None) + + def with_status(self, target: OfferRecord, status: OfferStatus) -> "OfferLedger": + updated = replace(target, status=status) + return replace(self, offers=tuple(updated if offer is target else offer for offer in self.offers)) + + @classmethod + def from_json(cls, raw: Any) -> "OfferLedger": + if not isinstance(raw, dict): + return cls() + raw_offers = raw.get("offers") + parsed = (OfferRecord.from_json(entry) for entry in raw_offers) if isinstance(raw_offers, list) else () + offers = tuple(offer for offer in parsed if offer is not None) + last_classified_turn = raw.get("last_classified_turn") + return cls( + offers=offers, + last_classified_turn=last_classified_turn if isinstance(last_classified_turn, int) else -1, + ) + + def to_json(self) -> dict[str, Any]: + return { + "offers": [offer.to_json() for offer in self.offers], + "last_classified_turn": self.last_classified_turn, + } + + def refusal(self, turn_index: int | None = None) -> ClaimRefusal | None: + """Why the conversation gets no card, checked before ``turn_index`` is known when it is ``None``.""" + if self.muted: + return ClaimRefusal.MUTED + if len(self.offers) >= MAX_OFFERS_PER_CONVERSATION: + return ClaimRefusal.BUDGET_SPENT + if turn_index is None: + return None + if any(offer.turn_index == turn_index - 1 for offer in self.offers): + return ClaimRefusal.FOLLOWS_AN_OFFER + if turn_index <= self.last_classified_turn: + return ClaimRefusal.ALREADY_CLASSIFIED + return None + + +def read_ledger(task_id: UUID | str, team_id: int) -> OfferLedger: + return OfferLedger.from_json(read_task_state_entry(task_id, team_id, STATE_KEY)) + + +def _update_ledger[R]( + task_id: UUID | str, team_id: int, update: Callable[[OfferLedger], tuple[OfferLedger | None, R]] +) -> R | None: + """Row-locked update of the ledger. ``update`` returns ``None`` in place of a ledger to skip the + write. Returns ``None`` without calling ``update`` when the task does not exist.""" + + def update_raw(raw: Any) -> tuple[Any, R]: + ledger, result = update(OfferLedger.from_json(raw)) + return (raw if ledger is None else ledger.to_json()), result + + return update_task_state_entry(task_id, team_id, STATE_KEY, update_raw) + + +def _claim( + task_id: UUID | str, team_id: int, decide: Callable[[OfferLedger], OfferLedger | ClaimRefusal] +) -> ClaimRefusal | None: + """Store the ledger ``decide`` returns, or return the refusal it gives instead.""" + + def update(ledger: OfferLedger) -> tuple[OfferLedger | None, ClaimRefusal | Literal[True]]: + decision = decide(ledger) + return (None, decision) if isinstance(decision, ClaimRefusal) else (decision, True) + + result = _update_ledger(task_id, team_id, update) + if result is None: + return ClaimRefusal.TASK_MISSING + return None if result is True else result + + +def claim_turn(task_id: UUID | str, team_id: int, turn_index: int) -> ClaimRefusal | None: + """Claim ``turn_index`` for classification, or say why it gets no card. + + The check and the claim share one row lock, so two reports of the same turn end in one claim. + """ + + def claim(ledger: OfferLedger) -> OfferLedger | ClaimRefusal: + refusal = ledger.refusal(turn_index) + if refusal is not None: + return refusal + return replace(ledger, last_classified_turn=turn_index) + + return _claim(task_id, team_id, claim) + + +def record_offer( + task_id: UUID | str, team_id: int, *, run_id: UUID | str, turn_index: int, kind: str +) -> ClaimRefusal | None: + """Record the card of ``turn_index`` before it is published, or say why it must not be. + + A later turn that claimed classification while this one was drafting supersedes it. The card + would arrive under a turn the thread already moved past, and it must not hold back the card of + the later turn. The check shares the row lock with ``claim_turn``, so of two consecutive turns + only one gets a card. The mute and the cap are checked again, because a dismissal can land + while this turn drafts. + """ + offer = OfferRecord(turn_index=turn_index, run_id=str(run_id), kind=kind, status=OfferStatus.OFFERED) + + def append(ledger: OfferLedger) -> OfferLedger | ClaimRefusal: + if ledger.last_classified_turn != turn_index: + return ClaimRefusal.SUPERSEDED + refusal = ledger.refusal() + if refusal is not None: + return refusal + return replace(ledger, offers=(*ledger.offers, offer)) + + return _claim(task_id, team_id, append) + + +def withdraw_offer(task_id: UUID | str, team_id: int, *, turn_index: int) -> None: + """Drop the recorded card of ``turn_index`` when it never reached the thread, so it spends no budget.""" + + def remove(ledger: OfferLedger) -> tuple[OfferLedger | None, None]: + offers = tuple(offer for offer in ledger.offers if offer.turn_index != turn_index) + if len(offers) == len(ledger.offers): + return None, None + return replace(ledger, offers=offers), None + + _update_ledger(task_id, team_id, remove) + + +def resolve_offer( + task_id: UUID | str, team_id: int, *, turn_index: int, resolution: TurnSuggestionResolution +) -> OfferRecord | None: + """Record what the user did with the card of ``turn_index``. A dismissal mutes the conversation. + + Returns the updated offer, or ``None`` when that turn got no card or its card was already + resolved. The first outcome stands, so a dismissal from another tab cannot overwrite an accept. + """ + status = OfferStatus(resolution.value) + + def resolve(ledger: OfferLedger) -> tuple[OfferLedger | None, OfferRecord | None]: + target = ledger.offer_at(turn_index) + if target is None or target.status != OfferStatus.OFFERED: + return None, None + resolved = ledger.with_status(target, status) + return resolved, resolved.offer_at(turn_index) + + return _update_ledger(task_id, team_id, resolve) diff --git a/products/posthog_ai/backend/turn_suggestions/service.py b/products/posthog_ai/backend/turn_suggestions/service.py new file mode 100644 index 000000000000..b7eef942ecae --- /dev/null +++ b/products/posthog_ai/backend/turn_suggestions/service.py @@ -0,0 +1,260 @@ +"""Builds the end-of-turn suggestion for a sandbox PostHog AI conversation. + +Runs off the request path once a turn completes. It checks the cheap gates first, reads the run's +stream, asks the classifier, and publishes a ``_posthog/turn_suggestion`` frame into the same +stream the thread renders from, so the card appears under the answer without the frontend polling. +""" + +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Literal + +import structlog + +from posthog.dataclasses import frozen +from posthog.ph_client import feature_enabled_or_false, ph_scoped_capture + +from products.posthog_ai.backend.turn_suggestions.classifier import card_copy, classify_turn +from products.posthog_ai.backend.turn_suggestions.drafter import DRAFT_MODEL +from products.posthog_ai.backend.turn_suggestions.judgment import JUDGE_MODEL, judge_configured +from products.posthog_ai.backend.turn_suggestions.offer_ledger import ( + TurnSuggestionResolution, + claim_turn, + read_ledger, + record_offer, + resolve_offer, + withdraw_offer, +) +from products.posthog_ai.backend.turn_suggestions.transcript import TurnTranscript, build_turn_transcript +from products.posthog_ai.backend.turn_suggestions.verdict import OfferKind, TurnVerdict +from products.signals.backend.facade.api import scout_creation_available +from products.tasks.backend.facade.api import ( + TaskClientProvenance, + get_latest_run_by_task, + publish_task_run_stream_notification, + read_task_run_history, +) +from products.tasks.backend.models import Task, TaskRun + +if TYPE_CHECKING: + from posthog.models.user import User + +logger = structlog.get_logger(__name__) + +TURN_SUGGESTIONS_FLAG = "posthog-ai-turn-suggestions" +TURN_SUGGESTION_METHOD = "_posthog/turn_suggestion" +TURN_SUGGESTION_RESOLVED_METHOD = "_posthog/turn_suggestion_resolved" + +# Past this many log bytes the conversation gets no card: the fold holds every log in worker memory. +MAX_TRANSCRIPT_LOG_BYTES = 16 * 1024 * 1024 + +# The offers whose text a language model writes after the judgment picks them. +_DRAFTED_OFFERS = frozenset({OfferKind.SCOUT, OfferKind.NOTEBOOK}) + +OutcomeStatus = Literal["emitted", "skipped", "failed"] + + +@frozen +class TurnSuggestionOutcome: + status: OutcomeStatus + reason: str + + +def _skipped(reason: str) -> TurnSuggestionOutcome: + return TurnSuggestionOutcome(status="skipped", reason=reason) + + +def _turn_suggestions_enabled(task_run: TaskRun, user: "User") -> bool: + organization_id = str(task_run.team.organization_id) + return feature_enabled_or_false( + TURN_SUGGESTIONS_FLAG, + str(user.distinct_id), + groups={"organization": organization_id}, + group_properties={"organization": {"id": organization_id}}, + send_feature_flag_events=False, + ) + + +def _load_transcript(task_run: TaskRun) -> TurnTranscript | None: + """Fold the whole resume chain, because the thread counts turns across it. Returns ``None`` + when the logs are over ``MAX_TRANSCRIPT_LOG_BYTES``.""" + entries = read_task_run_history(task_run.id, task_run.task_id, task_run.team_id, max_bytes=MAX_TRANSCRIPT_LOG_BYTES) + return None if entries is None else build_turn_transcript(entries) + + +def _turn_has_substance(transcript: TurnTranscript) -> bool: + """A notebook is only worth saving when the assistant actually queried something and answered.""" + return bool(transcript.tool_calls) and bool(transcript.assistant_text) + + +def available_offers(transcript: TurnTranscript, *, scouts_available: bool) -> frozenset[OfferKind]: + """The offers this turn and project can act on. The classifier picks one of these, or shows nothing.""" + offers = set() + if _turn_has_substance(transcript): + offers.add(OfferKind.NOTEBOOK) + if transcript.saved_insights: + offers.add(OfferKind.SUBSCRIPTION) + if any(ref.alertable for ref in transcript.saved_insights): + offers.add(OfferKind.ALERT) + if transcript.error_issues: + offers.add(OfferKind.ERROR_ALERT) + if scouts_available: + offers.add(OfferKind.SCOUT) + return frozenset(offers) + + +def _suggestion_params(verdict: TurnVerdict, turn_index: int) -> dict | None: + if verdict.draft is None: + return None + copy = card_copy(verdict.draft) + return { + "turnIndex": turn_index, + "kind": verdict.picked.value, + "intent": verdict.intent.value, + "confidence": verdict.show_probability, + "title": copy.title, + "description": copy.description, + verdict.draft.WIRE_KEY: verdict.draft.to_params(), + } + + +def _capture_classified( + task_run: TaskRun, user: "User", verdict: TurnVerdict | None, *, emitted: bool, turn_index: int +) -> None: + try: + with ph_scoped_capture() as capture: + capture( + distinct_id=str(user.distinct_id), + event="posthog ai turn suggestion classified", + properties={ + "team_id": task_run.team_id, + "task_id": str(task_run.task_id), + "run_id": str(task_run.id), + "turn_index": turn_index, + "model": JUDGE_MODEL, + "draft_model": DRAFT_MODEL if verdict and verdict.picked in _DRAFTED_OFFERS else None, + "intent": verdict.intent.value if verdict else None, + "picked": verdict.picked.value if verdict else None, + "show_probability": verdict.show_probability if verdict else None, + "offer_probabilities": dict(verdict.offer_probabilities) if verdict else None, + "offer": verdict.picked.value if verdict and verdict.draft else None, + "emitted": emitted, + }, + ) + except Exception: + # Analytics must not change the outcome of a suggestion that already reached the thread. + logger.warning("posthog_ai_turn_suggestion_capture_failed", run_id=str(task_run.id), exc_info=True) + + +def generate_turn_suggestion(run_id: str, team_id: int) -> TurnSuggestionOutcome: + try: + task_run = TaskRun.objects.select_related("task__created_by", "team__organization").get( + id=run_id, team_id=team_id + ) + except TaskRun.DoesNotExist: + return _skipped("run_missing") + + task = task_run.task + if task.origin_product != Task.OriginProduct.POSTHOG_AI or task_run.mode != "interactive": + return _skipped("not_posthog_ai_conversation") + # Cards render only in the PostHog AI web app. Slack conversations carry the Slack origin and + # fail the check above; PostHog Desktop ones carry the Desktop client provenance. + if task.client_provenance == TaskClientProvenance.POSTHOG_DESKTOP: + return _skipped("not_started_in_web") + user = task.created_by + if user is None: + return _skipped("no_user") + # The judgment and the drafter send conversation text to third-party AI services. + if task_run.team.organization.is_ai_data_processing_approved is not True: + return _skipped("ai_data_processing_not_approved") + if not _turn_suggestions_enabled(task_run, user): + return _skipped("flag_off") + if not judge_configured(): + return _skipped("judge_not_configured") + early_refusal = read_ledger(task.id, task_run.team_id).refusal() + if early_refusal is not None: + return _skipped(early_refusal.value) + + transcript = _load_transcript(task_run) + if transcript is None: + return _skipped("log_too_large") + if not transcript.human_messages: + return _skipped("no_user_message") + turn_index = len(transcript.human_messages) - 1 + # Claimed before the turn can bail out, so a card still drafting for the previous turn sees + # that the thread moved past it even when this turn offers nothing. + refusal = claim_turn(task.id, task_run.team_id, turn_index) + if refusal is not None: + return _skipped(refusal.value) + if not transcript.assistant_text and not transcript.tool_calls: + return _skipped("empty_turn") + available = available_offers(transcript, scouts_available=scout_creation_available(team=task_run.team, user=user)) + if not available: + return _skipped("no_offers_available") + + verdict = classify_turn(transcript, team_id=task_run.team_id, today=datetime.now(UTC).date(), available=available) + if verdict is None: + _capture_classified(task_run, user, None, emitted=False, turn_index=turn_index) + return TurnSuggestionOutcome(status="failed", reason="classifier_failed") + + params = _suggestion_params(verdict, turn_index) + if params is None: + _capture_classified(task_run, user, verdict, emitted=False, turn_index=turn_index) + if verdict.picked != OfferKind.NONE: + return TurnSuggestionOutcome(status="failed", reason="draft_failed") + return _skipped(f"no_offer:{verdict.intent.value}") + + offer = verdict.picked.value + # Record before publishing, so a next turn that completes while the frame is in flight already sees the card. + record_refusal = record_offer(task.id, task_run.team_id, run_id=task_run.id, turn_index=turn_index, kind=offer) + if record_refusal is not None: + _capture_classified(task_run, user, verdict, emitted=False, turn_index=turn_index) + return _skipped(record_refusal.value) + # Publishing also appends to the run's S3 log, a rewrite of the whole log; the offer ledger is + # what keeps that to a couple of times per conversation. + emitted = publish_task_run_stream_notification( + task_run.id, task_run.task_id, task_run.team_id, TURN_SUGGESTION_METHOD, params + ).delivered + _capture_classified(task_run, user, verdict, emitted=emitted, turn_index=turn_index) + if not emitted: + withdraw_offer(task.id, task_run.team_id, turn_index=turn_index) + return TurnSuggestionOutcome(status="failed", reason="publish_failed") + return TurnSuggestionOutcome(status="emitted", reason=offer) + + +def resolve_turn_suggestion( + task_id: str, team_id: int, *, turn_index: int, resolution: TurnSuggestionResolution +) -> bool: + """Record a dismissed or accepted card, and tell the conversation's other open tabs. + + The ledger is the one stored outcome: a reloaded thread reads it through + ``turn_suggestion_state``. Returns ``False`` when that turn got no card or its card was already + resolved. + """ + offer = resolve_offer(task_id, team_id, turn_index=turn_index, resolution=resolution) + if offer is None: + return False + # A warmed successor run can start before the next message, so a tab may stream a newer run + # than the one that carried the card. + latest_run = get_latest_run_by_task([task_id]).get(str(task_id)) + run_ids = {str(offer.run_id)} | ({str(latest_run.id)} if latest_run is not None else set()) + for run_id in sorted(run_ids): + publish_task_run_stream_notification( + run_id, + task_id, + team_id, + TURN_SUGGESTION_RESOLVED_METHOD, + {"turnIndex": turn_index, "outcome": resolution.value}, + persist=False, + ) + return True + + +@frozen +class TurnSuggestionState: + muted: bool + resolved_turns: tuple[int, ...] + + +def turn_suggestion_state(task_id: str, team_id: int) -> TurnSuggestionState: + ledger = read_ledger(task_id, team_id) + return TurnSuggestionState(muted=ledger.muted, resolved_turns=ledger.resolved_turns) diff --git a/products/posthog_ai/backend/turn_suggestions/transcript.py b/products/posthog_ai/backend/turn_suggestions/transcript.py new file mode 100644 index 000000000000..95cc789bbbe6 --- /dev/null +++ b/products/posthog_ai/backend/turn_suggestions/transcript.py @@ -0,0 +1,401 @@ +"""Compact transcript of a sandbox conversation's latest turn, folded from raw agent-run stream frames. + +The stream carries every ACP frame the agent-server emitted. The turn classifier only needs three +things from it: what the user asked, what the assistant answered, and which PostHog tools ran with +their inner names resolved out of the single-exec ``mcp__posthog__exec`` wrapper. +""" + +import re +import json +from collections import Counter +from collections.abc import Iterable, Iterator +from typing import Any + +from posthog.dataclasses import frozen + +from products.posthog_ai.backend.exec_commands import INFO_SYNTHETIC_PREFIX, normalize_tool_name, parse_exec_command +from products.posthog_ai.backend.wire_types import NotificationFrame, is_user_message_params, parse_log_entry + +POSTHOG_EXEC_TOOL_RE = re.compile(r"^mcp__(?:plugin_)?posthog(?:_[^_]+)*__exec$") + +# The send path prefixes user text with context blocks the user never sees. The legacy +# ```` wrapper still appears in older histories. +_CONTEXT_BLOCK_RE = re.compile( + r"\A\s*<(posthog_trusted_context|posthog_untrusted_context|posthog_context)>.*?\s*", + re.DOTALL, +) + +# Masks the values an answer reports while keeping its shape: "412 signups, down 8%" reads as +# " signups, down %". A lookbehind skips digits inside identifiers such as `p95` or `v2`. +_URL_RE = re.compile(r"https?://\S+") +_EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+") +_UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.IGNORECASE) +_NUMBER_RE = re.compile(r"(? bool: + """The alert card configures a series threshold, which only a trends insight carries.""" + return self.query_kind in _ALERTABLE_INSIGHT_KINDS + + def to_params(self) -> dict: + return {"insightShortId": self.short_id, "insightId": self.insight_id, "insightName": self.name} + + +@frozen +class ErrorIssueRef: + issue_id: str + name: str + + +@frozen +class TurnTranscript: + human_messages: tuple[str, ...] + assistant_text: str + tool_calls: tuple[TranscriptToolCall, ...] + earlier_turns: tuple[EarlierTurn, ...] + saved_insights: tuple[SavedInsightRef, ...] + error_issues: tuple[ErrorIssueRef, ...] + + @property + def last_human_message(self) -> str: + return self.human_messages[-1] + + +@frozen(frozen=False) +class _ToolCallAccumulator: + name: str | None + args_preview: str + status: str + from_posthog: bool + discovery: bool + output: Any + + +EARLIER_TURN_LIMIT = 5 +ANSWER_EXCERPT_LIMIT = 300 +_INSIGHT_TOOLS = frozenset({"insight-create", "insight-update", "insight-get", "insight-query"}) +_ERROR_ISSUE_TOOLS = frozenset({"query-error-tracking-issues-list", "error-tracking-issue-get"}) + + +def _is_hidden(content: Any) -> bool: + meta = content.get("_meta") if isinstance(content, dict) else None + ui = meta.get("ui") if isinstance(meta, dict) else None + return isinstance(ui, dict) and ui.get("hidden") is True + + +def _text_from_content(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts = [ + block.get("text") + for block in content + if isinstance(block, dict) + and block.get("type") == "text" + and isinstance(block.get("text"), str) + and not _is_hidden(block) + ] + return "".join(part for part in parts if part) + return "" + + +def strip_context_blocks(text: str) -> str: + stripped = text + while True: + without_block = _CONTEXT_BLOCK_RE.sub("", stripped, count=1) + if without_block == stripped: + return stripped.strip() + stripped = without_block + + +def redact_values(text: str) -> str: + """Replace the numbers, emails, links and ids in ``text`` with placeholders.""" + text = _URL_RE.sub("", text) + text = _EMAIL_RE.sub("", text) + text = _UUID_RE.sub("", text) + return _NUMBER_RE.sub("", text) + + +def truncate_text(value: str, limit: int, *, collapse_whitespace: bool = True) -> str: + text = " ".join(value.split()) if collapse_whitespace else value.strip() + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def _args_preview(raw_input: Any) -> str: + if not isinstance(raw_input, dict): + return "" + for key in _TOOL_ARGS_PREVIEW_KEYS: + value = raw_input.get(key) + if isinstance(value, str) and value.strip(): + return truncate_text(value, TOOL_ARGS_PREVIEW_LIMIT) + try: + return truncate_text(json.dumps(raw_input, ensure_ascii=False), TOOL_ARGS_PREVIEW_LIMIT) + except (TypeError, ValueError): + return "" + + +def _agent_tool_name(update: dict[str, Any]) -> str: + meta = update.get("_meta") + claude_meta = meta.get("claudeCode") if isinstance(meta, dict) else None + tool_name = claude_meta.get("toolName") if isinstance(claude_meta, dict) else None + if isinstance(tool_name, str) and tool_name: + return tool_name + server_name = update.get("serverName") + raw_name = update.get("toolName") or update.get("title") or "" + if isinstance(server_name, str) and server_name and isinstance(raw_name, str): + return f"mcp__{server_name}__{raw_name}" + return raw_name if isinstance(raw_name, str) else "" + + +def _resolve_tool(update: dict[str, Any], accumulator: _ToolCallAccumulator) -> None: + """Fill the accumulator's name and args from a tool_call or tool_call_update frame. + + An exec command streams in over updates, so a frame without a parseable ``command`` leaves the + name unresolved for a later frame to fill. + """ + raw_input = update.get("rawInput") + agent_tool_name = _agent_tool_name(update) + # A tool_call_update carries no tool identity, so an exec call identified by its first frame + # stays exec when its command arrives later. + if not accumulator.from_posthog and not POSTHOG_EXEC_TOOL_RE.match(agent_tool_name): + if accumulator.name is None and agent_tool_name: + accumulator.name = normalize_tool_name(agent_tool_name) + if raw_input and not accumulator.args_preview: + accumulator.args_preview = _args_preview(raw_input) + return + + accumulator.from_posthog = True + command = raw_input.get("command") if isinstance(raw_input, dict) else None + if not isinstance(command, str) or not command.strip(): + return + parsed = parse_exec_command(command) + if parsed is None or parsed[0].startswith(INFO_SYNTHETIC_PREFIX): + # `tools`, `search`, `info` and `schema` only read the catalog and say nothing about the turn. + accumulator.discovery = True + return + tool, inner_input = parsed + # A `call` still streaming in also fails to parse, so a later complete frame clears the mark. + accumulator.discovery = False + accumulator.name = tool + accumulator.args_preview = _args_preview(inner_input) if inner_input else "" + + +def _notifications(entries: Iterable[dict[str, Any]]) -> Iterator[tuple[str | None, dict[str, Any]]]: + for entry in entries: + frame = parse_log_entry(entry) + if isinstance(frame, NotificationFrame): + params = frame.notification.params + yield frame.notification.method, params if isinstance(params, dict) else {} + + +# The synthetic prompt a resumed run starts with. The thread never renders it, so it is not a turn. +RESUME_CONTEXT_PREFIX = "You are resuming a previous conversation." + + +def _human_text(method: str | None, params: dict[str, Any]) -> tuple[str, bool] | None: + """The user's text in a ``_posthog/user_message`` frame, paired with ``True``, or in a + ``session/update`` user message, paired with ``False``. ``None`` for any other frame. + + These are the two forms the thread renders as a human message, so turn indexes match its trailers. + """ + if is_user_message_params(params, method): + return strip_context_blocks(_text_from_content(params.get("content"))), True + if method != "session/update": + return None + update = params.get("update") + if not isinstance(update, dict) or update.get("sessionUpdate") not in {"user_message", "user_message_chunk"}: + return None + content = update.get("content") + if _is_hidden(content): + return None + text = content.get("text") if isinstance(content, dict) else None + if not isinstance(text, str): + text = update.get("text") + return strip_context_blocks(text if isinstance(text, str) else ""), False + + +def build_turn_transcript(entries: Iterable[dict[str, Any]]) -> TurnTranscript: + """Fold stream entries into the transcript of the turn after the last user message. + + Human messages are counted the way the thread counts them: a ``_posthog/user_message`` frame, or a + ``session/update`` user message that does not repeat one, so a turn persisted in both wire forms + counts once. + """ + human_messages: list[str] = [] + remembered_texts: Counter[str] = Counter() + earlier_turns: list[EarlierTurn] = [] + # Assistant text by message id, in arrival order. Chunks append; a closing `agent_message` + # carries the whole text and replaces them, matching how the thread fold finalizes a bubble. + assistant_messages: dict[str, str] = {} + tool_calls: dict[str, _ToolCallAccumulator] = {} + + def start_turn(text: str) -> None: + if human_messages: + # The finished turn stays available as context for classifying the one that follows. + earlier_turns.append( + EarlierTurn( + question=human_messages[-1], + tool_names=tuple( + accumulator.name or "unknown" + for accumulator in tool_calls.values() + if not accumulator.discovery + ), + answer_excerpt=truncate_text(_join_messages(assistant_messages), ANSWER_EXCERPT_LIMIT), + ) + ) + human_messages.append(text) + assistant_messages.clear() + tool_calls.clear() + + def record_assistant_text(update: dict[str, Any], text: str, *, final: bool) -> None: + message_id = update.get("messageId") + key = message_id if isinstance(message_id, str) and message_id else "current" + if final: + # The wire is not consistent about carrying the id on the chunks and on the finalize, so + # a finalize closes the buffer its chunks opened: the idless one, or the last open one. + if key not in assistant_messages and "current" in assistant_messages: + assistant_messages.pop("current") + elif key == "current" and assistant_messages: + key = next(reversed(assistant_messages)) + assistant_messages[key] = text + return + assistant_messages[key] = assistant_messages.get(key, "") + text + + for method, params in _notifications(entries): + human = _human_text(method, params) + if human is not None: + text, is_user_message = human + if not text or text.startswith(RESUME_CONTEXT_PREFIX): + continue + if is_user_message: + remembered_texts[text] += 1 + elif remembered_texts[text] > 0: + remembered_texts[text] -= 1 + continue + start_turn(text) + continue + if method != "session/update": + continue + + update = params.get("update") + if not isinstance(update, dict): + continue + kind = update.get("sessionUpdate") + if kind in {"agent_message", "agent_message_chunk"}: + content = update.get("content") + chunk = content.get("text") if isinstance(content, dict) and content.get("type") == "text" else None + if isinstance(chunk, str) and chunk: + record_assistant_text(update, chunk, final=kind == "agent_message") + continue + if kind not in {"tool_call", "tool_call_update"}: + continue + tool_call_id = update.get("toolCallId") + if not isinstance(tool_call_id, str): + continue + accumulator = tool_calls.get(tool_call_id) + if accumulator is None: + accumulator = _ToolCallAccumulator( + name=None, args_preview="", status="pending", from_posthog=False, discovery=False, output=None + ) + tool_calls[tool_call_id] = accumulator + if accumulator.name is None or (accumulator.from_posthog and not accumulator.args_preview): + _resolve_tool(update, accumulator) + status = update.get("status") + if isinstance(status, str) and status: + accumulator.status = status + if "rawOutput" in update: + accumulator.output = update.get("rawOutput") + + active = [accumulator for accumulator in tool_calls.values() if not accumulator.discovery] + return TurnTranscript( + human_messages=tuple(human_messages), + assistant_text=truncate_text( + _join_messages(assistant_messages), ASSISTANT_TEXT_LIMIT, collapse_whitespace=False + ), + tool_calls=tuple( + TranscriptToolCall( + name=accumulator.name or "unknown", + args_preview=accumulator.args_preview, + status=accumulator.status, + ) + for accumulator in active + ), + earlier_turns=tuple(earlier_turns[-EARLIER_TURN_LIMIT:]), + saved_insights=tuple(_saved_insights(active)), + error_issues=tuple(_error_issues(active)), + ) + + +def _join_messages(assistant_messages: dict[str, str]) -> str: + return "\n\n".join(text for text in assistant_messages.values() if text) + + +def _saved_insights(tool_calls: list[_ToolCallAccumulator]) -> list[SavedInsightRef]: + """Saved insights the turn created or read, from the REST payload the insight tools return.""" + refs: dict[str, SavedInsightRef] = {} + for accumulator in tool_calls: + if accumulator.name not in _INSIGHT_TOOLS or not isinstance(accumulator.output, dict): + continue + short_id = accumulator.output.get("short_id") + query = accumulator.output.get("query") + if not isinstance(short_id, str) or not short_id or not isinstance(query, dict): + continue + source = query.get("source") if isinstance(query.get("source"), dict) else query + kind = source.get("kind") if isinstance(source, dict) else None + insight_id = accumulator.output.get("id") + name = accumulator.output.get("name") + refs[short_id] = SavedInsightRef( + short_id=short_id, + insight_id=insight_id if isinstance(insight_id, int) else None, + name=name if isinstance(name, str) else "", + query_kind=kind if isinstance(kind, str) else "", + ) + return list(refs.values()) + + +def _error_issues(tool_calls: list[_ToolCallAccumulator]) -> list[ErrorIssueRef]: + """Error tracking issues the turn looked at, from the list and detail tool payloads.""" + refs: dict[str, ErrorIssueRef] = {} + for accumulator in tool_calls: + if accumulator.name not in _ERROR_ISSUE_TOOLS or not isinstance(accumulator.output, dict): + continue + results = accumulator.output.get("results") + candidates = results if isinstance(results, list) else [accumulator.output] + for candidate in candidates: + if not isinstance(candidate, dict): + continue + issue_id = candidate.get("id") + if not isinstance(issue_id, str) or not issue_id: + continue + name = candidate.get("name") + refs[issue_id] = ErrorIssueRef(issue_id=issue_id, name=name if isinstance(name, str) else "") + return list(refs.values()) diff --git a/products/posthog_ai/backend/turn_suggestions/verdict.py b/products/posthog_ai/backend/turn_suggestions/verdict.py new file mode 100644 index 000000000000..f6c885065a6d --- /dev/null +++ b/products/posthog_ai/backend/turn_suggestions/verdict.py @@ -0,0 +1,139 @@ +"""The verdict a finished PostHog AI turn gets: which follow-up to offer, and the draft that offer needs.""" + +from collections.abc import Mapping +from enum import StrEnum +from typing import ClassVar + +from posthog.dataclasses import frozen + +from products.posthog_ai.backend.turn_suggestions.transcript import ErrorIssueRef, SavedInsightRef + + +class TurnIntent(StrEnum): + METRIC_STATE = "metric_state" + DIAGNOSTIC = "diagnostic" + ACTION = "action" + KNOWLEDGE = "knowledge" + OTHER = "other" + + +class OfferKind(StrEnum): + NONE = "none" + SCOUT = "scout" + NOTEBOOK = "notebook" + ALERT = "alert" + SUBSCRIPTION = "subscription" + ERROR_ALERT = "error_alert" + + +class ScoutMode(StrEnum): + REPORT = "report" + WATCH = "watch" + INVESTIGATE = "investigate" + DIGEST = "digest" + + +class ScoutCadence(StrEnum): + DAILY = "daily" + WEEKLY = "weekly" + + +class NotebookTemplate(StrEnum): + CONVERSATION = "conversation" + INCIDENT = "incident" + + +class AlertDirection(StrEnum): + DECREASE = "decrease" + INCREASE = "increase" + + +@frozen +class ScoutDraft: + WIRE_KEY: ClassVar[str] = "scout" + + mode: ScoutMode + display_name: str + description: str + body: str + cadence: ScoutCadence + + def to_params(self) -> dict: + return { + "mode": self.mode.value, + "displayName": self.display_name, + "description": self.description, + "body": self.body, + "cadence": self.cadence.value, + } + + +@frozen +class IncidentOutline: + timeline: str + cause: str + fix: str + + +@frozen +class NotebookDraft: + WIRE_KEY: ClassVar[str] = "notebook" + + title: str + summary: str + incident: IncidentOutline | None + + def to_params(self) -> dict: + incident = self.incident + return { + "title": self.title, + "summary": self.summary, + "incident": ( + {"timeline": incident.timeline, "cause": incident.cause, "fix": incident.fix} if incident else None + ), + } + + +@frozen +class AlertDraft: + WIRE_KEY: ClassVar[str] = "alert" + + insight: SavedInsightRef + direction: AlertDirection + change_percent: int + + def to_params(self) -> dict: + return {**self.insight.to_params(), "direction": self.direction.value, "changePercent": self.change_percent} + + +@frozen +class SubscriptionDraft: + WIRE_KEY: ClassVar[str] = "subscription" + + insight: SavedInsightRef + cadence: ScoutCadence + + def to_params(self) -> dict: + return {**self.insight.to_params(), "cadence": self.cadence.value} + + +@frozen +class ErrorAlertDraft: + WIRE_KEY: ClassVar[str] = "errorAlert" + + issue: ErrorIssueRef + + def to_params(self) -> dict: + return {"issueId": self.issue.issue_id, "issueName": self.issue.name} + + +Draft = ScoutDraft | NotebookDraft | AlertDraft | SubscriptionDraft | ErrorAlertDraft + + +@frozen +class TurnVerdict: + intent: TurnIntent + show_probability: float + picked: OfferKind + offer_probabilities: Mapping[str, float] + draft: Draft | None diff --git a/products/posthog_ai/frontend/generated/api.schemas.ts b/products/posthog_ai/frontend/generated/api.schemas.ts index f46e46fa298e..93ea5ad1553f 100644 --- a/products/posthog_ai/frontend/generated/api.schemas.ts +++ b/products/posthog_ai/frontend/generated/api.schemas.ts @@ -769,6 +769,48 @@ export interface TerminalAIRequestApi { temperature?: number | null } +/** + * * `dismissed` - Dismissed + * * `accepted` - Accepted + */ +export type TurnSuggestionResolutionEnumApi = + (typeof TurnSuggestionResolutionEnumApi)[keyof typeof TurnSuggestionResolutionEnumApi] + +export const TurnSuggestionResolutionEnumApi = { + Dismissed: 'dismissed', + Accepted: 'accepted', +} as const + +export interface ResolveTurnSuggestionApi { + /** ID of the PostHog AI conversation (task) the suggestion card belongs to. */ + task_id: string + /** + * Zero-based index of the conversation turn the suggestion card was shown under. + * @minimum 0 + */ + turn_index: number + /** What the user did with the card: `dismissed` mutes suggestions for the rest of the conversation, `accepted` means the offered scout, notebook, alert or subscription was created. + * + * * `dismissed` - Dismissed + * * `accepted` - Accepted */ + resolution: TurnSuggestionResolutionEnumApi +} + +export interface ResolveTurnSuggestionResponseApi { + /** Whether a suggestion card existed for that turn and this call recorded its outcome. A card keeps the first outcome recorded for it. */ + recorded: boolean +} + +export interface TurnSuggestionStateApi { + /** Whether the user dismissed a suggestion card in this conversation, which stops further cards. */ + muted: boolean + /** + * Zero-based indexes of the turns whose suggestion card the user already dismissed or accepted. + * @items.minimum 0 + */ + resolved_turns: number[] +} + export type ConversationsListParams = { /** * Number of results to return per page. @@ -801,3 +843,10 @@ export const TerminalAiCreateFormat = { Json: 'json', Txt: 'txt', } as const + +export type TurnSuggestionsStateRetrieveParams = { + /** + * ID of the PostHog AI conversation (task) to read suggestion outcomes for. + */ + task_id: string +} diff --git a/products/posthog_ai/frontend/generated/api.ts b/products/posthog_ai/frontend/generated/api.ts index 4a5034df3edd..8613e2f70541 100644 --- a/products/posthog_ai/frontend/generated/api.ts +++ b/products/posthog_ai/frontend/generated/api.ts @@ -24,10 +24,14 @@ import type { PaginatedMaxCoreMemoryListApi, PatchedConversationApi, PatchedMaxCoreMemoryApi, + ResolveTurnSuggestionApi, + ResolveTurnSuggestionResponseApi, SandboxMessageResponseApi, SandboxOpenApi, TerminalAIRequestApi, TerminalAiCreateParams, + TurnSuggestionStateApi, + TurnSuggestionsStateRetrieveParams, } from './api.schemas' // https://stackoverflow.com/questions/49579094/typescript-conditional-types-filter-out-readonly-properties-pick-only-requir/49579497#49579497 @@ -476,3 +480,53 @@ export const terminalAiCreate = async ( body: JSON.stringify(terminalAIRequestApi), }) } + +export const getTurnSuggestionsResolveCreateUrl = (projectId: string) => { + return `/api/projects/${projectId}/turn_suggestions/resolve/` +} + +/** + * @summary Record what the user did with a PostHog AI turn suggestion card + */ +export const turnSuggestionsResolveCreate = async ( + projectId: string, + resolveTurnSuggestionApi: ResolveTurnSuggestionApi, + options?: RequestInit +): Promise => { + return apiMutator(getTurnSuggestionsResolveCreateUrl(projectId), { + ...options, + method: 'POST', + headers: { 'Content-Type': 'application/json', ...options?.headers }, + body: JSON.stringify(resolveTurnSuggestionApi), + }) +} + +export const getTurnSuggestionsStateRetrieveUrl = (projectId: string, params: TurnSuggestionsStateRetrieveParams) => { + const normalizedParams = new URLSearchParams() + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? 'null' : String(value)) + } + }) + + const stringifiedParams = normalizedParams.toString() + + return stringifiedParams.length > 0 + ? `/api/projects/${projectId}/turn_suggestions/state/?${stringifiedParams}` + : `/api/projects/${projectId}/turn_suggestions/state/` +} + +/** + * @summary Read which PostHog AI turn suggestion cards the user already resolved + */ +export const turnSuggestionsStateRetrieve = async ( + projectId: string, + params: TurnSuggestionsStateRetrieveParams, + options?: RequestInit +): Promise => { + return apiMutator(getTurnSuggestionsStateRetrieveUrl(projectId, params), { + ...options, + method: 'GET', + }) +} diff --git a/products/posthog_ai/frontend/generated/api.zod.ts b/products/posthog_ai/frontend/generated/api.zod.ts index 7744b5e84d5f..6adfff1644f2 100644 --- a/products/posthog_ai/frontend/generated/api.zod.ts +++ b/products/posthog_ai/frontend/generated/api.zod.ts @@ -318,3 +318,22 @@ export const TerminalAiCreateBody = /* @__PURE__ */ zod.object({ .optional() .describe('Sampling temperature.'), }) + +/** + * @summary Record what the user did with a PostHog AI turn suggestion card + */ +export const turnSuggestionsResolveCreateBodyTurnIndexMin = 0 + +export const TurnSuggestionsResolveCreateBody = /* @__PURE__ */ zod.object({ + task_id: zod.uuid().describe('ID of the PostHog AI conversation (task) the suggestion card belongs to.'), + turn_index: zod + .number() + .min(turnSuggestionsResolveCreateBodyTurnIndexMin) + .describe('Zero-based index of the conversation turn the suggestion card was shown under.'), + resolution: zod + .enum(['dismissed', 'accepted']) + .describe('\* `dismissed` - Dismissed\n\* `accepted` - Accepted') + .describe( + 'What the user did with the card: `dismissed` mutes suggestions for the rest of the conversation, `accepted` means the offered scout, notebook, alert or subscription was created.\n\n\* `dismissed` - Dismissed\n\* `accepted` - Accepted' + ), +}) diff --git a/products/posthog_ai/mcp/tools.yaml b/products/posthog_ai/mcp/tools.yaml index a98ec5887118..52ca43d98543 100644 --- a/products/posthog_ai/mcp/tools.yaml +++ b/products/posthog_ai/mcp/tools.yaml @@ -137,3 +137,9 @@ tools: terminal-ai-create: operation: terminal_ai_create enabled: false + turn-suggestions-resolve-create: + operation: turn_suggestions_resolve_create + enabled: false + turn-suggestions-state-retrieve: + operation: turn_suggestions_state_retrieve + enabled: false diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts index 0c96e32cb87d..5acbfa0d93df 100644 --- a/services/mcp/src/api/generated.ts +++ b/services/mcp/src/api/generated.ts @@ -90897,6 +90897,38 @@ export namespace Schemas { horizon_days?: number; } + /** + * * `dismissed` - Dismissed + * * `accepted` - Accepted + */ + export type TurnSuggestionResolutionEnum = typeof TurnSuggestionResolutionEnum[keyof typeof TurnSuggestionResolutionEnum]; + + + export const TurnSuggestionResolutionEnum = { + Dismissed: 'dismissed', + Accepted: 'accepted', + } as const; + + export interface ResolveTurnSuggestion { + /** ID of the PostHog AI conversation (task) the suggestion card belongs to. */ + task_id: string; + /** + * Zero-based index of the conversation turn the suggestion card was shown under. + * @minimum 0 + */ + turn_index: number; + /** What the user did with the card: `dismissed` mutes suggestions for the rest of the conversation, `accepted` means the offered scout, notebook, alert or subscription was created. + * + * * `dismissed` - Dismissed + * * `accepted` - Accepted */ + resolution: TurnSuggestionResolutionEnum; + } + + export interface ResolveTurnSuggestionResponse { + /** Whether a suggestion card existed for that turn and this call recorded its outcome. A card keeps the first outcome recorded for it. */ + recorded: boolean; + } + /** * Resolved training population filter. Pass as 'training_population' to autoresearch-create. */ @@ -105822,6 +105854,16 @@ export namespace Schemas { limit: number; } + export interface TurnSuggestionState { + /** Whether the user dismissed a suggestion card in this conversation, which stops further cards. */ + muted: boolean; + /** + * Zero-based indexes of the turns whose suggestion card the user already dismissed or accepted. + * @items.minimum 0 + */ + resolved_turns: number[]; + } + export interface TwoFactorStatus { /** Whether the user has any 2FA method enabled. */ is_enabled: boolean; @@ -123726,6 +123768,13 @@ export namespace Schemas { offset?: number; }; + export type TurnSuggestionsStateRetrieveParams = { + /** + * ID of the PostHog AI conversation (task) to read suggestion outcomes for. + */ + task_id: string; + }; + export type UploadedMediaListParams = { /** * Number of results to return per page. diff --git a/tach.toml b/tach.toml index 5285a43d46e8..b12d5417120f 100644 --- a/tach.toml +++ b/tach.toml @@ -715,6 +715,7 @@ depends_on = [ # two sides share one definition. The skills product has no facade, so this matches how # review_hog, signals and tasks reach it. "products.skills", + "products.signals", "products.tasks", ] layer = "modules" From 20de20495ee3a3eef2452707ab528af9892d0918 Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Fri, 25 Sep 2026 01:45:41 -0300 Subject: [PATCH 02/11] fix(growth): pass ids to the scout creation check Co-Authored-By: Claude Opus 5.5 (1M context) --- products/posthog_ai/backend/turn_suggestions/service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/products/posthog_ai/backend/turn_suggestions/service.py b/products/posthog_ai/backend/turn_suggestions/service.py index b7eef942ecae..b067393023f7 100644 --- a/products/posthog_ai/backend/turn_suggestions/service.py +++ b/products/posthog_ai/backend/turn_suggestions/service.py @@ -187,7 +187,9 @@ def generate_turn_suggestion(run_id: str, team_id: int) -> TurnSuggestionOutcome return _skipped(refusal.value) if not transcript.assistant_text and not transcript.tool_calls: return _skipped("empty_turn") - available = available_offers(transcript, scouts_available=scout_creation_available(team=task_run.team, user=user)) + available = available_offers( + transcript, scouts_available=scout_creation_available(team_id=task_run.team_id, user_id=user.id) + ) if not available: return _skipped("no_offers_available") From 21e10d28555450d636aae7ed29d0e42fceb77c00 Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Fri, 25 Sep 2026 09:15:25 -0300 Subject: [PATCH 03/11] fix(growth): harden turn suggestion turn selection, extraction and publishing - read insight and issue refs from the MCP app-data envelope, and resolve tool identity from the adapter's _meta.posthog stamp - note a follow-up that is still running instead of claiming its turn - skip conversations past the resume chain depth, where turn indexes slide - count a card only when it reached the run log - cap the insight and issue options Jev sees - release the enqueue dedup key when queueing fails, and keep it only for the settle wait - send the Luna draft cap as max_completion_tokens Co-Authored-By: Claude Opus 5.5 (1M context) --- .../backend/tests/test_turn_suggestions.py | 105 +++++++++++++++++- .../backend/turn_suggestions/dispatch.py | 26 ++++- .../backend/turn_suggestions/drafter.py | 2 +- .../backend/turn_suggestions/judgment.py | 7 +- .../backend/turn_suggestions/offer_ledger.py | 21 +++- .../backend/turn_suggestions/service.py | 26 ++++- .../backend/turn_suggestions/transcript.py | 59 ++++++++-- 7 files changed, 221 insertions(+), 25 deletions(-) diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions.py b/products/posthog_ai/backend/tests/test_turn_suggestions.py index 46152649d9d6..e6b2d3aa5249 100644 --- a/products/posthog_ai/backend/tests/test_turn_suggestions.py +++ b/products/posthog_ai/backend/tests/test_turn_suggestions.py @@ -27,6 +27,7 @@ from products.posthog_ai.backend.turn_suggestions.dispatch import TURN_SETTLE_SECONDS, enqueue_turn_suggestion from products.posthog_ai.backend.turn_suggestions.drafter import DRAFT_MODEL, draft_scout, render_turn_prompt from products.posthog_ai.backend.turn_suggestions.judgment import ( + MAX_REF_OPTIONS, TurnJudgment, build_judge_questions, build_judge_state, @@ -48,6 +49,7 @@ generate_turn_suggestion, ) from products.posthog_ai.backend.turn_suggestions.transcript import ( + MCP_APP_DATA_META_KEY, ErrorIssueRef, SavedInsightRef, build_turn_transcript, @@ -332,11 +334,52 @@ def test_earlier_turns_are_kept_as_context_and_only_the_last_turn_is_folded(self ) ] - def test_saved_insights_and_error_issues_come_from_the_tool_payloads(self): - assert build_turn_transcript(_saved_insight_turn()).saved_insights == (SAVED_INSIGHT,) - assert build_turn_transcript(_error_turn()).error_issues == (ERROR_ISSUE,) + @parameterized.expand( + [ + ("handler_payload", lambda payload: payload), + ( + "mcp_app_data", + lambda payload: { + "content": [{"type": "text", "text": "Saved."}], + "_meta": {MCP_APP_DATA_META_KEY: payload}, + }, + ), + ("mcp_structured_content", lambda payload: {"structuredContent": payload, "isError": False}), + ] + ) + def test_saved_insights_and_error_issues_come_from_the_tool_payloads(self, _name: str, wrap): + insight_turn = _saved_insight_turn() + insight_turn[1]["notification"]["params"]["update"]["rawOutput"] = wrap( + insight_turn[1]["notification"]["params"]["update"]["rawOutput"] + ) + error_turn = _error_turn() + error_turn[1]["notification"]["params"]["update"]["rawOutput"] = wrap( + error_turn[1]["notification"]["params"]["update"]["rawOutput"] + ) + + assert build_turn_transcript(insight_turn).saved_insights == (SAVED_INSIGHT,) + assert build_turn_transcript(error_turn).error_issues == (ERROR_ISSUE,) assert build_turn_transcript(_metric_turn()).saved_insights == () + def test_a_failed_insight_call_yields_no_reference(self): + insight_turn = _saved_insight_turn() + update = insight_turn[1]["notification"]["params"]["update"] + update["rawOutput"] = {"_meta": {MCP_APP_DATA_META_KEY: update["rawOutput"]}, "isError": True} + + assert build_turn_transcript(insight_turn).saved_insights == () + + def test_resolves_exec_identified_by_posthog_adapter_metadata(self): + entries = _saved_insight_turn() + update = entries[1]["notification"]["params"]["update"] + del update["serverName"], update["toolName"] + update["title"] = "exec" + update["_meta"] = {"posthog": {"toolName": "mcp__posthog__exec", "mcp": {"server": "posthog", "tool": "exec"}}} + + transcript = build_turn_transcript(entries) + + assert [call.name for call in transcript.tool_calls] == ["insight-create"] + assert transcript.saved_insights == (SAVED_INSIGHT,) + @parameterized.expand( [ ( @@ -502,6 +545,19 @@ def test_a_turn_with_one_possible_offer_skips_the_offer_question(self): assert "offer" not in build_judge_questions(transcript, only_scout) assert judgment is not None and judgment.offer == OfferKind.SCOUT + def test_a_turn_with_more_issues_than_a_choice_takes_is_still_judged(self): + issues = [{"id": f"issue-{index}", "name": f"Error {index}"} for index in range(300)] + entries = [ + _user_message("Which errors are new this week?"), + _exec_tool_call("t1", "call query-error-tracking-issues-list {}", "completed", {"results": issues}), + _agent_text("Here are the new errors."), + ] + transcript = build_turn_transcript(entries) + + question = build_judge_questions(transcript, ALL_OFFERS)["error_issue"] + + assert isinstance(question, ChoiceQuestion) and len(question.criteria) == MAX_REF_OPTIONS + 1 + def test_option_keys_map_back_to_the_refs_they_stand_for(self): answers = _answers( {"show_offer": 0.8, "issue_resolved": 0.1}, @@ -724,6 +780,18 @@ def test_a_turn_reported_twice_is_queued_once(self): apply_async.assert_called_once() + def test_a_failed_enqueue_leaves_the_turn_for_the_next_report(self): + task_run = self._run(Task.OriginProduct.POSTHOG_AI) + + with patch( + "products.posthog_ai.backend.tasks.generate_turn_suggestion_task.apply_async", + side_effect=[ConnectionError("broker down"), None], + ) as apply_async: + assert enqueue_turn_suggestion(task_run) is False + assert enqueue_turn_suggestion(task_run) is True + + assert apply_async.call_count == 2 + def test_the_task_runs_on_the_posthog_ai_queue(self): assert generate_turn_suggestion_task.queue == CeleryQueue.POSTHOG_AI.value @@ -911,6 +979,7 @@ def test_follow_up_turns_are_classified_with_their_own_turn_index(self): [ ("next_turn_claims", "superseded"), ("next_turn_offers_nothing", "superseded"), + ("follow_up_still_running", "superseded"), ("earlier_card_dismissed", "dismissed"), ] ) @@ -937,6 +1006,9 @@ def classify_while_interrupted(*_args: Any, **_kwargs: Any) -> TurnVerdict: ] self.mocks["scouts"].return_value = False assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="no_offers_available") + elif interruption == "follow_up_still_running": + self.mocks["history"].return_value = [*self.mocks["history"].return_value, _user_message("Thanks")] + assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="empty_turn") else: resolution = TurnSuggestionResolution.DISMISSED assert resolve_offer(self.task_run.task_id, self.team.id, turn_index=0, resolution=resolution) @@ -950,6 +1022,33 @@ def classify_while_interrupted(*_args: Any, **_kwargs: Any) -> TurnVerdict: self.mocks["publish"].assert_not_called() assert [offer.turn_index for offer in read_ledger(self.task_run.task_id, self.team.id).offers] == [0] + def test_a_follow_up_still_running_at_the_read_is_classified_when_it_completes(self): + follow_up = [*_metric_turn(), _user_message("Break that down by country")] + self.mocks["history"].return_value = follow_up + + assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="empty_turn") + + self.mocks["history"].return_value = [*follow_up, _agent_text("Most signups came from the US.")] + outcome = self._generate() + + assert outcome == TurnSuggestionOutcome(status="emitted", reason="scout") + assert self._published_params()["turnIndex"] == 1 + + def test_a_card_only_the_live_stream_got_spends_no_offer(self): + self.mocks["publish"].return_value = StreamNotificationDelivery(live=True, persisted=False) + + assert self._generate() == TurnSuggestionOutcome(status="failed", reason="publish_failed") + assert read_ledger(self.task_run.task_id, self.team.id).offers == () + + def test_a_conversation_past_the_resume_chain_depth_gets_no_card(self): + run = self.task_run + for _ in range(11): + run = run.task.create_run(mode="interactive", extra_state={"resume_from_run_id": str(run.id)}) + self.task_run = run + + assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="history_truncated") + self.mocks["history"].assert_not_called() + @parameterized.expand( [ ("muted", {"offers": [{**_offer(0), "status": "dismissed"}]}, "dismissed"), diff --git a/products/posthog_ai/backend/turn_suggestions/dispatch.py b/products/posthog_ai/backend/turn_suggestions/dispatch.py index ec2a92ae1229..7e43cd1df8de 100644 --- a/products/posthog_ai/backend/turn_suggestions/dispatch.py +++ b/products/posthog_ai/backend/turn_suggestions/dispatch.py @@ -11,12 +11,14 @@ logger = structlog.get_logger(__name__) -# The proxy callback, the event ingest and the sandbox relay can each report the same turn, within -# moments of each other. A person cannot finish two turns this close together. -ENQUEUE_DEDUP_SECONDS = 5 # The proxy callback reports over its own channel and can beat the ingest writes of the turn's last # frames, so the read waits for them instead of judging a turn whose answer has not landed. TURN_SETTLE_SECONDS = 2 +# The proxy callback, the event ingest and the sandbox relay can each report the same turn, within +# moments of each other. A report that lands while a job waits is covered by that job, which reads +# the conversation's latest turn when it runs. Past the wait, a report can be a later turn, so it +# queues its own job. +ENQUEUE_DEDUP_SECONDS = TURN_SETTLE_SECONDS def _enqueue_dedup_key(run_id: str) -> str: @@ -32,6 +34,13 @@ def _first_report_of_turn(run_id: str) -> bool: return True +def _release_report(run_id: str) -> None: + try: + get_client().delete(_enqueue_dedup_key(run_id)) + except Exception: + logger.warning("posthog_ai_turn_suggestion_dedup_release_failed", run_id=run_id, exc_info=True) + + def enqueue_turn_suggestion(task_run: "TaskRun") -> bool: """Queue the end-of-turn suggestion for a PostHog AI run. Never raises: the turn completion that calls this must not fail because a nudge could not be scheduled.""" @@ -45,9 +54,14 @@ def enqueue_turn_suggestion(task_run: "TaskRun") -> bool: generate_turn_suggestion_task, # noqa: PLC0415 — keeps the judge and drafter clients off the Django startup path ) - generate_turn_suggestion_task.apply_async( - kwargs={"run_id": run_id, "team_id": task_run.team_id}, countdown=TURN_SETTLE_SECONDS - ) + try: + generate_turn_suggestion_task.apply_async( + kwargs={"run_id": run_id, "team_id": task_run.team_id}, countdown=TURN_SETTLE_SECONDS + ) + except Exception: + # Another report of the turn can still queue it. + _release_report(run_id) + raise except Exception: logger.warning("posthog_ai_turn_suggestion_enqueue_failed", run_id=str(task_run.id), exc_info=True) return False diff --git a/products/posthog_ai/backend/turn_suggestions/drafter.py b/products/posthog_ai/backend/turn_suggestions/drafter.py index 9f735fa84f6d..b452bfc336a2 100644 --- a/products/posthog_ai/backend/turn_suggestions/drafter.py +++ b/products/posthog_ai/backend/turn_suggestions/drafter.py @@ -122,7 +122,7 @@ def _complete( response = client.chat.completions.create( model=DRAFT_MODEL, messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}], - max_tokens=DRAFT_MAX_TOKENS, + max_completion_tokens=DRAFT_MAX_TOKENS, response_format=_response_format(schema_name, reply_type), ) except Exception: diff --git a/products/posthog_ai/backend/turn_suggestions/judgment.py b/products/posthog_ai/backend/turn_suggestions/judgment.py index ab0a57d37186..812ce6fd32cb 100644 --- a/products/posthog_ai/backend/turn_suggestions/judgment.py +++ b/products/posthog_ai/backend/turn_suggestions/judgment.py @@ -202,8 +202,13 @@ def judge_configured() -> bool: return bool(settings.TYPESAFE_API_KEY) +# A turn that lists issues can collect hundreds, past what one choice question takes. The state, the +# questions and the answer lookup all number refs through `_numbered`, so the cap keeps them in step. +MAX_REF_OPTIONS = 20 + + def _numbered[T](prefix: str, refs: Sequence[T]) -> dict[str, T]: - return {f"{prefix}_{index}": ref for index, ref in enumerate(refs, start=1)} + return {f"{prefix}_{index}": ref for index, ref in enumerate(refs[:MAX_REF_OPTIONS], start=1)} def _insight_options(transcript: TurnTranscript) -> dict[str, SavedInsightRef]: diff --git a/products/posthog_ai/backend/turn_suggestions/offer_ledger.py b/products/posthog_ai/backend/turn_suggestions/offer_ledger.py index 32a1517a3cd7..6f107c00c590 100644 --- a/products/posthog_ai/backend/turn_suggestions/offer_ledger.py +++ b/products/posthog_ai/backend/turn_suggestions/offer_ledger.py @@ -67,6 +67,8 @@ def from_json(cls, raw: Any) -> "OfferRecord | None": class OfferLedger: offers: tuple[OfferRecord, ...] = () last_classified_turn: int = -1 + # The newest turn the conversation reached, classified or still running. + latest_turn: int = -1 @property def muted(self) -> bool: @@ -92,15 +94,18 @@ def from_json(cls, raw: Any) -> "OfferLedger": parsed = (OfferRecord.from_json(entry) for entry in raw_offers) if isinstance(raw_offers, list) else () offers = tuple(offer for offer in parsed if offer is not None) last_classified_turn = raw.get("last_classified_turn") + latest_turn = raw.get("latest_turn") return cls( offers=offers, last_classified_turn=last_classified_turn if isinstance(last_classified_turn, int) else -1, + latest_turn=latest_turn if isinstance(latest_turn, int) else -1, ) def to_json(self) -> dict[str, Any]: return { "offers": [offer.to_json() for offer in self.offers], "last_classified_turn": self.last_classified_turn, + "latest_turn": self.latest_turn, } def refusal(self, turn_index: int | None = None) -> ClaimRefusal | None: @@ -160,11 +165,23 @@ def claim(ledger: OfferLedger) -> OfferLedger | ClaimRefusal: refusal = ledger.refusal(turn_index) if refusal is not None: return refusal - return replace(ledger, last_classified_turn=turn_index) + return replace(ledger, last_classified_turn=turn_index, latest_turn=max(ledger.latest_turn, turn_index)) return _claim(task_id, team_id, claim) +def note_turn(task_id: UUID | str, team_id: int, turn_index: int) -> None: + """Record that the conversation reached ``turn_index`` without claiming it, so a card still + drafting for an earlier turn does not land under it. The turn stays open for classification.""" + + def note(ledger: OfferLedger) -> tuple[OfferLedger | None, None]: + if ledger.latest_turn >= turn_index: + return None, None + return replace(ledger, latest_turn=turn_index), None + + _update_ledger(task_id, team_id, note) + + def record_offer( task_id: UUID | str, team_id: int, *, run_id: UUID | str, turn_index: int, kind: str ) -> ClaimRefusal | None: @@ -179,7 +196,7 @@ def record_offer( offer = OfferRecord(turn_index=turn_index, run_id=str(run_id), kind=kind, status=OfferStatus.OFFERED) def append(ledger: OfferLedger) -> OfferLedger | ClaimRefusal: - if ledger.last_classified_turn != turn_index: + if ledger.last_classified_turn != turn_index or ledger.latest_turn > turn_index: return ClaimRefusal.SUPERSEDED refusal = ledger.refusal() if refusal is not None: diff --git a/products/posthog_ai/backend/turn_suggestions/service.py b/products/posthog_ai/backend/turn_suggestions/service.py index b067393023f7..161c6943fb19 100644 --- a/products/posthog_ai/backend/turn_suggestions/service.py +++ b/products/posthog_ai/backend/turn_suggestions/service.py @@ -19,6 +19,7 @@ from products.posthog_ai.backend.turn_suggestions.offer_ledger import ( TurnSuggestionResolution, claim_turn, + note_turn, read_ledger, record_offer, resolve_offer, @@ -74,6 +75,16 @@ def _turn_suggestions_enabled(task_run: TaskRun, user: "User") -> bool: ) +def _history_is_whole(task_run: TaskRun) -> bool: + """Whether the resume chain the history reads reaches the conversation's first run. + + The chain walk stops after a fixed depth, and the thread reads the same chain. Past that depth + turn indexes slide with every new run, so they no longer name one turn in the ledger. + """ + oldest = task_run.get_resume_chain()[0] + return not (oldest.state or {}).get("resume_from_run_id") + + def _load_transcript(task_run: TaskRun) -> TurnTranscript | None: """Fold the whole resume chain, because the thread counts turns across it. Returns ``None`` when the logs are over ``MAX_TRANSCRIPT_LOG_BYTES``.""" @@ -173,6 +184,8 @@ def generate_turn_suggestion(run_id: str, team_id: int) -> TurnSuggestionOutcome early_refusal = read_ledger(task.id, task_run.team_id).refusal() if early_refusal is not None: return _skipped(early_refusal.value) + if not _history_is_whole(task_run): + return _skipped("history_truncated") transcript = _load_transcript(task_run) if transcript is None: @@ -180,13 +193,17 @@ def generate_turn_suggestion(run_id: str, team_id: int) -> TurnSuggestionOutcome if not transcript.human_messages: return _skipped("no_user_message") turn_index = len(transcript.human_messages) - 1 + if not transcript.assistant_text and not transcript.tool_calls: + # A follow-up sent during the settle wait shows here as an empty latest turn. Its own + # completion classifies it, so it is only noted: a card still drafting for the previous + # turn must not land under it. + note_turn(task.id, task_run.team_id, turn_index) + return _skipped("empty_turn") # Claimed before the turn can bail out, so a card still drafting for the previous turn sees # that the thread moved past it even when this turn offers nothing. refusal = claim_turn(task.id, task_run.team_id, turn_index) if refusal is not None: return _skipped(refusal.value) - if not transcript.assistant_text and not transcript.tool_calls: - return _skipped("empty_turn") available = available_offers( transcript, scouts_available=scout_creation_available(team_id=task_run.team_id, user_id=user.id) ) @@ -212,10 +229,11 @@ def generate_turn_suggestion(run_id: str, team_id: int) -> TurnSuggestionOutcome _capture_classified(task_run, user, verdict, emitted=False, turn_index=turn_index) return _skipped(record_refusal.value) # Publishing also appends to the run's S3 log, a rewrite of the whole log; the offer ledger is - # what keeps that to a couple of times per conversation. + # what keeps that to a couple of times per conversation. Only a card in the log counts, because + # the log is what a reload replays it from. emitted = publish_task_run_stream_notification( task_run.id, task_run.task_id, task_run.team_id, TURN_SUGGESTION_METHOD, params - ).delivered + ).persisted _capture_classified(task_run, user, verdict, emitted=emitted, turn_index=turn_index) if not emitted: withdraw_offer(task.id, task_run.team_id, turn_index=turn_index) diff --git a/products/posthog_ai/backend/turn_suggestions/transcript.py b/products/posthog_ai/backend/turn_suggestions/transcript.py index 95cc789bbbe6..68e1ada1dfa3 100644 --- a/products/posthog_ai/backend/turn_suggestions/transcript.py +++ b/products/posthog_ai/backend/turn_suggestions/transcript.py @@ -17,6 +17,8 @@ from products.posthog_ai.backend.wire_types import NotificationFrame, is_user_message_params, parse_log_entry POSTHOG_EXEC_TOOL_RE = re.compile(r"^mcp__(?:plugin_)?posthog(?:_[^_]+)*__exec$") +# The PostHog MCP server puts a tool's handler payload here; its text content is a lossy rendering. +MCP_APP_DATA_META_KEY = "com.posthog.mcp/app_data" # The send path prefixes user text with context blocks the user never sees. The legacy # ```` wrapper still appears in older histories. @@ -163,8 +165,25 @@ def _args_preview(raw_input: Any) -> str: return "" +def _posthog_meta_tool_name(meta: Any) -> str | None: + """The tool identity an adapter stamps on ``_meta.posthog``, the way the thread's tool resolver reads it.""" + posthog_meta = meta.get("posthog") if isinstance(meta, dict) else None + if not isinstance(posthog_meta, dict): + return None + mcp = posthog_meta.get("mcp") + if isinstance(mcp, dict): + server, tool = mcp.get("server"), mcp.get("tool") + if isinstance(server, str) and server and isinstance(tool, str) and tool: + return f"mcp__{server}__{tool}" + tool_name = posthog_meta.get("toolName") + return tool_name if isinstance(tool_name, str) and tool_name else None + + def _agent_tool_name(update: dict[str, Any]) -> str: meta = update.get("_meta") + posthog_tool_name = _posthog_meta_tool_name(meta) + if posthog_tool_name: + return posthog_tool_name claude_meta = meta.get("claudeCode") if isinstance(meta, dict) else None tool_name = claude_meta.get("toolName") if isinstance(claude_meta, dict) else None if isinstance(tool_name, str) and tool_name: @@ -359,20 +378,41 @@ def _join_messages(assistant_messages: dict[str, str]) -> str: return "\n\n".join(text for text in assistant_messages.values() if text) +_MCP_ENVELOPE_KEYS = frozenset({"content", "structuredContent", "isError", "_meta", "__execBuiltPayload"}) + + +def _tool_output_record(accumulator: _ToolCallAccumulator) -> dict[str, Any] | None: + """The handler payload a tool returned, unwrapped from its MCP envelope the way the thread's widgets read it.""" + output = accumulator.output + if not isinstance(output, dict) or output.get("isError") is True or accumulator.status == "failed": + return None + meta = output.get("_meta") + app_data = meta.get(MCP_APP_DATA_META_KEY) if isinstance(meta, dict) else None + if isinstance(app_data, dict): + return app_data + structured = output.get("structuredContent") + if isinstance(structured, dict): + return structured + return None if output.keys() <= _MCP_ENVELOPE_KEYS else output + + def _saved_insights(tool_calls: list[_ToolCallAccumulator]) -> list[SavedInsightRef]: """Saved insights the turn created or read, from the REST payload the insight tools return.""" refs: dict[str, SavedInsightRef] = {} for accumulator in tool_calls: - if accumulator.name not in _INSIGHT_TOOLS or not isinstance(accumulator.output, dict): + if accumulator.name not in _INSIGHT_TOOLS: continue - short_id = accumulator.output.get("short_id") - query = accumulator.output.get("query") + output = _tool_output_record(accumulator) + if output is None: + continue + short_id = output.get("short_id") + query = output.get("query") if not isinstance(short_id, str) or not short_id or not isinstance(query, dict): continue source = query.get("source") if isinstance(query.get("source"), dict) else query kind = source.get("kind") if isinstance(source, dict) else None - insight_id = accumulator.output.get("id") - name = accumulator.output.get("name") + insight_id = output.get("id") + name = output.get("name") refs[short_id] = SavedInsightRef( short_id=short_id, insight_id=insight_id if isinstance(insight_id, int) else None, @@ -386,10 +426,13 @@ def _error_issues(tool_calls: list[_ToolCallAccumulator]) -> list[ErrorIssueRef] """Error tracking issues the turn looked at, from the list and detail tool payloads.""" refs: dict[str, ErrorIssueRef] = {} for accumulator in tool_calls: - if accumulator.name not in _ERROR_ISSUE_TOOLS or not isinstance(accumulator.output, dict): + if accumulator.name not in _ERROR_ISSUE_TOOLS: + continue + output = _tool_output_record(accumulator) + if output is None: continue - results = accumulator.output.get("results") - candidates = results if isinstance(results, list) else [accumulator.output] + results = output.get("results") + candidates = results if isinstance(results, list) else [output] for candidate in candidates: if not isinstance(candidate, dict): continue From 785c6c06527c736ed2d536ee11437bb9f75cc298 Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Fri, 25 Sep 2026 10:09:51 -0300 Subject: [PATCH 04/11] feat(growth): draft turn suggestions through the ai-gateway at posthog's cost The Luna drafter now uses build_openai_client, so it reaches the Go ai-gateway where AI_GATEWAY_URL is set and bills the wallet of the team that owns that key. The Python fallback moves from the posthog_ai route, which bills the customer's AI credits, to the unbilled growth route. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../backend/tests/test_turn_suggestions.py | 46 ++++++++++++++++--- .../backend/turn_suggestions/drafter.py | 16 ++++++- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions.py b/products/posthog_ai/backend/tests/test_turn_suggestions.py index e6b2d3aa5249..4a61a1b1ae4e 100644 --- a/products/posthog_ai/backend/tests/test_turn_suggestions.py +++ b/products/posthog_ai/backend/tests/test_turn_suggestions.py @@ -6,9 +6,10 @@ from posthog.test.base import BaseTest from unittest.mock import MagicMock, patch -from django.test import SimpleTestCase +from django.test import SimpleTestCase, override_settings import requests +from openai.resources.chat.completions import Completions from parameterized import parameterized from posthog.celery_queues import CeleryQueue @@ -20,7 +21,6 @@ SystemOneResult, TypeSafeRequestFailed, ) -from posthog.llm.gateway_client import GatewayNotConfiguredError from products.posthog_ai.backend.tasks import generate_turn_suggestion_task from products.posthog_ai.backend.turn_suggestions.classifier import CardCopy, card_copy, classify_turn, pick_offer @@ -690,7 +690,7 @@ class TestDraftScout(SimpleTestCase): def _draft(self, content: str) -> tuple[ScoutDraft | None, MagicMock]: client = _gateway_reply({}) client.chat.completions.create.return_value = MagicMock(choices=[MagicMock(message=MagicMock(content=content))]) - with patch("products.posthog_ai.backend.turn_suggestions.drafter.get_llm_client", return_value=client): + with patch("products.posthog_ai.backend.turn_suggestions.drafter.build_openai_client", return_value=client): draft = draft_scout( build_turn_transcript(_metric_turn()), team_id=1, @@ -729,11 +729,43 @@ def test_unusable_replies_return_none(self, _name: str, content: str): assert draft is None - def test_an_unconfigured_gateway_returns_none(self): - with patch( - "products.posthog_ai.backend.turn_suggestions.drafter.get_llm_client", - side_effect=GatewayNotConfiguredError("LLM_GATEWAY_URL and an API key must be configured"), + @parameterized.expand( + [ + ( + "go_gateway", + {"AI_GATEWAY_URL": "https://ai-gateway.example.com/v1", "AI_GATEWAY_API_KEY": "phs_test"}, + "https://ai-gateway.example.com/v1/", + ), + # The posthog_ai route bills the customer's AI credits, so the fallback must use an unbilled one. + ( + "python_fallback", + {"LLM_GATEWAY_URL": "http://llm-gateway.example.com", "LLM_GATEWAY_API_KEY": "phx_test"}, + "http://llm-gateway.example.com/growth/v1/", + ), + ] + ) + def test_drafts_are_billed_to_posthog_on_either_gateway(self, _name: str, gateway: dict, base_url: str): + unset = {"AI_GATEWAY_URL": "", "AI_GATEWAY_API_KEY": "", "LLM_GATEWAY_URL": "", "LLM_GATEWAY_API_KEY": ""} + reply = MagicMock(choices=[MagicMock(message=MagicMock(content="{}"))]) + with ( + override_settings(**{**unset, **gateway}), + patch.object(Completions, "create", autospec=True, return_value=reply) as create, ): + draft_scout( + build_turn_transcript(_metric_turn()), + team_id=7, + today=date(2026, 9, 16), + mode=ScoutMode.WATCH, + cadence=ScoutCadence.DAILY, + ) + + client = create.call_args.args[0]._client + assert str(client.base_url) == base_url + assert create.call_args.kwargs["user"] == "team-7" + + def test_an_unconfigured_gateway_returns_none(self): + unset = {"AI_GATEWAY_URL": "", "AI_GATEWAY_API_KEY": "", "LLM_GATEWAY_URL": "", "LLM_GATEWAY_API_KEY": ""} + with override_settings(**unset): draft = draft_scout( build_turn_transcript(_metric_turn()), team_id=1, diff --git a/products/posthog_ai/backend/turn_suggestions/drafter.py b/products/posthog_ai/backend/turn_suggestions/drafter.py index b452bfc336a2..cf35714acbe3 100644 --- a/products/posthog_ai/backend/turn_suggestions/drafter.py +++ b/products/posthog_ai/backend/turn_suggestions/drafter.py @@ -13,7 +13,7 @@ from openai.types.shared_params import ResponseFormatJSONSchema from pydantic import BaseModel, ConfigDict, ValidationError -from posthog.llm.gateway_client import get_llm_client +from posthog.llm.gateway_client import build_openai_client, team_distinct_id from posthog.llm.semantic_enrichment import extract_json_object from products.posthog_ai.backend.turn_suggestions.transcript import TurnTranscript, truncate_text @@ -30,6 +30,11 @@ DRAFT_MODEL = "gpt-5.6-luna" +# PostHog pays for drafts, not the customer's AI credits. The Go gateway bills the wallet of the team +# that owns its key, and the Python fallback route is one that bills no credit bucket. +DRAFT_FALLBACK_PRODUCT = "growth" +DRAFT_AI_PRODUCT = "posthog_ai_turn_suggestions" + # A reasoning model spends the same token budget on its thinking and on the reply, and the reply # carries a complete scout prompt, so the ceiling sits well above the JSON alone. DRAFT_MAX_TOKENS = 4096 @@ -115,13 +120,20 @@ def _complete( ) -> _ReplyT | None: try: # Inside the guard: an instance without a configured gateway raises here, and a failed draft offers nothing. - client = get_llm_client("posthog_ai", team_id=team_id).with_options( + distinct_id = team_distinct_id(team_id) + client = build_openai_client( + DRAFT_FALLBACK_PRODUCT, + ai_product=DRAFT_AI_PRODUCT, + properties={"team_id": str(team_id)}, + distinct_id=distinct_id, + ).with_options( timeout=DRAFT_TIMEOUT_SECONDS, max_retries=DRAFT_MAX_RETRIES, ) response = client.chat.completions.create( model=DRAFT_MODEL, messages=[{"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}], + user=distinct_id, max_completion_tokens=DRAFT_MAX_TOKENS, response_format=_response_format(schema_name, reply_type), ) From 4ecf438ca9b801203742e855677152f4e4dd799c Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Fri, 25 Sep 2026 10:14:39 -0300 Subject: [PATCH 05/11] feat(growth): draft turn suggestions with gpt-6-luna Co-Authored-By: Claude Opus 5.5 (1M context) --- products/posthog_ai/backend/turn_suggestions/drafter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/posthog_ai/backend/turn_suggestions/drafter.py b/products/posthog_ai/backend/turn_suggestions/drafter.py index cf35714acbe3..33d0f1dc3d56 100644 --- a/products/posthog_ai/backend/turn_suggestions/drafter.py +++ b/products/posthog_ai/backend/turn_suggestions/drafter.py @@ -28,7 +28,7 @@ logger = structlog.get_logger(__name__) -DRAFT_MODEL = "gpt-5.6-luna" +DRAFT_MODEL = "gpt-6-luna" # PostHog pays for drafts, not the customer's AI credits. The Go gateway bills the wallet of the team # that owns its key, and the Python fallback route is one that bills no credit bucket. From e36eb34e8c206d3cd32ab383d5b0f539dcbeab9f Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Fri, 25 Sep 2026 10:37:53 -0300 Subject: [PATCH 06/11] feat(growth): judge turns through the system one client The judge now asks posthog/hogference/jevk5-fp8-0.2 through the ai-gateway where it is configured, and jev-1.13.0 through TypeSafe elsewhere. Ref options cap at 15, so a choice fits the gateway model's 16 options. The classified event records the model the configured server was asked for. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../backend/tests/test_turn_suggestions.py | 62 +++++++++++--- .../backend/turn_suggestions/classifier.py | 2 +- .../backend/turn_suggestions/judgment.py | 83 ++++++++++++------- .../backend/turn_suggestions/service.py | 4 +- 4 files changed, 107 insertions(+), 44 deletions(-) diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions.py b/products/posthog_ai/backend/tests/test_turn_suggestions.py index 4a61a1b1ae4e..24331fe6153a 100644 --- a/products/posthog_ai/backend/tests/test_turn_suggestions.py +++ b/products/posthog_ai/backend/tests/test_turn_suggestions.py @@ -8,25 +8,20 @@ from django.test import SimpleTestCase, override_settings +import httpx import requests from openai.resources.chat.completions import Completions from parameterized import parameterized from posthog.celery_queues import CeleryQueue -from posthog.egress.typesafe import ( - Answer, - ChoiceAnswer, - ChoiceQuestion, - NoulAnswer, - SystemOneResult, - TypeSafeRequestFailed, -) +from posthog.llm.system_one import Answer, ChoiceAnswer, ChoiceQuestion, NoulAnswer, SystemOneResult from products.posthog_ai.backend.tasks import generate_turn_suggestion_task from products.posthog_ai.backend.turn_suggestions.classifier import CardCopy, card_copy, classify_turn, pick_offer from products.posthog_ai.backend.turn_suggestions.dispatch import TURN_SETTLE_SECONDS, enqueue_turn_suggestion from products.posthog_ai.backend.turn_suggestions.drafter import DRAFT_MODEL, draft_scout, render_turn_prompt from products.posthog_ai.backend.turn_suggestions.judgment import ( + JUDGE_MODELS, MAX_REF_OPTIONS, TurnJudgment, build_judge_questions, @@ -509,6 +504,9 @@ def _answers(nouls: dict[str, float], choices: dict[str, str]) -> SystemOneResul FUNNEL_INSIGHT = replace(SAVED_INSIGHT, query_kind="FunnelsQuery") JUDGMENT = "products.posthog_ai.backend.turn_suggestions.judgment" +_NO_SERVER = {"AI_GATEWAY_URL": "", "AI_GATEWAY_API_KEY": "", "TYPESAFE_API_KEY": ""} +_GATEWAY_ONLY = {**_NO_SERVER, "AI_GATEWAY_URL": "https://ai-gateway.example.com/v1", "AI_GATEWAY_API_KEY": "phs_test"} +_TYPESAFE_ONLY = {**_NO_SERVER, "TYPESAFE_API_KEY": "ts-test"} CLASSIFIER = "products.posthog_ai.backend.turn_suggestions.classifier" @@ -539,7 +537,9 @@ def test_a_turn_with_one_possible_offer_skips_the_offer_question(self): only_scout = frozenset({OfferKind.SCOUT}) answers = _answers({"show_offer": 0.8}, {"intent": "metric_state", "scout_mode": "report", "cadence": "weekly"}) - with patch(f"{JUDGMENT}.system_one", return_value=answers): + with patch( + f"{JUDGMENT}.build_system_one_client", return_value=MagicMock(decide=MagicMock(return_value=answers)) + ): judgment = judge_turn(transcript, available=only_scout) assert "offer" not in build_judge_questions(transcript, only_scout) @@ -573,7 +573,9 @@ def test_option_keys_map_back_to_the_refs_they_stand_for(self): }, ) - with patch(f"{JUDGMENT}.system_one", return_value=answers): + with patch( + f"{JUDGMENT}.build_system_one_client", return_value=MagicMock(decide=MagicMock(return_value=answers)) + ): judgment = judge_turn(build_turn_transcript(_saved_insight_turn()), available=ALL_OFFERS) assert judgment is not None @@ -584,11 +586,45 @@ def test_option_keys_map_back_to_the_refs_they_stand_for(self): assert judgment.cadence == ScoutCadence.DAILY assert judgment.error_issue is None + def test_the_gateway_answers_with_its_own_model_for_the_team(self): + picks = {"intent": "metric_state", "scout_mode": "report", "cadence": "weekly"} + + def answer(request: httpx.Request, **_kwargs: object) -> httpx.Response: + answers: dict[str, dict] = {} + for key, question in json.loads(request.content)["questions"].items(): + if question["type"] == "noul": + answers[key] = {"type": "noul", "noul": 0.8} + continue + choice = picks.get(key, next(iter(question["criteria"]))) + probabilities = {option: float(option == choice) for option in question["criteria"]} + answers[key] = {"type": "choice", "choice": choice, "confidence": 1.0, "probabilities": probabilities} + return httpx.Response(200, json={"model": JUDGE_MODELS.gateway, "answers": answers, "usage": {}}) + + only_scout = frozenset({OfferKind.SCOUT}) + with ( + override_settings(**_GATEWAY_ONLY), + patch.object(httpx.Client, "send", side_effect=answer) as send, + ): + judgment = judge_turn(build_turn_transcript(_metric_turn()), available=only_scout, team_id=7) + + request = send.call_args.args[0] + assert json.loads(request.content)["model"] == JUDGE_MODELS.gateway + assert request.headers["X-PostHog-Distinct-Id"] == "team-7" + assert judgment is not None and judgment.offer == OfferKind.SCOUT + @parameterized.expand( - [("typesafe_error", TypeSafeRequestFailed("HTTP 500")), ("network_error", requests.ConnectionError())] + [ + ("gateway_error", _GATEWAY_ONLY, "httpx", httpx.Response(500, json={})), + ("typesafe_network_error", _TYPESAFE_ONLY, "requests", requests.ConnectionError()), + ] ) - def test_a_failed_request_returns_none(self, _name: str, error: Exception): - with patch(f"{JUDGMENT}.system_one", side_effect=error): + def test_a_failed_request_returns_none(self, _name: str, configured: dict, transport: str, outcome: object): + target = patch.object(httpx.Client, "send") if transport == "httpx" else patch("requests.request") + with override_settings(**configured), target as send: + if isinstance(outcome, Exception): + send.side_effect = outcome + else: + send.return_value = outcome assert judge_turn(build_turn_transcript(_metric_turn()), available=ALL_OFFERS) is None diff --git a/products/posthog_ai/backend/turn_suggestions/classifier.py b/products/posthog_ai/backend/turn_suggestions/classifier.py index 218a6237827e..a53667c4a768 100644 --- a/products/posthog_ai/backend/turn_suggestions/classifier.py +++ b/products/posthog_ai/backend/turn_suggestions/classifier.py @@ -147,7 +147,7 @@ def classify_turn( ) -> TurnVerdict | None: """``None`` means the judgment failed. A verdict whose draft is ``None`` offers nothing, either because the policy picked nothing or because drafting the picked offer failed.""" - judgment = judge_turn(transcript, available=available) + judgment = judge_turn(transcript, available=available, team_id=team_id) if judgment is None: return None picked = pick_offer(judgment, available) diff --git a/products/posthog_ai/backend/turn_suggestions/judgment.py b/products/posthog_ai/backend/turn_suggestions/judgment.py index 812ce6fd32cb..6894cef56ddc 100644 --- a/products/posthog_ai/backend/turn_suggestions/judgment.py +++ b/products/posthog_ai/backend/turn_suggestions/judgment.py @@ -1,4 +1,7 @@ -"""Asks Jev, TypeSafe's System One model, whether a finished PostHog AI turn deserves a follow-up and which. +"""Asks Jev, a System One model, whether a finished PostHog AI turn deserves a follow-up and which. + +The ai-gateway answers with the Jev build PostHog hosts where it is configured, and TypeSafe answers +elsewhere (see ``posthog.llm.system_one_client``). Jev answers typed questions instead of writing text, so one request carries every judgment the policy might need: whether to show anything now, which offer, and the speculative parameters of each @@ -12,24 +15,29 @@ from collections.abc import Mapping, Sequence -from django.conf import settings - import structlog from posthog.dataclasses import frozen from posthog.egress.limiter.policies import Priority -from posthog.egress.typesafe import ( +from posthog.egress.typesafe import TypeSafeEgressBudgetExhausted +from posthog.llm.gateway_client import team_distinct_id +from posthog.llm.system_one import ( ChoiceAnswer, ChoiceQuestion, JsonValue, NoulAnswer, NoulQuestion, Question, + SystemOneNotConfigured, + SystemOneRequestFailed, SystemOneResult, - TypeSafeEgressBudgetExhausted, - TypeSafeNotConfigured, - TypeSafeRequestFailed, - system_one, +) +from posthog.llm.system_one_client import ( + GATEWAY_MAX_CHOICE_OPTIONS, + SystemOneClient, + SystemOneModels, + build_system_one_client, + system_one_configured, ) from products.posthog_ai.backend.turn_suggestions.transcript import ( @@ -49,11 +57,11 @@ logger = structlog.get_logger(__name__) -# Pinned rather than `jev-latest`, because the thresholds in classifier.py are tuned against this -# version's probabilities and an alias moves on each release. -JUDGE_MODEL = "jev-1.13.0" +# Pinned rather than `jev-latest`, because an alias moves on each release. The thresholds in +# classifier.py are tuned against the TypeSafe model's probabilities. +JUDGE_MODELS = SystemOneModels(gateway="posthog/hogference/jevk5-fp8-0.2", typesafe="jev-1.13.0") JUDGE_SOURCE = "posthog_ai_turn_suggestions" -JUDGE_TIMEOUT: tuple[float, float] = (3.0, 10.0) +JUDGE_TIMEOUT_SECONDS = 10.0 _NO_MATCH = "none" @@ -199,12 +207,33 @@ class TurnJudgment: def judge_configured() -> bool: - return bool(settings.TYPESAFE_API_KEY) + return system_one_configured() + + +def _judge_client(team_id: int | None) -> SystemOneClient: + return build_system_one_client( + models=JUDGE_MODELS, + ai_product=JUDGE_SOURCE, + typesafe_source=JUDGE_SOURCE, + priority=Priority.BATCH, + distinct_id=team_distinct_id(team_id) if team_id is not None else None, + properties={"team_id": str(team_id)} if team_id is not None else None, + timeout=JUDGE_TIMEOUT_SECONDS, + ) + + +def judge_model() -> str | None: + """The model the configured server is asked for, or ``None`` when no server is configured.""" + try: + return _judge_client(None).model + except SystemOneNotConfigured: + return None -# A turn that lists issues can collect hundreds, past what one choice question takes. The state, the -# questions and the answer lookup all number refs through `_numbered`, so the cap keeps them in step. -MAX_REF_OPTIONS = 20 +# A turn that lists issues can collect hundreds, past what one choice question takes: the gateway's +# Jev takes 16 options, one of them `_NO_MATCH`. The state, the questions and the answer lookup all +# number refs through `_numbered`, so the cap keeps them in step. +MAX_REF_OPTIONS = GATEWAY_MAX_CHOICE_OPTIONS - 1 def _numbered[T](prefix: str, refs: Sequence[T]) -> dict[str, T]: @@ -289,22 +318,20 @@ def build_judge_questions(transcript: TurnTranscript, available: frozenset[Offer return questions -def judge_turn(transcript: TurnTranscript, *, available: frozenset[OfferKind]) -> TurnJudgment | None: - """One Jev request. ``None`` means the call failed, was shed, or the instance has no key. - ``available`` must hold at least one offer, because the offer question needs an option.""" +def judge_turn( + transcript: TurnTranscript, *, available: frozenset[OfferKind], team_id: int | None = None +) -> TurnJudgment | None: + """One Jev request. ``None`` means the call failed, was shed, or no server is configured. + ``available`` must hold at least one offer, because the offer question needs an option. + ``team_id`` labels the gateway's event with the team the turn belongs to.""" try: - result = system_one( - state=build_judge_state(transcript), - questions=build_judge_questions(transcript, available), - source=JUDGE_SOURCE, - model=JUDGE_MODEL, - priority=Priority.BATCH, - timeout=JUDGE_TIMEOUT, + result = _judge_client(team_id).decide( + state=build_judge_state(transcript), questions=build_judge_questions(transcript, available) ) - except (TypeSafeNotConfigured, TypeSafeEgressBudgetExhausted) as error: + except (SystemOneNotConfigured, TypeSafeEgressBudgetExhausted) as error: logger.info("posthog_ai_turn_suggestion_judge_skipped", reason=type(error).__name__) return None - except (TypeSafeRequestFailed, OSError): + except (SystemOneRequestFailed, OSError): logger.warning("posthog_ai_turn_suggestion_judge_failed", exc_info=True) return None return read_judgment(result, transcript, available) diff --git a/products/posthog_ai/backend/turn_suggestions/service.py b/products/posthog_ai/backend/turn_suggestions/service.py index 161c6943fb19..ef308890055c 100644 --- a/products/posthog_ai/backend/turn_suggestions/service.py +++ b/products/posthog_ai/backend/turn_suggestions/service.py @@ -15,7 +15,7 @@ from products.posthog_ai.backend.turn_suggestions.classifier import card_copy, classify_turn from products.posthog_ai.backend.turn_suggestions.drafter import DRAFT_MODEL -from products.posthog_ai.backend.turn_suggestions.judgment import JUDGE_MODEL, judge_configured +from products.posthog_ai.backend.turn_suggestions.judgment import judge_configured, judge_model from products.posthog_ai.backend.turn_suggestions.offer_ledger import ( TurnSuggestionResolution, claim_turn, @@ -141,7 +141,7 @@ def _capture_classified( "task_id": str(task_run.task_id), "run_id": str(task_run.id), "turn_index": turn_index, - "model": JUDGE_MODEL, + "model": judge_model(), "draft_model": DRAFT_MODEL if verdict and verdict.picked in _DRAFTED_OFFERS else None, "intent": verdict.intent.value if verdict else None, "picked": verdict.picked.value if verdict else None, From 71e198f123cb7cbcd2e9b6bde96fe48ea748fddc Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Fri, 25 Sep 2026 10:51:54 -0300 Subject: [PATCH 07/11] feat(growth): opt the turn suggestion judge into the typesafe fallback Co-Authored-By: Claude Opus 5.5 (1M context) --- .../backend/tests/test_turn_suggestions.py | 6 +++--- .../backend/turn_suggestions/judgment.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions.py b/products/posthog_ai/backend/tests/test_turn_suggestions.py index 24331fe6153a..b7a25ebfd1f6 100644 --- a/products/posthog_ai/backend/tests/test_turn_suggestions.py +++ b/products/posthog_ai/backend/tests/test_turn_suggestions.py @@ -21,7 +21,7 @@ from products.posthog_ai.backend.turn_suggestions.dispatch import TURN_SETTLE_SECONDS, enqueue_turn_suggestion from products.posthog_ai.backend.turn_suggestions.drafter import DRAFT_MODEL, draft_scout, render_turn_prompt from products.posthog_ai.backend.turn_suggestions.judgment import ( - JUDGE_MODELS, + JUDGE_MODEL, MAX_REF_OPTIONS, TurnJudgment, build_judge_questions, @@ -598,7 +598,7 @@ def answer(request: httpx.Request, **_kwargs: object) -> httpx.Response: choice = picks.get(key, next(iter(question["criteria"]))) probabilities = {option: float(option == choice) for option in question["criteria"]} answers[key] = {"type": "choice", "choice": choice, "confidence": 1.0, "probabilities": probabilities} - return httpx.Response(200, json={"model": JUDGE_MODELS.gateway, "answers": answers, "usage": {}}) + return httpx.Response(200, json={"model": JUDGE_MODEL, "answers": answers, "usage": {}}) only_scout = frozenset({OfferKind.SCOUT}) with ( @@ -608,7 +608,7 @@ def answer(request: httpx.Request, **_kwargs: object) -> httpx.Response: judgment = judge_turn(build_turn_transcript(_metric_turn()), available=only_scout, team_id=7) request = send.call_args.args[0] - assert json.loads(request.content)["model"] == JUDGE_MODELS.gateway + assert json.loads(request.content)["model"] == JUDGE_MODEL assert request.headers["X-PostHog-Distinct-Id"] == "team-7" assert judgment is not None and judgment.offer == OfferKind.SCOUT diff --git a/products/posthog_ai/backend/turn_suggestions/judgment.py b/products/posthog_ai/backend/turn_suggestions/judgment.py index 6894cef56ddc..6ca85e3558fc 100644 --- a/products/posthog_ai/backend/turn_suggestions/judgment.py +++ b/products/posthog_ai/backend/turn_suggestions/judgment.py @@ -35,7 +35,7 @@ from posthog.llm.system_one_client import ( GATEWAY_MAX_CHOICE_OPTIONS, SystemOneClient, - SystemOneModels, + TypeSafeFallback, build_system_one_client, system_one_configured, ) @@ -57,10 +57,11 @@ logger = structlog.get_logger(__name__) -# Pinned rather than `jev-latest`, because an alias moves on each release. The thresholds in -# classifier.py are tuned against the TypeSafe model's probabilities. -JUDGE_MODELS = SystemOneModels(gateway="posthog/hogference/jevk5-fp8-0.2", typesafe="jev-1.13.0") JUDGE_SOURCE = "posthog_ai_turn_suggestions" +JUDGE_MODEL = "posthog/hogference/jevk5-fp8-0.2" +# Pinned rather than `jev-latest`, because an alias moves on each release. The thresholds in +# classifier.py are tuned against this model's probabilities. +JUDGE_TYPESAFE_FALLBACK = TypeSafeFallback(model="jev-1.13.0", source=JUDGE_SOURCE, priority=Priority.BATCH) JUDGE_TIMEOUT_SECONDS = 10.0 _NO_MATCH = "none" @@ -207,15 +208,14 @@ class TurnJudgment: def judge_configured() -> bool: - return system_one_configured() + return system_one_configured(JUDGE_TYPESAFE_FALLBACK) def _judge_client(team_id: int | None) -> SystemOneClient: return build_system_one_client( - models=JUDGE_MODELS, + model=JUDGE_MODEL, ai_product=JUDGE_SOURCE, - typesafe_source=JUDGE_SOURCE, - priority=Priority.BATCH, + typesafe_fallback=JUDGE_TYPESAFE_FALLBACK, distinct_id=team_distinct_id(team_id) if team_id is not None else None, properties={"team_id": str(team_id)} if team_id is not None else None, timeout=JUDGE_TIMEOUT_SECONDS, From 9d1aad090a3f568a10c5f3dbdeaab2e0599456d3 Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Fri, 25 Sep 2026 11:19:19 -0300 Subject: [PATCH 08/11] fix(growth): type the drafter's fallback route as a gateway product Co-Authored-By: Claude Opus 5.5 (1M context) --- products/posthog_ai/backend/turn_suggestions/drafter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/products/posthog_ai/backend/turn_suggestions/drafter.py b/products/posthog_ai/backend/turn_suggestions/drafter.py index 33d0f1dc3d56..2790a350fb53 100644 --- a/products/posthog_ai/backend/turn_suggestions/drafter.py +++ b/products/posthog_ai/backend/turn_suggestions/drafter.py @@ -13,7 +13,7 @@ from openai.types.shared_params import ResponseFormatJSONSchema from pydantic import BaseModel, ConfigDict, ValidationError -from posthog.llm.gateway_client import build_openai_client, team_distinct_id +from posthog.llm.gateway_client import Product, build_openai_client, team_distinct_id from posthog.llm.semantic_enrichment import extract_json_object from products.posthog_ai.backend.turn_suggestions.transcript import TurnTranscript, truncate_text @@ -32,7 +32,7 @@ # PostHog pays for drafts, not the customer's AI credits. The Go gateway bills the wallet of the team # that owns its key, and the Python fallback route is one that bills no credit bucket. -DRAFT_FALLBACK_PRODUCT = "growth" +DRAFT_FALLBACK_PRODUCT: Product = "growth" DRAFT_AI_PRODUCT = "posthog_ai_turn_suggestions" # A reasoning model spends the same token budget on its thinking and on the reply, and the reply From c65c4c1728cb48e5d560489a51cc1d55f81e229f Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Sat, 3 Oct 2026 00:19:48 +0200 Subject: [PATCH 09/11] fix(growth): ask jev only through the ai-gateway Master now blocks TypeSafe on PostHog Cloud and its README says a Cloud feature must not pass a TypeSafeFallback. The judge state carries the user's question, so the judge drops the jev-1.13.0 TypeSafe fallback and asks only the PostHog-hosted Jev build on the ai-gateway. Team attribution uses the new team_id argument of build_system_one_client instead of setting the distinct id and team_id property by hand. A TypeSafe key on its own no longer configures the judge, and the failure test checks that no request leaves in that case. Co-Authored-By: Claude Opus 5.5 --- .../backend/tests/test_turn_suggestions.py | 11 ++++++---- .../backend/turn_suggestions/judgment.py | 21 +++++++------------ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions.py b/products/posthog_ai/backend/tests/test_turn_suggestions.py index b7a25ebfd1f6..c09ec21b86a8 100644 --- a/products/posthog_ai/backend/tests/test_turn_suggestions.py +++ b/products/posthog_ai/backend/tests/test_turn_suggestions.py @@ -499,7 +499,7 @@ def _answers(nouls: dict[str, float], choices: dict[str, str]) -> SystemOneResul answers.update( {key: ChoiceAnswer(choice=value, probabilities={value: 1.0}, confidence=1.0) for key, value in choices.items()} ) - return SystemOneResult(model="jev-1.13.0", answers=answers, input_tokens=300) + return SystemOneResult(model=JUDGE_MODEL, answers=answers, input_tokens=300) FUNNEL_INSIGHT = replace(SAVED_INSIGHT, query_kind="FunnelsQuery") @@ -614,11 +614,13 @@ def answer(request: httpx.Request, **_kwargs: object) -> httpx.Response: @parameterized.expand( [ - ("gateway_error", _GATEWAY_ONLY, "httpx", httpx.Response(500, json={})), - ("typesafe_network_error", _TYPESAFE_ONLY, "requests", requests.ConnectionError()), + ("gateway_error", _GATEWAY_ONLY, "httpx", httpx.Response(500, json={}), 1), + ("typesafe_key_only", _TYPESAFE_ONLY, "requests", requests.ConnectionError(), 0), ] ) - def test_a_failed_request_returns_none(self, _name: str, configured: dict, transport: str, outcome: object): + def test_a_failed_request_returns_none( + self, _name: str, configured: dict, transport: str, outcome: object, expected_calls: int + ): target = patch.object(httpx.Client, "send") if transport == "httpx" else patch("requests.request") with override_settings(**configured), target as send: if isinstance(outcome, Exception): @@ -626,6 +628,7 @@ def test_a_failed_request_returns_none(self, _name: str, configured: dict, trans else: send.return_value = outcome assert judge_turn(build_turn_transcript(_metric_turn()), available=ALL_OFFERS) is None + assert send.call_count == expected_calls class TestPickOffer(SimpleTestCase): diff --git a/products/posthog_ai/backend/turn_suggestions/judgment.py b/products/posthog_ai/backend/turn_suggestions/judgment.py index 6ca85e3558fc..efa55a1d757b 100644 --- a/products/posthog_ai/backend/turn_suggestions/judgment.py +++ b/products/posthog_ai/backend/turn_suggestions/judgment.py @@ -1,7 +1,7 @@ """Asks Jev, a System One model, whether a finished PostHog AI turn deserves a follow-up and which. -The ai-gateway answers with the Jev build PostHog hosts where it is configured, and TypeSafe answers -elsewhere (see ``posthog.llm.system_one_client``). +Only the ai-gateway answers, with the Jev build PostHog hosts. TypeSafe is never a fallback here, +because the state carries customer text (see ``posthog/egress/typesafe/README.md``). Jev answers typed questions instead of writing text, so one request carries every judgment the policy might need: whether to show anything now, which offer, and the speculative parameters of each @@ -18,9 +18,6 @@ import structlog from posthog.dataclasses import frozen -from posthog.egress.limiter.policies import Priority -from posthog.egress.typesafe import TypeSafeEgressBudgetExhausted -from posthog.llm.gateway_client import team_distinct_id from posthog.llm.system_one import ( ChoiceAnswer, ChoiceQuestion, @@ -35,7 +32,6 @@ from posthog.llm.system_one_client import ( GATEWAY_MAX_CHOICE_OPTIONS, SystemOneClient, - TypeSafeFallback, build_system_one_client, system_one_configured, ) @@ -58,10 +54,9 @@ logger = structlog.get_logger(__name__) JUDGE_SOURCE = "posthog_ai_turn_suggestions" +# Pinned to a build rather than an alias, because the thresholds in classifier.py read this model's +# probabilities. JUDGE_MODEL = "posthog/hogference/jevk5-fp8-0.2" -# Pinned rather than `jev-latest`, because an alias moves on each release. The thresholds in -# classifier.py are tuned against this model's probabilities. -JUDGE_TYPESAFE_FALLBACK = TypeSafeFallback(model="jev-1.13.0", source=JUDGE_SOURCE, priority=Priority.BATCH) JUDGE_TIMEOUT_SECONDS = 10.0 _NO_MATCH = "none" @@ -208,16 +203,14 @@ class TurnJudgment: def judge_configured() -> bool: - return system_one_configured(JUDGE_TYPESAFE_FALLBACK) + return system_one_configured() def _judge_client(team_id: int | None) -> SystemOneClient: return build_system_one_client( model=JUDGE_MODEL, ai_product=JUDGE_SOURCE, - typesafe_fallback=JUDGE_TYPESAFE_FALLBACK, - distinct_id=team_distinct_id(team_id) if team_id is not None else None, - properties={"team_id": str(team_id)} if team_id is not None else None, + team_id=team_id, timeout=JUDGE_TIMEOUT_SECONDS, ) @@ -328,7 +321,7 @@ def judge_turn( result = _judge_client(team_id).decide( state=build_judge_state(transcript), questions=build_judge_questions(transcript, available) ) - except (SystemOneNotConfigured, TypeSafeEgressBudgetExhausted) as error: + except SystemOneNotConfigured as error: logger.info("posthog_ai_turn_suggestion_judge_skipped", reason=type(error).__name__) return None except (SystemOneRequestFailed, OSError): From 46f65080edba054fef08553a6702c54eb6bf394f Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Sat, 3 Oct 2026 00:52:18 +0200 Subject: [PATCH 10/11] fix(growth): mask insight names and keep named refs under the jev option cap A failed enqueue now releases only its own dedup reservation. Co-Authored-By: Claude Opus 5.5 --- .../backend/tests/test_turn_suggestions.py | 47 ++++++++++++++++--- .../backend/turn_suggestions/dispatch.py | 29 +++++++++--- .../backend/turn_suggestions/judgment.py | 12 +++-- 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions.py b/products/posthog_ai/backend/tests/test_turn_suggestions.py index c09ec21b86a8..96eaa60700a8 100644 --- a/products/posthog_ai/backend/tests/test_turn_suggestions.py +++ b/products/posthog_ai/backend/tests/test_turn_suggestions.py @@ -15,6 +15,7 @@ from posthog.celery_queues import CeleryQueue from posthog.llm.system_one import Answer, ChoiceAnswer, ChoiceQuestion, NoulAnswer, SystemOneResult +from posthog.redis import get_client from products.posthog_ai.backend.tasks import generate_turn_suggestion_task from products.posthog_ai.backend.turn_suggestions.classifier import CardCopy, card_copy, classify_turn, pick_offer @@ -171,14 +172,14 @@ def _metric_turn() -> list[dict]: ERROR_ISSUE = ErrorIssueRef(issue_id="0199c0de-1111-7000-8000-0000000000aa", name="Checkout error") -def _saved_insight_turn(query_kind: str = "TrendsQuery") -> list[dict]: +def _saved_insight_turn(query_kind: str = "TrendsQuery", name: str = "Signups") -> list[dict]: return [ _user_message("How many signups did we get this week? Save it."), _exec_tool_call( "t1", 'call insight-create {"name":"Signups"}', "completed", - {"id": 42, "short_id": "abc123", "name": "Signups", "query": {"kind": query_kind, "series": []}}, + {"id": 42, "short_id": "abc123", "name": name, "query": {"kind": query_kind, "series": []}}, ), _agent_text("Saved. You had 412 signups this week."), ] @@ -512,7 +513,11 @@ def _answers(nouls: dict[str, float], choices: dict[str, str]) -> SystemOneResul class TestJudgeTurn(SimpleTestCase): @parameterized.expand( - [("saved_insight", _saved_insight_turn(), "abc123"), ("error_issue", _error_turn(), ERROR_ISSUE.issue_id)] + [ + ("saved_insight", _saved_insight_turn(), "abc123"), + ("insight_name", _saved_insight_turn(name="Signups for owner@example.com"), "owner@example.com"), + ("error_issue", _error_turn(), ERROR_ISSUE.issue_id), + ] ) def test_jev_sees_the_question_and_a_masked_answer_but_no_outputs_or_ids( self, _name: str, entries: list[dict], hidden_id: str @@ -545,18 +550,26 @@ def test_a_turn_with_one_possible_offer_skips_the_offer_question(self): assert "offer" not in build_judge_questions(transcript, only_scout) assert judgment is not None and judgment.offer == OfferKind.SCOUT - def test_a_turn_with_more_issues_than_a_choice_takes_is_still_judged(self): - issues = [{"id": f"issue-{index}", "name": f"Error {index}"} for index in range(300)] + def test_a_turn_with_more_issues_than_a_choice_takes_still_offers_the_one_the_answer_names(self): + issues = [{"id": f"issue-{index}", "name": f"Error {index:03d}"} for index in range(300)] entries = [ _user_message("Which errors are new this week?"), _exec_tool_call("t1", "call query-error-tracking-issues-list {}", "completed", {"results": issues}), - _agent_text("Here are the new errors."), + _agent_text("Error 250 is the new one this week."), ] transcript = build_turn_transcript(entries) + answers = _answers( + {"show_offer": 0.8}, {"intent": "metric_state", "offer": "error_alert", "error_issue": "issue_1"} + ) question = build_judge_questions(transcript, ALL_OFFERS)["error_issue"] + with patch( + f"{JUDGMENT}.build_system_one_client", return_value=MagicMock(decide=MagicMock(return_value=answers)) + ): + judgment = judge_turn(transcript, available=ALL_OFFERS) assert isinstance(question, ChoiceQuestion) and len(question.criteria) == MAX_REF_OPTIONS + 1 + assert judgment is not None and judgment.error_issue == ErrorIssueRef(issue_id="issue-250", name="Error 250") def test_option_keys_map_back_to_the_refs_they_stand_for(self): answers = _answers( @@ -863,6 +876,28 @@ def test_a_failed_enqueue_leaves_the_turn_for_the_next_report(self): assert apply_async.call_count == 2 + def test_a_failed_enqueue_keeps_the_reservation_a_later_report_took(self): + task_run = self._run(Task.OriginProduct.POSTHOG_AI) + calls: list[str] = [] + later_report: list[bool] = [] + + def stall_past_the_reservation(**_kwargs: object) -> None: + calls.append("enqueue") + if len(calls) == 1: + get_client().delete(f"turn_suggestion:{task_run.id}") + later_report.append(enqueue_turn_suggestion(task_run)) + raise ConnectionError("broker timed out") + + with patch( + "products.posthog_ai.backend.tasks.generate_turn_suggestion_task.apply_async", + side_effect=stall_past_the_reservation, + ) as apply_async: + assert enqueue_turn_suggestion(task_run) is False + assert enqueue_turn_suggestion(task_run) is False + + assert later_report == [True] + assert apply_async.call_count == 2 + def test_the_task_runs_on_the_posthog_ai_queue(self): assert generate_turn_suggestion_task.queue == CeleryQueue.POSTHOG_AI.value diff --git a/products/posthog_ai/backend/turn_suggestions/dispatch.py b/products/posthog_ai/backend/turn_suggestions/dispatch.py index 7e43cd1df8de..ea746567fbad 100644 --- a/products/posthog_ai/backend/turn_suggestions/dispatch.py +++ b/products/posthog_ai/backend/turn_suggestions/dispatch.py @@ -1,3 +1,4 @@ +import uuid from typing import TYPE_CHECKING, Any import structlog @@ -21,22 +22,35 @@ ENQUEUE_DEDUP_SECONDS = TURN_SETTLE_SECONDS +# A broker call can outlast the reservation, so a failed call releases only the reservation it made, +# never one a later report took after it expired. +_RELEASE_OWN_RESERVATION_SCRIPT = """ +if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("del", KEYS[1]) +end +return 0 +""" + + def _enqueue_dedup_key(run_id: str) -> str: return f"turn_suggestion:{run_id}" -def _first_report_of_turn(run_id: str) -> bool: +def _reserve_report(run_id: str) -> str | None: + """A token for this report's reservation of the turn, or ``None`` when another report holds it.""" + token = uuid.uuid4().hex try: - return bool(get_client().set(_enqueue_dedup_key(run_id), 1, nx=True, ex=ENQUEUE_DEDUP_SECONDS)) + reserved = get_client().set(_enqueue_dedup_key(run_id), token, nx=True, ex=ENQUEUE_DEDUP_SECONDS) except Exception: # The offer ledger still refuses a second claim of the same turn, so a duplicate only costs a read. logger.warning("posthog_ai_turn_suggestion_dedup_failed", run_id=run_id, exc_info=True) - return True + return token + return token if reserved else None -def _release_report(run_id: str) -> None: +def _release_report(run_id: str, token: str) -> None: try: - get_client().delete(_enqueue_dedup_key(run_id)) + get_client().eval(_RELEASE_OWN_RESERVATION_SCRIPT, 1, _enqueue_dedup_key(run_id), token) except Exception: logger.warning("posthog_ai_turn_suggestion_dedup_release_failed", run_id=run_id, exc_info=True) @@ -48,7 +62,8 @@ def enqueue_turn_suggestion(task_run: "TaskRun") -> bool: if task_run.origin_product != TaskOriginProduct.POSTHOG_AI: return False run_id = str(task_run.id) - if not _first_report_of_turn(run_id): + token = _reserve_report(run_id) + if token is None: return False from products.posthog_ai.backend.tasks import ( generate_turn_suggestion_task, # noqa: PLC0415 — keeps the judge and drafter clients off the Django startup path @@ -60,7 +75,7 @@ def enqueue_turn_suggestion(task_run: "TaskRun") -> bool: ) except Exception: # Another report of the turn can still queue it. - _release_report(run_id) + _release_report(run_id, token) raise except Exception: logger.warning("posthog_ai_turn_suggestion_enqueue_failed", run_id=str(task_run.id), exc_info=True) diff --git a/products/posthog_ai/backend/turn_suggestions/judgment.py b/products/posthog_ai/backend/turn_suggestions/judgment.py index efa55a1d757b..75160f11aaa4 100644 --- a/products/posthog_ai/backend/turn_suggestions/judgment.py +++ b/products/posthog_ai/backend/turn_suggestions/judgment.py @@ -233,16 +233,22 @@ def _numbered[T](prefix: str, refs: Sequence[T]) -> dict[str, T]: return {f"{prefix}_{index}": ref for index, ref in enumerate(refs[:MAX_REF_OPTIONS], start=1)} +def _named_in_answer_first[T: (SavedInsightRef, ErrorIssueRef)](refs: Sequence[T], answer: str) -> list[T]: + """Puts the refs the answer names ahead of the rest, so the cap drops refs the answer never mentions.""" + answer = answer.casefold() + return sorted(refs, key=lambda ref: not (ref.name and ref.name.casefold() in answer)) + + def _insight_options(transcript: TurnTranscript) -> dict[str, SavedInsightRef]: - return _numbered("insight", transcript.saved_insights) + return _numbered("insight", _named_in_answer_first(transcript.saved_insights, transcript.assistant_text)) def _issue_options(transcript: TurnTranscript) -> dict[str, ErrorIssueRef]: - return _numbered("issue", transcript.error_issues) + return _numbered("issue", _named_in_answer_first(transcript.error_issues, transcript.assistant_text)) def _insight_label(ref: SavedInsightRef) -> str: - return f"{ref.name or 'Untitled insight'} ({ref.query_kind or 'unknown chart type'})" + return f"{redact_values(ref.name) or 'Untitled insight'} ({ref.query_kind or 'unknown chart type'})" def _issue_label(ref: ErrorIssueRef) -> str: From 12f5841789d418ed1b4f1fc1fe8fac8f12a4cdd0 Mon Sep 17 00:00:00 2001 From: Rafa Audibert Date: Tue, 6 Oct 2026 04:12:49 +0100 Subject: [PATCH 11/11] fix(growth): judge only completed turns and expire stale suggestion jobs - skip a latest turn that has no turn_complete frame yet, so a follow-up still streaming is noted instead of judged and claimed - expire the celery job shortly after the settle wait, so a stalled posthog_ai worker drains no backlog of late cards - count and log every job outcome, and warn when the judge is not configured - mark the insight or issue the answer names, so masked labels that read the same stay distinguishable for jev Co-Authored-By: Claude Opus 5.5 --- products/posthog_ai/backend/tasks.py | 20 +++++++- .../backend/tests/test_turn_suggestions.py | 49 +++++++++++++++---- .../backend/turn_suggestions/dispatch.py | 6 ++- .../backend/turn_suggestions/judgment.py | 31 +++++++++--- .../backend/turn_suggestions/service.py | 9 ++-- .../backend/turn_suggestions/transcript.py | 10 ++++ 6 files changed, 102 insertions(+), 23 deletions(-) diff --git a/products/posthog_ai/backend/tasks.py b/products/posthog_ai/backend/tasks.py index 1c1a4079ffbc..0809fea530af 100644 --- a/products/posthog_ai/backend/tasks.py +++ b/products/posthog_ai/backend/tasks.py @@ -1,11 +1,29 @@ +import structlog from celery import shared_task +from prometheus_client import Counter from posthog.celery_queues import CeleryQueue from products.posthog_ai.backend.turn_suggestions.service import generate_turn_suggestion +logger = structlog.get_logger(__name__) + +TURN_SUGGESTION_OUTCOMES_TOTAL = Counter( + "posthog_ai_turn_suggestion_outcomes_total", + "End-of-turn suggestion jobs by outcome, so a gate that skips every turn in a region shows up", + labelnames=["status", "reason"], +) + # No retries: a suggestion that misses its turn is worth less than a stale one arriving mid-conversation. @shared_task(ignore_result=True, queue=CeleryQueue.POSTHOG_AI.value, soft_time_limit=90, time_limit=120) def generate_turn_suggestion_task(*, run_id: str, team_id: int) -> None: - generate_turn_suggestion(run_id, team_id) + outcome = generate_turn_suggestion(run_id, team_id) + TURN_SUGGESTION_OUTCOMES_TOTAL.labels(status=outcome.status, reason=outcome.reason).inc() + logger.info( + "posthog_ai_turn_suggestion_outcome", + run_id=run_id, + team_id=team_id, + status=outcome.status, + reason=outcome.reason, + ) diff --git a/products/posthog_ai/backend/tests/test_turn_suggestions.py b/products/posthog_ai/backend/tests/test_turn_suggestions.py index 96eaa60700a8..56d1171bf2de 100644 --- a/products/posthog_ai/backend/tests/test_turn_suggestions.py +++ b/products/posthog_ai/backend/tests/test_turn_suggestions.py @@ -19,7 +19,11 @@ from products.posthog_ai.backend.tasks import generate_turn_suggestion_task from products.posthog_ai.backend.turn_suggestions.classifier import CardCopy, card_copy, classify_turn, pick_offer -from products.posthog_ai.backend.turn_suggestions.dispatch import TURN_SETTLE_SECONDS, enqueue_turn_suggestion +from products.posthog_ai.backend.turn_suggestions.dispatch import ( + TURN_SETTLE_SECONDS, + TURN_SUGGESTION_EXPIRES_SECONDS, + enqueue_turn_suggestion, +) from products.posthog_ai.backend.turn_suggestions.drafter import DRAFT_MODEL, draft_scout, render_turn_prompt from products.posthog_ai.backend.turn_suggestions.judgment import ( JUDGE_MODEL, @@ -111,6 +115,10 @@ def _agent_final(text: str, message_id: str | None = "m1") -> dict: return _notification("session/update", {"update": update}) +def _turn_complete() -> dict: + return _notification("_posthog/turn_complete", {"stopReason": "end_turn"}) + + def _builtin_tool_call(tool_call_id: str, tool_name: str, raw_input: dict) -> dict: return _notification( "session/update", @@ -164,7 +172,7 @@ def _metric_turn() -> list[dict]: ), _tool_call_update("t1", "completed"), _agent_text("You had 412 signups this week, up 8% on last week."), - _notification("_posthog/turn_complete", {"stopReason": "end_turn"}), + _turn_complete(), ] @@ -182,6 +190,7 @@ def _saved_insight_turn(query_kind: str = "TrendsQuery", name: str = "Signups") {"id": 42, "short_id": "abc123", "name": name, "query": {"kind": query_kind, "series": []}}, ), _agent_text("Saved. You had 412 signups this week."), + _turn_complete(), ] @@ -195,6 +204,7 @@ def _error_turn() -> list[dict]: {"results": [{"id": ERROR_ISSUE.issue_id, "name": "Checkout error"}], "hasMore": False}, ), _agent_text("A checkout error started at 14:10."), + _turn_complete(), ] @@ -563,12 +573,15 @@ def test_a_turn_with_more_issues_than_a_choice_takes_still_offers_the_one_the_an ) question = build_judge_questions(transcript, ALL_OFFERS)["error_issue"] + labels = build_judge_state(transcript)["error_issues"] with patch( f"{JUDGMENT}.build_system_one_client", return_value=MagicMock(decide=MagicMock(return_value=answers)) ): judgment = judge_turn(transcript, available=ALL_OFFERS) assert isinstance(question, ChoiceQuestion) and len(question.criteria) == MAX_REF_OPTIONS + 1 + assert isinstance(labels, dict) + assert [key for key, label in labels.items() if "named in the answer" in str(label)] == ["issue_1"] assert judgment is not None and judgment.error_issue == ErrorIssueRef(issue_id="issue-250", name="Error 250") def test_option_keys_map_back_to_the_refs_they_stand_for(self): @@ -853,6 +866,7 @@ def test_only_posthog_ai_runs_are_classified(self, _name: str, origin: str, expe assert apply_async.call_args.kwargs == { "kwargs": {"run_id": str(task_run.id), "team_id": self.team.id}, "countdown": TURN_SETTLE_SECONDS, + "expires": TURN_SUGGESTION_EXPIRES_SECONDS, } def test_a_turn_reported_twice_is_queued_once(self): @@ -1031,7 +1045,11 @@ def test_offers_follow_what_the_turn_and_project_make_possible(self, _name: str, def test_a_turn_with_nothing_to_offer_never_reaches_the_classifier(self): self.mocks["scouts"].return_value = False - self.mocks["history"].return_value = [_user_message("Why did signups drop?"), _agent_text("Hard to say.")] + self.mocks["history"].return_value = [ + _user_message("Why did signups drop?"), + _agent_text("Hard to say."), + _turn_complete(), + ] outcome = self._generate() @@ -1068,6 +1086,7 @@ def test_follow_up_turns_are_classified_with_their_own_turn_index(self): "t2", 'call query-trends {"breakdownFilter":{"breakdown":"$geoip_country_code"}}', "completed" ), _agent_text("Most signups came from the US."), + _turn_complete(), ] outcome = self._generate() @@ -1095,7 +1114,7 @@ def test_a_card_that_lost_its_turn_while_drafting_is_not_published(self, interru *( entry for question in ("By country?", "By plan?") - for entry in (_user_message(question), _agent_text("A.")) + for entry in (_user_message(question), _agent_text("A."), _turn_complete()) ), ] earlier = _offer(0, run_id=str(self.task_run.id)) @@ -1109,12 +1128,13 @@ def classify_while_interrupted(*_args: Any, **_kwargs: Any) -> TurnVerdict: *self.mocks["history"].return_value, _user_message("Thanks"), _agent_text("You're welcome."), + _turn_complete(), ] self.mocks["scouts"].return_value = False assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="no_offers_available") elif interruption == "follow_up_still_running": self.mocks["history"].return_value = [*self.mocks["history"].return_value, _user_message("Thanks")] - assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="empty_turn") + assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="turn_in_progress") else: resolution = TurnSuggestionResolution.DISMISSED assert resolve_offer(self.task_run.task_id, self.team.id, turn_index=0, resolution=resolution) @@ -1128,13 +1148,17 @@ def classify_while_interrupted(*_args: Any, **_kwargs: Any) -> TurnVerdict: self.mocks["publish"].assert_not_called() assert [offer.turn_index for offer in read_ledger(self.task_run.task_id, self.team.id).offers] == [0] - def test_a_follow_up_still_running_at_the_read_is_classified_when_it_completes(self): - follow_up = [*_metric_turn(), _user_message("Break that down by country")] + @parameterized.expand([("no_frames_yet", []), ("answer_streaming", [_agent_text("Most signups")])]) + def test_a_follow_up_still_running_at_the_read_is_classified_when_it_completes( + self, _name: str, streamed: list[dict] + ): + follow_up = [*_metric_turn(), _user_message("Break that down by country"), *streamed] self.mocks["history"].return_value = follow_up - assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="empty_turn") + assert self._generate() == TurnSuggestionOutcome(status="skipped", reason="turn_in_progress") + self.mocks["classify"].assert_not_called() - self.mocks["history"].return_value = [*follow_up, _agent_text("Most signups came from the US.")] + self.mocks["history"].return_value = [*follow_up, _agent_text(" came from the US."), _turn_complete()] outcome = self._generate() assert outcome == TurnSuggestionOutcome(status="emitted", reason="scout") @@ -1171,7 +1195,12 @@ def test_a_conversation_past_the_resume_chain_depth_gets_no_card(self): def test_the_offer_ledger_holds_back_a_card(self, _name: str, ledger: dict, reason: str): Task.objects.filter(id=self.task_run.task_id).update(state={STATE_KEY: ledger}) updated_at = Task.objects.get(id=self.task_run.task_id).updated_at - self.mocks["history"].return_value = [*_metric_turn(), _user_message("And by country?"), _agent_text("US.")] + self.mocks["history"].return_value = [ + *_metric_turn(), + _user_message("And by country?"), + _agent_text("US."), + _turn_complete(), + ] outcome = self._generate() diff --git a/products/posthog_ai/backend/turn_suggestions/dispatch.py b/products/posthog_ai/backend/turn_suggestions/dispatch.py index ea746567fbad..d26e84612158 100644 --- a/products/posthog_ai/backend/turn_suggestions/dispatch.py +++ b/products/posthog_ai/backend/turn_suggestions/dispatch.py @@ -20,6 +20,8 @@ # the conversation's latest turn when it runs. Past the wait, a report can be a later turn, so it # queues its own job. ENQUEUE_DEDUP_SECONDS = TURN_SETTLE_SECONDS +# Every PostHog AI turn queues a job, so a stalled worker must not drain a backlog of late cards. +TURN_SUGGESTION_EXPIRES_SECONDS = TURN_SETTLE_SECONDS + 30 # A broker call can outlast the reservation, so a failed call releases only the reservation it made, @@ -71,7 +73,9 @@ def enqueue_turn_suggestion(task_run: "TaskRun") -> bool: try: generate_turn_suggestion_task.apply_async( - kwargs={"run_id": run_id, "team_id": task_run.team_id}, countdown=TURN_SETTLE_SECONDS + kwargs={"run_id": run_id, "team_id": task_run.team_id}, + countdown=TURN_SETTLE_SECONDS, + expires=TURN_SUGGESTION_EXPIRES_SECONDS, ) except Exception: # Another report of the turn can still queue it. diff --git a/products/posthog_ai/backend/turn_suggestions/judgment.py b/products/posthog_ai/backend/turn_suggestions/judgment.py index 75160f11aaa4..0e3199415db2 100644 --- a/products/posthog_ai/backend/turn_suggestions/judgment.py +++ b/products/posthog_ai/backend/turn_suggestions/judgment.py @@ -233,10 +233,13 @@ def _numbered[T](prefix: str, refs: Sequence[T]) -> dict[str, T]: return {f"{prefix}_{index}": ref for index, ref in enumerate(refs[:MAX_REF_OPTIONS], start=1)} +def _named_in(ref: SavedInsightRef | ErrorIssueRef, answer: str) -> bool: + return bool(ref.name) and ref.name.casefold() in answer.casefold() + + def _named_in_answer_first[T: (SavedInsightRef, ErrorIssueRef)](refs: Sequence[T], answer: str) -> list[T]: """Puts the refs the answer names ahead of the rest, so the cap drops refs the answer never mentions.""" - answer = answer.casefold() - return sorted(refs, key=lambda ref: not (ref.name and ref.name.casefold() in answer)) + return sorted(refs, key=lambda ref: not _named_in(ref, answer)) def _insight_options(transcript: TurnTranscript) -> dict[str, SavedInsightRef]: @@ -247,12 +250,20 @@ def _issue_options(transcript: TurnTranscript) -> dict[str, ErrorIssueRef]: return _numbered("issue", _named_in_answer_first(transcript.error_issues, transcript.assistant_text)) -def _insight_label(ref: SavedInsightRef) -> str: - return f"{redact_values(ref.name) or 'Untitled insight'} ({ref.query_kind or 'unknown chart type'})" +# Masking can give two refs the same label ("Revenue 2025" and "Revenue 2026" both read "Revenue "). +_NAMED_IN_ANSWER_MARK = "named in the answer" + +def _insight_label(ref: SavedInsightRef, answer: str) -> str: + details = [ref.query_kind or "unknown chart type"] + if _named_in(ref, answer): + details.append(_NAMED_IN_ANSWER_MARK) + return f"{redact_values(ref.name) or 'Untitled insight'} ({', '.join(details)})" -def _issue_label(ref: ErrorIssueRef) -> str: - return redact_values(ref.name) or "Unnamed issue" + +def _issue_label(ref: ErrorIssueRef, answer: str) -> str: + label = redact_values(ref.name) or "Unnamed issue" + return f"{label} ({_NAMED_IN_ANSWER_MARK})" if _named_in(ref, answer) else label def build_judge_state(transcript: TurnTranscript) -> dict[str, JsonValue]: @@ -274,9 +285,13 @@ def build_judge_state(transcript: TurnTranscript) -> dict[str, JsonValue]: for turn in transcript.earlier_turns ] if transcript.saved_insights: - state["saved_insights"] = {key: _insight_label(ref) for key, ref in _insight_options(transcript).items()} + state["saved_insights"] = { + key: _insight_label(ref, transcript.assistant_text) for key, ref in _insight_options(transcript).items() + } if transcript.error_issues: - state["error_issues"] = {key: _issue_label(ref) for key, ref in _issue_options(transcript).items()} + state["error_issues"] = { + key: _issue_label(ref, transcript.assistant_text) for key, ref in _issue_options(transcript).items() + } return state diff --git a/products/posthog_ai/backend/turn_suggestions/service.py b/products/posthog_ai/backend/turn_suggestions/service.py index ef308890055c..835e79ca542a 100644 --- a/products/posthog_ai/backend/turn_suggestions/service.py +++ b/products/posthog_ai/backend/turn_suggestions/service.py @@ -180,6 +180,7 @@ def generate_turn_suggestion(run_id: str, team_id: int) -> TurnSuggestionOutcome if not _turn_suggestions_enabled(task_run, user): return _skipped("flag_off") if not judge_configured(): + logger.warning("posthog_ai_turn_suggestion_judge_not_configured", run_id=str(task_run.id)) return _skipped("judge_not_configured") early_refusal = read_ledger(task.id, task_run.team_id).refusal() if early_refusal is not None: @@ -193,10 +194,12 @@ def generate_turn_suggestion(run_id: str, team_id: int) -> TurnSuggestionOutcome if not transcript.human_messages: return _skipped("no_user_message") turn_index = len(transcript.human_messages) - 1 + if not transcript.latest_turn_complete: + # A follow-up sent during the settle wait is still streaming here. Its own completion + # classifies it, and noting it keeps a card still drafting for the previous turn away. + note_turn(task.id, task_run.team_id, turn_index) + return _skipped("turn_in_progress") if not transcript.assistant_text and not transcript.tool_calls: - # A follow-up sent during the settle wait shows here as an empty latest turn. Its own - # completion classifies it, so it is only noted: a card still drafting for the previous - # turn must not land under it. note_turn(task.id, task_run.team_id, turn_index) return _skipped("empty_turn") # Claimed before the turn can bail out, so a card still drafting for the previous turn sees diff --git a/products/posthog_ai/backend/turn_suggestions/transcript.py b/products/posthog_ai/backend/turn_suggestions/transcript.py index 68e1ada1dfa3..629ac42d0d11 100644 --- a/products/posthog_ai/backend/turn_suggestions/transcript.py +++ b/products/posthog_ai/backend/turn_suggestions/transcript.py @@ -81,6 +81,7 @@ class ErrorIssueRef: @frozen class TurnTranscript: human_messages: tuple[str, ...] + latest_turn_complete: bool assistant_text: str tool_calls: tuple[TranscriptToolCall, ...] earlier_turns: tuple[EarlierTurn, ...] @@ -236,6 +237,8 @@ def _notifications(entries: Iterable[dict[str, Any]]) -> Iterator[tuple[str | No yield frame.notification.method, params if isinstance(params, dict) else {} +TURN_COMPLETE_METHOD = "_posthog/turn_complete" + # The synthetic prompt a resumed run starts with. The thread never renders it, so it is not a turn. RESUME_CONTEXT_PREFIX = "You are resuming a previous conversation." @@ -276,8 +279,10 @@ def build_turn_transcript(entries: Iterable[dict[str, Any]]) -> TurnTranscript: # carries the whole text and replaces them, matching how the thread fold finalizes a bubble. assistant_messages: dict[str, str] = {} tool_calls: dict[str, _ToolCallAccumulator] = {} + latest_turn_complete = False def start_turn(text: str) -> None: + nonlocal latest_turn_complete if human_messages: # The finished turn stays available as context for classifying the one that follows. earlier_turns.append( @@ -292,6 +297,7 @@ def start_turn(text: str) -> None: ) ) human_messages.append(text) + latest_turn_complete = False assistant_messages.clear() tool_calls.clear() @@ -322,6 +328,9 @@ def record_assistant_text(update: dict[str, Any], text: str, *, final: bool) -> continue start_turn(text) continue + if method == TURN_COMPLETE_METHOD: + latest_turn_complete = bool(human_messages) + continue if method != "session/update": continue @@ -357,6 +366,7 @@ def record_assistant_text(update: dict[str, Any], text: str, *, final: bool) -> active = [accumulator for accumulator in tool_calls.values() if not accumulator.discovery] return TurnTranscript( human_messages=tuple(human_messages), + latest_turn_complete=latest_turn_complete, assistant_text=truncate_text( _join_messages(assistant_messages), ASSISTANT_TEXT_LIMIT, collapse_whitespace=False ),