feat(mcp): trial goal-first intent prompts behind a flag

docs/published/handbook/engineering/ai/implementing-mcp-tools.md

@@ -90,6 +90,31 @@ Primarily oriented toward coding agents (PostHog Desktop, PostHog AI, Claude Cod
 
 When the MCP server advertises Business Knowledge or documentation search tools, its agent instructions require one search of each available source before the first answer to a request. Business Knowledge is searched first, followed by current PostHog documentation. If a search fails or returns no relevant result, the agent continues with the other available evidence. In exec sessions with skill discovery enabled, the agent loads the relevant skill before it runs these searches.
 
+## Intent prompt trials
+
+The MCP Analytics SDK adds a `context` argument asking the agent to explain why it calls a tool.
+The `mcp-intent-prompt` flag can select a different description for ordinary external MCP requests.
+Disabled flags, missing or invalid payloads, impersonation, and analytics-suppressed requests retain the SDK default.
+
+Set the description directly in the flag payload:
+
+```json
+{
+  "prompt": "The intent instruction, including its privacy constraints."
+}
+```
+
+The server reads the text from the existing flag evaluation and accepts only nonblank text of at most 2,000 characters.
+Keep the instruction abstract and preserve the default prompt's constraints against identifying or sensitive information.
+Start with a release condition matching one authenticated user's distinct ID; do not add an unconditional release condition.
+
+Clients cache tool schemas. Refresh the client's tool list or reconnect after deployment before collecting trial data.
+Use the targeted user's calls after that refresh for offline evaluation, and record when the payload changes.
+Existing intent analytics stay unchanged; the server does not capture the prompt text separately.
+
+This collection trial changes no tool results and adds no recommendations or elicitation.
+Evaluate intent against independently recorded abstract task goals, checking invented goals, identifying details, and appropriate uncertainty.
+
 ## Interview recruitment with URL elicitation
 
 > [!CAUTION]

services/mcp/src/hono/intent-prompt.ts

@@ -0,0 +1,17 @@
+import { z } from 'zod'
+
+import type { FlagValue } from '@/lib/posthog/flags'
+
+export const MCP_INTENT_PROMPT_FLAG = 'mcp-intent-prompt'
+
+const intentPromptSchema = z.object({
+    prompt: z.string().trim().min(1).max(2000),
+})
+
+export function resolveIntentPrompt(value: FlagValue, payload: unknown): string | undefined {
+    if (value !== true && (typeof value !== 'string' || !value)) {
+        return undefined
+    }
+    const result = intentPromptSchema.safeParse(payload)
+    return result.success ? result.data.prompt : undefined
+}

services/mcp/src/hono/request-state-resolver.ts

@@ -27,6 +27,7 @@ import type { Context, Env, PinnedActiveContext, Tool, ZodObjectAny } from '@/to
 import { McpSessionRedisStore } from './cache/McpSessionRedisStore'
 import type { RedisLike } from './cache/RedisCache'
 import { MCP_EXEC_SKILLS_FEATURE_FLAG } from './constants'
+import { MCP_INTENT_PROMPT_FLAG, resolveIntentPrompt } from './intent-prompt'
 import {
     type InterviewRecruitmentConfig,
     resolveInterviewRecruitmentConfig,
@@ -50,6 +51,7 @@ export interface ResolvedState {
     useSingleExec: boolean
     toolFeatureFlags: EvaluatedFlags | undefined
     interviewRecruitmentConfig?: InterviewRecruitmentConfig
+    intentPrompt?: string
     apiKeyScopes: string[]
     isImpersonated?: boolean
     suppressAnalytics?: boolean
@@ -170,6 +172,7 @@ export class RequestStateResolver {
                 MCP_GATEWAY_FLAG,
                 MCP_EXEC_SKILLS_FEATURE_FLAG,
                 MCP_INTERVIEW_RECRUITMENT_FLAG,
+                MCP_INTENT_PROMPT_FLAG,
             ]),
         ]
 
@@ -223,6 +226,10 @@ export class RequestStateResolver {
         // even when no catalog tool referenced it.
         const flagKeysForState = [...new Set([...allFlagKeys, ...Object.keys(overrides)])]
         const toolFeatureFlags = Object.fromEntries(flagKeysForState.map((k) => [k, mergedFlags[k]]))
+        const intentPrompt =
+            source === EVENT_SOURCE.MCP && !_apiKey?.is_impersonated && !props.suppressAnalytics
+                ? resolveIntentPrompt(mergedFlags[MCP_INTENT_PROMPT_FLAG], payloads[MCP_INTENT_PROMPT_FLAG])
+                : undefined
 
         const clientProfile = new MCPClientProfile({
             clientName: clientContext.mcpClientName,
@@ -289,6 +296,7 @@ export class RequestStateResolver {
             useSingleExec,
             toolFeatureFlags,
             interviewRecruitmentConfig,
+            intentPrompt,
             apiKeyScopes,
             isImpersonated: _apiKey?.is_impersonated === true,
             suppressAnalytics: props.suppressAnalytics,
@@ -429,10 +437,15 @@ export class RequestStateResolver {
             return { flags: {}, payloads: {} }
         }
         // Enable product surfaces locally because the analytics flag client is disabled there.
-        // Recruitment still needs an explicit override so development tasks do not ask for interviews.
+        // Recruitment and intent experiments stay off so ordinary development keeps the default behavior.
         if (isLocalApi()) {
             return {
-                flags: Object.fromEntries(flagKeys.map((key) => [key, key !== MCP_INTERVIEW_RECRUITMENT_FLAG])),
+                flags: Object.fromEntries(
+                    flagKeys.map((key) => [
+                        key,
+                        key !== MCP_INTERVIEW_RECRUITMENT_FLAG && key !== MCP_INTENT_PROMPT_FLAG,
+                    ])
+                ),
                 payloads: {},
             }
         }

services/mcp/src/hono/tool-executor.ts

@@ -154,7 +154,7 @@ export class ToolExecutor {
     }
 
     async handleToolsList(state: ResolvedState): Promise<ListToolsResult> {
-        const tools = this.injectAnalyticsParameters(this.buildAdvertisedTools(state))
+        const tools = this.injectAnalyticsParameters(this.buildAdvertisedTools(state), state.intentPrompt)
 
         void trackToolsList(
             tools.map((t) => t.name),
@@ -165,9 +165,9 @@ export class ToolExecutor {
     }
 
     // Guarded because analytics must never break `tools/list`.
-    private injectAnalyticsParameters(tools: ListToolsResult['tools']): ListToolsResult['tools'] {
+    private injectAnalyticsParameters(tools: ListToolsResult['tools'], description?: string): ListToolsResult['tools'] {
         try {
-            return getPostHogClient().prepareToolList(tools)
+            return getPostHogClient().prepareToolList(tools, description ? { context: { description } } : undefined)
         } catch {
             return tools
         }

services/mcp/tests/hono/request-state-resolver.test.ts

@@ -143,6 +143,7 @@ vi.mock('@/hono/request-context', () => {
 
 import type { RedisLike } from '@/hono/cache/RedisCache'
 import { MCP_EXEC_SKILLS_FEATURE_FLAG } from '@/hono/constants'
+import { MCP_INTENT_PROMPT_FLAG } from '@/hono/intent-prompt'
 import { MCP_INTERVIEW_RECRUITMENT_FLAG } from '@/hono/interview-recruitment'
 import { RequestStateResolver } from '@/hono/request-state-resolver'
 import { ToolCatalog } from '@/hono/tool-catalog'
@@ -490,7 +491,7 @@ describe('RequestStateResolver MCP client contexts', () => {
         expect(props.mode).toBe('cli')
     })
 
-    it.each([MCP_EXEC_SKILLS_FEATURE_FLAG, MCP_INTERVIEW_RECRUITMENT_FLAG])(
+    it.each([MCP_EXEC_SKILLS_FEATURE_FLAG, MCP_INTERVIEW_RECRUITMENT_FLAG, MCP_INTENT_PROMPT_FLAG])(
         'evaluates %s even though no generated tool declares it',
         async (flag) => {
             vi.mocked(evaluateFeatureFlags).mockResolvedValueOnce({ flags: { [flag]: true }, payloads: {} })
@@ -602,6 +603,35 @@ describe('RequestStateResolver MCP client contexts', () => {
         }
     )
 
+    it.each([
+        { value: true, enabled: true },
+        { value: 'test-1', enabled: true },
+        { value: false, enabled: false },
+        { value: undefined, enabled: false },
+        { value: '', enabled: false },
+        { value: true, missingPayload: true, enabled: false },
+        { value: true, payload: { prompt: ' ' }, enabled: false },
+        { value: true, payload: { prompt: 'x'.repeat(2001) }, enabled: false },
+        { value: true, impersonated: true, enabled: false },
+        { value: true, suppressed: true, enabled: false },
+        { value: true, consumer: 'posthog-cli', enabled: false },
+    ])('resolves the intent prompt only for an eligible configured request: %j', async (input) => {
+        const prompt = {
+            prompt: 'State the task objective before the tool action. Omit identifying details.',
+        }
+        mockApiKey.is_impersonated = input.impersonated
+        mockApiKey.suppress_analytics = input.suppressed
+        vi.mocked(evaluateFeatureFlags).mockResolvedValueOnce({
+            flags: { [MCP_INTENT_PROMPT_FLAG]: input.value },
+            payloads: {
+                [MCP_INTENT_PROMPT_FLAG]: input.missingPayload ? undefined : { ...prompt, ...input.payload },
+            },
+        })
+        const state = await makeResolver().resolve(makeProps({ mcpConsumer: input.consumer }))
+
+        expect(state.intentPrompt).toEqual(input.enabled ? prompt.prompt : undefined)
+    })
+
     it('honors a dev/test flag override even when evaluation returns nothing', async () => {
         // Evaluation stays empty (analytics client disabled, as in local dev/evals);
         // the override seam is what flips a tool flag on so it reaches the tool layer.

services/mcp/tests/hono/tool-executor-intent.test.ts

@@ -197,20 +197,36 @@ describe('ToolExecutor analytics capture', () => {
         expect(state.context.api.config.onPrivateResponse).toBeUndefined()
     })
 
-    it('injects the analytics arguments into advertised tools', async () => {
-        const state = makeToolExecutorState([], { useSingleExec: true })
+    it.each([true, false])('isolates intent descriptions with single exec=%s', async (useSingleExec) => {
+        const tool = catalog.getFilteredTools({ scopes: ['*'] }).find((entry) => entry.name === 'projects-get')!
+        const state = makeToolExecutorState([tool], { useSingleExec })
+        const prompt = 'State the task objective before the tool action. Omit identifying details.'
 
         const result = await executor.handleToolsList(state)
+        const treatment = await executor.handleToolsList({
+            ...state,
+            distinctId: 'trial-user',
+            intentPrompt: prompt,
+        })
+        const other = await executor.handleToolsList({ ...state, distinctId: 'another-user' })
+        vi.spyOn(getPostHogClient(), 'prepareToolList').mockImplementationOnce(() => {
+            throw new Error('Synthetic SDK preparation failure')
+        })
+        const fallback = await executor.handleToolsList({ ...state, intentPrompt: prompt })
 
-        const execEntry = result.tools.find((t) => t.name === 'exec')!
-        const properties = execEntry.inputSchema.properties as Record<string, unknown>
-        // The SDK injects `context` (required, to nudge the agent to state intent)
-        // while leaving the existing `command` arg untouched.
+        const entry = result.tools[0]!
+        const properties = entry.inputSchema.properties as Record<string, unknown>
         expect(properties).toHaveProperty('context')
         expect(properties).toHaveProperty('llm_model')
-        expect(properties).toHaveProperty('command')
-        expect(execEntry.inputSchema.required).toContain('context')
-        expect(execEntry.inputSchema.required).toContain('llm_model')
+        expect(entry.inputSchema.required).toContain('context')
+        expect(entry.inputSchema.required).toContain('llm_model')
+        expect(treatment.tools[0]!.inputSchema.properties?.context).toMatchObject({ description: prompt })
+        expect(other.tools[0]!.inputSchema.properties?.context).toEqual(properties.context)
+        expect(properties.context).not.toEqual(treatment.tools[0]!.inputSchema.properties?.context)
+        expect(fallback.tools[0]!.inputSchema.properties).not.toHaveProperty('context')
+        if (useSingleExec) {
+            expect(properties).toHaveProperty('command')
+        }
     })
 
     it.each([