diff --git a/docs/published/handbook/engineering/ai/implementing-mcp-tools.md b/docs/published/handbook/engineering/ai/implementing-mcp-tools.md index e8d55cf51ced..db9b1337b250 100644 --- a/docs/published/handbook/engineering/ai/implementing-mcp-tools.md +++ b/docs/published/handbook/engineering/ai/implementing-mcp-tools.md @@ -90,6 +90,31 @@ Primarily oriented toward coding agents (PostHog Desktop, PostHog AI, Claude Cod When the MCP server advertises Business Knowledge or documentation search tools, its agent instructions require one search of each available source before the first answer to a request. Business Knowledge is searched first, followed by current PostHog documentation. If a search fails or returns no relevant result, the agent continues with the other available evidence. In exec sessions with skill discovery enabled, the agent loads the relevant skill before it runs these searches. +## Intent prompt trials + +The MCP Analytics SDK adds a `context` argument asking the agent to explain why it calls a tool. +The `mcp-intent-prompt` flag can select a different description for ordinary external MCP requests. +Disabled flags, missing or invalid payloads, impersonation, and analytics-suppressed requests retain the SDK default. + +Set the description directly in the flag payload: + +```json +{ + "prompt": "The intent instruction, including its privacy constraints." +} +``` + +The server reads the text from the existing flag evaluation and accepts only nonblank text of at most 2,000 characters. +Keep the instruction abstract and preserve the default prompt's constraints against identifying or sensitive information. +Start with a release condition matching one authenticated user's distinct ID; do not add an unconditional release condition. + +Clients cache tool schemas. Refresh the client's tool list or reconnect after deployment before collecting trial data. +Use the targeted user's calls after that refresh for offline evaluation, and record when the payload changes. +Existing intent analytics stay unchanged; the server does not capture the prompt text separately. + +This collection trial changes no tool results and adds no recommendations or elicitation. +Evaluate intent against independently recorded abstract task goals, checking invented goals, identifying details, and appropriate uncertainty. + ## Interview recruitment with URL elicitation > [!CAUTION] diff --git a/services/mcp/src/hono/intent-prompt.ts b/services/mcp/src/hono/intent-prompt.ts new file mode 100644 index 000000000000..a278c811a478 --- /dev/null +++ b/services/mcp/src/hono/intent-prompt.ts @@ -0,0 +1,17 @@ +import { z } from 'zod' + +import type { FlagValue } from '@/lib/posthog/flags' + +export const MCP_INTENT_PROMPT_FLAG = 'mcp-intent-prompt' + +const intentPromptSchema = z.object({ + prompt: z.string().trim().min(1).max(2000), +}) + +export function resolveIntentPrompt(value: FlagValue, payload: unknown): string | undefined { + if (value !== true && (typeof value !== 'string' || !value)) { + return undefined + } + const result = intentPromptSchema.safeParse(payload) + return result.success ? result.data.prompt : undefined +} diff --git a/services/mcp/src/hono/request-state-resolver.ts b/services/mcp/src/hono/request-state-resolver.ts index 66ec2bc9ccaa..08cfbd19d962 100644 --- a/services/mcp/src/hono/request-state-resolver.ts +++ b/services/mcp/src/hono/request-state-resolver.ts @@ -27,6 +27,7 @@ import type { Context, Env, PinnedActiveContext, Tool, ZodObjectAny } from '@/to import { McpSessionRedisStore } from './cache/McpSessionRedisStore' import type { RedisLike } from './cache/RedisCache' import { MCP_EXEC_SKILLS_FEATURE_FLAG } from './constants' +import { MCP_INTENT_PROMPT_FLAG, resolveIntentPrompt } from './intent-prompt' import { type InterviewRecruitmentConfig, resolveInterviewRecruitmentConfig, @@ -50,6 +51,7 @@ export interface ResolvedState { useSingleExec: boolean toolFeatureFlags: EvaluatedFlags | undefined interviewRecruitmentConfig?: InterviewRecruitmentConfig + intentPrompt?: string apiKeyScopes: string[] isImpersonated?: boolean suppressAnalytics?: boolean @@ -170,6 +172,7 @@ export class RequestStateResolver { MCP_GATEWAY_FLAG, MCP_EXEC_SKILLS_FEATURE_FLAG, MCP_INTERVIEW_RECRUITMENT_FLAG, + MCP_INTENT_PROMPT_FLAG, ]), ] @@ -223,6 +226,10 @@ export class RequestStateResolver { // even when no catalog tool referenced it. const flagKeysForState = [...new Set([...allFlagKeys, ...Object.keys(overrides)])] const toolFeatureFlags = Object.fromEntries(flagKeysForState.map((k) => [k, mergedFlags[k]])) + const intentPrompt = + source === EVENT_SOURCE.MCP && !_apiKey?.is_impersonated && !props.suppressAnalytics + ? resolveIntentPrompt(mergedFlags[MCP_INTENT_PROMPT_FLAG], payloads[MCP_INTENT_PROMPT_FLAG]) + : undefined const clientProfile = new MCPClientProfile({ clientName: clientContext.mcpClientName, @@ -289,6 +296,7 @@ export class RequestStateResolver { useSingleExec, toolFeatureFlags, interviewRecruitmentConfig, + intentPrompt, apiKeyScopes, isImpersonated: _apiKey?.is_impersonated === true, suppressAnalytics: props.suppressAnalytics, @@ -429,10 +437,15 @@ export class RequestStateResolver { return { flags: {}, payloads: {} } } // Enable product surfaces locally because the analytics flag client is disabled there. - // Recruitment still needs an explicit override so development tasks do not ask for interviews. + // Recruitment and intent experiments stay off so ordinary development keeps the default behavior. if (isLocalApi()) { return { - flags: Object.fromEntries(flagKeys.map((key) => [key, key !== MCP_INTERVIEW_RECRUITMENT_FLAG])), + flags: Object.fromEntries( + flagKeys.map((key) => [ + key, + key !== MCP_INTERVIEW_RECRUITMENT_FLAG && key !== MCP_INTENT_PROMPT_FLAG, + ]) + ), payloads: {}, } } diff --git a/services/mcp/src/hono/tool-executor.ts b/services/mcp/src/hono/tool-executor.ts index 54debc93409d..0fd0547119dd 100644 --- a/services/mcp/src/hono/tool-executor.ts +++ b/services/mcp/src/hono/tool-executor.ts @@ -154,7 +154,7 @@ export class ToolExecutor { } async handleToolsList(state: ResolvedState): Promise { - const tools = this.injectAnalyticsParameters(this.buildAdvertisedTools(state)) + const tools = this.injectAnalyticsParameters(this.buildAdvertisedTools(state), state.intentPrompt) void trackToolsList( tools.map((t) => t.name), @@ -165,9 +165,9 @@ export class ToolExecutor { } // Guarded because analytics must never break `tools/list`. - private injectAnalyticsParameters(tools: ListToolsResult['tools']): ListToolsResult['tools'] { + private injectAnalyticsParameters(tools: ListToolsResult['tools'], description?: string): ListToolsResult['tools'] { try { - return getPostHogClient().prepareToolList(tools) + return getPostHogClient().prepareToolList(tools, description ? { context: { description } } : undefined) } catch { return tools } diff --git a/services/mcp/tests/hono/request-state-resolver.test.ts b/services/mcp/tests/hono/request-state-resolver.test.ts index 94ed886b7030..deb0ad349995 100644 --- a/services/mcp/tests/hono/request-state-resolver.test.ts +++ b/services/mcp/tests/hono/request-state-resolver.test.ts @@ -143,6 +143,7 @@ vi.mock('@/hono/request-context', () => { import type { RedisLike } from '@/hono/cache/RedisCache' import { MCP_EXEC_SKILLS_FEATURE_FLAG } from '@/hono/constants' +import { MCP_INTENT_PROMPT_FLAG } from '@/hono/intent-prompt' import { MCP_INTERVIEW_RECRUITMENT_FLAG } from '@/hono/interview-recruitment' import { RequestStateResolver } from '@/hono/request-state-resolver' import { ToolCatalog } from '@/hono/tool-catalog' @@ -490,7 +491,7 @@ describe('RequestStateResolver MCP client contexts', () => { expect(props.mode).toBe('cli') }) - it.each([MCP_EXEC_SKILLS_FEATURE_FLAG, MCP_INTERVIEW_RECRUITMENT_FLAG])( + it.each([MCP_EXEC_SKILLS_FEATURE_FLAG, MCP_INTERVIEW_RECRUITMENT_FLAG, MCP_INTENT_PROMPT_FLAG])( 'evaluates %s even though no generated tool declares it', async (flag) => { vi.mocked(evaluateFeatureFlags).mockResolvedValueOnce({ flags: { [flag]: true }, payloads: {} }) @@ -602,6 +603,35 @@ describe('RequestStateResolver MCP client contexts', () => { } ) + it.each([ + { value: true, enabled: true }, + { value: 'test-1', enabled: true }, + { value: false, enabled: false }, + { value: undefined, enabled: false }, + { value: '', enabled: false }, + { value: true, missingPayload: true, enabled: false }, + { value: true, payload: { prompt: ' ' }, enabled: false }, + { value: true, payload: { prompt: 'x'.repeat(2001) }, enabled: false }, + { value: true, impersonated: true, enabled: false }, + { value: true, suppressed: true, enabled: false }, + { value: true, consumer: 'posthog-cli', enabled: false }, + ])('resolves the intent prompt only for an eligible configured request: %j', async (input) => { + const prompt = { + prompt: 'State the task objective before the tool action. Omit identifying details.', + } + mockApiKey.is_impersonated = input.impersonated + mockApiKey.suppress_analytics = input.suppressed + vi.mocked(evaluateFeatureFlags).mockResolvedValueOnce({ + flags: { [MCP_INTENT_PROMPT_FLAG]: input.value }, + payloads: { + [MCP_INTENT_PROMPT_FLAG]: input.missingPayload ? undefined : { ...prompt, ...input.payload }, + }, + }) + const state = await makeResolver().resolve(makeProps({ mcpConsumer: input.consumer })) + + expect(state.intentPrompt).toEqual(input.enabled ? prompt.prompt : undefined) + }) + it('honors a dev/test flag override even when evaluation returns nothing', async () => { // Evaluation stays empty (analytics client disabled, as in local dev/evals); // the override seam is what flips a tool flag on so it reaches the tool layer. diff --git a/services/mcp/tests/hono/tool-executor-intent.test.ts b/services/mcp/tests/hono/tool-executor-intent.test.ts index 858a4b6c462b..01562529c401 100644 --- a/services/mcp/tests/hono/tool-executor-intent.test.ts +++ b/services/mcp/tests/hono/tool-executor-intent.test.ts @@ -197,20 +197,36 @@ describe('ToolExecutor analytics capture', () => { expect(state.context.api.config.onPrivateResponse).toBeUndefined() }) - it('injects the analytics arguments into advertised tools', async () => { - const state = makeToolExecutorState([], { useSingleExec: true }) + it.each([true, false])('isolates intent descriptions with single exec=%s', async (useSingleExec) => { + const tool = catalog.getFilteredTools({ scopes: ['*'] }).find((entry) => entry.name === 'projects-get')! + const state = makeToolExecutorState([tool], { useSingleExec }) + const prompt = 'State the task objective before the tool action. Omit identifying details.' const result = await executor.handleToolsList(state) + const treatment = await executor.handleToolsList({ + ...state, + distinctId: 'trial-user', + intentPrompt: prompt, + }) + const other = await executor.handleToolsList({ ...state, distinctId: 'another-user' }) + vi.spyOn(getPostHogClient(), 'prepareToolList').mockImplementationOnce(() => { + throw new Error('Synthetic SDK preparation failure') + }) + const fallback = await executor.handleToolsList({ ...state, intentPrompt: prompt }) - const execEntry = result.tools.find((t) => t.name === 'exec')! - const properties = execEntry.inputSchema.properties as Record - // The SDK injects `context` (required, to nudge the agent to state intent) - // while leaving the existing `command` arg untouched. + const entry = result.tools[0]! + const properties = entry.inputSchema.properties as Record expect(properties).toHaveProperty('context') expect(properties).toHaveProperty('llm_model') - expect(properties).toHaveProperty('command') - expect(execEntry.inputSchema.required).toContain('context') - expect(execEntry.inputSchema.required).toContain('llm_model') + expect(entry.inputSchema.required).toContain('context') + expect(entry.inputSchema.required).toContain('llm_model') + expect(treatment.tools[0]!.inputSchema.properties?.context).toMatchObject({ description: prompt }) + expect(other.tools[0]!.inputSchema.properties?.context).toEqual(properties.context) + expect(properties.context).not.toEqual(treatment.tools[0]!.inputSchema.properties?.context) + expect(fallback.tools[0]!.inputSchema.properties).not.toHaveProperty('context') + if (useSingleExec) { + expect(properties).toHaveProperty('command') + } }) it.each([