diff --git a/products/replay_vision/mcp/tools.yaml b/products/replay_vision/mcp/tools.yaml index e7279ad57ab1..ec4a57e9e96f 100644 --- a/products/replay_vision/mcp/tools.yaml +++ b/products/replay_vision/mcp/tools.yaml @@ -450,15 +450,34 @@ tools: failed — useful for triaging why a scan produced no result), `triggered_by` (`schedule` or `on_demand`), `workflow_id`, the `scanner_result` (the LLM output validated against the scanner's response schema for that scanner_type), and the `session_id` of the recording it analysed. Use this to inspect what a scanner has - found over time; to see every scanner's observations for one session, use `vision-observations-list`. + found over time; to see every scanner's observations for one session, use `vision-observations-list`. Rows + are trimmed to keep a page small. `scanner_snapshot.scanner_config.prompt` and `scanner_snapshot.query` + repeat on every row of one scanner version, so neither is on a row. The citation segments restate the flat + text they were split from and are dropped too: `model_output.summary_segments` on a summarizer observation, + `model_output.reasoning_segments` on a monitor, classifier, or scorer one. Editing a scanner's prompt or + query bumps `scanner_snapshot.scanner_version`, and a page can hold rows from either side of that edit. Call + `vision-scanners-observations-get` for one observation's full prompt, query, and citation segments. When a + page spans more than one `scanner_version`, read one observation per version before you compare findings. + Call `vision-scanners-get` for the scanner's current prompt. agent_note: > - Each observation's `_posthogUrl` opens the recording it analysed. - `scanner_result.model_output.reasoning_segments` interleaves prose with `chip` segments, and a chip's - `timestamp_ms` is the recording-relative offset of the moment being cited — append `?t=` - (`timestamp_ms` / 1000, rounded down) to that URL to seek straight to it. When you report a finding to - someone, deep-link the one or two moments it turns on rather than only describing them. + Each observation's `_posthogUrl` opens the recording it analysed. When you report a finding to someone, + deep-link the one or two moments it turns on rather than only describing them: read that observation with + `vision-scanners-observations-get`, take a `chip` segment's `timestamp_ms` from its citation segments, and + append `?t=` (`timestamp_ms` / 1000, rounded down) to the recording URL. The citation segments are + `scanner_result.model_output.summary_segments` on a summarizer observation, and + `scanner_result.model_output.reasoning_segments` on a monitor, classifier, or scorer one. + param_overrides: + limit: + description: Number of observations to return per page (default 20). + default: 20 ui_app: vision-observation-list url_prefix: /replay + response: + exclude: + - scanner_snapshot.scanner_config.prompt + - scanner_snapshot.query + - scanner_result.model_output.reasoning_segments + - scanner_result.model_output.summary_segments vision-scanners-observations-retry-create: operation: vision_scanners_observations_retry_create enabled: false diff --git a/products/replay_vision/skills/exploring-replay-vision-observations/SKILL.md b/products/replay_vision/skills/exploring-replay-vision-observations/SKILL.md index 036d98030954..2da774d3f7c6 100644 --- a/products/replay_vision/skills/exploring-replay-vision-observations/SKILL.md +++ b/products/replay_vision/skills/exploring-replay-vision-observations/SKILL.md @@ -99,10 +99,14 @@ and, like all observations, runs at most once per `(scanner, session)`. ### Cite moments, not just sessions -`scanner_result.model_output.reasoning_segments` is the same prose as `reasoning`, pre-split into `text` segments and `chip` segments. +An observation's citation segments are the same prose as its flat text, pre-split into `text` segments and `chip` segments. +The field is `scanner_result.model_output.summary_segments` on a summarizer, and `scanner_result.model_output.reasoning_segments` on a monitor, classifier, or scorer. Each chip carries a `timestamp_ms`: the recording-relative offset of the moment the model is pointing at. That's what makes a finding checkable — it turns "the user hit a paywall" into a link that opens on the paywall. +`vision-scanners-observations-list` leaves the segments off its rows to keep a page small, so read the observation you want to cite with `vision-scanners-observations-get` first. +`vision-observations-list` and `vision-observations-retrieve` still return them, so a session-scoped page needs no extra call. + The observation's `_posthogUrl` is its recording; append `?t=` (`timestamp_ms` / 1000, rounded down) to seek there. ```text diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json index 05d3dbfca5b7..3f4a3bcc3c2f 100644 --- a/services/mcp/schema/generated-tool-definitions.json +++ b/services/mcp/schema/generated-tool-definitions.json @@ -11673,7 +11673,7 @@ } }, "vision-scanners-observations-list": { - "description": "List the observations recorded by a single scanner. Each observation is one scan of one session and includes its `status` (pending/running/succeeded/failed/ineligible), `error_reason` (set on terminal non-success statuses, e.g. `too_short`/`no_recording` for ineligible or `provider_rejected`/`validation_failed` for failed — useful for triaging why a scan produced no result), `triggered_by` (`schedule` or `on_demand`), `workflow_id`, the `scanner_result` (the LLM output validated against the scanner's response schema for that scanner_type), and the `session_id` of the recording it analysed. Use this to inspect what a scanner has found over time; to see every scanner's observations for one session, use `vision-observations-list`.", + "description": "List the observations recorded by a single scanner. Each observation is one scan of one session and includes its `status` (pending/running/succeeded/failed/ineligible), `error_reason` (set on terminal non-success statuses, e.g. `too_short`/`no_recording` for ineligible or `provider_rejected`/`validation_failed` for failed — useful for triaging why a scan produced no result), `triggered_by` (`schedule` or `on_demand`), `workflow_id`, the `scanner_result` (the LLM output validated against the scanner's response schema for that scanner_type), and the `session_id` of the recording it analysed. Use this to inspect what a scanner has found over time; to see every scanner's observations for one session, use `vision-observations-list`. Rows are trimmed to keep a page small. `scanner_snapshot.scanner_config.prompt` and `scanner_snapshot.query` repeat on every row of one scanner version, so neither is on a row. The citation segments restate the flat text they were split from and are dropped too: `model_output.summary_segments` on a summarizer observation, `model_output.reasoning_segments` on a monitor, classifier, or scorer one. Editing a scanner's prompt or query bumps `scanner_snapshot.scanner_version`, and a page can hold rows from either side of that edit. Call `vision-scanners-observations-get` for one observation's full prompt, query, and citation segments. When a page spans more than one `scanner_version`, read one observation per version before you compare findings. Call `vision-scanners-get` for the scanner's current prompt.", "category": "Replay vision", "feature": "replay_vision", "summary": "List scanner observations", diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json index 112be5747f6f..43cc56a8e700 100644 --- a/services/mcp/schema/tool-definitions-all.json +++ b/services/mcp/schema/tool-definitions-all.json @@ -12301,7 +12301,7 @@ } }, "vision-scanners-observations-list": { - "description": "List the observations recorded by a single scanner. Each observation is one scan of one session and includes its `status` (pending/running/succeeded/failed/ineligible), `error_reason` (set on terminal non-success statuses, e.g. `too_short`/`no_recording` for ineligible or `provider_rejected`/`validation_failed` for failed — useful for triaging why a scan produced no result), `triggered_by` (`schedule` or `on_demand`), `workflow_id`, the `scanner_result` (the LLM output validated against the scanner's response schema for that scanner_type), and the `session_id` of the recording it analysed. Use this to inspect what a scanner has found over time; to see every scanner's observations for one session, use `vision-observations-list`.", + "description": "List the observations recorded by a single scanner. Each observation is one scan of one session and includes its `status` (pending/running/succeeded/failed/ineligible), `error_reason` (set on terminal non-success statuses, e.g. `too_short`/`no_recording` for ineligible or `provider_rejected`/`validation_failed` for failed — useful for triaging why a scan produced no result), `triggered_by` (`schedule` or `on_demand`), `workflow_id`, the `scanner_result` (the LLM output validated against the scanner's response schema for that scanner_type), and the `session_id` of the recording it analysed. Use this to inspect what a scanner has found over time; to see every scanner's observations for one session, use `vision-observations-list`. Rows are trimmed to keep a page small. `scanner_snapshot.scanner_config.prompt` and `scanner_snapshot.query` repeat on every row of one scanner version, so neither is on a row. The citation segments restate the flat text they were split from and are dropped too: `model_output.summary_segments` on a summarizer observation, `model_output.reasoning_segments` on a monitor, classifier, or scorer one. Editing a scanner's prompt or query bumps `scanner_snapshot.scanner_version`, and a page can hold rows from either side of that edit. Call `vision-scanners-observations-get` for one observation's full prompt, query, and citation segments. When a page spans more than one `scanner_version`, read one observation per version before you compare findings. Call `vision-scanners-get` for the scanner's current prompt.", "category": "Replay vision", "feature": "replay_vision", "summary": "List scanner observations", diff --git a/services/mcp/src/tools/generated/replay_vision.ts b/services/mcp/src/tools/generated/replay_vision.ts index 3d162ec65094..acbb6b840a16 100644 --- a/services/mcp/src/tools/generated/replay_vision.ts +++ b/services/mcp/src/tools/generated/replay_vision.ts @@ -4,7 +4,13 @@ import { z } from 'zod' import type { Schemas } from '@/api/generated' import * as orvalSchemas from '@/generated/replay_vision/api' import { withUiApp } from '@/resources/ui-apps' -import { withPostHogUrl, withAgentNote, type WithPostHogUrl, type WithAgentNote } from '@/tools/tool-utils' +import { + withPostHogUrl, + withAgentNote, + omitResponseFields, + type WithPostHogUrl, + type WithAgentNote, +} from '@/tools/tool-utils' import type { Context, ToolBase, ZodObjectAny } from '@/tools/types' const VisionObservationsLabelCreateSchema = () => { @@ -515,9 +521,14 @@ const visionScannersObservationsGet = (): ToolBase< const VisionScannersObservationsListSchema = () => { const VisionScannersObservationsListParams = orvalSchemas.VisionScannersObservationsListParams() const VisionScannersObservationsListQueryParams = orvalSchemas.VisionScannersObservationsListQueryParams() - return VisionScannersObservationsListParams.omit({ project_id: true }).extend( - VisionScannersObservationsListQueryParams.shape - ) + return VisionScannersObservationsListParams.omit({ project_id: true }) + .extend(VisionScannersObservationsListQueryParams.shape) + .extend({ + limit: VisionScannersObservationsListQueryParams.shape['limit'] + .default(20) + .optional() + .describe('Number of observations to return per page (default 20).'), + }) } const visionScannersObservationsList = (): ToolBase< @@ -550,20 +561,31 @@ const visionScannersObservationsList = (): ToolBase< verdict: params.verdict, }, }) + const filtered = { + ...result, + results: (result.results ?? []).map((item: any) => + omitResponseFields(item, [ + 'scanner_snapshot.scanner_config.prompt', + 'scanner_snapshot.query', + 'scanner_result.model_output.reasoning_segments', + 'scanner_result.model_output.summary_segments', + ]) + ), + } as typeof result return withAgentNote( await withPostHogUrl( context, { - ...result, + ...filtered, results: await Promise.all( - (result.results ?? []).map((item) => + (filtered.results ?? []).map((item) => withPostHogUrl(context, item, `/replay/${item.session_id}`) ) ), }, '/replay' ), - "Each observation's `_posthogUrl` opens the recording it analysed. `scanner_result.model_output.reasoning_segments` interleaves prose with `chip` segments, and a chip's `timestamp_ms` is the recording-relative offset of the moment being cited — append `?t=` (`timestamp_ms` / 1000, rounded down) to that URL to seek straight to it. When you report a finding to someone, deep-link the one or two moments it turns on rather than only describing them.\n" + "Each observation's `_posthogUrl` opens the recording it analysed. When you report a finding to someone, deep-link the one or two moments it turns on rather than only describing them: read that observation with `vision-scanners-observations-get`, take a `chip` segment's `timestamp_ms` from its citation segments, and append `?t=` (`timestamp_ms` / 1000, rounded down) to the recording URL. The citation segments are `scanner_result.model_output.summary_segments` on a summarizer observation, and `scanner_result.model_output.reasoning_segments` on a monitor, classifier, or scorer one.\n" ) }, }) diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/vision-scanners-observations-list.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/vision-scanners-observations-list.json index 78c86de5b216..51b399219eef 100644 --- a/services/mcp/tests/unit/__snapshots__/tool-schemas/vision-scanners-observations-list.json +++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/vision-scanners-observations-list.json @@ -18,7 +18,8 @@ "type": "boolean" }, "limit": { - "description": "Number of results to return per page.", + "default": 20, + "description": "Number of observations to return per page (default 20).", "type": "number" }, "max_score": { diff --git a/services/mcp/tests/unit/vision-scanner-observations-list-trimming.test.ts b/services/mcp/tests/unit/vision-scanner-observations-list-trimming.test.ts new file mode 100644 index 000000000000..8a72ffd22252 --- /dev/null +++ b/services/mcp/tests/unit/vision-scanner-observations-list-trimming.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from 'vitest' + +import { GENERATED_TOOLS } from '@/tools/generated/replay_vision' +import type { Context, ToolBase, ZodObjectAny } from '@/tools/types' + +// One observation row carrying every field the tool's `response.exclude` drops, alongside the +// siblings it must keep, so a broken path (silent no-op -> full payload back) is visible. +const FULL_ROW = { + id: 'obs-1', + session_id: 'sess-1', + status: 'succeeded', + scanner_snapshot: { + name: 'Checkout drop-offs', + scanner_type: 'monitor', + scanner_version: 3, + // Identical on every row of a scanner's page, and the largest field on it. + scanner_config: { prompt: 'Did the shopper abandon the cart?', allow_inconclusive: true }, + query: { kind: 'RecordingsQuery', events: [{ id: '$pageview' }] }, + }, + scanner_result: { + model_output: { + scanner_type: 'monitor', + verdict: 'yes', + reasoning: 'The shopper opened the cart and left without paying.', + // Restates `reasoning` interleaved with timestamp chips. + reasoning_segments: [ + { kind: 'text', value: 'The shopper opened the cart and left without paying.' }, + { kind: 'chip', timestamp_ms: 42000 }, + ], + }, + signals_count: 0, + }, +} + +// A summarizer row: same duplication, under `summary_segments`, because segments are written as +// `{citation_field}_segments` and summarizer cites `summary` rather than `reasoning`. +const SUMMARIZER_ROW = { + id: 'obs-2', + session_id: 'sess-2', + status: 'succeeded', + scanner_snapshot: { + name: 'Session summaries', + scanner_type: 'summarizer', + scanner_version: 1, + scanner_config: { prompt: 'Summarize what the person did.', length: 'long' }, + query: { kind: 'RecordingsQuery', events: [{ id: '$pageview' }] }, + }, + scanner_result: { + model_output: { + scanner_type: 'summarizer', + title: 'Renamed a project, then opened billing', + summary: 'The person renamed a project and then opened the billing page.', + summary_segments: [ + { kind: 'text', value: 'The person renamed a project and then opened the billing page.' }, + { kind: 'chip', timestamp_ms: 91000 }, + ], + }, + signals_count: 0, + }, +} + +function mockContext(row: object = FULL_ROW): Context { + return { + stateManager: { getProjectId: async () => 1 }, + api: { + getProjectBaseUrl: () => 'https://us.posthog.com/project/1', + request: async () => ({ count: 1, next: null, previous: null, results: [structuredClone(row)] }), + }, + } as unknown as Context +} + +describe('vision-scanners-observations-list row trimming', () => { + const listTool = GENERATED_TOOLS['vision-scanners-observations-list'] as () => ToolBase + + it('drops the repeated snapshot fields and the duplicated reasoning segments', async () => { + const result: any = await listTool().handler(mockContext(), { scanner_id: 'scanner-1' }) + const [row] = result.results + + expect(row.scanner_snapshot.scanner_config).not.toHaveProperty('prompt') + expect(row.scanner_snapshot).not.toHaveProperty('query') + expect(row.scanner_result.model_output).not.toHaveProperty('reasoning_segments') + }) + + it('keeps the per-row output a caller reads, and the version tag beside it', async () => { + const result: any = await listTool().handler(mockContext(), { scanner_id: 'scanner-1' }) + const [row] = result.results + + expect(row.scanner_result.model_output.reasoning).toBe('The shopper opened the cart and left without paying.') + expect(row.scanner_result.model_output.verdict).toBe('yes') + expect(row.scanner_snapshot.scanner_version).toBe(3) + // Trimming one config key must not take the rest of the config with it. + expect(row.scanner_snapshot.scanner_config.allow_inconclusive).toBe(true) + // Enrichment runs on the trimmed rows, so each row still deep-links its recording. + expect(row._posthogUrl).toBe('https://us.posthog.com/project/1/replay/sess-1') + }) + + it('drops the duplicated summary segments on a summarizer row, and keeps the summary', async () => { + const result: any = await listTool().handler(mockContext(SUMMARIZER_ROW), { scanner_id: 'scanner-2' }) + const [row] = result.results + + expect(row.scanner_result.model_output).not.toHaveProperty('summary_segments') + expect(row.scanner_result.model_output.summary).toBe( + 'The person renamed a project and then opened the billing page.' + ) + }) + + it('defaults to a page a caller can read without naming a limit', () => { + expect(listTool().schema.parse({ scanner_id: 'scanner-1' }).limit).toBe(20) + }) +})