diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml index 08e8e37f127e..586db1f84852 100644 --- a/frontend/snapshots.yml +++ b/frontend/snapshots.yml @@ -5996,6 +5996,18 @@ snapshots: hash: v1.k794b7964.e21450a1b2054a36fab2a183fd4485190e7161c07830a04008b734f83ecabb1c.Hj2ygumZlfd2zAIz7tC6m7XoM-G15hY_amDmI_m-cJo replay-vision-analysis-nudge--default--light: hash: v1.k794b7964.7eff340ad050dbd7cf07813207e989e33fccb29ef9767e38f89d16b1d567017c.Kr8YaJd8YlDg3tPWTMYWt4vAxTb_HKQuhGUe4IKACDU + replay-vision-cited-markdown--hostile-links--dark: + hash: v1.k794b7964.d98b1851e8c4c695b87856379b85fd225d554c843086706f8d51fe5c060ad2d9.Nu73wfxltRa0Loo3i2L0DYIY_5NgIZwuP5RMTMyQGts + replay-vision-cited-markdown--hostile-links--light: + hash: v1.k794b7964.362d39ed28755489dbf7a698e8115a98141f601b8db1364bc44c8558d488ee1c.AIbcwFSazoZiB1_qtsq7cOH1rV6XPryL3zckEZWetEA + replay-vision-cited-markdown--plain-prose--dark: + hash: v1.k794b7964.c31ceb73ee19d59911337cca2b583f30b7bfdc69349088088811998d357c87f3.y2qdje-cpHJgB9yhNVGjvi8_bsTMVyuR98LWJrnT9Lk + replay-vision-cited-markdown--plain-prose--light: + hash: v1.k794b7964.1a196e580088555e900e84c0162ae4c523feef638540838b541827654ae507ca.beJnsabMlBBuFJexvb0HtCu-cO0W1Md_3b2ucWGA72I + replay-vision-cited-markdown--structured--dark: + hash: v1.k794b7964.b8136fc7ff4b2794158921d793938bdd536bccbc0a66eeadf2d988386e27a2f0.5BXQa3xxRK-9zihaQbWZ4rHtM0lmb0uFKZaNcLbKiX8 + replay-vision-cited-markdown--structured--light: + hash: v1.k794b7964.380bf20693c1029dce9ff64afc117eb5fd831449e81d541f4463ade2ec32b1e0.BBhrMSTk8RqH29BEuHYQeMRRlEos0ZQxgB1wPgvSMxQ scenes-app-ai-observability-clustercard--default--dark: hash: v1.k794b7964.745c8ac41b251f329b890d56d7b07fb31b4939a440a40eea7932f1081d3de702.ESnQVAOfySBfRJWfJsURYu-prgU3EUvQZCD62MyLlX4 scenes-app-ai-observability-clustercard--default--light: diff --git a/frontend/src/lib/lemon-ui/LemonMarkdown/LemonMarkdown.tsx b/frontend/src/lib/lemon-ui/LemonMarkdown/LemonMarkdown.tsx index 5d1ef9f9c9ae..72cddb8c8df9 100644 --- a/frontend/src/lib/lemon-ui/LemonMarkdown/LemonMarkdown.tsx +++ b/frontend/src/lib/lemon-ui/LemonMarkdown/LemonMarkdown.tsx @@ -31,6 +31,17 @@ function chartRefId(href: unknown): string | null { return typeof href === 'string' ? (CHART_REF_TARGET.exec(href)?.[1] ?? null) : null } +/** Link target for a moment in a recording, e.g. `[01:32](t:92000)`, the offset in milliseconds. */ +export const TIMESTAMP_REF_PREFIX = 't:' + +// Bounded digits, so an overlong target stays an ordinary link and keeps the usual URL sanitizing. +const TIMESTAMP_REF_TARGET = new RegExp(`^${TIMESTAMP_REF_PREFIX}(\\d{1,12})$`) + +function timestampRefMs(href: unknown): number | null { + const digits = typeof href === 'string' ? TIMESTAMP_REF_TARGET.exec(href)?.[1] : undefined + return digits === undefined ? null : parseInt(digits, 10) +} + /** Whether a paragraph's child node is a chart reference. */ function isChartRefNode(node: any): boolean { return node?.tagName === 'a' && chartRefId(node.properties?.href) !== null @@ -70,8 +81,16 @@ export interface LemonMarkdownProps { * URL — leaking the viewer's IP, acting as a tracking pixel, or probing internal addresses. * When set, only images served from PostHog (same-origin or a `posthog.com` host) render inline * as ; every other image is rendered as a plain click-to-open link instead. + * `'all'` emits no at all: a same-origin `src` is still a credentialed GET on this host. */ - disableImages?: boolean + disableImages?: boolean | 'all' + /** Whether `@member:` / `@role:` stay plain text instead of chips naming a real colleague. */ + disableMentions?: boolean + /** + * Whether every link renders as its label rather than an anchor. Preferred over stripping link syntax + * from the source, which has to out-guess the inline, reference, autolink and image-fallback forms. + */ + disableLinks?: boolean className?: string wrapCode?: boolean /** @@ -97,6 +116,8 @@ export interface LemonMarkdownProps { * is a scheme no browser can follow. */ renderChartRef?: (chartId: string, sourceOffset?: number) => React.ReactNode + /** Without it, or on a nullish return, a `t:` target renders as its label. */ + renderTimestampRef?: (timestampMs: number) => React.ReactNode } const HEADING_TAGS = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] as const @@ -132,15 +153,22 @@ const LemonMarkdownRenderer = memo(function LemonMarkdownRenderer({ lowKeyHeadings = false, disableDocsRedirect = false, disableImages = false, + disableLinks = false, + disableMentions = false, wrapCode = false, codeMaxLines, generateHeadingIds = false, renderMermaid, renderChartRef, + renderTimestampRef, }: LemonMarkdownProps): JSX.Element { const components = useMemo( () => ({ a: ({ href, children, node }: any): JSX.Element => { + const timestampMs = timestampRefMs(href) + if (timestampMs !== null) { + return <>{renderTimestampRef?.(timestampMs) ?? children} + } const chartId = chartRefId(href) if (chartId !== null) { // Falls back to its own label as plain text: for a consumer that can't draw @@ -149,6 +177,9 @@ const LemonMarkdownRenderer = memo(function LemonMarkdownRenderer({ // instead would point the reader at a scheme no browser can follow. return <>{renderChartRef?.(chartId, node?.position?.start?.offset) ?? children} } + if (disableLinks) { + return <>{children} + } return ( {children} @@ -226,14 +257,19 @@ const LemonMarkdownRenderer = memo(function LemonMarkdownRenderer({ }, ...(disableImages ? { - img: ({ src, alt }: any): JSX.Element => - isTrustedPostHogUrl(src) ? ( - {alt} + img: ({ src, alt }: any): JSX.Element => { + if (disableImages !== 'all' && isTrustedPostHogUrl(src)) { + return {alt} + } + // The click-to-open fallback is an anchor, so it answers to `disableLinks` too. + return disableLinks ? ( + <>{alt || src} ) : ( {alt || src} - ), + ) + }, } : {}), li: ({ children, node }: any): JSX.Element => { @@ -282,39 +318,38 @@ const LemonMarkdownRenderer = memo(function LemonMarkdownRenderer({ [ disableDocsRedirect, disableImages, + disableLinks, lowKeyHeadings, wrapCode, codeMaxLines, generateHeadingIds, renderMermaid, renderChartRef, + renderTimestampRef, ] ) - // `chart:` is not a protocol the default URL sanitizer knows, so it strips the href before the - // `a` override ever sees it — and an emptied href is what `Link` turns into a focusable - // target-blank stub that goes nowhere. Let the scheme through whether or not this caller can - // draw charts, so the `a` override can recognize the reference and render its label as text. - // Only on `href`: the same transform runs for an image `src`, where nothing downstream reads a - // chart reference, so `![x](chart:y)` would carry an unsanitized scheme into a broken image. + // The default sanitizer strips both schemes, so let them through, on `href` only. const urlTransform = useMemo( () => (url: string, key: string): string => - key === 'href' && chartRefId(url) !== null ? url : defaultUrlTransform(url), + key === 'href' && (chartRefId(url) !== null || timestampRefMs(url) !== null) + ? url + : defaultUrlTransform(url), [] ) // remark-breaks: a single newline becomes a line break, so prose authored without the arcane // two-trailing-spaces hard-break rule (e.g. agent-written report summaries) renders with the // line breaks the author intended. + const remarkPlugins = useMemo( + () => (disableMentions ? [remarkGfm, remarkBreaks] : [remarkGfm, remarkBreaks, remarkMentions]), + [disableMentions] + ) + return ( /* eslint-disable-next-line react/forbid-elements */ - + {children} ) @@ -326,11 +361,14 @@ function LemonMarkdownComponent({ lowKeyHeadings = false, disableDocsRedirect = false, disableImages = false, + disableLinks = false, + disableMentions = false, wrapCode = false, codeMaxLines, generateHeadingIds = false, renderMermaid, renderChartRef, + renderTimestampRef, className, }: LemonMarkdownProps): JSX.Element { return ( @@ -339,11 +377,14 @@ function LemonMarkdownComponent({ lowKeyHeadings={lowKeyHeadings} disableDocsRedirect={disableDocsRedirect} disableImages={disableImages} + disableLinks={disableLinks} + disableMentions={disableMentions} wrapCode={wrapCode} codeMaxLines={codeMaxLines} generateHeadingIds={generateHeadingIds} renderMermaid={renderMermaid} renderChartRef={renderChartRef} + renderTimestampRef={renderTimestampRef} > {children} diff --git a/frontend/src/lib/lemon-ui/LemonMarkdown/index.ts b/frontend/src/lib/lemon-ui/LemonMarkdown/index.ts index 04f49fa6afbd..68705c4e8d5d 100644 --- a/frontend/src/lib/lemon-ui/LemonMarkdown/index.ts +++ b/frontend/src/lib/lemon-ui/LemonMarkdown/index.ts @@ -1,4 +1,10 @@ -export { CHART_REF_PREFIX, LemonMarkdown, type LemonMarkdownProps, slugifyHeading } from './LemonMarkdown' +export { + CHART_REF_PREFIX, + LemonMarkdown, + type LemonMarkdownProps, + TIMESTAMP_REF_PREFIX, + slugifyHeading, +} from './LemonMarkdown' // LemonMarkdownWithMermaid is deliberately not re-exported: this barrel is imported by lib // components shipped in the toolbar bundle, and the mermaid variant must stay out of that graph. // Import it from 'lib/lemon-ui/LemonMarkdown/LemonMarkdownWithMermaid' directly. diff --git a/products/posthog_ai/frontend/components/tool/widgets/ReplayVisionScanWidget.tsx b/products/posthog_ai/frontend/components/tool/widgets/ReplayVisionScanWidget.tsx index eaa97fa98d23..65432b376c13 100644 --- a/products/posthog_ai/frontend/components/tool/widgets/ReplayVisionScanWidget.tsx +++ b/products/posthog_ai/frontend/components/tool/widgets/ReplayVisionScanWidget.tsx @@ -5,6 +5,7 @@ import { LemonBanner, Link, Spinner } from '@posthog/lemon-ui' import { urls } from 'scenes/urls' import type { ReplayObservationApi } from 'products/replay_vision/frontend/generated/api.schemas' +import { flattenMarkdownToLine } from 'products/replay_vision/frontend/utils/markdown' import { readErrorMessage, readReasoning, @@ -98,9 +99,10 @@ function ObservationRow({ observation }: { observation: ReplayObservationApi }): ) } - // Summarizers write a title and a body; a monitor scan on the same widget has reasoning instead. + // Summarizers write a title and a body; a monitor scan has reasoning, which can carry markdown. const title = readTitle(observation) - const body = readSummary(observation) ?? readReasoning(observation) + const reasoning = readReasoning(observation) + const body = readSummary(observation) ?? (reasoning ? flattenMarkdownToLine(reasoning) : null) return (
diff --git a/products/replay_vision/backend/observation_formatting.py b/products/replay_vision/backend/observation_formatting.py index 9e0a23eb67d2..065b780e4483 100644 --- a/products/replay_vision/backend/observation_formatting.py +++ b/products/replay_vision/backend/observation_formatting.py @@ -10,6 +10,73 @@ SEARCH_SNIPPET_LIMIT = 600 +_FENCE_RE = re.compile(r"^ {0,3}(?:```|~~~)") +_RULE_RE = re.compile(r"^ {0,3}(?:[-*_]\s*){3,}$") +_HEADING_RE = re.compile(r"^ {0,3}#{1,6}\s+") +_QUOTE_RE = re.compile(r"^ {0,3}>\s?") +_BULLET_RE = re.compile(r"^\s*(?:[-*+]|\d{1,9}[.)])\s+") +_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)") +_LINK_RE = re.compile(r"\[([^\]]*)\]\([^)]*\)") +_LINK_DEFINITION_RE = re.compile(r"^ {0,3}\[[^\]]+\]:\s*\S+(?:\s+(?:\"[^\"]*\"|'[^']*'|\([^)]*\)))?\s*$") +_REFERENCE_RE = re.compile(r"!?\[([^\]]*)\]\[[^\]]*\]") +_CODE_SPAN_RE = re.compile(r"`+([^`]*)`+") +_STRONG_RE = re.compile(r"(\*\*|__)(?=\S)(.+?)(?<=\S)\1") +_STRIKE_RE = re.compile(r"~~(?=\S)(.+?)(?<=\S)~~") +_EMPHASIS_RE = re.compile(r"(?~|])") +_SENTENCE_ENDINGS = ".!?:;," + + +def flatten_markdown(text: str) -> str: + """Markdown-bearing model text as plain prose, keeping the line structure. + + Scanner reasoning can carry a bold lead-in, a heading, or bullets. Every consumer outside the UI reads + it as prose, where the syntax is noise at best (`**` and `##` inside a search snippet) and a forged list + row at worst. Flattening here rather than at each consumer keeps the stored field the one canonical + copy. `plain_snippet` is what folds the result onto a single line. + + The regexes above read a line the way a markdown parser does: block markers only at the start of a + line and only space-indented, inline markers anywhere. Three carry a reason worth knowing: + `_LINK_DEFINITION_RE` has to match a whole line, or it eats a sentence that merely opens the same way + ("[Save]: clicked twice before it took"); bare `[text]` is deliberately left alone, being far more + often a real bracket in prose than a shortcut reference; and `_EMPHASIS_RE`'s lookarounds keep + `snake_case` and `2 * 3 * 4` intact. The frontend's `flattenMarkdownToLine` + (products/replay_vision/frontend/utils/markdown.ts) is the same table and has to stay in step, or a + snippet flattens differently in a search result and on the seekbar. + """ + lines: list[str] = [] + for raw in text.split("\n"): + if _FENCE_RE.match(raw) or _RULE_RE.match(raw) or _LINK_DEFINITION_RE.match(raw): + continue + line = _BULLET_RE.sub("", _QUOTE_RE.sub("", _HEADING_RE.sub("", raw))) + line = _IMAGE_RE.sub(r"\1", line) + line = _LINK_RE.sub(r"\1", line) + line = _REFERENCE_RE.sub(r"\1", line) + line = _CODE_SPAN_RE.sub(r"\1", line) + line = _STRONG_RE.sub(r"\2", line) + line = _STRIKE_RE.sub(r"\1", line) + line = _EMPHASIS_RE.sub(r"\2", line) + lines.append(_ESCAPE_RE.sub(r"\1", line).rstrip()) + return re.sub(r"\n{3,}", "\n\n", "\n".join(lines)).strip() + + +def plain_snippet(text: str, *, limit: int | None = SEARCH_SNIPPET_LIMIT) -> str: + """One line of readable plain text from a model's free-text field. + + The single line is a safety property, not tidiness. These snippets go into markdown someone else reads + as structure — the search-result bullet list inside a Max tool's untrusted-data fence — where a newline + lets recording-derived text forge a row or a header. Flattening runs first so a markdown bullet cannot + survive as a literal `-` at the start of the folded line either. Blocks are joined with a sentence + break so a flattened list still reads as prose rather than one run-on clause. + """ + flat = flatten_markdown(EVENT_ID_CITATION_RE.sub("", text)) + out = "" + for block in (b for line in flat.split("\n") if (b := " ".join(line.split()))): + if out: + out += " " if out[-1] in _SENTENCE_ENDINGS else ". " + out += block + return out if limit is None else out[:limit] + def read_output(obs: "ReplayObservation") -> dict[str, Any] | None: scanner_result = obs.scanner_result if isinstance(obs.scanner_result, dict) else None @@ -36,11 +103,12 @@ def describe_output(output: dict[str, Any]) -> str | None: def explanation_text(output: dict[str, Any]) -> str: """The free text a scanner wrote about the session: monitors, scorers and classifiers reason, summarizers - summarize. Event-id citations are for the UI, so they are stripped here.""" + summarize. Event-id citations are for the UI, so they are stripped here, and any markdown the model wrote + is flattened: every caller reads this as one line of prose.""" explanation = output.get("reasoning") or output.get("summary") if not isinstance(explanation, str): return "" - return re.sub(r"\s+", " ", EVENT_ID_CITATION_RE.sub("", explanation)).strip() + return plain_snippet(explanation, limit=None) def format_line(obs: "ReplayObservation", output: dict[str, Any], *, show_scanner: bool) -> str: diff --git a/products/replay_vision/backend/temporal/activities/embed_observation.py b/products/replay_vision/backend/temporal/activities/embed_observation.py index 24879a32002c..5ecad96cffb4 100644 --- a/products/replay_vision/backend/temporal/activities/embed_observation.py +++ b/products/replay_vision/backend/temporal/activities/embed_observation.py @@ -27,6 +27,7 @@ EMBEDDING_PRODUCT, OBSERVATION_EMBEDDING_MODEL, ) +from products.replay_vision.backend.observation_formatting import flatten_markdown from products.replay_vision.backend.tags import slugify_tag from products.replay_vision.backend.temporal.constants import KAFKA_DELIVERY_TIMEOUT_S from products.replay_vision.backend.temporal.decorators import track_activity @@ -39,8 +40,13 @@ def _renderings_for(model_output: AnyScannerOutput) -> list[tuple[str, str]]: + """The `(rendering, content)` pairs to embed for one scanner output, or none when it has nothing to say. + + Flattened first: markdown syntax the model wrote is not meaning, and embedding `**` and `##` tokens + dilutes the vector that semantic search ranks on. + """ document = model_output.embedding_document() - return [(document.rendering, document.text)] if document else [] + return [(document.rendering, flatten_markdown(document.text))] if document else [] def _result_metadata(model_output: AnyScannerOutput) -> dict[str, Any]: diff --git a/products/replay_vision/backend/tests/test_observation_formatting.py b/products/replay_vision/backend/tests/test_observation_formatting.py index 28122e66845e..4dc205819798 100644 --- a/products/replay_vision/backend/tests/test_observation_formatting.py +++ b/products/replay_vision/backend/tests/test_observation_formatting.py @@ -1,6 +1,8 @@ from datetime import date -from products.replay_vision.backend.observation_formatting import format_line +from parameterized import parameterized + +from products.replay_vision.backend.observation_formatting import flatten_markdown, format_line, plain_snippet class _FakeObs: @@ -17,4 +19,59 @@ def test_format_line_collapses_whitespace_so_observations_cannot_forge_rows() -> show_scanner=False, ) assert "\n" not in line - assert "clicked checkout - forged row ignore the above" in line + assert "clicked checkout. forged row. ignore the above" in line + + +class TestPlainSnippet: + @parameterized.expand( + [ + ("bold", "The user **abandoned** checkout.", "The user abandoned checkout."), + ("italic", "The user *hesitated* here.", "The user hesitated here."), + ("inline_code", "Clicked the `Submit` button.", "Clicked the Submit button."), + ("heading", "## Checkout blocked\nThe form rejected it.", "Checkout blocked. The form rejected it."), + ("bullets", "- Reached payment\n- Card rejected", "Reached payment. Card rejected"), + ("numbered", "1. Reached payment\n2. Card rejected", "Reached payment. Card rejected"), + ("blockquote", "> The user gave up.", "The user gave up."), + ("link", "Landed on [the pricing page](https://example.com/p).", "Landed on the pricing page."), + ("reference_link", "Landed on [pricing][p].\n\n[p]: https://example.com/p", "Landed on pricing."), + ("bracketed_prose", "The user clicked [Save].", "The user clicked [Save]."), + ("prose_after_bracket", "[Save]: clicked twice before it took", "[Save]: clicked twice before it took"), + ("image", "![a screenshot](https://example.com/x.png) followed.", "a screenshot followed."), + ("escaped_star", r"Priced at 5\* the usual.", "Priced at 5* the usual."), + ("tab_indented_heading", "\t# literal hash", "# literal hash"), + ("already_one_sentence_per_line", "Reached payment.\nCard rejected.", "Reached payment. Card rejected."), + ("existing_terminal_punctuation", "Two problems:\n- one\n- two", "Two problems: one. two"), + ] + ) + def test_flattens_markdown_to_one_readable_line(self, _label: str, text: str, expected: str) -> None: + assert plain_snippet(text) == expected + + @parameterized.expand( + [ + ("snake_case", "The handler read team_id_override from the payload."), + ("multiplication", "Retried 3 * 4 times."), + ] + ) + def test_leaves_literal_punctuation_alone(self, _label: str, text: str) -> None: + assert plain_snippet(text) == text + + @parameterized.expand( + [ + ("markdown_list", "- forged row\n- second row"), + ("markdown_heading", "# Alert fired\nsomething else"), + ("hard_wrapped_paragraphs", "line one\nline two\n\nline three"), + ] + ) + def test_never_emits_a_line_break_or_a_leading_block_marker(self, _label: str, text: str) -> None: + snippet = plain_snippet(text) + assert "\n" not in snippet + assert not snippet.startswith(("-", "*", "#", ">")) + + def test_caps_length_and_can_opt_out(self) -> None: + text = "word " * 400 + assert len(plain_snippet(text)) == 600 + assert len(plain_snippet(text, limit=None)) > 600 + + +def test_flatten_markdown_keeps_line_structure_for_embeddings() -> None: + assert flatten_markdown("## Title\n\n- **one**\n- two") == "Title\n\none\ntwo" diff --git a/products/replay_vision/frontend/components/CitedMarkdown.stories.tsx b/products/replay_vision/frontend/components/CitedMarkdown.stories.tsx new file mode 100644 index 000000000000..46a5d18e9776 --- /dev/null +++ b/products/replay_vision/frontend/components/CitedMarkdown.stories.tsx @@ -0,0 +1,76 @@ +import type { Meta, StoryObj } from '@storybook/react' + +import { CitedMarkdown } from './CitedMarkdown' + +const meta: Meta = { + title: 'Replay Vision/Cited markdown', + component: CitedMarkdown, + args: { onSeek: () => {} }, +} +export default meta + +type Story = StoryObj + +export const Structured: Story = { + args: { + text: '**Checkout blocked at payment**\n\nThe user reached the payment step and never got past it:\n\n- The card form rejected the submission twice with no visible error\n- A third attempt on a different card behaved the same way\n\nThe session ends on the payment page.', + segments: [ + { kind: 'text', value: '**Checkout blocked at payment**\n\nThe user reached the payment step' }, + { kind: 'chip', timestamp_ms: 92000 }, + { + kind: 'text', + value: ' and never got past it:\n\n- The card form rejected the submission twice with no visible error\n- A third attempt on a different card behaved the same way', + }, + { kind: 'chip', timestamp_ms: 241000 }, + { kind: 'text', value: '\n\nThe session ends on the payment page.' }, + ], + }, +} + +export const PlainProse: Story = { + args: { + text: 'The user opened the pricing page, scrolled to the comparison table, and left without starting a trial.', + segments: [], + }, +} + +/** Only `test-storybook` checks this: `react-markdown` is ESM-only and mocked out under Jest. */ +export const HostileLinks: Story = { + args: { + text: [ + 'Inline [click here](https://evil.example/phish).', + 'Reference [click here][ref] and collapsed [click here][].', + 'Image by reference ![a banner][img].', + 'Same-origin image ![a probe](/api/projects/@current/session_recordings) fires a credentialed GET.', + 'Autolink and bare https://evil.example/bare.', + 'Scheme [click here](javascript:alert(1)).', + 'Mentions @member:1 and @role:1 name whoever holds those ids.', + '', + '[ref]: https://evil.example/phish', + '[img]: https://evil.example/banner.png', + ].join('\n'), + segments: [], + }, + // `storybook/test` belongs to the frontend workspace and does not resolve from `products/`, so the + // wait and the assertions are by hand. An empty canvas passes the element counts vacuously, so the + // prose has to arrive before anything is asserted. + play: async ({ canvasElement }) => { + let text = '' + for (let attempt = 0; attempt < 40; attempt++) { + text = canvasElement.textContent ?? '' + if (text.includes('click here')) { + break + } + await new Promise((resolve) => setTimeout(resolve, 50)) + } + const check = (ok: boolean, failure: string): void => { + if (!ok) { + throw new Error(failure) + } + } + check(text.includes('click here'), 'the prose never rendered, or a link label was dropped') + check(text.includes('@member:1 and @role:1'), 'a mention resolved into a chip') + check(canvasElement.querySelectorAll('a').length === 0, 'a clickable link reached the reader') + check(canvasElement.querySelectorAll('img').length === 0, 'an image fired a request') + }, +} diff --git a/products/replay_vision/frontend/components/CitedMarkdown.tsx b/products/replay_vision/frontend/components/CitedMarkdown.tsx new file mode 100644 index 000000000000..93c4a0066bd1 --- /dev/null +++ b/products/replay_vision/frontend/components/CitedMarkdown.tsx @@ -0,0 +1,32 @@ +import { useMemo } from 'react' + +import { LemonMarkdown } from 'lib/lemon-ui/LemonMarkdown' + +import { citedMarkdown } from '../utils/citations' +import { TimestampCitation } from './TimestampCitation' + +/** Cited scanner output as markdown. The text is hostile, so it renders no request, link or mention. */ +export function CitedMarkdown({ + text, + segments, + onSeek, +}: { + text: string + segments: unknown + onSeek?: (timestampMs: number) => void +}): JSX.Element { + const markdown = useMemo(() => citedMarkdown(text, segments), [text, segments]) + return ( + } + > + {markdown} + + ) +} diff --git a/products/replay_vision/frontend/components/ObservationCard.tsx b/products/replay_vision/frontend/components/ObservationCard.tsx index 0602838399bf..c72782431da3 100644 --- a/products/replay_vision/frontend/components/ObservationCard.tsx +++ b/products/replay_vision/frontend/components/ObservationCard.tsx @@ -1,8 +1,7 @@ -import { IconCopy, IconRewindPlay, IconSparkles } from '@posthog/icons' +import { IconCopy, IconSparkles } from '@posthog/icons' import { LemonButton, LemonTag, Link, Spinner, Tooltip } from '@posthog/lemon-ui' import { copyToClipboard } from 'lib/utils/copyToClipboard' -import { colonDelimitedDuration } from 'lib/utils/durations' import { urls } from 'scenes/urls' import type { ReplayObservationApi } from '../generated/api.schemas' @@ -18,10 +17,12 @@ import { } from '../replay_scanners/types' import { citedTextToPlainText, parseCitedSegments } from '../utils/citations' import { readReasoning, scannerLabel } from '../utils/observation' +import { CitedMarkdown } from './CitedMarkdown' import { LabeledRow } from './LabeledRow' import { ObservationProgressBar } from './ObservationProgressBar' import { ObservationRetryButton } from './ObservationRetryButton' import { ScannerTypeBadge } from './ScannerTypeBadge' +import { TimestampCitation } from './TimestampCitation' export function ObservationStatusTag({ status, @@ -99,26 +100,13 @@ export function CitedText({ } return ( <> - {list.map((segment, i) => { - if (segment.kind === 'text') { - return {segment.value} - } - const seconds = Math.max(0, Math.floor(segment.timestamp_ms / 1000)) - const label = colonDelimitedDuration(seconds, null) - if (onSeek) { - return ( - onSeek(segment.timestamp_ms)} className="ml-0.5"> - - {label} - - ) - } - return ( - - {label} - + {list.map((segment, i) => + segment.kind === 'text' ? ( + {segment.value} + ) : ( + ) - })} + )} ) } @@ -496,9 +484,7 @@ export function ObservationDockCard({ {reasoning && ( -

- -

+ )} diff --git a/products/replay_vision/frontend/components/TimestampCitation.tsx b/products/replay_vision/frontend/components/TimestampCitation.tsx new file mode 100644 index 000000000000..7254e0bfffc0 --- /dev/null +++ b/products/replay_vision/frontend/components/TimestampCitation.tsx @@ -0,0 +1,24 @@ +import { IconRewindPlay } from '@posthog/icons' +import { Link } from '@posthog/lemon-ui' + +import { colonDelimitedDuration } from 'lib/utils/durations' + +/** One cited moment: a seek control into the recording, or a plain timestamp where no player is on screen. */ +export function TimestampCitation({ + timestampMs, + onSeek, +}: { + timestampMs: number + onSeek?: (timestampMs: number) => void +}): JSX.Element { + const label = colonDelimitedDuration(Math.max(0, Math.floor(timestampMs / 1000)), null) + if (!onSeek) { + return {label} + } + return ( + onSeek(timestampMs)} className="ml-0.5" data-attr="vision-observation-citation"> + + {label} + + ) +} diff --git a/products/replay_vision/frontend/observations/ReplayObservation.tsx b/products/replay_vision/frontend/observations/ReplayObservation.tsx index 935fae29ef98..1c1f54abe6c8 100644 --- a/products/replay_vision/frontend/observations/ReplayObservation.tsx +++ b/products/replay_vision/frontend/observations/ReplayObservation.tsx @@ -40,9 +40,9 @@ import { ProductKey } from '~/queries/schema/schema-general' import { BooleanTag } from '../components/BooleanTag' import { CardHeader } from '../components/CardHeader' +import { CitedMarkdown } from '../components/CitedMarkdown' import { LabeledRow } from '../components/LabeledRow' import { - CitedText, ObservationConfidence, ObservationPrimaryOutput, ObservationStatusTag, @@ -493,9 +493,7 @@ export function ReplayObservationSceneComponent(): JSX.Element { > } title="Model reasoning" /> {reasoning ? ( -

- -

+ ) : (

{observation.status === 'ineligible' diff --git a/products/replay_vision/frontend/utils/citations.test.ts b/products/replay_vision/frontend/utils/citations.test.ts index bc80bbb1e391..a17717091076 100644 --- a/products/replay_vision/frontend/utils/citations.test.ts +++ b/products/replay_vision/frontend/utils/citations.test.ts @@ -1,4 +1,4 @@ -import { type Segment, citedTextToPlainText, parseCitedSegments } from './citations' +import { type Segment, citedMarkdown, citedTextToPlainText, parseCitedSegments } from './citations' describe('parseCitedSegments', () => { const text = (value: string): Segment => ({ kind: 'text', value }) @@ -74,3 +74,63 @@ describe('citedTextToPlainText', () => { expect(citedTextToPlainText(text, segments)).toBe(expected) }) }) + +describe('citedMarkdown', () => { + it.each<{ name: string; text: string; segments: unknown; expected: string }>([ + { + name: 'writes a chip back as a t: link target', + text: 'ignored when segments exist', + segments: [ + { kind: 'text', value: 'Abandoned the cart' }, + { kind: 'chip', timestamp_ms: 161_000 }, + { kind: 'text', value: ' and left.' }, + ], + expected: 'Abandoned the cart[02:41](t:161000) and left.', + }, + { + name: 'keeps a block that spans a citation in one document', + text: '', + segments: [ + { kind: 'text', value: '- **Reached payment**' }, + { kind: 'chip', timestamp_ms: 92_000 }, + { kind: 'text', value: '\n- Card rejected' }, + ], + expected: '- **Reached payment**[01:32](t:92000)\n- Card rejected', + }, + { + name: 'does not let a trailing exclamation mark turn the citation into an image', + text: '', + segments: [ + { kind: 'text', value: 'Something broke!' }, + { kind: 'chip', timestamp_ms: 5_000 }, + ], + expected: 'Something broke! [00:05](t:5000)', + }, + { + name: 'passes the prose through untouched, including anything link-shaped', + text: '', + segments: [ + { kind: 'text', value: 'Saw [an offer](https://evil.example)' }, + { kind: 'chip', timestamp_ms: 5_000 }, + ], + expected: 'Saw [an offer](https://evil.example)[00:05](t:5000)', + }, + { + name: 'rounds a fractional offset and clamps a negative one so the target stays parseable', + text: '', + segments: [ + { kind: 'chip', timestamp_ms: 92_000.6 }, + { kind: 'chip', timestamp_ms: -5 }, + ], + expected: '[01:32](t:92001)[00:00](t:0)', + }, + { + name: 'keeps text without citations untouched', + text: 'No citations here.', + segments: [], + expected: 'No citations here.', + }, + ])('$name', ({ text, segments, expected }) => { + expect(citedMarkdown(text, segments)).toBe(expected) + }) +}) diff --git a/products/replay_vision/frontend/utils/citations.ts b/products/replay_vision/frontend/utils/citations.ts index cf330d850fe0..f7f0941165a2 100644 --- a/products/replay_vision/frontend/utils/citations.ts +++ b/products/replay_vision/frontend/utils/citations.ts @@ -1,3 +1,4 @@ +import { TIMESTAMP_REF_PREFIX } from 'lib/lemon-ui/LemonMarkdown' import { colonDelimitedDuration } from 'lib/utils/durations' // `uuid` is legacy (only old event-uuid citations carry it). Timestamp citations use `timestamp_ms` alone. @@ -56,6 +57,30 @@ export function parseCitedSegments(text: string, segments: unknown): Segment[] { return persisted.flatMap((segment) => (segment.kind === 'text' ? splitLeakedCitations(segment.value) : [segment])) } +/** + * Markdown source for a cited field, each chip written back as a `t:` target. One string rather than + * a segment list, or a bullet spanning a citation parses as two documents. The renderer sanitizes. + */ +export function citedMarkdown(text: string, segments: unknown): string { + const list = parseCitedSegments(text, segments) + if (list.length === 0) { + return text + } + let out = '' + for (const segment of list) { + if (segment.kind === 'text') { + out += segment.value + continue + } + // Glued to the preceding word, except after a `!`, where that would make the pair an image. + // Whole non-negative digits only: LemonMarkdown reads nothing else, and would leave a dead label. + const timestampMs = Math.max(0, Math.round(segment.timestamp_ms)) + const label = colonDelimitedDuration(Math.floor(timestampMs / 1000), null) + out += `${out.endsWith('!') ? ' ' : ''}[${label}](${TIMESTAMP_REF_PREFIX}${timestampMs})` + } + return out +} + /** Plain-text rendering of a cited field for the clipboard: citation chips become readable `(mm:ss)` timestamps. */ export function citedTextToPlainText(text: string, segments: unknown): string { const list = parseCitedSegments(text, segments) diff --git a/products/replay_vision/frontend/utils/markdown.test.ts b/products/replay_vision/frontend/utils/markdown.test.ts new file mode 100644 index 000000000000..4120afcd1dd2 --- /dev/null +++ b/products/replay_vision/frontend/utils/markdown.test.ts @@ -0,0 +1,58 @@ +import { flattenMarkdownToLine } from './markdown' + +describe('flattenMarkdownToLine', () => { + it.each<{ name: string; text: string; expected: string }>([ + { name: 'strips bold', text: 'The user **abandoned** checkout.', expected: 'The user abandoned checkout.' }, + { name: 'strips italics', text: 'The user *hesitated* here.', expected: 'The user hesitated here.' }, + { name: 'strips inline code', text: 'Clicked `Submit`.', expected: 'Clicked Submit.' }, + { + name: 'turns a heading and its body into one sentence pair', + text: '## Checkout blocked\nThe form rejected it.', + expected: 'Checkout blocked. The form rejected it.', + }, + { + name: 'leaves a tab-indented hash as text', + text: '\t# literal hash', + expected: '# literal hash', + }, + { + name: 'reads a bullet list as prose', + text: '- Reached payment\n- Card rejected', + expected: 'Reached payment. Card rejected', + }, + { + name: 'does not double up punctuation the model already wrote', + text: 'Two problems:\n- one\n- two', + expected: 'Two problems: one. two', + }, + { + name: 'strips a link to its label', + text: 'Landed on [pricing](https://e.com).', + expected: 'Landed on pricing.', + }, + { + name: 'strips a reference link to its label and drops the definition line', + text: 'Landed on [pricing][p].\n\n[p]: https://e.example/pricing', + expected: 'Landed on pricing.', + }, + { + name: 'leaves bracketed prose alone', + text: 'The user clicked [Save].', + expected: 'The user clicked [Save].', + }, + { + name: 'keeps a sentence that merely opens like a reference definition', + text: '[Save]: clicked twice before it took', + expected: '[Save]: clicked twice before it took', + }, + { name: 'leaves snake_case alone', text: 'Read team_id_override.', expected: 'Read team_id_override.' }, + { name: 'leaves multiplication alone', text: 'Retried 3 * 4 times.', expected: 'Retried 3 * 4 times.' }, + { name: 'renders plain prose unchanged', text: 'The user gave up.', expected: 'The user gave up.' }, + ])('$name', ({ text, expected }) => { + expect(flattenMarkdownToLine(text)).toBe(expected) + }) + + it('never emits a line break, so a one-line surface stays one line', () => { + expect(flattenMarkdownToLine('# Title\n\n- one\n- two\n\nEnd.')).not.toContain('\n') + }) +}) diff --git a/products/replay_vision/frontend/utils/markdown.ts b/products/replay_vision/frontend/utils/markdown.ts new file mode 100644 index 000000000000..2aaff7711460 --- /dev/null +++ b/products/replay_vision/frontend/utils/markdown.ts @@ -0,0 +1,47 @@ +// Kept in step with `flatten_markdown` (backend/observation_formatting.py), which documents the table. +const FENCE_RE = /^ {0,3}(?:```|~~~)/ +const RULE_RE = /^ {0,3}(?:[-*_]\s*){3,}$/ +const HEADING_RE = /^ {0,3}#{1,6}\s+/ +const QUOTE_RE = /^ {0,3}>\s?/ +const BULLET_RE = /^\s*(?:[-*+]|\d{1,9}[.)])\s+/ +const IMAGE_RE = /!\[([^\]]*)\]\([^)]*\)/g +const LINK_RE = /\[([^\]]*)\]\([^)]*\)/g +const LINK_DEFINITION_RE = /^ {0,3}\[[^\]]+\]:\s*\S+(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*$/ +const REFERENCE_RE = /!?\[([^\]]*)\]\[[^\]]*\]/g +const CODE_SPAN_RE = /`+([^`]*)`+/g +const STRONG_RE = /(\*\*|__)(\S(?:.*?\S)?)\1/g +const STRIKE_RE = /~~(\S(?:.*?\S)?)~~/g +// Captured rather than a lookbehind, which breaks chunk parsing. +const EMPHASIS_RE = /(^|[^\w*_])([*_])(\S(?:.*?\S)?)\2(?![\w*_])/g +const ESCAPE_RE = /\\([\\`*_{}[\]()#+\-.!>~|])/g +const SENTENCE_ENDINGS = '.!?:;,' + +/** Markdown-bearing model text on one line, for surfaces with room for a fragment, not a rendered block. */ +export function flattenMarkdownToLine(text: string): string { + let out = '' + for (const raw of text.split('\n')) { + if (FENCE_RE.test(raw) || RULE_RE.test(raw) || LINK_DEFINITION_RE.test(raw)) { + continue + } + const block = raw + .replace(HEADING_RE, '') + .replace(QUOTE_RE, '') + .replace(BULLET_RE, '') + .replace(IMAGE_RE, '$1') + .replace(LINK_RE, '$1') + .replace(REFERENCE_RE, '$1') + .replace(CODE_SPAN_RE, '$1') + .replace(STRONG_RE, '$2') + .replace(STRIKE_RE, '$1') + .replace(EMPHASIS_RE, '$1$3') + .replace(ESCAPE_RE, '$1') + .split(/\s+/) + .filter(Boolean) + .join(' ') + if (!block) { + continue + } + out += out ? `${SENTENCE_ENDINGS.includes(out.slice(-1)) ? ' ' : '. '}${block}` : block + } + return out +} diff --git a/products/replay_vision/frontend/utils/observation.ts b/products/replay_vision/frontend/utils/observation.ts index c2c667c66683..b687d8cbf98b 100644 --- a/products/replay_vision/frontend/utils/observation.ts +++ b/products/replay_vision/frontend/utils/observation.ts @@ -2,6 +2,7 @@ import { dayjs } from 'lib/dayjs' import { type ReplayObservationApi, ScannerOriginEnumApi, ScannerTypeEnumApi } from '../generated/api.schemas' import { citedTextToPlainText, parseCitedSegments } from './citations' +import { flattenMarkdownToLine } from './markdown' /** The dock's built-in prompt and the observations it produces have to answer to one name. */ export const BUILT_IN_SUMMARY_LABEL = 'Quick summary' @@ -131,16 +132,17 @@ export interface ObservationSeekbarMark { const SNIPPET_MAX_LENGTH = 160 function lastSentence(text: string): string | null { - const trimmed = text.trim() - if (!trimmed) { + // Flattened first: a mark has room for one line, where a bullet marker reads as the sentence start. + const flat = flattenMarkdownToLine(text) + if (!flat) { return null } // No lookbehind regex, it breaks chunk parsing on older browsers. let start = 0 - for (const match of trimmed.matchAll(/[.!?]+\s+/g)) { + for (const match of flat.matchAll(/[.!?]+\s+/g)) { start = match.index + match[0].length } - const last = trimmed + const last = flat .slice(start) .replace(/[.!?]+$/, '') .trim()