diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml index 7e80e882780e..2b39b50eea33 100644 --- a/frontend/snapshots.yml +++ b/frontend/snapshots.yml @@ -6796,6 +6796,10 @@ snapshots: hash: v1.k794b7964.6f9f209d5daaf8573020fb50615421d426055c8090d0ee73a2629487d30cbf29.stsWopH6BI50ASXjhDxCWn-Xa46oqy_cH4v0xAO4VIQ scenes-app-ai-observability-instrumentation-checklist-card--warnings-with-a-dismissal--light: hash: v1.k794b7964.b0e06944ee7afe4af8cb39363280135111e71ba215aab56a3e410c79de62238b.WtW8suUOWanaTrvjxhV2Z_ANVDRmEW3bqHpTrTe5GoI + scenes-app-ai-observability-promptreferencetags--label-and-version-references--dark: + hash: v1.k794b7964.4cc699d7489063a64cf4ec1f930d11fc062eb0d9c9e46150f939503945bb8841.jKmH2ZmjZex98fImM00HE_xBBayalWKHtqXbVlhE_VM + scenes-app-ai-observability-promptreferencetags--label-and-version-references--light: + hash: v1.k794b7964.b406b4c5f2cc872222ab4156f8179c50db3a873b1b75d332e826ea9dc37eaa12.BHAFV67Jt-fG7F3xlWVwu0Th-yvY_DepLdasKhNQA30 scenes-app-ai-observability-prompts--empty-state--dark: hash: v1.k794b7964.3881018cfbf85cf1beed0dd610d747267571381277a983c7b8239acc5d372b86.yIUKSXpWrhD6XV_uZ1Sk-kb3tJbbBLfdcYML7kWkw40 scenes-app-ai-observability-prompts--empty-state--light: diff --git a/products/ai_observability/frontend/prompts/InsertPromptReferenceButton.tsx b/products/ai_observability/frontend/prompts/InsertPromptReferenceButton.tsx new file mode 100644 index 000000000000..0f9eea92815b --- /dev/null +++ b/products/ai_observability/frontend/prompts/InsertPromptReferenceButton.tsx @@ -0,0 +1,57 @@ +import { useActions, useValues } from 'kea' +import { useState } from 'react' + +import { IconPlusSmall } from '@posthog/icons' +import { LemonButton, LemonModal } from '@posthog/lemon-ui' + +import { insertPromptReferenceLogic } from './insertPromptReferenceLogic' +import { PromptReferencePicker } from './PromptReferencePicker' + +export function InsertPromptReferenceButton({ + currentPromptName, + onInsert, +}: { + currentPromptName: string + onInsert: (tag: string) => void +}): JSX.Element { + const { promptOptionsLoading } = useValues(insertPromptReferenceLogic) + const { loadPromptOptions } = useActions(insertPromptReferenceLogic) + const [isOpen, setIsOpen] = useState(false) + + return ( + <> + } + loading={!isOpen && promptOptionsLoading} + tooltip="Insert a reference to another prompt. Its content replaces the tag when the prompt is fetched." + data-attr="llma-prompt-insert-reference-button" + onClick={(e) => { + e.preventDefault() + if (promptOptionsLoading) { + return + } + setIsOpen(true) + loadPromptOptions('') + }} + > + Insert reference + + setIsOpen(false)} + title="Insert prompt reference" + description="Reference a label to pick up its future moves, or pin a version to freeze the content." + width={480} + > + { + onInsert(tag) + setIsOpen(false) + }} + /> + + + ) +} diff --git a/products/ai_observability/frontend/prompts/PromptReferencePicker.tsx b/products/ai_observability/frontend/prompts/PromptReferencePicker.tsx new file mode 100644 index 000000000000..af588ef35525 --- /dev/null +++ b/products/ai_observability/frontend/prompts/PromptReferencePicker.tsx @@ -0,0 +1,89 @@ +import { useActions, useValues } from 'kea' +import { useState } from 'react' + +import { Combobox, ComboboxContent, ComboboxEmpty, ComboboxInput, ComboboxItem, ComboboxList } from 'lib/ui/quill' + +import { insertPromptReferenceLogic } from './insertPromptReferenceLogic' +import { buildPromptReferenceTag } from './promptReferences' + +interface ReferenceOption { + tag: string + promptName: string + selectorLabel: string +} + +export function PromptReferencePicker({ + currentPromptName, + onSelect, +}: { + currentPromptName: string + onSelect: (tag: string) => void +}): JSX.Element { + const { promptOptions, promptOptionsLoading } = useValues(insertPromptReferenceLogic) + const { loadPromptOptions } = useActions(insertPromptReferenceLogic) + const [inputValue, setInputValue] = useState('') + + // Keys double as display strings so the combobox's text filtering matches + // what the item shows. The "(pinned)" suffix cannot collide with a label + // row: label names allow no spaces or parentheses. + const optionsByKey = new Map() + for (const prompt of promptOptions) { + if (prompt.name === currentPromptName) { + continue + } + for (const label of prompt.all_labels) { + optionsByKey.set(`${prompt.name} @ ${label.name}`, { + tag: buildPromptReferenceTag(prompt.name, { label: label.name }), + promptName: prompt.name, + selectorLabel: `@ ${label.name} (follows the label)`, + }) + } + optionsByKey.set(`${prompt.name} @ v${prompt.latest_version} (pinned)`, { + tag: buildPromptReferenceTag(prompt.name, { version: prompt.latest_version }), + promptName: prompt.name, + selectorLabel: `@ v${prompt.latest_version} (pinned)`, + }) + } + const items = [...optionsByKey.keys()] + + return ( + { + const option = next ? optionsByKey.get(next) : undefined + if (option) { + onSelect(option.tag) + } + }} + inputValue={inputValue} + onInputValueChange={(value: string) => { + setInputValue(value) + loadPromptOptions(value) + }} + defaultOpen + > + + + {promptOptionsLoading ? 'Loading prompts...' : 'No matching prompts'} + + {(item: string) => { + const option = optionsByKey.get(item) + return ( + + {option?.promptName} + {option?.selectorLabel} + + ) + }} + + + + ) +} diff --git a/products/ai_observability/frontend/prompts/PromptReferenceTags.stories.tsx b/products/ai_observability/frontend/prompts/PromptReferenceTags.stories.tsx new file mode 100644 index 000000000000..31adafc4ba87 --- /dev/null +++ b/products/ai_observability/frontend/prompts/PromptReferenceTags.stories.tsx @@ -0,0 +1,17 @@ +import type { Meta, StoryObj } from '@storybook/react' + +import { PromptReferenceTags } from './PromptReferenceTags' + +const meta: Meta = { + title: 'Scenes-App/AI Observability/PromptReferenceTags', + component: PromptReferenceTags, +} +export default meta + +type Story = StoryObj + +export const LabelAndVersionReferences: Story = { + args: { + text: 'Intro.\n@@@prompt:name=guardrails|label=production@@@\nMore text.\n@@@prompt:name=tone|version=3@@@', + }, +} diff --git a/products/ai_observability/frontend/prompts/PromptReferenceTags.tsx b/products/ai_observability/frontend/prompts/PromptReferenceTags.tsx new file mode 100644 index 000000000000..e0f61825a7b3 --- /dev/null +++ b/products/ai_observability/frontend/prompts/PromptReferenceTags.tsx @@ -0,0 +1,28 @@ +import { LemonTag, Link } from '@posthog/lemon-ui' + +import { urls } from 'scenes/urls' + +import { extractPromptReferences } from './promptReferences' + +export function PromptReferenceTags({ text }: { text: string }): JSX.Element | null { + const references = extractPromptReferences(text) + if (references.length === 0) { + return null + } + return ( +
+ References: + {references.map((reference) => ( + + + {reference.name} @ {reference.label ?? `v${reference.version}`} + + + ))} +
+ ) +} diff --git a/products/ai_observability/frontend/prompts/insertPromptReferenceLogic.ts b/products/ai_observability/frontend/prompts/insertPromptReferenceLogic.ts new file mode 100644 index 000000000000..fc14ea048462 --- /dev/null +++ b/products/ai_observability/frontend/prompts/insertPromptReferenceLogic.ts @@ -0,0 +1,61 @@ +import { MakeLogicType, kea, path } from 'kea' +import { loaders } from 'kea-loaders' + +import { ApiConfig } from '~/lib/api' + +import { llmPromptsList } from '../generated/api' +import type { LLMPromptListApi } from '../generated/api.schemas' + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface insertPromptReferenceLogicValues { + promptOptions: LLMPromptListApi[] + promptOptionsLoading: boolean +} + +// Generated by kea-typegen. Update if you're an agent, ignore if you're human. +export interface insertPromptReferenceLogicActions { + loadPromptOptions: (search: string) => string + loadPromptOptionsFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadPromptOptionsSuccess: ( + promptOptions: LLMPromptListApi[], + payload?: string + ) => { + promptOptions: LLMPromptListApi[] + payload?: string + } +} + +export type insertPromptReferenceLogicType = MakeLogicType< + insertPromptReferenceLogicValues, + insertPromptReferenceLogicActions +> + +export const insertPromptReferenceLogic = kea([ + path(['scenes', 'ai-observability', 'insertPromptReferenceLogic']), + loaders({ + promptOptions: [ + [] as LLMPromptListApi[], + { + loadPromptOptions: async (search: string, breakpoint) => { + if (search) { + await breakpoint(250) + } + const response = await llmPromptsList(String(ApiConfig.getCurrentTeamId()), { + content: 'none', + order_by: 'name', + limit: 100, + ...(search ? { search } : {}), + }) + breakpoint() + return response.results + }, + }, + ], + }), +]) diff --git a/products/ai_observability/frontend/prompts/llmPromptLogic.ts b/products/ai_observability/frontend/prompts/llmPromptLogic.ts index 08db0f43a06a..221fc2dc23f4 100644 --- a/products/ai_observability/frontend/prompts/llmPromptLogic.ts +++ b/products/ai_observability/frontend/prompts/llmPromptLogic.ts @@ -41,9 +41,10 @@ import { llmPromptsNameLabelsDestroy, llmPromptsNameLabelsUpdate, llmPromptsNamePartialUpdate, + llmPromptsNameRetrieve, llmPromptsResolveNameRetrieve, } from '../generated/api' -import type { LLMPromptLabelApi, LLMPromptResolveResponseApi } from '../generated/api.schemas' +import type { LLMPromptLabelApi, LLMPromptPublicApi, LLMPromptResolveResponseApi } from '../generated/api.schemas' import { llmPromptsLogic } from './llmPromptsLogic' import { LLM_PROMPTS_FORCE_RELOAD_PARAM } from './llmPromptsLogic' import { LLMPrompt, LLMPromptVersionSummary } from './types' @@ -232,6 +233,7 @@ export interface llmPromptLogicValues { isPromptMissing: boolean isPublishReviewOpen: boolean isRenderingMarkdown: boolean + isShowingResolvedPreview: boolean isViewMode: boolean labelPickerVersion: number | null labelsByVersion: Record @@ -256,6 +258,8 @@ export interface llmPromptLogicValues { publishConflict: PublishConflict | null relatedTracesQuery: DataTableNode | null relatedTracesQueryOverride: DataTableNode | null + resolvedPreview: LLMPromptPublicApi | null + resolvedPreviewLoading: boolean shouldDisplaySkeleton: boolean showPromptFormErrors: boolean snippetLanguage: PromptSnippetLanguage @@ -321,6 +325,21 @@ export interface llmPromptLogicActions { prompt: ResolvedLLMPrompt payload?: any } + loadResolvedPreview: () => any + loadResolvedPreviewFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadResolvedPreviewSuccess: ( + resolvedPreview: LLMPromptPublicApi, + payload?: any + ) => { + resolvedPreview: LLMPromptPublicApi + payload?: any + } openLabelPicker: (version: number) => { version: number } @@ -421,6 +440,9 @@ export interface llmPromptLogicActions { toggleOutlineExpanded: () => { value: true } + toggleResolvedPreview: () => { + value: true + } touchPromptFormField: (key: string) => { key: string } @@ -522,6 +544,7 @@ export const llmPromptLogic = kea([ setAnalyticsScope: (analyticsScope: PromptAnalyticsScope) => ({ analyticsScope }), setRelatedTracesQuery: (query: DataTableNode) => ({ query }), toggleMarkdownRendering: true, + toggleResolvedPreview: true, setCompareVersion: (compareVersion: number | null) => ({ compareVersion }), toggleOutlineExpanded: true, showConfigEditor: true, @@ -575,6 +598,24 @@ export const llmPromptLogic = kea([ setRelatedTracesQuery: (_, { query }) => query, }, ], + // Cleared the moment a prompt load starts: every path to different + // content (navigation, version switch, refresh, failure) begins with + // loadPrompt, so clearing on the trigger closes the whole class of + // stale-preview states instead of patching individual outcomes. + resolvedPreview: { + loadPrompt: () => null, + setMode: () => null, + }, + isShowingResolvedPreview: [ + false, + { + toggleResolvedPreview: (state: boolean) => !state, + loadPrompt: () => false, + setMode: () => false, + // A failed resolution must not present the raw source as resolved content. + loadResolvedPreviewFailure: () => false, + }, + ], isRenderingMarkdown: [ props.promptName === 'new' ? false : (props.mode ?? PromptMode.View) !== PromptMode.Edit, { @@ -678,6 +719,22 @@ export const llmPromptLogic = kea([ }, })), + loaders(({ props }) => ({ + resolvedPreview: { + __default: null as LLMPromptPublicApi | null, + loadResolvedPreview: async () => { + // Version from the router, like loadPrompt: values.prompt still holds + // the previous version while a back/forward navigation is loading. + const urlVersion = getSelectedVersionFromUrl() + return await llmPromptsNameRetrieve( + String(ApiConfig.getCurrentTeamId()), + props.promptName, + urlVersion !== undefined ? { version: urlVersion } : undefined + ) + }, + }, + })), + forms(({ actions, props, values }) => ({ promptForm: { defaults: DEFAULT_PROMPT_FORM_VALUES, @@ -1191,6 +1248,14 @@ export const llmPromptLogic = kea([ }), listeners(({ actions, asyncActions, props, values }) => ({ + toggleResolvedPreview: () => { + if (values.isShowingResolvedPreview) { + actions.loadResolvedPreview() + } + }, + loadResolvedPreviewFailure: ({ errorObject }) => { + lemonToast.error(getApiErrorDetail(errorObject) ?? 'Could not resolve this prompt. Try again.') + }, requestSetLabel: ({ labelName, version }) => { const existing = values.promptLabels.find((label) => label.name === labelName) if (existing?.version === version) { diff --git a/products/ai_observability/frontend/prompts/promptReferences.ts b/products/ai_observability/frontend/prompts/promptReferences.ts new file mode 100644 index 000000000000..5c31e4493cc9 --- /dev/null +++ b/products/ai_observability/frontend/prompts/promptReferences.ts @@ -0,0 +1,34 @@ +// Mirrors the tag grammar in products/ai_observability/backend/prompt_references.py; +// the backend is authoritative, keep the two in sync. +export const PROMPT_REFERENCE_REGEX = + /@@@prompt:name=([A-Za-z0-9_-]{1,255})\|(?:version=([1-9][0-9]{0,8})|label=([a-z0-9](?:[a-z0-9._-]{0,126}[a-z0-9])?))@@@/g + +export interface PromptReferenceTag { + raw: string + name: string + version: number | null + label: string | null +} + +export function extractPromptReferences(text: string): PromptReferenceTag[] { + const seen = new Set() + const references: PromptReferenceTag[] = [] + for (const match of text.matchAll(PROMPT_REFERENCE_REGEX)) { + if (seen.has(match[0])) { + continue + } + seen.add(match[0]) + references.push({ + raw: match[0], + name: match[1], + version: match[2] ? parseInt(match[2], 10) : null, + label: match[3] ?? null, + }) + } + return references +} + +export function buildPromptReferenceTag(name: string, selector: { version: number } | { label: string }): string { + const selectorPart = 'version' in selector ? `version=${selector.version}` : `label=${selector.label}` + return `@@@prompt:name=${name}|${selectorPart}@@@` +} diff --git a/products/ai_observability/frontend/prompts/promptSceneComponents.tsx b/products/ai_observability/frontend/prompts/promptSceneComponents.tsx index 88bd912be322..8aaf1e59c924 100644 --- a/products/ai_observability/frontend/prompts/promptSceneComponents.tsx +++ b/products/ai_observability/frontend/prompts/promptSceneComponents.tsx @@ -38,10 +38,13 @@ import { useTracesQueryContext } from '../AIObservabilityTracesScene' import { MarkdownOutline } from '../components/MarkdownOutline' import { CreatePromptExperimentModal } from './CreatePromptExperimentModal' import { createPromptExperimentModalLogic } from './createPromptExperimentModalLogic' +import { InsertPromptReferenceButton } from './InsertPromptReferenceButton' import { PromptAnalyticsScope, formatPromptConfig, isPrompt, llmPromptLogic } from './llmPromptLogic' import { promptExperimentsLogic } from './promptExperimentsLogic' import { PromptLabelChip } from './PromptLabelChip' import { PromptLabelPicker } from './PromptLabelPicker' +import { extractPromptReferences } from './promptReferences' +import { PromptReferenceTags } from './PromptReferenceTags' import { LLMPrompt, LLMPromptVersionSummary } from './types' import { PROMPT_NAME_MAX_LENGTH } from './utils' @@ -74,9 +77,17 @@ function PromptOutline({ } export function PromptViewDetails(): JSX.Element { - const { prompt, isRenderingMarkdown, isDiffVisible, canCompareVersions, compareVersionOptions } = - useValues(llmPromptLogic) - const { toggleMarkdownRendering, setCompareVersion } = useActions(llmPromptLogic) + const { + prompt, + isRenderingMarkdown, + isDiffVisible, + canCompareVersions, + compareVersionOptions, + isShowingResolvedPreview, + resolvedPreview, + resolvedPreviewLoading, + } = useValues(llmPromptLogic) + const { toggleMarkdownRendering, setCompareVersion, toggleResolvedPreview } = useActions(llmPromptLogic) const markdownContainerRef = useRef(null) if (!prompt || !isPrompt(prompt)) { @@ -86,7 +97,11 @@ export function PromptViewDetails(): JSX.Element { const configJson = prompt.config != null ? formatPromptConfig(prompt.config) : null const promptText = prompt.prompt - const variableMatches = promptText.match(/\{\{([^}]+)\}\}/g) + const references = extractPromptReferences(promptText) + const resolvedText = + isShowingResolvedPreview && typeof resolvedPreview?.prompt === 'string' ? resolvedPreview.prompt : null + const displayText = resolvedText ?? promptText + const variableMatches = displayText.match(/\{\{([^}]+)\}\}/g) const variables = variableMatches ? [...new Set(variableMatches.map((match: string) => match.slice(2, -2).trim()))] : [] @@ -97,6 +112,18 @@ export function PromptViewDetails(): JSX.Element {

Prompt

+ {!isDiffVisible && references.length > 0 && ( + + Show resolved + + )} {!isDiffVisible && ( ) : isRenderingMarkdown ? ( <> - +
- {promptText} + {displayText}
) : ( - 
{prompt.prompt}
+ 
{displayText}
)}
+ {resolvedText !== null && (resolvedPreview?.resolved_references?.length ?? 0) > 0 && ( +
+ Includes: + {resolvedPreview?.resolved_references?.map((reference) => ( + + {reference.name} v{reference.version} + {reference.label ? ` (via ${reference.label})` : ''} + + ))} +
+ )} + + {!isShowingResolvedPreview && } + {variables.length > 0 && (
Variables: @@ -905,7 +946,16 @@ export function PromptEditForm({ selectedVersion: number | null }): JSX.Element { const { promptVariables, isNewPrompt, isRenderingMarkdown, promptForm, publishConflict } = useValues(llmPromptLogic) - const { toggleMarkdownRendering } = useActions(llmPromptLogic) + const { toggleMarkdownRendering, setPromptFormValue } = useActions(llmPromptLogic) + const promptTextAreaRef = useRef(null) + + const insertReferenceTag = (tag: string): void => { + const textarea = promptTextAreaRef.current + const current = promptForm.prompt || '' + const start = textarea?.selectionStart ?? current.length + const end = textarea?.selectionEnd ?? current.length + setPromptFormValue('prompt', current.slice(0, start) + tag + current.slice(end)) + } return (
@@ -957,6 +1007,12 @@ export function PromptEditForm({ toggleMarkdownRendering() }} /> + {!isRenderingMarkdown && ( + + )}
} help="Use {{variable_name}} to define variables that will be replaced when fetching the prompt from your backend." @@ -967,6 +1023,7 @@ export function PromptEditForm({ ) : ( )} + +
)