feat(sql-editor): browse nested BI connections

docs/published/handbook/engineering/data-warehouse.md

@@ -236,3 +236,18 @@ You'll need to install MS SQL drivers for the PostHog app to connect to a MS SQL
 ```text
 symbol not found in flat namespace '_bcp_batch'
 ```
+
+## Connected fields in BI mode
+
+Below Dimensions and Measures, **Connections** lists the selected table's linked tables and views.
+Expand a connection to load its dimensions, measures, and nested connections. Inside connections,
+fields and further links appear without section headings.
+
+Drag connected fields onto a shelf, double-click them, or press Enter or Space to add dimensions to Rows and measures to Values.
+Measures are aggregated automatically. The worksheet keeps its original source table and uses the full
+connection path in queries and shelf labels, such as `person.company.name`.
+Connections expand on demand, including repeated links to the same table. Search filters the fields
+inside expanded connections, and a failed field load has a Retry button.
+Aliases load any intermediate tables automatically. Loading and failed connections stay visible during search.
+Virtual connections expose the field names provided by the existing schema as dimensions. The schema does
+not include their field types or nested link definitions, so these connections do not infer measures or further links.

frontend/snapshots.yml

@@ -9470,6 +9470,10 @@ snapshots:
         hash: v1.k794b7964.25fd8a22fa490bf3e3e600a984b131b28202b8172e46e1a35e090694dfe63e2f.UBgi2bw3KLPn9F1RDJIVKYgfFlzVDD2yZj4iUPufRVA
     scenes-app-data-warehouse-sql-editor--bi-calculated-measure-editor--light:
         hash: v1.k794b7964.66016e766d5d3b78aae28be29d15e9bbf7ebb60b48de6d488eb7b10ea4676afd.wpa1Nkk9kYDByTPyA-MjGrtArnLDfzP4S6zkO34CMfo
+    scenes-app-data-warehouse-sql-editor--bi-connections--dark:
+        hash: v1.k794b7964.1f2ab2237c64512591fa64ede2d01c957272510b58e8d280d03f972947127f18.FGHAY9evifk4UKdQC7B9nSpLfbdJH9G26TeeACebtLY
+    scenes-app-data-warehouse-sql-editor--bi-connections--light:
+        hash: v1.k794b7964.6cc3f998d8be9bf398eaaa215c3f8eaa9dc6557a2008361d3ba9bf3bf9d91da2.ddofcjJWgAQ_QZYP6WkNgJIp_TeW5k1Wmc_NTG1M7ZQ
     scenes-app-data-warehouse-sql-editor--bi-data-source-picker--dark:
         hash: v1.k794b7964.5a7a0a5291dae3a67320c33fe0d7e087ab35dcd5738ee99f101735cd7fe6895d.v84-mZ0JtjVV_28syxCq0lC0DPMk7Z4LPSKq_seARBg
     scenes-app-data-warehouse-sql-editor--bi-data-source-picker--light:
@@ -9483,9 +9487,9 @@ snapshots:
     scenes-app-data-warehouse-sql-editor--bi-quick-filters--light:
         hash: v1.k794b7964.7d055020d8b9cdc001efc8b5ca146f2d22e7c916af54b7361f2d8f61d59b940f.11bvglysenXe98m_innE4VvTdCUKMhRuSS62QGmFscM
     scenes-app-data-warehouse-sql-editor--bi-quick-filters-narrow--dark:
-        hash: v1.k794b7964.3a3d179dc8ec4fed037e1e0737ddad5dea4f828b262e35d07ae5c0dcfec1447d.dqy-xY-ApF2QSGtDMf_HIMrtyrPbFAnhhWHB8cmi4M8
+        hash: v1.k794b7964.9438ec9789fe83709762909669ffa2b2377eab421066004a257c7467ea941ad1.zz_SahLmEDK9GozyYfPrB5ydS01sT-n0L9kpHjIPCd0
     scenes-app-data-warehouse-sql-editor--bi-quick-filters-narrow--light:
-        hash: v1.k794b7964.46ca9e07669ba124d6d051b6b0cf1dbbf30b441533a9eaa064c3492577893e93.KhY5WMXImePUoV5bH-nrNztp1axD08hcOl4kfH-uaYQ
+        hash: v1.k794b7964.27d42f80122f06e2159d402c8887260e3a057bae29102b96b66f6de97c4116c3.04u1T8VmDQZWOP90QHr9sO19BElmuaemVAxK-394fHc
     scenes-app-data-warehouse-sql-editor--edited-insight--dark:
         hash: v1.k794b7964.94eba9d351ec4c6b649fc4b09f5d4a69fed2995448c0cc1dac5d63d3df0ad20c.U6hnpHxuk9RKFoPU8bc536a9dTuzluwHE0pF4J0nPYc
     scenes-app-data-warehouse-sql-editor--edited-insight--light:

frontend/src/scenes/data-management/database/DatabaseTable.tsx

@@ -117,8 +117,8 @@ export function DatabaseTable({ table, tables, inEditSchemaMode, schemaOnChange
                     title: 'Column',
                     key: 'key',
                     dataIndex: 'name',
-                    render: function RenderColumn(column) {
-                        return <code>{column}</code>
+                    render: function RenderColumn(_, { name }) {
+                        return <code>{name}</code>
                     },
                 },
                 {

frontend/src/scenes/data-warehouse/editor/OutputPane.tsx

@@ -1348,7 +1348,7 @@ const Content = ({
         if (!response && !responseLoading && !insightLoading) {
             return (
                 <div
-                    className="flex flex-1 flex-col justify-center items-center border-t gap-4 p-4"
+                    className="flex min-w-0 flex-1 flex-col justify-center items-center border-t gap-4 p-4"
                     data-attr="sql-editor-output-pane-empty-state"
                 >
                     <span className="text-secondary">

frontend/src/scenes/data-warehouse/editor/SQLEditorScene.stories.tsx

@@ -867,3 +867,104 @@ export const JevEarlySelection: Story = {
     parameters: chartExperimentParameters(true, 100, 3000),
     decorators: [withAIConsent(true)],
 }
+const BI_CONNECTIONS_CONFIG: BIConfig = {
+    ...BI_WORKSHEET_CONFIG,
+    chartType: ChartDisplayType.ActionsTable,
+    rows: [],
+    columns: [],
+    filters: [],
+}
+
+export const BIConnections: Story = {
+    ...BIModeWorksheet,
+    parameters: {
+        ...BIModeWorksheet.parameters,
+        pageUrl: `${urls.sqlEditor()}#${new URLSearchParams({
+            q: buildBIQuery(BI_CONNECTIONS_CONFIG)?.query ?? '',
+            mode: 'bi',
+            bi: JSON.stringify(BI_CONNECTIONS_CONFIG),
+        })}`,
+        testOptions: {
+            waitForSelector: '[data-attr="bi-editor-connection-fields"] [data-attr="bi-editor-connection-fields"]',
+            viewport: { width: 1280, height: 900 },
+        },
+        msw: {
+            mocks: {
+                ...BIModeWorksheet.parameters?.msw.mocks,
+                post: {
+                    ...BIModeWorksheet.parameters?.msw.mocks.post,
+                    '/api/environments/:team_id/query/DatabaseSchemaQuery/': {
+                        tables: {
+                            events: {
+                                id: 'events',
+                                name: 'events',
+                                type: 'posthog',
+                                fields: {
+                                    event: BI_EVENTS_FIELDS.event,
+                                    revenue: BI_EVENTS_FIELDS.revenue,
+                                    person: {
+                                        name: 'person',
+                                        type: 'lazy_table',
+                                        table: 'persons',
+                                        schema_valid: true,
+                                        hogql_value: 'person',
+                                    },
+                                },
+                            },
+                            persons: {
+                                id: 'persons',
+                                name: 'persons',
+                                type: 'posthog',
+                                fields: {
+                                    email: { name: 'email', type: 'string', schema_valid: true, hogql_value: 'email' },
+                                    lifetime_value: {
+                                        name: 'lifetime_value',
+                                        type: 'float',
+                                        schema_valid: true,
+                                        hogql_value: 'lifetime_value',
+                                    },
+                                    company: {
+                                        name: 'company',
+                                        type: 'lazy_table',
+                                        table: 'companies',
+                                        schema_valid: true,
+                                        hogql_value: 'company',
+                                    },
+                                },
+                            },
+                            companies: {
+                                id: 'companies',
+                                name: 'companies',
+                                type: 'data_warehouse',
+                                fields: {
+                                    name: { name: 'name', type: 'string', schema_valid: true, hogql_value: 'name' },
+                                    annual_revenue: {
+                                        name: 'annual_revenue',
+                                        type: 'decimal',
+                                        schema_valid: true,
+                                        hogql_value: 'annual_revenue',
+                                    },
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    },
+    play: async ({ canvasElement }) => {
+        const canvas = within(canvasElement)
+        await waitFor(() => expect(canvasElement.querySelector('[data-attr="bi-editor-data-source"]')).toBeVisible(), {
+            timeout: 15000,
+        })
+        await userEvent.click(canvas.getByRole('button', { name: 'SQL' }))
+        await userEvent.click(canvas.getByRole('button', { name: 'BI' }))
+        const autoUpdate = canvasElement.querySelector('[data-attr="bi-editor-auto-update"]')!
+        if (autoUpdate.getAttribute('aria-checked') === 'true') {
+            await userEvent.click(autoUpdate)
+        }
+        await userEvent.click(await canvas.findByRole('button', { name: 'person' }))
+        await userEvent.click(await canvas.findByRole('button', { name: 'person.company' }))
+        await waitFor(() => expect(canvas.getByText('annual_revenue')).toBeVisible())
+    },
+}

frontend/src/scenes/data-warehouse/editor/bi/biEditorTypes.ts

@@ -298,12 +298,17 @@ export interface BIDataPaneFields {
     measures: BIField[]
 }
 
-export function getBIDataPaneFields(table: DatabaseSchemaTable | undefined, source: BIDataSource): BIDataPaneFields {
+export function getBIDataPaneFields(
+    table: Pick<DatabaseSchemaTable, 'fields'> | undefined,
+    source: BIDataSource,
+    path: string[] = []
+): BIDataPaneFields {
     const fields = Object.values(table?.fields ?? {})
         .filter((field) => DATA_PANE_FIELD_TYPES.has(field.type))
         .map((field): BIField => {
-            const expression = escapeDottedHogQLIdentifier(field.name)
-            return { id: getBIFieldId(source, expression), name: field.name, expression, type: field.type, source }
+            const name = [...path, field.name].join('.')
+            const expression = escapeDottedHogQLIdentifier(name)
+            return { id: getBIFieldId(source, expression), name, expression, type: field.type, source }
         })
         .sort((first, second) => first.name.localeCompare(second.name))
 

frontend/src/scenes/data-warehouse/editor/bi/components/BIDataPane.tsx

@@ -1,93 +1,15 @@
 import { useActions, useValues } from 'kea'
 
-import { IconCalendar, IconPlus } from '@posthog/icons'
+import { IconPlus } from '@posthog/icons'
 import { LemonButton, LemonInput, Spinner } from '@posthog/lemon-ui'
 
-import { cn } from 'lib/utils/css-classes'
-
+import { BIConnections } from 'products/data_warehouse/frontend/bi/BIConnections'
+import { BIDataPaneSection } from 'products/data_warehouse/frontend/bi/BIDataPaneSection'
 import { BIDataSourcePicker } from 'products/data_warehouse/frontend/bi/BIDataSourcePicker'
 
 import { editorSizingLogic } from '../../editorSizingLogic'
 import { queryDatabaseLogic } from '../../sidebar/queryDatabaseLogic'
 import { biEditorLogic } from '../biEditorLogic'
-import { BIField, BI_FIELD_DRAG_MIME_TYPE, getBIDropTarget, serializeBIField } from '../biEditorTypes'
-
-function FieldTypeGlyph({ field, measure }: { field: BIField; measure: boolean }): JSX.Element {
-    const glyph =
-        field.type === 'date' || field.type === 'datetime' ? (
-            <IconCalendar />
-        ) : measure || ['integer', 'float', 'decimal'].includes(field.type) ? (
-            '#'
-        ) : field.type === 'boolean' ? (
-            'T|F'
-        ) : field.type === 'json' || field.type === 'array' ? (
-            '{ }'
-        ) : (
-            'Abc'
-        )
-    return (
-        <span
-            className={cn(
-                'flex w-7 shrink-0 justify-center font-mono text-[10px] font-semibold',
-                measure ? 'text-success' : 'text-brand-blue'
-            )}
-        >
-            {glyph}
-        </span>
-    )
-}
-
-function DataPaneSection({
-    title,
-    fields,
-    measure,
-    emptyText,
-}: {
-    title: string
-    fields: BIField[]
-    measure: boolean
-    emptyText: string
-}): JSX.Element {
-    const { addFieldToShelf } = useActions(biEditorLogic)
-
-    const addField = (field: BIField): void => {
-        const target = getBIDropTarget(field, 'rows')
-        addFieldToShelf(target.field, target.shelf)
-    }
-
-    return (
-        <div className="flex flex-col">
-            <div className="px-2 pb-1 pt-2 text-xs font-semibold text-secondary">{title}</div>
-            {fields.length === 0 ? <span className="px-2 text-xs text-tertiary">{emptyText}</span> : null}
-            {fields.map((field) => (
-                <button
-                    key={field.id}
-                    type="button"
-                    draggable
-                    title={field.name}
-                    className={cn(
-                        'flex h-6 w-full cursor-grab items-center gap-1 rounded px-1 text-left text-xs hover:bg-fill-highlight-100',
-                        'focus-visible:outline focus-visible:outline-2 focus-visible:outline-accent'
-                    )}
-                    onDragStart={(event) => {
-                        event.dataTransfer.effectAllowed = 'copy'
-                        event.dataTransfer.setData(BI_FIELD_DRAG_MIME_TYPE, serializeBIField(field))
-                    }}
-                    onDoubleClick={() => addField(field)}
-                    onKeyDown={(event) => {
-                        if (event.key === 'Enter') {
-                            addField(field)
-                        }
-                    }}
-                    data-attr={measure ? 'bi-editor-data-pane-measure' : 'bi-editor-data-pane-dimension'}
-                >
-                    <FieldTypeGlyph field={field} measure={measure} />
-                    <span className="truncate">{field.name}</span>
-                </button>
-            ))}
-        </div>
-    )
-}
 
 /** Lists the selected table's fields as dimensions and measures, ready to drag onto shelves. */
 export function BIDataPane(): JSX.Element {
@@ -161,20 +83,18 @@ export function BIDataPane(): JSX.Element {
                         </LemonButton>
                     </div>
                 ) : !hasFields ? (
-                    <p className="px-2 text-xs text-secondary">
-                        No fields found. Drag columns from the database tree instead.
-                    </p>
+                    <p className="px-2 text-xs text-secondary">No dimensions or measures on this table.</p>
                 ) : !hasMatches ? (
-                    <p className="px-2 text-xs text-secondary">No matching fields</p>
+                    <p className="px-2 text-xs text-secondary">No matching fields in this table</p>
                 ) : (
                     <>
-                        <DataPaneSection
+                        <BIDataPaneSection
                             title="Dimensions"
                             fields={filteredDataPaneFields.dimensions}
                             measure={false}
                             emptyText="No matching dimensions"
                         />
-                        <DataPaneSection
+                        <BIDataPaneSection
                             title="Measures"
                             fields={filteredDataPaneFields.measures}
                             measure
@@ -186,6 +106,7 @@ export function BIDataPane(): JSX.Element {
                         />
                     </>
                 )}
+                {config.source && !dataPaneFieldsError ? <BIConnections /> : null}
             </div>
             <div className="border-t p-2">
                 <LemonButton

frontend/src/scenes/data-warehouse/editor/sidebar/queryDatabaseLogic.tsx

@@ -583,16 +583,23 @@ const formatTraversalChain = (chain?: (string | number)[]): string | null => {
     return chain.map((segment) => String(segment)).join('.')
 }
 
-const resolveFieldTraverserTarget = (
+export const resolveFieldTraverserTarget = (
     tableName: string,
     field: DatabaseSchemaField,
     tableLookup?: TableLookup,
-    visitedChains: Set<string> = new Set()
+    visitedChains: Set<string> = new Set(),
+    onUnloadedTable?: (tableName: string) => void
 ): DatabaseSchemaField | null => {
     if (!field.chain || !tableLookup) {
         return null
     }
 
+    const traversalKey = JSON.stringify([tableName, field.chain])
+    if (visitedChains.has(traversalKey)) {
+        return null
+    }
+    visitedChains.add(traversalKey)
+
     const baseTable = tableLookup[tableName]
     if (!baseTable) {
         return null
@@ -613,6 +620,9 @@ const resolveFieldTraverserTarget = (
         if (!currentField) {
             const nextField: DatabaseSchemaField | undefined = currentTable?.fields?.[segmentKey]
             if (!nextField) {
+                if (currentTable && Object.keys(currentTable.fields ?? {}).length === 0) {
+                    onUnloadedTable?.(currentTable.name)
+                }
                 return null
             }
             currentField = nextField
@@ -641,7 +651,13 @@ const resolveFieldTraverserTarget = (
                 return null
             }
             visitedChains.add(chainKey)
-            currentField = resolveFieldTraverserTarget(tableName, currentField, tableLookup, visitedChains)
+            currentField = resolveFieldTraverserTarget(
+                tableName,
+                currentField,
+                tableLookup,
+                visitedChains,
+                onUnloadedTable
+            )
             if (!currentField) {
                 return null
             }
@@ -652,7 +668,10 @@ const resolveFieldTraverserTarget = (
     }
 
     if (currentField?.type === 'field_traverser') {
-        return resolveFieldTraverserTarget(tableName, currentField, tableLookup, visitedChains) ?? currentField
+        return (
+            resolveFieldTraverserTarget(tableName, currentField, tableLookup, visitedChains, onUnloadedTable) ??
+            currentField
+        )
     }
 
     return currentField