From 44d87d4fa56dd7a0d69a2d0b76407634eb1de07d Mon Sep 17 00:00:00 2001 From: sarahxsanders Date: Wed, 9 Sep 2026 10:14:30 -0400 Subject: [PATCH 1/9] feat: allow AIO and Logs skills in default integrations --- .../__tests__/__snapshots__/commandments.test.ts.snap | 4 ++-- src/lib/agent/runner/harness/pi/README.md | 2 +- src/lib/agent/runner/harness/pi/runtime-notes.ts | 2 +- .../programs/__tests__/warehouse-suggestion.test.ts | 10 ++++++++++ src/lib/programs/posthog-integration/index.ts | 4 ++-- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/lib/agent/__tests__/__snapshots__/commandments.test.ts.snap b/src/lib/agent/__tests__/__snapshots__/commandments.test.ts.snap index 9135f7aab..a1c395e11 100644 --- a/src/lib/agent/__tests__/__snapshots__/commandments.test.ts.snap +++ b/src/lib/agent/__tests__/__snapshots__/commandments.test.ts.snap @@ -93,7 +93,7 @@ Below are important guidance on the harness constraints you are bound to. Follow - If a \`bash\` command is blocked, do NOT retry it or a reworded variant — the fence is deterministic and will block it again. Change approach: inspect with \`read\`/\`grep\`, fix the \`edit\` and continue, or skip a step that is not essential. Retrying blocked commands only wastes turns. - If you get stuck on something outside your control — a package install that keeps failing, a command you are not permitted to run, or a fix outside the scope of this integration — do NOT spiral retrying it. Note it in the setup report for the user to resolve, and move on with the rest of the work. - A \`[YARA]\` block from the security scanner is on YOUR side — it caught a real problem in the edit you just tried (PII in a \`capture()\`, a hardcoded secret or host URL). Read the block reason, understand exactly what it flagged, and change the CODE to comply — e.g. a PII block means move that field off the event and onto the person via \`identify()\`/\`$set\`, keeping the event itself. Retrying the same edit will just block again, and dropping the step loses the instrumentation — so fix it to satisfy the scanner, then continue. -- Call \`load_skill_menu\` once to choose the skill, then \`install_skill\`. Do not call \`load_skill_menu\` again this session. +- Call \`load_skill_menu\` once per category needed by the workflow, then \`install_skill\` with the chosen ID. Reuse a category’s menu instead of loading it again. - Follow the skill's steps in order. Finish the SDK setup — install it, import it at the top of the module, and INITIALIZE it at the framework's entry point for every runtime the integration targets (typically both client and server) — BEFORE adding any event capture. A capture against an uninitialized SDK silently no-ops, so initialization comes first. If you're stuck and cannot install an SDK, add capture calls and add a clear note at the top of the integration report. Never guard a capture behind a runtime "if the SDK happens to be installed" check or a dynamic \`require\`; that ships an uninitialized SDK and no events fire. Do not jump ahead to the fix/revise step just to get a build passing. - Never write a PostHog URL or token as a literal in source (e.g. 'https://us.i.posthog.com') — it is blocked. Read them from environment variables (process.env.POSTHOG_HOST, os.environ['POSTHOG_HOST'], etc.). - To inspect or change a project's \`.env\` files, go straight to the wizard-tools MCP: \`check_env_keys\` to see which keys are present, \`set_env_values\` to write them. A plain \`read\`, \`edit\`, or \`write\` of any \`.env*\` file is blocked — reach for those tools first rather than discovering the block. @@ -192,7 +192,7 @@ Below are important guidance on the harness constraints you are bound to. Follow - If a \`bash\` command is blocked, do NOT retry it or a reworded variant — the fence is deterministic and will block it again. Change approach: inspect with \`read\`/\`grep\`, fix the \`edit\` and continue, or skip a step that is not essential. Retrying blocked commands only wastes turns. - If you get stuck on something outside your control — a package install that keeps failing, a command you are not permitted to run, or a fix outside the scope of this integration — do NOT spiral retrying it. Note it in the setup report for the user to resolve, and move on with the rest of the work. - A \`[YARA]\` block from the security scanner is on YOUR side — it caught a real problem in the edit you just tried (PII in a \`capture()\`, a hardcoded secret or host URL). Read the block reason, understand exactly what it flagged, and change the CODE to comply — e.g. a PII block means move that field off the event and onto the person via \`identify()\`/\`$set\`, keeping the event itself. Retrying the same edit will just block again, and dropping the step loses the instrumentation — so fix it to satisfy the scanner, then continue. -- Call \`load_skill_menu\` once to choose the skill, then \`install_skill\`. Do not call \`load_skill_menu\` again this session. +- Call \`load_skill_menu\` once per category needed by the workflow, then \`install_skill\` with the chosen ID. Reuse a category’s menu instead of loading it again. - Follow the skill's steps in order. Finish the SDK setup — install it, import it at the top of the module, and INITIALIZE it at the framework's entry point for every runtime the integration targets (typically both client and server) — BEFORE adding any event capture. A capture against an uninitialized SDK silently no-ops, so initialization comes first. If you're stuck and cannot install an SDK, add capture calls and add a clear note at the top of the integration report. Never guard a capture behind a runtime "if the SDK happens to be installed" check or a dynamic \`require\`; that ships an uninitialized SDK and no events fire. Do not jump ahead to the fix/revise step just to get a build passing. - Never write a PostHog URL or token as a literal in source (e.g. 'https://us.i.posthog.com') — it is blocked. Read them from environment variables (process.env.POSTHOG_HOST, os.environ['POSTHOG_HOST'], etc.). - To inspect or change a project's \`.env\` files, go straight to the wizard-tools MCP: \`check_env_keys\` to see which keys are present, \`set_env_values\` to write them. A plain \`read\`, \`edit\`, or \`write\` of any \`.env*\` file is blocked — reach for those tools first rather than discovering the block. diff --git a/src/lib/agent/runner/harness/pi/README.md b/src/lib/agent/runner/harness/pi/README.md index 0142954ef..bae0e2d17 100644 --- a/src/lib/agent/runner/harness/pi/README.md +++ b/src/lib/agent/runner/harness/pi/README.md @@ -76,7 +76,7 @@ extension: Because pi doesn't have Claude Code's built-in guidance, the wizard appends a long `PI_RUNTIME_NOTES` block to the shared commandments — batching rules, "use `ls`/`find`/`grep` not `bash ls`", "don't retry blocked commands", "call -`load_skill_menu` once", "no literal PostHog URLs in source", and the +`load_skill_menu` once per category", "no literal PostHog URLs in source", and the `posthog_exec` command grammar (`info` before `call`). These close the anti-spiral gaps that showed up in profiling before they became prompt engineering. diff --git a/src/lib/agent/runner/harness/pi/runtime-notes.ts b/src/lib/agent/runner/harness/pi/runtime-notes.ts index 0d9dabfa3..5350d72ab 100644 --- a/src/lib/agent/runner/harness/pi/runtime-notes.ts +++ b/src/lib/agent/runner/harness/pi/runtime-notes.ts @@ -69,7 +69,7 @@ const DONT_SPIRAL = '- If you get stuck on something outside your control — a package install that keeps failing, a command you are not permitted to run, or a fix outside the scope of this integration — do NOT spiral retrying it. Note it in the setup report for the user to resolve, and move on with the rest of the work.'; const SKILL_MENU = - '- Call `load_skill_menu` once to choose the skill, then `install_skill`. Do not call `load_skill_menu` again this session.'; + '- Call `load_skill_menu` once per category needed by the workflow, then `install_skill` with the chosen ID. Reuse a category’s menu instead of loading it again.'; const SKILL_STEPS = "- Follow the skill's steps in order. Finish the SDK setup — install it, import it at the top of the module, and INITIALIZE it at the framework's entry point for every runtime the integration targets (typically both client and server) — BEFORE adding any event capture. A capture against an uninitialized SDK silently no-ops, so initialization comes first. If you're stuck and cannot install an SDK, add capture calls and add a clear note at the top of the integration report. Never guard a capture behind a runtime \"if the SDK happens to be installed\" check or a dynamic `require`; that ships an uninitialized SDK and no events fire. Do not jump ahead to the fix/revise step just to get a build passing."; diff --git a/src/lib/programs/__tests__/warehouse-suggestion.test.ts b/src/lib/programs/__tests__/warehouse-suggestion.test.ts index 97257b228..945195ce2 100644 --- a/src/lib/programs/__tests__/warehouse-suggestion.test.ts +++ b/src/lib/programs/__tests__/warehouse-suggestion.test.ts @@ -139,6 +139,16 @@ const promptFor = async (sources: DetectedSource[]) => { }); }; +describe('default integration skill workflow', () => { + it('loads the framework category first and delegates observability to its workflow', async () => { + const prompt = await promptFor([]); + expect(prompt).toContain('category: "integration"'); + expect(prompt).toContain('AI Observability and Logs skills'); + expect(prompt).toContain('before verification and the setup report'); + expect(prompt).not.toContain('Do NOT pick skills from other categories'); + }); +}); + describe('report instruction', () => { it('asks the agent to note the sources in the report checklist', async () => { const prompt = await promptFor([POSTGRES]); diff --git a/src/lib/programs/posthog-integration/index.ts b/src/lib/programs/posthog-integration/index.ts index 7a90f313e..e60474ee9 100644 --- a/src/lib/programs/posthog-integration/index.ts +++ b/src/lib/programs/posthog-integration/index.ts @@ -325,12 +325,12 @@ Project context: Instructions (follow these steps IN ORDER - do not skip or reorder): -STEP 1: Call load_skill_menu (from the wizard-tools MCP server) to see available skills. +STEP 1: Call load_skill_menu (from the wizard-tools MCP server) with category: "integration" to see available framework skills. If the tool fails, emit: ${ AgentSignals.ERROR_MCP_MISSING } Could not load skill menu and halt. - Choose a skill from the \`integration\` category that matches this project's framework. Do NOT pick skills from other categories (llm-analytics, error-tracking, feature-flags, omnibus, etc.) — those are handled separately. + Choose a skill from the \`integration\` category that matches this project's framework. Start with this framework skill; load the AI Observability and Logs skills when its workflow calls for them, before verification and the setup report. Both are included by default where applicable; the skills define applicability and how to report skipped work. If no suitable integration skill is found, emit: ${ AgentSignals.ERROR_RESOURCE_MISSING } Could not find a suitable skill for this project. From b8df80d75cbf9e6906152d598962b0557286944a Mon Sep 17 00:00:00 2001 From: Edwin Lim Date: Fri, 18 Sep 2026 17:26:25 -0700 Subject: [PATCH 2/9] tasks as optional from frontmatter --- src/lib/agent/__tests__/agent-prompt-loader.test.ts | 12 +++++++++--- src/lib/agent/agent-prompt-loader.ts | 6 ++++++ .../orchestrator/__tests__/queue-tools.test.ts | 8 ++++++++ .../sequence/orchestrator/orchestrator-runner.ts | 2 ++ .../runner/sequence/orchestrator/queue-tools.ts | 8 ++++++++ src/lib/agent/runner/sequence/orchestrator/queue.ts | 2 +- 6 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index ab23e6a49..9bb529006 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -143,15 +143,17 @@ Connect the sources. it('marks the sink and the runner-seeded task from frontmatter', () => { const p = parseAgentPrompt( - '---\nsink: true\nrunnerSeeded: true\n---\nx', + '---\nsink: true\nrunnerSeeded: true\noptional: true\n---\nx', 't', ); expect(p.sink).toBe(true); expect(p.runnerSeeded).toBe(true); + expect(p.optional).toBe(true); const plain = parseAgentPrompt('---\nmodel: x\n---\nx', 't'); expect(plain.sink).toBe(false); expect(plain.runnerSeeded).toBe(false); + expect(plain.optional).toBe(false); }); it('defaults missing array fields to empty and models to undefined', () => { @@ -212,6 +214,7 @@ describe('buildRegistry', () => { seed: false, sink: false, runnerSeeded: false, + optional: false, skills: [], allowedTools: [], disallowedTools: [], @@ -243,15 +246,17 @@ describe('buildRegistry', () => { prompt({ type: 'plan', flow: 'f', seed: true }), prompt({ type: 'install', flow: 'f' }), prompt({ type: 'warehouse', flow: 'f', runnerSeeded: true }), + prompt({ type: 'logs', flow: 'f', optional: true }), prompt({ type: 'report', flow: 'f', sink: true }), ], 'f', ); // The type still runs — it is only the planner that cannot reach it. - expect(registry.types).toEqual(['install', 'warehouse', 'report']); - expect(registry.enqueueableTypes).toEqual(['install', 'report']); + expect(registry.types).toEqual(['install', 'warehouse', 'logs', 'report']); + expect(registry.enqueueableTypes).toEqual(['install', 'logs', 'report']); expect(registry.runnerSeededTypes).toEqual(['warehouse']); + expect(registry.optionalTypes).toEqual(['logs']); expect(registry.sinkTypes).toEqual(['report']); }); @@ -322,6 +327,7 @@ describe('resolveTask', () => { seed: false, sink: false, runnerSeeded: false, + optional: false, modelPi: 'openai/gpt-5.6-luna', effortPi: 'low', modelSdk: 'claude-haiku-4-5', diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index 576dcc2be..9ce06ce87 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -214,6 +214,8 @@ export interface AgentPrompt { * one an agent can reach — it can neither invent the task nor forget it. */ runnerSeeded: boolean; + /** Marks a supplementary task: terminal failure unblocks dependents and never fails the run. */ + optional: boolean; /** Per-profile model + effort. `pi` = the gpt/pi harness, `sdk` = the anthropic * harness. The mapping is not 1:1 across providers, so each agent names both. */ modelPi?: string; @@ -260,6 +262,8 @@ export interface AgentRegistry { readonly sinkTypes: string[]; /** The types only the wizard queues, from what it detected before the run. */ readonly runnerSeededTypes: string[]; + /** The types whose terminal failure must not block dependents or fail the run. */ + readonly optionalTypes: string[]; /** The flow's planner, the one prompt marked `seed: true` in its frontmatter. */ readonly seed?: AgentPrompt; get(type: string): AgentPrompt | undefined; @@ -300,6 +304,7 @@ export function buildRegistry( enqueueableTypes: tasks.filter((p) => !p.runnerSeeded).map((p) => p.type), sinkTypes: tasks.filter((p) => p.sink).map((p) => p.type), runnerSeededTypes: tasks.filter((p) => p.runnerSeeded).map((p) => p.type), + optionalTypes: tasks.filter((p) => p.optional).map((p) => p.type), seed: inFlow.find((p) => p.seed), get: (type) => byType.get(type), }; @@ -393,6 +398,7 @@ export function parseAgentPrompt( seed: fields.seed === 'true', sink: fields.sink === 'true', runnerSeeded: fields.runnerSeeded === 'true', + optional: fields.optional === 'true', modelPi: str(fields.model_pi), effortPi: effort(fields.effort_pi, 'effort_pi'), modelSdk: str(fields.model_sdk), diff --git a/src/lib/agent/runner/sequence/orchestrator/__tests__/queue-tools.test.ts b/src/lib/agent/runner/sequence/orchestrator/__tests__/queue-tools.test.ts index 3ee223d9e..fb9f38311 100644 --- a/src/lib/agent/runner/sequence/orchestrator/__tests__/queue-tools.test.ts +++ b/src/lib/agent/runner/sequence/orchestrator/__tests__/queue-tools.test.ts @@ -132,6 +132,14 @@ describe('apply functions', () => { expect(r.task.enqueuedBy).toBe('orchestrator'); }); + it('stamps optional from the registry list, not the caller', () => { + ctx.optionalTypes = ['capture']; + const opt = applyEnqueue(ctx, { type: 'capture', reason: 'x' }); + const req = applyEnqueue(ctx, { type: 'install', reason: 'x' }); + expect(opt.ok && opt.task.optional).toBe(true); + expect(req.ok && !req.task.optional).toBe(true); + }); + it('attributes a follow-up enqueue to the running task', () => { const parent = store.enqueue({ type: 'init' }); ctx.currentTaskId = parent.id; diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index d6a75a592..ec86b4745 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -640,6 +640,8 @@ export async function runOrchestrator( // — a single planner edge is enough to pull the task back to the front of // the drain and put its prompt in front of the code work again. runnerSeededTypes: registry.runnerSeededTypes, + // Optionality comes from the task's frontmatter, never from the enqueue call. + optionalTypes: registry.optionalTypes, currentTaskId, }); diff --git a/src/lib/agent/runner/sequence/orchestrator/queue-tools.ts b/src/lib/agent/runner/sequence/orchestrator/queue-tools.ts index 5f1fa3444..11f84eace 100644 --- a/src/lib/agent/runner/sequence/orchestrator/queue-tools.ts +++ b/src/lib/agent/runner/sequence/orchestrator/queue-tools.ts @@ -70,6 +70,13 @@ export interface OrchestratorToolsContext { * {@link seededDepViolations}. */ runnerSeededTypes?: readonly string[]; + /** + * Types marked `optional: true` in their frontmatter. Enqueue stamps the flag + * from this list — the task's definition decides, never the enqueuing agent — + * so terminal failure of such a task unblocks dependents and never fails the + * run. + */ + optionalTypes?: readonly string[]; /** * The id of the task this tool server is bound to. Each task agent gets its * own wizard-tools server, so attribution holds when independent tasks run @@ -295,6 +302,7 @@ export function applyEnqueue( inputs: args.inputs ?? {}, dependsOn: args.dependsOn ?? [], model: args.model, + optional: (ctx.optionalTypes ?? []).includes(args.type) || undefined, enqueuedBy: ctx.currentTaskId ?? 'orchestrator', }); return { ok: true, task }; diff --git a/src/lib/agent/runner/sequence/orchestrator/queue.ts b/src/lib/agent/runner/sequence/orchestrator/queue.ts index 5896ef07e..ad2c842ce 100644 --- a/src/lib/agent/runner/sequence/orchestrator/queue.ts +++ b/src/lib/agent/runner/sequence/orchestrator/queue.ts @@ -114,7 +114,7 @@ export interface QueuedTask { handoff?: TaskHandoff; /** 'orchestrator' for seeded tasks, or the id of the task that enqueued this one. */ enqueuedBy: string; - /** Wizard-seeded only: terminal failure unblocks dependents and never fails the run. */ + /** Terminal failure unblocks dependents and never fails the run. */ optional?: boolean; createdAt: string; startedAt?: string; From 63c6f0a58f3cca4e733d5fa455018ba80aa8f48e Mon Sep 17 00:00:00 2001 From: Edwin Lim Date: Fri, 18 Sep 2026 18:56:04 -0700 Subject: [PATCH 3/9] linear prompt for skill categories --- src/lib/programs/__tests__/warehouse-suggestion.test.ts | 4 +++- src/lib/programs/posthog-integration/index.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/programs/__tests__/warehouse-suggestion.test.ts b/src/lib/programs/__tests__/warehouse-suggestion.test.ts index 945195ce2..f7120a9dd 100644 --- a/src/lib/programs/__tests__/warehouse-suggestion.test.ts +++ b/src/lib/programs/__tests__/warehouse-suggestion.test.ts @@ -145,7 +145,9 @@ describe('default integration skill workflow', () => { expect(prompt).toContain('category: "integration"'); expect(prompt).toContain('AI Observability and Logs skills'); expect(prompt).toContain('before verification and the setup report'); - expect(prompt).not.toContain('Do NOT pick skills from other categories'); + expect(prompt).toContain( + 'Do NOT load or install skills from any other category', + ); }); }); diff --git a/src/lib/programs/posthog-integration/index.ts b/src/lib/programs/posthog-integration/index.ts index e60474ee9..83bb50921 100644 --- a/src/lib/programs/posthog-integration/index.ts +++ b/src/lib/programs/posthog-integration/index.ts @@ -330,7 +330,7 @@ STEP 1: Call load_skill_menu (from the wizard-tools MCP server) with category: " AgentSignals.ERROR_MCP_MISSING } Could not load skill menu and halt. - Choose a skill from the \`integration\` category that matches this project's framework. Start with this framework skill; load the AI Observability and Logs skills when its workflow calls for them, before verification and the setup report. Both are included by default where applicable; the skills define applicability and how to report skipped work. + Choose a skill from the \`integration\` category that matches this project's framework. Start with this framework skill; load the AI Observability and Logs skills when its workflow calls for them, before verification and the setup report. Both are included by default where applicable; the skills define applicability and how to report skipped work. These three categories — \`integration\`, \`ai-observability\`, \`logs\` — are the only ones this run uses. Do NOT load or install skills from any other category (llm-analytics, error-tracking, feature-flags, audit, etc.) — those are handled separately. In particular, \`ai-observability\` is the category for AI Observability; do not substitute \`llm-analytics\`. If no suitable integration skill is found, emit: ${ AgentSignals.ERROR_RESOURCE_MISSING } Could not find a suitable skill for this project. From 5b30cf19ef4070426c1bf2151822a77aa79e3467 Mon Sep 17 00:00:00 2001 From: Edwin Lim Date: Fri, 18 Sep 2026 19:36:19 -0700 Subject: [PATCH 4/9] moving tests --- e2e-harness/__tests__/e2e-result.test.ts | 16 +++++ e2e-harness/e2e-result.ts | 23 ++++++ .../orchestrator/orchestrator-runner.ts | 8 +++ .../runner/sequence/orchestrator/queue.ts | 10 +++ .../__tests__/helpers/integration-prompt.ts | 61 ++++++++++++++++ .../posthog-integration-prompt.test.ts | 27 +++++++ .../__tests__/warehouse-suggestion.test.ts | 72 ++----------------- 7 files changed, 151 insertions(+), 66 deletions(-) create mode 100644 src/lib/programs/__tests__/helpers/integration-prompt.ts create mode 100644 src/lib/programs/__tests__/posthog-integration-prompt.test.ts diff --git a/e2e-harness/__tests__/e2e-result.test.ts b/e2e-harness/__tests__/e2e-result.test.ts index cb03ec971..f64e9a118 100644 --- a/e2e-harness/__tests__/e2e-result.test.ts +++ b/e2e-harness/__tests__/e2e-result.test.ts @@ -15,12 +15,14 @@ import { OutroKind, RunPhase } from '@lib/wizard-session'; import type { AskQuestion, WizardSession } from '@lib/wizard-session'; import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; import { Overlay } from '@ui/tui/router'; +import { TASK_OUTCOMES_KEY } from '@lib/agent/runner/sequence/orchestrator/queue'; import { E2eRunRecorder, abortReasonFrom, buildE2eResult, detectedSourcesFrom, readReportFile, + taskOutcomesFrom, } from '../e2e-result'; import { DEFAULT_E2E_PROFILE, decideE2eAction } from '../e2e-profile'; import type { CiState } from '../wizard-ci-driver'; @@ -422,6 +424,9 @@ describe('buildE2eResult', () => { matchedSignal: 'found DATABASE_URL', }, ], + [TASK_OUTCOMES_KEY]: [ + { type: 'ai-observability', status: 'not needed', optional: true }, + ], }, outroData: null, }, @@ -447,6 +452,7 @@ describe('buildE2eResult', () => { 'runPhase', 'screenPath', 'skillsComplete', + 'taskOutcomes', 'tasks', 'unansweredAsks', ].sort(), @@ -463,6 +469,16 @@ describe('buildE2eResult', () => { ]); }); + it("reports the queue's terminal outcomes by type", () => { + expect(build().taskOutcomes).toEqual([ + { type: 'ai-observability', status: 'not needed', optional: true }, + ]); + }); + + it('has no outcomes for a linear run, which drains no queue', () => { + expect(taskOutcomesFrom({ frameworkContext: {} })).toEqual([]); + }); + it('reports the sources detection found', () => { expect(build().detectedSources).toEqual([ { diff --git a/e2e-harness/e2e-result.ts b/e2e-harness/e2e-result.ts index e34a52dfd..9dd819251 100644 --- a/e2e-harness/e2e-result.ts +++ b/e2e-harness/e2e-result.ts @@ -22,6 +22,10 @@ import fs from 'fs'; import path from 'path'; import { OutroKind, type WizardSession } from '@lib/wizard-session'; +import { + TASK_OUTCOMES_KEY, + type TaskOutcome, +} from '@lib/agent/runner/sequence/orchestrator/queue'; import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; import type { DetectedSource } from '@lib/warehouse-sources/types'; import type { E2eDecisionReport } from './e2e-profile.js'; @@ -219,6 +223,20 @@ export function detectedSourcesFrom( return Array.isArray(raw) ? (raw as DetectedSource[]) : []; } +/** + * The drained queue's final outcomes the orchestrator wrote into + * frameworkContext. Unlike the UI `tasks` rows (label + display status, where + * done and failed both render `completed`), these carry the task `type` and + * the queue's real terminal status — the stable vocabulary e2e expectations + * assert on. Empty on linear runs, which have no queue. + */ +export function taskOutcomesFrom( + session: Pick, +): TaskOutcome[] { + const raw = session.frameworkContext[TASK_OUTCOMES_KEY]; + return Array.isArray(raw) ? (raw as TaskOutcome[]) : []; +} + /** The keys the result payload carried before the warehouse work. */ export interface E2eResultBase { runPhase: string; @@ -248,6 +266,11 @@ export function buildE2eResult(args: { refusedAsks: recorder.refusedAsks, notices: recorder.notices, tasks: tasks.map((t) => ({ label: t.label, status: t.status })), + taskOutcomes: taskOutcomesFrom(session).map((t) => ({ + type: t.type, + status: t.status, + optional: t.optional, + })), detectedSources: detectedSourcesFrom(session).map((s) => ({ kind: s.kind, label: s.label, diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index ec86b4745..144b18847 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -58,8 +58,10 @@ import { QueueStore, QUEUE_DIR_NAME, SkipReason, + TASK_OUTCOMES_KEY, TaskStatus, type QueuedTask, + type TaskOutcome, } from './queue'; import { drainQueue, type RunTask } from './executor'; import { RunMetrics } from './run-metrics'; @@ -1041,6 +1043,12 @@ export async function runOrchestrator( try { await drainQueue(store, runTask); } finally { + // The queue file is wiped below; the e2e harness reads outcomes from here. + session.frameworkContext[TASK_OUTCOMES_KEY] = store.list().map((t) => ({ + type: t.type, + status: t.status, + optional: t.optional === true, + })) satisfies TaskOutcome[]; try { if (referenceSkillId && referenceInstallPath) { promoteReferenceSkill( diff --git a/src/lib/agent/runner/sequence/orchestrator/queue.ts b/src/lib/agent/runner/sequence/orchestrator/queue.ts index ad2c842ce..08c66ccde 100644 --- a/src/lib/agent/runner/sequence/orchestrator/queue.ts +++ b/src/lib/agent/runner/sequence/orchestrator/queue.ts @@ -132,6 +132,16 @@ export interface QueueFile { tasks: QueuedTask[]; } +/** Session frameworkContext key holding the drained queue's final outcomes — + * written by the runner before the cache wipe, read by the e2e harness. */ +export const TASK_OUTCOMES_KEY = 'orchestrator-task-outcomes'; + +export interface TaskOutcome { + type: string; + status: TaskStatus; + optional: boolean; +} + /** The structured handoff a task leaves for the next agent. */ export interface TaskHandoff { goals: string; diff --git a/src/lib/programs/__tests__/helpers/integration-prompt.ts b/src/lib/programs/__tests__/helpers/integration-prompt.ts new file mode 100644 index 000000000..72758ac55 --- /dev/null +++ b/src/lib/programs/__tests__/helpers/integration-prompt.ts @@ -0,0 +1,61 @@ +/** + * Shared fixtures for tests that build the default integration's run + * definition and prompt (`warehouse-suggestion.test.ts`, + * `posthog-integration-prompt.test.ts`). + */ + +import { posthogIntegrationConfig } from '@lib/programs/posthog-integration/index'; +import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; +import { buildSession, type WizardSession } from '@lib/wizard-session'; +import type { DetectedSource } from '@lib/warehouse-sources/types'; + +export const CREDENTIALS = { + accessToken: 'tok', + projectApiKey: 'phc_test', + projectId: '1', + host: { + apiHost: 'https://us.i.posthog.com', + appHost: 'https://us.posthog.com', + }, +}; + +const FRAMEWORK_CONFIG = { + metadata: { name: 'Next.js', docsUrl: 'https://posthog.com/docs' }, + environment: { getEnvVars: () => ({ POSTHOG_KEY: 'phc_test' }) }, + ui: { getOutroChanges: () => ['Added PostHog provider'] }, + detection: { + usesPackageJson: false, + getVersion: () => '15.0.0', + packageName: 'next', + packageDisplayName: 'Next.js', + }, + analytics: { getTags: () => ({}) }, + prompts: { projectTypeDetection: 'app router' }, +}; + +export function sessionWith(sources: DetectedSource[]): WizardSession { + const s = buildSession({ installDir: '/tmp/app' }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + s.frameworkConfig = FRAMEWORK_CONFIG as any; + if (sources.length > 0) { + s.frameworkContext[DETECTED_WAREHOUSE_SOURCES_KEY] = sources; + } + return s; +} + +export async function resolveRun(session: WizardSession) { + const { run } = posthogIntegrationConfig; + if (typeof run !== 'function') throw new Error('expected a run function'); + return run(session); +} + +export const promptFor = async (sources: DetectedSource[]) => { + const s = sessionWith(sources); + const runDef = await resolveRun(s); + return runDef.customPrompt!({ + projectId: 1, + projectApiKey: 'phc_test', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + host: CREDENTIALS.host as any, + }); +}; diff --git a/src/lib/programs/__tests__/posthog-integration-prompt.test.ts b/src/lib/programs/__tests__/posthog-integration-prompt.test.ts new file mode 100644 index 000000000..9cb84c7bb --- /dev/null +++ b/src/lib/programs/__tests__/posthog-integration-prompt.test.ts @@ -0,0 +1,27 @@ +/** + * The default integration prompt's skill-workflow instructions. + * + * STEP 1 scopes the linear agent to exactly three skill categories: the + * framework skill first, then AI Observability and Logs when its workflow + * calls for them. These tests pin that scoping — the allowlist, the delegation + * to the skill's workflow, and the hard guard against every other category. + */ + +import { promptFor } from './helpers/integration-prompt'; + +describe('default integration skill workflow', () => { + it('loads the framework category first and delegates observability to its workflow', async () => { + const prompt = await promptFor([]); + expect(prompt).toContain('category: "integration"'); + expect(prompt).toContain('AI Observability and Logs skills'); + expect(prompt).toContain('before verification and the setup report'); + }); + + it('forbids every category outside the three the run uses', async () => { + const prompt = await promptFor([]); + expect(prompt).toContain( + 'Do NOT load or install skills from any other category', + ); + expect(prompt).toContain('do not substitute `llm-analytics`'); + }); +}); diff --git a/src/lib/programs/__tests__/warehouse-suggestion.test.ts b/src/lib/programs/__tests__/warehouse-suggestion.test.ts index f7120a9dd..800228c50 100644 --- a/src/lib/programs/__tests__/warehouse-suggestion.test.ts +++ b/src/lib/programs/__tests__/warehouse-suggestion.test.ts @@ -8,11 +8,14 @@ * into an inline run. */ -import { posthogIntegrationConfig } from '@lib/programs/posthog-integration/index'; import { POSTHOG_INTEGRATION_PROGRAM } from '@lib/programs/posthog-integration/steps'; -import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; -import { buildSession, type WizardSession } from '@lib/wizard-session'; import type { DetectedSource } from '@lib/warehouse-sources/types'; +import { + CREDENTIALS, + promptFor, + resolveRun, + sessionWith, +} from './helpers/integration-prompt'; const POSTGRES: DetectedSource = { kind: 'postgres', @@ -28,46 +31,6 @@ const STRIPE: DetectedSource = { matchedSignal: 'stripe in package.json', }; -const CREDENTIALS = { - accessToken: 'tok', - projectApiKey: 'phc_test', - projectId: '1', - host: { - apiHost: 'https://us.i.posthog.com', - appHost: 'https://us.posthog.com', - }, -}; - -const FRAMEWORK_CONFIG = { - metadata: { name: 'Next.js', docsUrl: 'https://posthog.com/docs' }, - environment: { getEnvVars: () => ({ POSTHOG_KEY: 'phc_test' }) }, - ui: { getOutroChanges: () => ['Added PostHog provider'] }, - detection: { - usesPackageJson: false, - getVersion: () => '15.0.0', - packageName: 'next', - packageDisplayName: 'Next.js', - }, - analytics: { getTags: () => ({}) }, - prompts: { projectTypeDetection: 'app router' }, -}; - -function sessionWith(sources: DetectedSource[]): WizardSession { - const s = buildSession({ installDir: '/tmp/app' }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - s.frameworkConfig = FRAMEWORK_CONFIG as any; - if (sources.length > 0) { - s.frameworkContext[DETECTED_WAREHOUSE_SOURCES_KEY] = sources; - } - return s; -} - -async function resolveRun(session: WizardSession) { - const { run } = posthogIntegrationConfig; - if (typeof run !== 'function') throw new Error('expected a run function'); - return run(session); -} - describe('outro suggestion', () => { it('gives every detected source its own pre-filled link', async () => { const s = sessionWith([POSTGRES, STRIPE]); @@ -128,29 +91,6 @@ describe('outro suggestion', () => { }); }); -const promptFor = async (sources: DetectedSource[]) => { - const s = sessionWith(sources); - const runDef = await resolveRun(s); - return runDef.customPrompt!({ - projectId: 1, - projectApiKey: 'phc_test', - // eslint-disable-next-line @typescript-eslint/no-explicit-any - host: CREDENTIALS.host as any, - }); -}; - -describe('default integration skill workflow', () => { - it('loads the framework category first and delegates observability to its workflow', async () => { - const prompt = await promptFor([]); - expect(prompt).toContain('category: "integration"'); - expect(prompt).toContain('AI Observability and Logs skills'); - expect(prompt).toContain('before verification and the setup report'); - expect(prompt).toContain( - 'Do NOT load or install skills from any other category', - ); - }); -}); - describe('report instruction', () => { it('asks the agent to note the sources in the report checklist', async () => { const prompt = await promptFor([POSTGRES]); From 27467a82b4893d3753f42732268b6ea10934e07e Mon Sep 17 00:00:00 2001 From: Edwin Lim Date: Sat, 19 Sep 2026 00:04:25 -0700 Subject: [PATCH 5/9] update tool description --- src/lib/agent/agent-prompt-loader.ts | 2 +- src/lib/agent/runner/harness/pi/orchestrator-tools.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index 4dec317e6..f9c58f93d 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -89,7 +89,7 @@ export function renderToolInventory(toolNames: readonly string[]): string { const TASK_BASICS = `You are one step in a larger PostHog workflow made of several tasks, run as a fresh agent with no memory of the other tasks beyond the context you are given. Other tasks — before and after you — own the rest of the work, so stay strictly on your own task: do not do a neighbouring step's job, redo what an upstream handoff already did, or reach beyond what you were asked. Do only your task, then report exactly once by calling complete_task with a structured handoff: what your goal was, what you did, and what the next agent should know. When you are given context from previous steps, trust it — those agents already did their work, so do not re-verify or re-read what their handoffs tell you. Build on it and move fast. Read a file before you edit it, so your own changes do not duplicate what is already there. Work only inside this project's own directory: never read, list, or search (find, ls, grep, glob) outside it — not the OS, not other projects, not global package caches. If your task seems to need something outside this directory, it does not — skip that part and say so in your handoff rather than hunting across the filesystem. If your task does not apply to this project — there is genuinely nothing for it to do — report it with status \`not needed\` and say why, rather than marking it done.`; -const SEED_BASICS = `You are the orchestrator. Plan the work and seed the queue with enqueue_task — each call returns an id you can pass as a dependency to a later task. Give each task a short label for the UI — the action in a few words, not file names, class names, or other specifics. The last task you queue, the one that reports on the run, must depend on every other task in the queue — directly, or through a task it already depends on. You are not a task yourself: do not call complete_task and do not edit the project.`; +const SEED_BASICS = `You are the orchestrator. Plan the work and seed the queue with enqueue_task — each call names the task kind in its \`type\` field (the tool has no \`task\` field) and returns an id you can pass as a dependency to a later task. Give each task a short label for the UI — the action in a few words, not file names, class names, or other specifics. The last task you queue, the one that reports on the run, must depend on every other task in the queue — directly, or through a task it already depends on. You are not a task yourself: do not call complete_task and do not edit the project.`; /** * Tasks the wizard queued before the planner ran. It has to see them: they are diff --git a/src/lib/agent/runner/harness/pi/orchestrator-tools.ts b/src/lib/agent/runner/harness/pi/orchestrator-tools.ts index 83ebbadc3..dfb81dd93 100644 --- a/src/lib/agent/runner/harness/pi/orchestrator-tools.ts +++ b/src/lib/agent/runner/harness/pi/orchestrator-tools.ts @@ -99,7 +99,7 @@ export function createPiOrchestratorTools( description: 'Add a task to the orchestrator queue. Use it to seed work and to enqueue follow-up work you discover. Keep tasks small and discrete.', promptSnippet: - 'enqueue_task(type, label, dependsOn, reason) — add a task to the queue; returns its id', + 'enqueue_task({type, label, dependsOn, reason}) — add a task to the queue; the task type goes in the "type" field, and it returns the task id', parameters: Type.Object({ type: Type.String({ description: `The task type. One of: ${ctx.validTypes.join(', ')}.`, From 50039c95f686c3911cf6f0cd66832aa4224092e2 Mon Sep 17 00:00:00 2001 From: Edwin Lim Date: Mon, 21 Sep 2026 16:29:24 -0700 Subject: [PATCH 6/9] flag for planner enqueued tasks; adding flag for logs + aio in posthog-integration --- .../__tests__/agent-prompt-loader.test.ts | 30 +++++++++++++++++++ src/lib/agent/agent-prompt-loader.ts | 23 +++++++++++++- .../orchestrator/orchestrator-runner.ts | 15 ++++++++-- src/lib/constants.ts | 3 ++ .../posthog-integration-prompt.test.ts | 21 +++++++++++++ src/lib/programs/posthog-integration/index.ts | 12 +++++++- src/lib/programs/program-step.ts | 7 +++++ 7 files changed, 107 insertions(+), 4 deletions(-) diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index 9bb529006..2e45abbbf 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -566,6 +566,36 @@ describe('renderToolInventory', () => { }); describe('assembleSeedPrompt', () => { + it('is byte-identical to a no-exclusions prompt when nothing is excluded', () => { + const ctx = { + projectId: 1, + projectApiKey: 'k', + host: { apiHost: 'https://h' }, + } as Parameters[0]; + + expect(assembleSeedPrompt(ctx, 'plan it', [], [])).toBe( + assembleSeedPrompt(ctx, 'plan it', []), + ); + expect(assembleSeedPrompt(ctx, 'plan it')).not.toContain('excludes them'); + }); + + it('names excluded types so the plan mentioning them reads as a skip', () => { + const ctx = { + projectId: 1, + projectApiKey: 'k', + host: { apiHost: 'https://h' }, + } as Parameters[0]; + + const prompt = assembleSeedPrompt( + ctx, + 'plan it', + [], + ['ai-observability', 'logs'], + ); + expect(prompt).toContain('this run excludes them: ai-observability, logs'); + expect(prompt).toContain('do not queue them, do not retry them'); + }); + it('names the tasks the wizard queued before the planner ran', () => { const ctx = { projectId: 1, diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index f9c58f93d..65dacffb3 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -91,6 +91,20 @@ const TASK_BASICS = `You are one step in a larger PostHog workflow made of sever const SEED_BASICS = `You are the orchestrator. Plan the work and seed the queue with enqueue_task — each call names the task kind in its \`type\` field (the tool has no \`task\` field) and returns an id you can pass as a dependency to a later task. Give each task a short label for the UI — the action in a few words, not file names, class names, or other specifics. The last task you queue, the one that reports on the run, must depend on every other task in the queue — directly, or through a task it already depends on. You are not a task yourself: do not call complete_task and do not edit the project.`; +/** + * Injected only when the run excludes task types (a flag or CI gate), so the + * seed prompt of an ordinary run is byte-identical to one with no exclusions. + * Naming the types beats a "check the tool's list" rule: the planner has + * nothing to infer, and the pre-queued (runner-seeded) tasks — absent from the + * enqueue list by design — cannot be mistaken for exclusions. + */ +function excludedTypesNote(types: readonly string[]): string | null { + if (types.length === 0) return null; + return `The plan below may mention these task types, but this run excludes them: ${types.join( + ', ', + )}. enqueue_task will not accept them — do not queue them, do not retry them, and hang nothing off them.`; +} + /** * Tasks the wizard queued before the planner ran. It has to see them: they are * part of the run it is planning around, and the task that reports last has to @@ -147,8 +161,15 @@ export function assembleSeedPrompt( ctx: OrchestratorPromptContext, body: string, preQueued: readonly { id: string; type: string }[] = [], + excludedTypes: readonly string[] = [], ): string { - return [projectContext(ctx), SEED_BASICS, preQueuedTasks(preQueued), body] + return [ + projectContext(ctx), + SEED_BASICS, + excludedTypesNote(excludedTypes), + preQueuedTasks(preQueued), + body, + ] .filter(Boolean) .join('\n\n'); } diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index 83ba0b6cf..14831aaa5 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -451,8 +451,14 @@ export async function runOrchestrator( // once up front: its types drive enqueue validation, and resolving a task to // its run config is then synchronous, with no mid-drain network latency. const flow = programConfig.agentFlow ?? programConfig.id; + // Also named to the planner in its seed prompt, so the plan's mention of an + // excluded type reads as an instructed skip instead of a guard round-trip. + const excludedTaskTypes = [ + ...ciExcludedTaskTypes(), + ...(programConfig.excludedTaskTypes?.(boot.wizardFlags) ?? []), + ]; const registry = await loadAgentRegistry(boot.skillsBaseUrl, flow, { - exclude: ciExcludedTaskTypes(), + exclude: excludedTaskTypes, // Baked into the prompts at load, so enqueue, dispatch, and telemetry all read one effective spec. overrides: resolveStageOverrides( programConfig.id, @@ -833,7 +839,12 @@ export async function runOrchestrator( session, programConfig, boot, - prompt: assembleSeedPrompt(promptContext, seedPrompt.body, store.list()), + prompt: assembleSeedPrompt( + promptContext, + seedPrompt.body, + store.list(), + excludedTaskTypes, + ), spinner, model: requireKnownModel(seedModel.model, seedPick.model), effort: seedModel.effort, diff --git a/src/lib/constants.ts b/src/lib/constants.ts index a0162e708..62a80d434 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -279,6 +279,8 @@ export const WIZARD_BASIC_INTEGRATION_AGENTIC_DETECTION_FLAG_KEY = /** Boolean flag: the orchestrator queues the program's runner-seeded tasks (the warehouse step). Off, no task is queued and the run is byte-identical to a no-sources project. */ export const WIZARD_ORCHESTRATOR_SEEDED_TASKS_FLAG_KEY = 'wizard-orchestrator-seeded-tasks'; +/** Kill switch over a shipped default: ONLY an explicit 'false' excludes the AI Observability and Logs tasks from default integration runs — absent or fetch-failed means include, so the default never vanishes on a network hiccup. */ +export const WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY = 'wizard-default-aio-logs'; // Reading a flag enters this run into that flag's experiment, so a closed set — not a // `wizard-` prefix anyone can name into — decides what a run evaluates. Test-pinned exhaustive. export const WIZARD_FLAG_KEYS = [ @@ -287,6 +289,7 @@ export const WIZARD_FLAG_KEYS = [ WIZARD_ORCHESTRATOR_SEEDED_TASKS_FLAG_KEY, WIZARD_SELF_DRIVING_USE_PI_HARNESS_FLAG_KEY, WIZARD_BASIC_INTEGRATION_AGENTIC_DETECTION_FLAG_KEY, + WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY, ] as const; /** User-Agent for wizard HTTP requests and MCP server identification. */ export const WIZARD_USER_AGENT = `posthog/wizard; version: ${VERSION}`; diff --git a/src/lib/programs/__tests__/posthog-integration-prompt.test.ts b/src/lib/programs/__tests__/posthog-integration-prompt.test.ts index 38b81818b..75085bcd1 100644 --- a/src/lib/programs/__tests__/posthog-integration-prompt.test.ts +++ b/src/lib/programs/__tests__/posthog-integration-prompt.test.ts @@ -7,6 +7,8 @@ * to the skill's workflow, and the hard guard against every other category. */ +import { WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY } from '@lib/constants'; +import { posthogIntegrationConfig } from '@lib/programs/posthog-integration/index'; import { promptFor } from './helpers/integration-prompt.no-jest'; describe('default integration skill workflow', () => { @@ -25,3 +27,22 @@ describe('default integration skill workflow', () => { expect(prompt).toContain('do not substitute `llm-analytics`'); }); }); + +describe('default observability flag gating', () => { + const excluded = (flags: Record) => + posthogIntegrationConfig.excludedTaskTypes!(flags); + + it("excludes AIO and Logs only on an explicit 'false'", () => { + expect(excluded({ [WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY]: 'false' })).toEqual([ + 'ai-observability', + 'logs', + ]); + }); + + it('includes them when the flag is true, absent, or the fetch failed', () => { + expect(excluded({ [WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY]: 'true' })).toEqual( + [], + ); + expect(excluded({})).toEqual([]); + }); +}); diff --git a/src/lib/programs/posthog-integration/index.ts b/src/lib/programs/posthog-integration/index.ts index 7bb6c0a93..5d4624307 100644 --- a/src/lib/programs/posthog-integration/index.ts +++ b/src/lib/programs/posthog-integration/index.ts @@ -15,7 +15,10 @@ import { scopeInstallDirToProject } from '@lib/detection/project-scope'; import { FRAMEWORK_REGISTRY } from '@lib/registry'; import { wizardAbort } from '@utils/wizard-abort'; import { ErrorCodes } from '@lib/errors'; -import { WIZARD_INTERACTION_EVENT_NAME } from '@lib/constants'; +import { + WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY, + WIZARD_INTERACTION_EVENT_NAME, +} from '@lib/constants'; import { getUI } from '@ui/index'; import { requestDeepLink } from '@utils/provisioning'; import { openTrackedLink, withUtm } from '@utils/links'; @@ -212,6 +215,13 @@ export const posthogIntegrationConfig: ProgramConfig = { seedTasks: warehouseSeedTasks, + // Kill switch over the shipped default: only an explicit 'false' excludes, + // so a failed flag fetch keeps AI Observability and Logs in the run. + excludedTaskTypes: (flags) => + flags[WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY] === 'false' + ? ['ai-observability', 'logs'] + : [], + // CI-mode prerequisite work: the headless equivalent of the detect step's // onReady hook. Auto-detect the framework, then gather context. ciPreRun: async (session: WizardSession): Promise => { diff --git a/src/lib/programs/program-step.ts b/src/lib/programs/program-step.ts index 9388bee79..590f223a1 100644 --- a/src/lib/programs/program-step.ts +++ b/src/lib/programs/program-step.ts @@ -268,6 +268,13 @@ export interface ProgramConfig { */ notice?: TaskNotice; }>; + /** + * Task types this run excludes, decided from the run's wizard flags. An + * excluded type does not exist for the run: the planner cannot enqueue it + * and no agent boots for it. The program owns the flag→type mapping; the + * runner only applies it. + */ + excludedTaskTypes?: (flags: Record) => readonly string[]; /** Prerequisites: other program ids that must have run first */ requires?: string[]; /** From 6f85957d12da2b08da4692d26c08d36a09c982a7 Mon Sep 17 00:00:00 2001 From: Edwin Lim Date: Tue, 22 Sep 2026 15:52:19 -0700 Subject: [PATCH 7/9] adding flagged prompt for excluded tasks in anthropic harness --- .../posthog-integration-prompt.test.ts | 27 +++++++++++++++++++ .../__tests__/warehouse-suggestion.test.ts | 11 ++++++++ src/lib/programs/posthog-integration/index.ts | 13 ++++++++- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/lib/programs/__tests__/posthog-integration-prompt.test.ts b/src/lib/programs/__tests__/posthog-integration-prompt.test.ts index 75085bcd1..cb2484360 100644 --- a/src/lib/programs/__tests__/posthog-integration-prompt.test.ts +++ b/src/lib/programs/__tests__/posthog-integration-prompt.test.ts @@ -9,8 +9,18 @@ import { WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY } from '@lib/constants'; import { posthogIntegrationConfig } from '@lib/programs/posthog-integration/index'; +import { analytics } from '@utils/analytics'; import { promptFor } from './helpers/integration-prompt.no-jest'; +// The run builder reads the run's wizard flags; keep tests hermetic and let +// each case choose the flag state. Empty map = flags unreadable = include. +beforeEach(() => { + vi.spyOn(analytics, 'getAllFlagsForWizard').mockResolvedValue({}); +}); +afterEach(() => { + vi.restoreAllMocks(); +}); + describe('default integration skill workflow', () => { it('loads the framework category first and delegates observability to its workflow', async () => { const prompt = await promptFor([]); @@ -28,6 +38,23 @@ describe('default integration skill workflow', () => { }); }); +describe('linear-run flag gate', () => { + it('drops AIO and Logs from the prompt when the flag is false', async () => { + vi.spyOn(analytics, 'getAllFlagsForWizard').mockResolvedValue({ + [WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY]: 'false', + }); + const prompt = await promptFor([]); + expect(prompt).toContain('The `integration` category is the ONLY one'); + expect(prompt).toContain('skip that entire section'); + expect(prompt).not.toContain('load the AI Observability and Logs skills'); + }); + + it('keeps the three-category workflow when the flag is unreadable', async () => { + const prompt = await promptFor([]); + expect(prompt).toContain('load the AI Observability and Logs skills'); + }); +}); + describe('default observability flag gating', () => { const excluded = (flags: Record) => posthogIntegrationConfig.excludedTaskTypes!(flags); diff --git a/src/lib/programs/__tests__/warehouse-suggestion.test.ts b/src/lib/programs/__tests__/warehouse-suggestion.test.ts index 144b5f36b..8ee33679a 100644 --- a/src/lib/programs/__tests__/warehouse-suggestion.test.ts +++ b/src/lib/programs/__tests__/warehouse-suggestion.test.ts @@ -16,6 +16,8 @@ import { POSTHOG_INTEGRATION_PROGRAM } from '@lib/programs/posthog-integration/steps'; import type { WizardSession } from '@lib/wizard-session'; import type { DetectedSource } from '@lib/warehouse-sources/types'; +import { analytics } from '@utils/analytics'; + import { CREDENTIALS, promptFor, @@ -23,6 +25,15 @@ import { sessionWith, } from './helpers/integration-prompt.no-jest'; +// The run builder reads the run's wizard flags; pin them empty so these +// tests stay hermetic (empty map = flags unreadable = shipped default). +beforeEach(() => { + vi.spyOn(analytics, 'getAllFlagsForWizard').mockResolvedValue({}); +}); +afterEach(() => { + vi.restoreAllMocks(); +}); + const POSTGRES: DetectedSource = { kind: 'postgres', label: 'Postgres', diff --git a/src/lib/programs/posthog-integration/index.ts b/src/lib/programs/posthog-integration/index.ts index 5d4624307..d508f44d1 100644 --- a/src/lib/programs/posthog-integration/index.ts +++ b/src/lib/programs/posthog-integration/index.ts @@ -295,6 +295,17 @@ export const posthogIntegrationConfig: ProgramConfig = { const versionBucket = config.detection.getVersionBucket(frameworkVersion); analytics.setTag(`${config.metadata.integration}-version`, versionBucket); } + // The same kill switch the orchestrator applies via excludedTaskTypes, + // gated here for linear/composed runs, which assemble their own prompt — + // without this, disabling the flag would not reach this path and the + // prompt would still instruct installing the AIO and Logs skills. Only an + // explicit 'false' excludes, so a failed flag fetch keeps the default. + const wizardFlags = await analytics.getAllFlagsForWizard(); + const skillCategoryInstruction = + wizardFlags[WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY] !== 'false' + ? `Choose a skill from the \`integration\` category that matches this project's framework. Start with this framework skill; load the AI Observability and Logs skills when its workflow calls for them, before verification and the setup report. Both are included by default where applicable; the skills define applicability and how to report skipped work. These three categories — \`integration\`, \`ai-observability\`, \`logs\` — are the only ones this run uses. Do NOT load or install skills from any other category (llm-analytics, error-tracking, feature-flags, audit, etc.) — those are handled separately. In particular, \`ai-observability\` is the category for AI Observability; do not substitute \`llm-analytics\`.` + : `Choose a skill from the \`integration\` category that matches this project's framework. The \`integration\` category is the ONLY one this run uses. Do NOT load or install skills from any other category (ai-observability, logs, llm-analytics, error-tracking, feature-flags, audit, etc.) — those are handled separately. If the installed skill's workflow contains an "AI Observability and Logs" section, skip that entire section: this run excludes both products.`; + const frameworkContext = session.frameworkContext; const contextTags = config.analytics.getTags(frameworkContext); Object.entries(contextTags).forEach(([key, value]) => { @@ -348,7 +359,7 @@ STEP 1: Call load_skill_menu (from the wizard-tools MCP server) with category: " AgentSignals.ERROR_MCP_MISSING } Could not load skill menu and halt. - Choose a skill from the \`integration\` category that matches this project's framework. Start with this framework skill; load the AI Observability and Logs skills when its workflow calls for them, before verification and the setup report. Both are included by default where applicable; the skills define applicability and how to report skipped work. These three categories — \`integration\`, \`ai-observability\`, \`logs\` — are the only ones this run uses. Do NOT load or install skills from any other category (llm-analytics, error-tracking, feature-flags, audit, etc.) — those are handled separately. In particular, \`ai-observability\` is the category for AI Observability; do not substitute \`llm-analytics\`. + ${skillCategoryInstruction} If no suitable integration skill is found, emit: ${ AgentSignals.ERROR_RESOURCE_MISSING } Could not find a suitable skill for this project. From b8f2631317321a65c650370f7d88cb516947d5de Mon Sep 17 00:00:00 2001 From: Edwin Lim Date: Tue, 22 Sep 2026 16:18:37 -0700 Subject: [PATCH 8/9] address vincent comments --- e2e-harness/__tests__/e2e-result.test.ts | 10 ++++-- e2e-harness/e2e-result.ts | 12 ++++--- .../__tests__/agent-prompt-loader.test.ts | 15 ++++++++ src/lib/agent/agent-prompt-loader.ts | 11 ++++++ .../__tests__/excluded-task-types.test.ts | 36 +++++++++++++++++++ .../orchestrator/orchestrator-runner.ts | 28 ++++++++++----- .../__tests__/detect.test.ts | 2 ++ .../__tests__/index.test.ts | 2 ++ src/lib/programs/posthog-integration/index.ts | 2 +- 9 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 src/lib/agent/runner/sequence/orchestrator/__tests__/excluded-task-types.test.ts diff --git a/e2e-harness/__tests__/e2e-result.test.ts b/e2e-harness/__tests__/e2e-result.test.ts index f64e9a118..96d1f5c29 100644 --- a/e2e-harness/__tests__/e2e-result.test.ts +++ b/e2e-harness/__tests__/e2e-result.test.ts @@ -475,8 +475,14 @@ describe('buildE2eResult', () => { ]); }); - it('has no outcomes for a linear run, which drains no queue', () => { - expect(taskOutcomesFrom({ frameworkContext: {} })).toEqual([]); + it('distinguishes never-recorded from an orchestrator run with no tasks', () => { + // Linear runs (or a run that died pre-drain) never set the key → null, + // and the payload drops the field; an orchestrator run that drained an + // empty queue records [] — graders must not conflate the two. + expect(taskOutcomesFrom({ frameworkContext: {} })).toBeNull(); + expect( + taskOutcomesFrom({ frameworkContext: { [TASK_OUTCOMES_KEY]: [] } }), + ).toEqual([]); }); it('reports the sources detection found', () => { diff --git a/e2e-harness/e2e-result.ts b/e2e-harness/e2e-result.ts index 9dd819251..f862fa2c5 100644 --- a/e2e-harness/e2e-result.ts +++ b/e2e-harness/e2e-result.ts @@ -228,13 +228,15 @@ export function detectedSourcesFrom( * frameworkContext. Unlike the UI `tasks` rows (label + display status, where * done and failed both render `completed`), these carry the task `type` and * the queue's real terminal status — the stable vocabulary e2e expectations - * assert on. Empty on linear runs, which have no queue. + * assert on. Null when nothing was recorded (a linear run, or a run that died + * before the drain settled) — distinct from an orchestrator run whose queue + * held no tasks, which records `[]`. */ export function taskOutcomesFrom( session: Pick, -): TaskOutcome[] { +): TaskOutcome[] | null { const raw = session.frameworkContext[TASK_OUTCOMES_KEY]; - return Array.isArray(raw) ? (raw as TaskOutcome[]) : []; + return Array.isArray(raw) ? (raw as TaskOutcome[]) : null; } /** The keys the result payload carried before the warehouse work. */ @@ -266,7 +268,9 @@ export function buildE2eResult(args: { refusedAsks: recorder.refusedAsks, notices: recorder.notices, tasks: tasks.map((t) => ({ label: t.label, status: t.status })), - taskOutcomes: taskOutcomesFrom(session).map((t) => ({ + // Key absent = never recorded (linear run); present-but-empty = an + // orchestrator run that drained no tasks. Graders must not conflate them. + taskOutcomes: taskOutcomesFrom(session)?.map((t) => ({ type: t.type, status: t.status, optional: t.optional, diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index 2e45abbbf..a00baf6fa 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -272,6 +272,21 @@ describe('buildRegistry', () => { expect(buildRegistry(prompts, 'f').types).toEqual(['build', 'dashboard']); }); + it('reports only excluded types the flow actually had, deduped', () => { + const prompts = [ + prompt({ type: 'plan', flow: 'f', seed: true }), + prompt({ type: 'build', flow: 'f' }), + prompt({ type: 'logs', flow: 'f' }), + ]; + const registry = buildRegistry(prompts, 'f', { + // 'logs' twice (two exclusion sources overlap) and a type this flow + // never carried — the note must name 'logs' once and nothing else. + exclude: ['logs', 'logs', 'ghost-type'], + }); + expect(registry.excludedTypes).toEqual(['logs']); + expect(buildRegistry(prompts, 'f').excludedTypes).toEqual([]); + }); + it('bakes stage overrides into the pi frontmatter at load; unnamed stages and sdk fields untouched', () => { const prompts = [ prompt({ diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index 65dacffb3..19afc034a 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -283,6 +283,13 @@ export interface AgentRegistry { readonly runnerSeededTypes: string[]; /** The types whose terminal failure must not block dependents or fail the run. */ readonly optionalTypes: string[]; + /** + * The excluded types this flow actually had, deduped — the honest list for + * the seed prompt's exclusion note. An exclusion naming a type the flow + * never carried is dropped: telling the planner it was "excluded" would + * describe work that was never available. + */ + readonly excludedTypes: string[]; /** The flow's planner, the one prompt marked `seed: true` in its frontmatter. */ readonly seed?: AgentPrompt; get(type: string): AgentPrompt | undefined; @@ -318,12 +325,16 @@ export function buildRegistry( }); const byType = new Map(inFlow.map((p) => [p.type, p])); const tasks = inFlow.filter((p) => !p.seed); + const flowTypes = new Set( + prompts.filter((p) => p.flow === flow && !p.seed).map((p) => p.type), + ); return { types: tasks.map((p) => p.type), enqueueableTypes: tasks.filter((p) => !p.runnerSeeded).map((p) => p.type), sinkTypes: tasks.filter((p) => p.sink).map((p) => p.type), runnerSeededTypes: tasks.filter((p) => p.runnerSeeded).map((p) => p.type), optionalTypes: tasks.filter((p) => p.optional).map((p) => p.type), + excludedTypes: [...excluded].filter((t) => flowTypes.has(t)), seed: inFlow.find((p) => p.seed), get: (type) => byType.get(type), }; diff --git a/src/lib/agent/runner/sequence/orchestrator/__tests__/excluded-task-types.test.ts b/src/lib/agent/runner/sequence/orchestrator/__tests__/excluded-task-types.test.ts new file mode 100644 index 000000000..02240e173 --- /dev/null +++ b/src/lib/agent/runner/sequence/orchestrator/__tests__/excluded-task-types.test.ts @@ -0,0 +1,36 @@ +/** + * Pins the flag→exclusion hookup against the REAL posthog-integration config, + * through the same helper the runner feeds to the registry and the seed note. + * A refactor that disconnects the program's mapping from the run turns these + * red — the mapping test alone cannot see that. + */ + +import { WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY } from '@lib/constants'; +import { posthogIntegrationConfig } from '@lib/programs/posthog-integration/index'; +import { effectiveExcludedTaskTypes } from '../orchestrator-runner'; + +describe('effectiveExcludedTaskTypes', () => { + it("excludes both observability types when the real config sees flag 'false'", () => { + const excluded = effectiveExcludedTaskTypes(posthogIntegrationConfig, { + [WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY]: 'false', + }); + expect(excluded).toEqual( + expect.arrayContaining(['ai-observability', 'logs']), + ); + }); + + it('excludes neither for the shipped default — flag true or unreadable', () => { + const cases: Record[] = [ + {}, + { [WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY]: 'true' }, + ]; + for (const flags of cases) { + const excluded = effectiveExcludedTaskTypes( + posthogIntegrationConfig, + flags, + ); + expect(excluded).not.toContain('ai-observability'); + expect(excluded).not.toContain('logs'); + } + }); +}); diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index 14831aaa5..33030c28d 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -429,6 +429,21 @@ export function displayOrder( .map((entry) => entry.task); } +/** + * The run's excluded task types: CI gates plus the program's flag mapping. + * Exported so a test can pin the flag→exclusion hookup against the real + * program config — the registry and seed note both read this one list. + */ +export function effectiveExcludedTaskTypes( + programConfig: ProgramConfig, + flags: Record, +): string[] { + return [ + ...ciExcludedTaskTypes(), + ...(programConfig.excludedTaskTypes?.(flags) ?? []), + ]; +} + export async function runOrchestrator( session: WizardSession, config: ProgramRun, @@ -451,14 +466,8 @@ export async function runOrchestrator( // once up front: its types drive enqueue validation, and resolving a task to // its run config is then synchronous, with no mid-drain network latency. const flow = programConfig.agentFlow ?? programConfig.id; - // Also named to the planner in its seed prompt, so the plan's mention of an - // excluded type reads as an instructed skip instead of a guard round-trip. - const excludedTaskTypes = [ - ...ciExcludedTaskTypes(), - ...(programConfig.excludedTaskTypes?.(boot.wizardFlags) ?? []), - ]; const registry = await loadAgentRegistry(boot.skillsBaseUrl, flow, { - exclude: excludedTaskTypes, + exclude: effectiveExcludedTaskTypes(programConfig, boot.wizardFlags), // Baked into the prompts at load, so enqueue, dispatch, and telemetry all read one effective spec. overrides: resolveStageOverrides( programConfig.id, @@ -839,11 +848,14 @@ export async function runOrchestrator( session, programConfig, boot, + // The exclusion note names `registry.excludedTypes` — the excluded types + // this flow actually had — so the planner never hears about work that was + // never available, and overlapping exclusion sources cannot double-list. prompt: assembleSeedPrompt( promptContext, seedPrompt.body, store.list(), - excludedTaskTypes, + registry.excludedTypes, ), spinner, model: requireKnownModel(seedModel.model, seedPick.model), diff --git a/src/lib/programs/posthog-integration/__tests__/detect.test.ts b/src/lib/programs/posthog-integration/__tests__/detect.test.ts index 3e4a67f56..456abdfd8 100644 --- a/src/lib/programs/posthog-integration/__tests__/detect.test.ts +++ b/src/lib/programs/posthog-integration/__tests__/detect.test.ts @@ -11,6 +11,8 @@ vi.mock('@utils/analytics', () => ({ capture: vi.fn(), captureException: vi.fn(), groupIdentify: vi.fn(), + // Empty map = flags unreadable = the shipped default (AIO + Logs on). + getAllFlagsForWizard: vi.fn().mockResolvedValue({}), }, })); diff --git a/src/lib/programs/posthog-integration/__tests__/index.test.ts b/src/lib/programs/posthog-integration/__tests__/index.test.ts index ec55f6efe..576125180 100644 --- a/src/lib/programs/posthog-integration/__tests__/index.test.ts +++ b/src/lib/programs/posthog-integration/__tests__/index.test.ts @@ -16,6 +16,8 @@ vi.mock('@utils/analytics', () => ({ wizardCapture: vi.fn(), setTag: vi.fn(), capture: vi.fn(), + // Empty map = flags unreadable = the shipped default (AIO + Logs on). + getAllFlagsForWizard: vi.fn().mockResolvedValue({}), }, })); diff --git a/src/lib/programs/posthog-integration/index.ts b/src/lib/programs/posthog-integration/index.ts index d508f44d1..2b9c6e480 100644 --- a/src/lib/programs/posthog-integration/index.ts +++ b/src/lib/programs/posthog-integration/index.ts @@ -303,7 +303,7 @@ export const posthogIntegrationConfig: ProgramConfig = { const wizardFlags = await analytics.getAllFlagsForWizard(); const skillCategoryInstruction = wizardFlags[WIZARD_DEFAULT_AIO_LOGS_FLAG_KEY] !== 'false' - ? `Choose a skill from the \`integration\` category that matches this project's framework. Start with this framework skill; load the AI Observability and Logs skills when its workflow calls for them, before verification and the setup report. Both are included by default where applicable; the skills define applicability and how to report skipped work. These three categories — \`integration\`, \`ai-observability\`, \`logs\` — are the only ones this run uses. Do NOT load or install skills from any other category (llm-analytics, error-tracking, feature-flags, audit, etc.) — those are handled separately. In particular, \`ai-observability\` is the category for AI Observability; do not substitute \`llm-analytics\`.` + ? `Choose a skill from the \`integration\` category that matches this project's framework. Start with this framework skill; load the AI Observability and Logs skills when its workflow calls for them, and follow each installed skill's own steps to completion — framework first, then AI Observability, then Logs — before verification and the setup report. Both are included by default where applicable; the skills define applicability and how to report skipped work. These three categories — \`integration\`, \`ai-observability\`, \`logs\` — are the only ones this run uses. Do NOT load or install skills from any other category (llm-analytics, error-tracking, feature-flags, audit, etc.) — those are handled separately. In particular, \`ai-observability\` is the category for AI Observability; do not substitute \`llm-analytics\`.` : `Choose a skill from the \`integration\` category that matches this project's framework. The \`integration\` category is the ONLY one this run uses. Do NOT load or install skills from any other category (ai-observability, logs, llm-analytics, error-tracking, feature-flags, audit, etc.) — those are handled separately. If the installed skill's workflow contains an "AI Observability and Logs" section, skip that entire section: this run excludes both products.`; const frameworkContext = session.frameworkContext; From 378a23bcc4b45b8d210ada63b86ce48e820dd6fb Mon Sep 17 00:00:00 2001 From: Edwin Lim Date: Wed, 23 Sep 2026 11:03:09 -0700 Subject: [PATCH 9/9] copy --- .../content/product-suite.tsx | 2 +- src/ui/tui/screens/OutroScreen.tsx | 48 ++++++++++--------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/lib/programs/posthog-integration/content/product-suite.tsx b/src/lib/programs/posthog-integration/content/product-suite.tsx index d2efd2e75..b640a14f8 100644 --- a/src/lib/programs/posthog-integration/content/product-suite.tsx +++ b/src/lib/programs/posthog-integration/content/product-suite.tsx @@ -62,7 +62,7 @@ export const PRODUCT_SUITE_BLOCK: ContentBlock = { {' ◆ '} {'Customer Analytics '} {'◆ '} - {'PostHog Code'} + {'PostHog Desktop'} , ], }; diff --git a/src/ui/tui/screens/OutroScreen.tsx b/src/ui/tui/screens/OutroScreen.tsx index 626f2b7d1..0ce4ca01e 100644 --- a/src/ui/tui/screens/OutroScreen.tsx +++ b/src/ui/tui/screens/OutroScreen.tsx @@ -56,40 +56,23 @@ export const OutroScreen = ({ store }: OutroScreenProps) => { )} - {outroData.nextSteps && outroData.nextSteps.items.length > 0 && ( - - - {outroData.nextSteps.heading} - - {/* Items can carry a URL, so render through LinkText: it keeps the - full address as the click target and shortens the visible label - instead of wrapping it across lines. */} - {outroData.nextSteps.items.map((item, i) => ( - - • - - - ))} - - )} - - {outroData.dashboardUrl && ( + {outroData.notebookUrl && ( - Dashboard:{' '} + Notebook:{' '} - {withUtm(outroData.dashboardUrl, 'outro-dashboard')} + {withUtm(outroData.notebookUrl, 'outro-notebook')} )} - {outroData.notebookUrl && ( + {outroData.dashboardUrl && ( - Notebook:{' '} + Dashboard:{' '} - {withUtm(outroData.notebookUrl, 'outro-notebook')} + {withUtm(outroData.dashboardUrl, 'outro-dashboard')} @@ -156,6 +139,25 @@ export const OutroScreen = ({ store }: OutroScreenProps) => { )} + {/* Suggestions (e.g. unconnected warehouse sources) go after + everything the run actually produced. */} + {outroData.nextSteps && outroData.nextSteps.items.length > 0 && ( + + + {outroData.nextSteps.heading} + + {/* Items can carry a URL, so render through LinkText: it keeps the + full address as the click target and shortens the visible label + instead of wrapping it across lines. */} + {outroData.nextSteps.items.map((item, i) => ( + + • + + + ))} + + )} + Note: This wizard uses an LLM agent to analyze and modify your