From 51e03cb6e8022a20df3a21d73722b0649fa92594 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Wed, 23 Sep 2026 14:08:56 +0100 Subject: [PATCH] fix(wizard-ask): let one call carry a whole connector form The per-call question limit was eight, below the widest data-warehouse connector's advertised field set. The databases this step is mostly offered for sat at or over it, so the agent could not collect one in the single call the guidance asks for, and handed the user a browser link instead. Both harness facades now read one shared ASK_MAX_QUESTIONS_PER_CALL, and the tool description and the batching nudge quote it, so the limit and what the agent is told about it cannot drift. The description also says what to do with a subject wider than one call: split it over consecutive calls reusing the same `subject`, which the batching guard already allows, rather than abandoning it. Co-Authored-By: Claude Opus 5 Generated-By: PostHog Desktop Task-Id: 4c9a85df-9b83-4a2f-9a8e-6a127402705f --- src/lib/__tests__/wizard-tools.test.ts | 26 +++++++++++++++++ .../runner/harness/pi/__tests__/tools.test.ts | 17 +++++++++++ src/lib/agent/runner/harness/pi/tools.ts | 3 +- src/lib/programs/self-driving/ARCHITECTURE.md | 12 ++++---- src/lib/wizard-tools/mcp.ts | 6 +++- src/lib/wizard-tools/tools.ts | 28 +++++++++++++++---- 6 files changed, 79 insertions(+), 13 deletions(-) diff --git a/src/lib/__tests__/wizard-tools.test.ts b/src/lib/__tests__/wizard-tools.test.ts index 943e67998..84acef33b 100644 --- a/src/lib/__tests__/wizard-tools.test.ts +++ b/src/lib/__tests__/wizard-tools.test.ts @@ -8,6 +8,7 @@ import { analytics } from '@utils/analytics'; import { ASK_BATCH_THRESHOLD, ASK_CANCELLED_NOTE, + ASK_MAX_QUESTIONS_PER_CALL, ASK_SUBJECT_UNSPECIFIED, ASK_TIMED_OUT_NOTE, DEFAULT_ASK_MAX_QUESTIONS, @@ -653,6 +654,28 @@ describe('normaliseAskSubject', () => { }); }); +describe('ASK_MAX_QUESTIONS_PER_CALL', () => { + it('carries the widest connector form in one call', () => { + // Nine is what the widest data-warehouse connector advertises; a cap below + // it makes those sources uncollectable in the single call the guidance + // asks for, and the agent falls back to a browser link. + expect(ASK_MAX_QUESTIONS_PER_CALL).toBeGreaterThanOrEqual(9); + }); + + it('is the number the tool description quotes', () => { + expect(WIZARD_ASK_TOOL_DESCRIPTION).toContain( + `up to ${ASK_MAX_QUESTIONS_PER_CALL}`, + ); + }); + + it('tells the agent to split a wider subject rather than give up on it', () => { + expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch( + /consecutive calls reusing the same `subject`/, + ); + expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(/never a reason to abandon/i); + }); +}); + describe('evaluateAskCap', () => { const MAX = DEFAULT_ASK_MAX_QUESTIONS; const at = (over: Partial[0]>) => @@ -710,6 +733,9 @@ describe('evaluateAskCap', () => { // impossible, so the agent fell back to browser links instead. const decision = at({ subjectRunLength: ASK_BATCH_THRESHOLD }); if (decision.kind !== 'capped') throw new Error('expected capped'); + expect(decision.message).toContain( + `up to ${ASK_MAX_QUESTIONS_PER_CALL} questions`, + ); expect(decision.message).toMatch(/different `subject`/); expect(decision.message).toMatch(/per subject/i); expect(decision.message).toMatch(/one call per source is never blocked/i); diff --git a/src/lib/agent/runner/harness/pi/__tests__/tools.test.ts b/src/lib/agent/runner/harness/pi/__tests__/tools.test.ts index c3ef60dc1..0156a0cd5 100644 --- a/src/lib/agent/runner/harness/pi/__tests__/tools.test.ts +++ b/src/lib/agent/runner/harness/pi/__tests__/tools.test.ts @@ -19,6 +19,7 @@ import { allowedPiCodingTools, allowedOrchestratorTools } from '../task'; import { ASK_BATCH_THRESHOLD, ASK_CANCELLED_NOTE, + ASK_MAX_QUESTIONS_PER_CALL, ASK_TIMED_OUT_NOTE, WIZARD_ASK_KIND_DESCRIPTION, WIZARD_ASK_SENSITIVE_DESCRIPTION, @@ -370,6 +371,22 @@ describe('pi wizard_ask — the batching guard counts per subject', () => { ); }); + it('caps one call at the shared question limit, not a harness-local number', () => { + // The orchestrator runs on pi, so this schema is the one in effect: a + // limit below the widest connector form is what sends a database source + // to the browser fallback. + const { wizardAsk } = makeTools({}); + const questions = ( + wizardAsk as unknown as { + parameters: { + properties: { questions: { minItems?: number; maxItems?: number } }; + }; + } + ).parameters.properties.questions; + expect(questions.maxItems).toBe(ASK_MAX_QUESTIONS_PER_CALL); + expect(questions.minItems).toBe(1); + }); + it('shares one tool description with the MCP server', () => { const { wizardAsk } = makeTools({}); expect((wizardAsk as unknown as { description: string }).description).toBe( diff --git a/src/lib/agent/runner/harness/pi/tools.ts b/src/lib/agent/runner/harness/pi/tools.ts index b194cc79a..13c8f126b 100644 --- a/src/lib/agent/runner/harness/pi/tools.ts +++ b/src/lib/agent/runner/harness/pi/tools.ts @@ -19,6 +19,7 @@ import type { ToolDefinition } from '@earendil-works/pi-coding-agent'; import { analytics } from '@utils/analytics'; import { logToFile } from '@utils/debug'; import { + ASK_MAX_QUESTIONS_PER_CALL, AUDIT_ADD_CHECKS_DESCRIPTION, AUDIT_ADD_CHECKS_PARAM_DESCRIPTION, AUDIT_RESOLVE_CHECKS_DESCRIPTION, @@ -439,7 +440,7 @@ export function createWizardPiTools(ctx: PiToolsContext): ToolDefinition[] { }), ), }), - { minItems: 1, maxItems: 8 }, + { minItems: 1, maxItems: ASK_MAX_QUESTIONS_PER_CALL }, ), subject: Type.Optional( Type.String({ description: WIZARD_ASK_SUBJECT_DESCRIPTION }), diff --git a/src/lib/programs/self-driving/ARCHITECTURE.md b/src/lib/programs/self-driving/ARCHITECTURE.md index 96408a2d0..c24073007 100644 --- a/src/lib/programs/self-driving/ARCHITECTURE.md +++ b/src/lib/programs/self-driving/ARCHITECTURE.md @@ -182,12 +182,12 @@ against `config.abortCases`. `PromptContext` (project/host + AI-consent **`wizard-tools` MCP + `wizard_ask`** (`src/lib/wizard-tools.ts`). `check_env_keys` / `set_env_values` are the only sanctioned `.env` access (value-safe, `.gitignore`-guarded, secret-vault aware). `wizard_ask` is the -**only** way to ask the user anything — 1–8 questions, capped at `maxQuestions` -(13), batched. Each call carries an optional `subject` tag; the one-time -batch-your-questions nudge counts consecutive calls **per subject**, so a step -that walks a list (one call per detected source) is never interrupted, while -repeated prompting about one thing still gets nudged. Each `single`/`multi` -option is `{ label, value, description? }`. +**only** way to ask the user anything — questions batched per call, with +`maxQuestions` (13) calls per run. Each call carries an optional `subject` tag; +the one-time batch-your-questions nudge counts consecutive calls **per +subject**, so a step that walks a list (one call per detected source) is never +interrupted, while repeated prompting about one thing still gets nudged. Each +`single`/`multi` option is `{ label, value, description? }`. `description` is **optional and additive** (added for STEP 7): rendered dimmed and wrapped beneath the label, and **only in the multi-select render path** (`PickerMenu` `MultiPickerMenu` + `WizardAskScreen`); when a question omits it, diff --git a/src/lib/wizard-tools/mcp.ts b/src/lib/wizard-tools/mcp.ts index 4ad43e235..ea2460490 100644 --- a/src/lib/wizard-tools/mcp.ts +++ b/src/lib/wizard-tools/mcp.ts @@ -34,6 +34,7 @@ import { } from '../agent/runner/sequence/orchestrator/queue-tools'; import type { LLMProvider } from '@posthog/warlock'; import { + ASK_MAX_QUESTIONS_PER_CALL, DEFAULT_ASK_MAX_QUESTIONS, CHECK_ENV_KEYS_DESCRIPTION, CHECK_ENV_KEYS_FILE_PATH_DESCRIPTION, @@ -587,7 +588,10 @@ export async function createWizardToolsServer(options: WizardToolsOptions) { 'wizard_ask', WIZARD_ASK_TOOL_DESCRIPTION, { - questions: z.array(askQuestionSchema).min(1).max(8), + questions: z + .array(askQuestionSchema) + .min(1) + .max(ASK_MAX_QUESTIONS_PER_CALL), subject: z.string().optional().describe(WIZARD_ASK_SUBJECT_DESCRIPTION), }, async (args: { diff --git a/src/lib/wizard-tools/tools.ts b/src/lib/wizard-tools/tools.ts index 77d99c9e3..e02645682 100644 --- a/src/lib/wizard-tools/tools.ts +++ b/src/lib/wizard-tools/tools.ts @@ -312,13 +312,27 @@ export async function installSkillById( } export const DEFAULT_ASK_MAX_QUESTIONS = 10; + +/** + * Most questions one `wizard_ask` call may carry, enforced by both harness + * facades' schemas and quoted by the guidance below, so the limit and what the + * agent is told about it cannot drift apart. + * + * Sized against the forms this tool exists to collect: the widest + * data-warehouse connector advertises nine configuration fields, and a + * database whose form includes an SSH tunnel expands past that again. At eight + * the databases the step is mostly offered for did not fit, and an agent that + * reads "one subject, one call" as the contract treats a form it cannot fit in + * one call as one it cannot collect at all — and hands over a browser link. + */ +export const ASK_MAX_QUESTIONS_PER_CALL = 12; /** * Consecutive calls about the *same subject* before the one-time * batch-your-questions nudge fires. Adjacency is per subject, not per run: * the guard exists to stop an agent firing many small prompts about one * thing, not to stop a flow that legitimately walks a list — the warehouse * task asks one call per detected source, and 5–8 sources need 15–25 fields, - * far more than the 8-question schema limit allows in a single call. + * far more than {@link ASK_MAX_QUESTIONS_PER_CALL} allows in a single call. */ export const ASK_BATCH_THRESHOLD = 3; @@ -425,8 +439,12 @@ export const WIZARD_ASK_SUBJECT_DESCRIPTION = export const WIZARD_ASK_TOOL_DESCRIPTION = 'Ask the user one or more structured questions and wait for their answers. ' + 'Use this whenever you would otherwise inline a question in your text output. ' + - 'Batch every question about one subject into a single call (up to 8) rather ' + - 'than asking one at a time, and tag the call with `subject`. Walking a list — ' + + `Batch every question about one subject into a single call (up to ${ASK_MAX_QUESTIONS_PER_CALL}) rather ` + + 'than asking one at a time, and tag the call with `subject`. A subject needing ' + + 'more fields than one call carries — a connection form wider than the limit — ' + + 'is collected over consecutive calls reusing the same `subject`: that is ' + + 'expected, and is never a reason to abandon it for a link the user has to ' + + 'follow themselves. Walking a list — ' + 'one call per data-warehouse source, one call per integration step — is ' + 'expected and is never blocked, because the batching guard counts consecutive ' + 'calls per subject. A fully cancelled or timed-out response does NOT count ' + @@ -585,10 +603,10 @@ export function evaluateAskCap({ message: `Not an error — this ask was not sent (a one-time nudge). ` + `You have sent ${subjectRunLength} wizard_ask calls in a row about the same subject (${subjectNote}). ` + - `Batch every question you still need for that subject into one call (up to 8 questions) and send wizard_ask again now. ` + + `Batch every question you still need for that subject into one call (up to ${ASK_MAX_QUESTIONS_PER_CALL} questions) and send wizard_ask again now. ` + `If your next questions are about something else — another data-warehouse source, another integration step — ` + `set a different \`subject\` on the call. Adjacency is counted per subject, so one call per source is never blocked, ` + - `and you must not try to squeeze several sources into one 8-question call. ` + + `and you must not try to squeeze several sources into one ${ASK_MAX_QUESTIONS_PER_CALL}-question call. ` + `Either way the next call is sent. Do not abandon the task, and do not fall back to browser setup because of this message.`, }; }