Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions src/lib/__tests__/wizard-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { analytics } from '@utils/analytics';
import {
ASK_BATCH_THRESHOLD,
ASK_CANCELLED_NOTE,
ASK_MAX_QUESTIONS_PER_CALL,
ASK_SUBJECT_UNSPECIFIED,
ASK_TIMED_OUT_NOTE,
DEFAULT_ASK_MAX_QUESTIONS,
Expand Down Expand Up @@ -653,6 +654,28 @@ describe('normaliseAskSubject', () => {
});
});

describe('ASK_MAX_QUESTIONS_PER_CALL', () => {
it('carries the widest connector form in one call', () => {
// Nine is what the widest data-warehouse connector advertises; a cap below
// it makes those sources uncollectable in the single call the guidance
// asks for, and the agent falls back to a browser link.
expect(ASK_MAX_QUESTIONS_PER_CALL).toBeGreaterThanOrEqual(9);
});

it('is the number the tool description quotes', () => {
expect(WIZARD_ASK_TOOL_DESCRIPTION).toContain(
`up to ${ASK_MAX_QUESTIONS_PER_CALL}`,
);
});

it('tells the agent to split a wider subject rather than give up on it', () => {
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(
/consecutive calls reusing the same `subject`/,
);
expect(WIZARD_ASK_TOOL_DESCRIPTION).toMatch(/never a reason to abandon/i);
});
});

describe('evaluateAskCap', () => {
const MAX = DEFAULT_ASK_MAX_QUESTIONS;
const at = (over: Partial<Parameters<typeof evaluateAskCap>[0]>) =>
Expand Down Expand Up @@ -710,6 +733,9 @@ describe('evaluateAskCap', () => {
// impossible, so the agent fell back to browser links instead.
const decision = at({ subjectRunLength: ASK_BATCH_THRESHOLD });
if (decision.kind !== 'capped') throw new Error('expected capped');
expect(decision.message).toContain(
`up to ${ASK_MAX_QUESTIONS_PER_CALL} questions`,
);
expect(decision.message).toMatch(/different `subject`/);
expect(decision.message).toMatch(/per subject/i);
expect(decision.message).toMatch(/one call per source is never blocked/i);
Expand Down
17 changes: 17 additions & 0 deletions src/lib/agent/runner/harness/pi/__tests__/tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { allowedPiCodingTools, allowedOrchestratorTools } from '../task';
import {
ASK_BATCH_THRESHOLD,
ASK_CANCELLED_NOTE,
ASK_MAX_QUESTIONS_PER_CALL,
ASK_TIMED_OUT_NOTE,
WIZARD_ASK_KIND_DESCRIPTION,
WIZARD_ASK_SENSITIVE_DESCRIPTION,
Expand Down Expand Up @@ -370,6 +371,22 @@ describe('pi wizard_ask — the batching guard counts per subject', () => {
);
});

it('caps one call at the shared question limit, not a harness-local number', () => {
// The orchestrator runs on pi, so this schema is the one in effect: a
// limit below the widest connector form is what sends a database source
// to the browser fallback.
const { wizardAsk } = makeTools({});
const questions = (
wizardAsk as unknown as {
parameters: {
properties: { questions: { minItems?: number; maxItems?: number } };
};
}
).parameters.properties.questions;
expect(questions.maxItems).toBe(ASK_MAX_QUESTIONS_PER_CALL);
expect(questions.minItems).toBe(1);
});

it('shares one tool description with the MCP server', () => {
const { wizardAsk } = makeTools({});
expect((wizardAsk as unknown as { description: string }).description).toBe(
Expand Down
3 changes: 2 additions & 1 deletion src/lib/agent/runner/harness/pi/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import type { ToolDefinition } from '@earendil-works/pi-coding-agent';
import { analytics } from '@utils/analytics';
import { logToFile } from '@utils/debug';
import {
ASK_MAX_QUESTIONS_PER_CALL,
AUDIT_ADD_CHECKS_DESCRIPTION,
AUDIT_ADD_CHECKS_PARAM_DESCRIPTION,
AUDIT_RESOLVE_CHECKS_DESCRIPTION,
Expand Down Expand Up @@ -439,7 +440,7 @@ export function createWizardPiTools(ctx: PiToolsContext): ToolDefinition[] {
}),
),
}),
{ minItems: 1, maxItems: 8 },
{ minItems: 1, maxItems: ASK_MAX_QUESTIONS_PER_CALL },
),
subject: Type.Optional(
Type.String({ description: WIZARD_ASK_SUBJECT_DESCRIPTION }),
Expand Down
12 changes: 6 additions & 6 deletions src/lib/programs/self-driving/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,12 +182,12 @@ against `config.abortCases`. `PromptContext` (project/host + AI-consent
**`wizard-tools` MCP + `wizard_ask`** (`src/lib/wizard-tools.ts`).
`check_env_keys` / `set_env_values` are the only sanctioned `.env` access
(value-safe, `.gitignore`-guarded, secret-vault aware). `wizard_ask` is the
**only** way to ask the user anything — 1–8 questions, capped at `maxQuestions`
(13), batched. Each call carries an optional `subject` tag; the one-time
batch-your-questions nudge counts consecutive calls **per subject**, so a step
that walks a list (one call per detected source) is never interrupted, while
repeated prompting about one thing still gets nudged. Each `single`/`multi`
option is `{ label, value, description? }`.
**only** way to ask the user anything — questions batched per call, with
`maxQuestions` (13) calls per run. Each call carries an optional `subject` tag;
the one-time batch-your-questions nudge counts consecutive calls **per
subject**, so a step that walks a list (one call per detected source) is never
interrupted, while repeated prompting about one thing still gets nudged. Each
`single`/`multi` option is `{ label, value, description? }`.
`description` is **optional and additive** (added for STEP 7): rendered dimmed
and wrapped beneath the label, and **only in the multi-select render path**
(`PickerMenu` `MultiPickerMenu` + `WizardAskScreen`); when a question omits it,
Expand Down
6 changes: 5 additions & 1 deletion src/lib/wizard-tools/mcp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import {
} from '../agent/runner/sequence/orchestrator/queue-tools';
import type { LLMProvider } from '@posthog/warlock';
import {
ASK_MAX_QUESTIONS_PER_CALL,
DEFAULT_ASK_MAX_QUESTIONS,
CHECK_ENV_KEYS_DESCRIPTION,
CHECK_ENV_KEYS_FILE_PATH_DESCRIPTION,
Expand Down Expand Up @@ -587,7 +588,10 @@ export async function createWizardToolsServer(options: WizardToolsOptions) {
'wizard_ask',
WIZARD_ASK_TOOL_DESCRIPTION,
{
questions: z.array(askQuestionSchema).min(1).max(8),
questions: z
.array(askQuestionSchema)
.min(1)
.max(ASK_MAX_QUESTIONS_PER_CALL),
subject: z.string().optional().describe(WIZARD_ASK_SUBJECT_DESCRIPTION),
},
async (args: {
Expand Down
28 changes: 23 additions & 5 deletions src/lib/wizard-tools/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -312,13 +312,27 @@ export async function installSkillById(
}

export const DEFAULT_ASK_MAX_QUESTIONS = 10;

/**
* Most questions one `wizard_ask` call may carry, enforced by both harness
* facades' schemas and quoted by the guidance below, so the limit and what the
* agent is told about it cannot drift apart.
*
* Sized against the forms this tool exists to collect: the widest
* data-warehouse connector advertises nine configuration fields, and a
* database whose form includes an SSH tunnel expands past that again. At eight
* the databases the step is mostly offered for did not fit, and an agent that
* reads "one subject, one call" as the contract treats a form it cannot fit in
* one call as one it cannot collect at all — and hands over a browser link.
*/
export const ASK_MAX_QUESTIONS_PER_CALL = 12;
/**
* Consecutive calls about the *same subject* before the one-time
* batch-your-questions nudge fires. Adjacency is per subject, not per run:
* the guard exists to stop an agent firing many small prompts about one
* thing, not to stop a flow that legitimately walks a list — the warehouse
* task asks one call per detected source, and 5–8 sources need 15–25 fields,
* far more than the 8-question schema limit allows in a single call.
* far more than {@link ASK_MAX_QUESTIONS_PER_CALL} allows in a single call.
*/
export const ASK_BATCH_THRESHOLD = 3;

Expand Down Expand Up @@ -425,8 +439,12 @@ export const WIZARD_ASK_SUBJECT_DESCRIPTION =
export const WIZARD_ASK_TOOL_DESCRIPTION =
'Ask the user one or more structured questions and wait for their answers. ' +
'Use this whenever you would otherwise inline a question in your text output. ' +
'Batch every question about one subject into a single call (up to 8) rather ' +
'than asking one at a time, and tag the call with `subject`. Walking a list — ' +
`Batch every question about one subject into a single call (up to ${ASK_MAX_QUESTIONS_PER_CALL}) rather ` +
'than asking one at a time, and tag the call with `subject`. A subject needing ' +
'more fields than one call carries — a connection form wider than the limit — ' +
'is collected over consecutive calls reusing the same `subject`: that is ' +
'expected, and is never a reason to abandon it for a link the user has to ' +
'follow themselves. Walking a list — ' +
'one call per data-warehouse source, one call per integration step — is ' +
'expected and is never blocked, because the batching guard counts consecutive ' +
'calls per subject. A fully cancelled or timed-out response does NOT count ' +
Expand Down Expand Up @@ -585,10 +603,10 @@ export function evaluateAskCap({
message:
`Not an error — this ask was not sent (a one-time nudge). ` +
`You have sent ${subjectRunLength} wizard_ask calls in a row about the same subject (${subjectNote}). ` +
`Batch every question you still need for that subject into one call (up to 8 questions) and send wizard_ask again now. ` +
`Batch every question you still need for that subject into one call (up to ${ASK_MAX_QUESTIONS_PER_CALL} questions) and send wizard_ask again now. ` +
`If your next questions are about something else — another data-warehouse source, another integration step — ` +
`set a different \`subject\` on the call. Adjacency is counted per subject, so one call per source is never blocked, ` +
`and you must not try to squeeze several sources into one 8-question call. ` +
`and you must not try to squeeze several sources into one ${ASK_MAX_QUESTIONS_PER_CALL}-question call. ` +
`Either way the next call is sent. Do not abandon the task, and do not fall back to browser setup because of this message.`,
};
}
Expand Down
Loading