From f11b1e264ffea41aa16ce2f67e63c8b3d0c59993 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 14:07:31 -0800 Subject: [PATCH 1/7] Show explicit agent sign-out facts before sending a chat --- .../acp/__fixtures__/grok-catalog-signin.json | 134 ++++++++++++++++ src/main/acp/acp-launch-specs.ts | 4 + src/main/acp/acp-model-catalog-probe.test.ts | 45 ++++++ src/main/acp/acp-model-catalog-probe.ts | 45 +++++- .../claude/__fixtures__/catalog-signin.json | 143 ++++++++++++++++++ .../claude/claude-command-catalog-probe.ts | 35 +---- src/main/claude/claude-control-responses.ts | 30 ++++ .../claude/claude-model-catalog-probe.test.ts | 72 ++++++++- src/main/claude/claude-model-catalog-probe.ts | 105 ++++++++----- .../agent-model-catalog-probe-runner.test.ts | 20 +++ .../agent-model-catalog-probe-runner.ts | 6 + .../agent-model-catalog-scaling.test.ts | 81 ++++++++++ src/main/pi/rpc-model-catalog-probe.test.ts | 30 ++++ src/main/pi/rpc-model-catalog-probe.ts | 22 ++- src/shared/grok-model-list-probe.ts | 4 + 15 files changed, 701 insertions(+), 75 deletions(-) create mode 100644 src/main/acp/__fixtures__/grok-catalog-signin.json create mode 100644 src/main/claude/__fixtures__/catalog-signin.json create mode 100644 src/main/claude/claude-control-responses.ts diff --git a/src/main/acp/__fixtures__/grok-catalog-signin.json b/src/main/acp/__fixtures__/grok-catalog-signin.json new file mode 100644 index 000000000000..bcad83d1d35b --- /dev/null +++ b/src/main/acp/__fixtures__/grok-catalog-signin.json @@ -0,0 +1,134 @@ +{ + "version": "1.0.70", + "initialize": { + "protocolVersion": 1, + "agentCapabilities": { + "loadSession": true, + "promptCapabilities": { + "image": true, + "audio": false, + "embeddedContext": true + }, + "mcpCapabilities": { + "http": true, + "sse": true + }, + "sessionCapabilities": { + "list": {}, + "resume": {}, + "close": {} + }, + "auth": {}, + "_meta": { + "x.ai/fs_notify": true, + "x.ai/hooks": { + "blockingEvents": ["pre_tool_use", "stop", "subagent_stop"], + "decisions": ["deny", "block"], + "stopSignals": ["continue", "stopReason", "additionalContext"] + }, + "x.ai/capabilities": { + "toolOverrides": { + "x_keyword_search": true, + "x_semantic_search": true, + "x_user_search": false, + "x_thread_fetch": false + } + } + } + }, + "authMethods": [ + { + "id": "grok.com", + "name": "Grok", + "description": "Sign in with Grok" + } + ], + "_meta": { + "modelState": { + "currentModelId": "grok-4.6", + "availableModels": [ + { + "modelId": "grok-4.6", + "name": "Grok 4.6", + "description": "SpaceXAI's latest frontier model", + "_meta": { + "totalContextTokens": 500000, + "agentType": "grok-build-plan", + "supportsReasoningEffort": true, + "reasoningEffort": "high", + "reasoningEfforts": [ + { + "id": "xhigh", + "value": "xhigh", + "label": "Extra High Effort", + "description": "Highest effort and reasoning level", + "default": false + }, + { + "id": "high", + "value": "high", + "label": "High Effort", + "description": "Higher implementation quality with extensive reasoning", + "default": true + }, + { + "id": "medium", + "value": "medium", + "label": "Medium Effort", + "description": "Balanced effort with standard implementation and testing", + "default": false + }, + { + "id": "low", + "value": "low", + "label": "Low Effort", + "description": "Quick, fast implementations", + "default": false + } + ] + } + }, + { + "modelId": "grok-4.5", + "name": "Grok 4.5", + "_meta": { + "totalContextTokens": 500000, + "agentType": "grok-build-plan", + "supportsReasoningEffort": true, + "reasoningEffort": "high", + "reasoningEfforts": [ + { + "id": "high", + "value": "high", + "label": "High Effort", + "description": "Highest implementation quality with extensive reasoning", + "default": true + }, + { + "id": "medium", + "value": "medium", + "label": "Medium Effort", + "description": "Balanced effort with standard implementation and testing", + "default": false + }, + { + "id": "low", + "value": "low", + "label": "Low Effort", + "description": "Quick, fast implementations", + "default": false + } + ] + } + } + ] + } + } + }, + "authError": { + "code": -32000, + "message": "Authentication required", + "data": "no auth method id provided" + }, + "models": "You are not authenticated.\n\nDefault model: grok-4.6\n\nAvailable models:\n * grok-4.6 (default)\n - grok-4.5\n" +} diff --git a/src/main/acp/acp-launch-specs.ts b/src/main/acp/acp-launch-specs.ts index 5c3e4a861764..ee0151166635 100644 --- a/src/main/acp/acp-launch-specs.ts +++ b/src/main/acp/acp-launch-specs.ts @@ -15,6 +15,7 @@ import type { InitializeResponse } from './generated/acp-protocol.generated' import { compareAppVersions } from '../../shared/app-version' import type { ManagedRuntime } from '../node-runtime/managed-runtime-release' import { readGrokModelCatalog } from './acp-dialects/grok-model-catalog' +import { GROK_MODEL_LIST_ARGS, grokModelListSignedOut } from '../../shared/grok-model-list-probe' import { OPENCODE_MODEL_LISTING_ARGS, parseOpenCodeModelListing @@ -65,6 +66,8 @@ export type AcpModelDiscovery = } ): Promise listingNamesConfiguredModel: boolean + /** Read after initialize, which may silently refresh credentials. */ + signInStatus?: { args: readonly string[]; signedOut(stdout: string): boolean } } | { kind: 'command' @@ -166,6 +169,7 @@ const GROK_LAUNCH_SPEC: AcpLaunchSpec = { modelDiscovery: { kind: 'initialize', read: readGrokModelCatalog, + signInStatus: { args: GROK_MODEL_LIST_ARGS, signedOut: grokModelListSignedOut }, listingNamesConfiguredModel: false }, visualsSkill: loadGrokVisualsSkill, diff --git a/src/main/acp/acp-model-catalog-probe.test.ts b/src/main/acp/acp-model-catalog-probe.test.ts index e0c357e57cd4..89d5902e7e32 100644 --- a/src/main/acp/acp-model-catalog-probe.test.ts +++ b/src/main/acp/acp-model-catalog-probe.test.ts @@ -12,6 +12,7 @@ import { AcpScriptedAgent } from './acp-scripted-agent.test-support' import { AgentModelCatalogListingStoppedError } from '../native-chat/agent-model-catalog/agent-model-catalog-failures' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' import { openCodeAcpAccountBinding } from '../opencode/opencode-structured-account-home' +import signinCapture from './__fixtures__/grok-catalog-signin.json' const PREFIX = ['run', '--', 'agent'] const launcherSettings = (agent: string) => ({ @@ -82,6 +83,7 @@ function scriptedGrok(initializeMeta: Record): { resolveLaunchEnv: () => ({ XAI_API_KEY: 'from-settings' }), resolveCommand: () => '/opt/grok/bin/grok', homePath: '/home/user', + runListing: vi.fn(async () => ''), connect: (launch) => { launches.push(launch) const runtime = new AcpSessionRuntime(agent.stdout, agent.stdin) @@ -167,6 +169,49 @@ function scriptedOpenCode2(canDelete: boolean): { } describe('ACP model catalog probes', () => { + it('retains Grok models with its recorded sign-out banner after initialize, without creating a session', async () => { + const { deps, agent, launches, closed } = scriptedGrok({}) + agent.on('initialize', (frame) => agent.reply(frame, signinCapture.initialize)) + deps.resolveCommandSettings = () => launcherSettings('grok') + const runListing = vi.fn>(async () => { + expect(agent.frames.map((frame) => frame.method)).toEqual(['initialize']) + return signinCapture.models + }) + deps.runListing = runListing + const success = await createAcpModelCatalogProbe(GROK, deps)(GROK_HOME) + expect(success.unavailable).toEqual({ reason: 'notSignedIn' }) + expect(success.models.map((model) => model.id)).toContain('grok-4.6') + expect(runListing).toHaveBeenCalledExactlyOnceWith( + { ...launches[0], args: [...PREFIX, 'models'] }, + expect.objectContaining({ timeoutMs: 2_000 }) + ) + expect(agent.frames.map((frame) => frame.method)).toEqual(['initialize']) + expect(closed()).toBe(1) + }) + + it('keeps usable Grok inventory when the optional status command fails', async () => { + const { deps } = scriptedGrok({ modelState: MODEL_STATE }) + deps.runListing = async () => { + throw new Error('unsupported command') + } + const result = await createAcpModelCatalogProbe(GROK, deps)(GROK_HOME) + expect(result.models.length).toBeGreaterThan(0) + expect(result.unavailable).toBeUndefined() + }) + + it('projects an agent-authored auth-required error, retaining unknown for other failures', async () => { + const { deps, agent } = scriptedGrok({}) + const { code, message, data } = signinCapture.authError + agent.on('x.ai/models/list', (frame) => agent.fail(frame, code, message, data)) + await expect(createAcpModelCatalogProbe(GROK, deps)(GROK_HOME)).rejects.toMatchObject({ + unavailable: { reason: 'notSignedIn' } + }) + agent.on('x.ai/models/list', (frame) => agent.fail(frame, -32603, 'Internal error')) + await expect(createAcpModelCatalogProbe(GROK, deps)(GROK_HOME)).rejects.toMatchObject({ + code: -32603 + }) + }) + it('lists Grok from initialize alone, under its pinned account, and closes the child', async () => { const { agent, deps, launches, closed } = scriptedGrok({ modelState: MODEL_STATE }) deps.resolveCommandSettings = () => launcherSettings('grok') diff --git a/src/main/acp/acp-model-catalog-probe.ts b/src/main/acp/acp-model-catalog-probe.ts index 361920e7134e..02b8ed5b0234 100644 --- a/src/main/acp/acp-model-catalog-probe.ts +++ b/src/main/acp/acp-model-catalog-probe.ts @@ -32,6 +32,8 @@ import { } from './acp-structured-launch-resolution' import { AcpStructuredOptions } from './acp-structured-options' import type { InitializeResponse } from './generated/acp-protocol.generated' +import { AcpAgentError } from './acp-errors' +import { AgentModelCatalogUnavailableError } from '../native-chat/agent-model-catalog/agent-model-catalog-unavailable' /** Session-free capability boundary: no session methods are exposed. */ export type AcpModelCatalogConnection = { @@ -60,7 +62,7 @@ export function createAcpModelCatalogProbe( spec: AcpLaunchSpec, deps: AcpModelCatalogProbeDeps ): AgentModelCatalogProbe { - return async (accountHome, options): Promise => { + const probe: AgentModelCatalogProbe = async (accountHome, options) => { const operation = options?.operation ?? { signal: options?.signal } assertStructuredAgentResolutionCurrent(operation) const discovery = spec.modelDiscovery @@ -107,6 +109,7 @@ export function createAcpModelCatalogProbe( } const run = { label: spec.agent, signal: options?.signal } let models: AgentSessionModelOption[] + let unavailable: AgentModelCatalogSuccess['unavailable'] if (listing.kind === 'command') { const stdout = await (deps.runListing ?? runAgentModelCatalogListing)( { @@ -146,15 +149,51 @@ export function createAcpModelCatalogProbe( async (connection) => { const initialized = await connection.initialize() assertStructuredAgentResolutionCurrent(operation) - return listing.read(initialized, connection) + const listed = listing.read(initialized, connection).then( + (models) => ({ models }), + (error: unknown) => ({ error }) + ) + const signedOut = listing.signInStatus + ? await (deps.runListing ?? runAgentModelCatalogListing)( + { ...launch, args: [...prefixArgs, ...listing.signInStatus.args] }, + { site: `${spec.agent}-signin-probe`, signal: options?.signal, timeoutMs: 2_000 } + ) + .then(listing.signInStatus.signedOut) + .catch(() => false) + : false + assertStructuredAgentResolutionCurrent(operation) + const result = await listed + if (signedOut) { + unavailable = { reason: 'notSignedIn' } + } + if ('error' in result) { + if (unavailable) { + throw new AgentModelCatalogUnavailableError(unavailable) + } + throw result.error + } + return result.models }, run ) } if (models.length === 0) { + if (unavailable) { + throw new AgentModelCatalogUnavailableError(unavailable) + } throw new Error(`${spec.agent} listed no models`) } - return { models, origin: 'probe' } + return { models, origin: 'probe', ...(unavailable ? { unavailable } : {}) } + } + return async (home, options) => { + try { + return await probe(home, options) + } catch (error) { + if (error instanceof AcpAgentError && error.code === -32000) { + throw new AgentModelCatalogUnavailableError({ reason: 'notSignedIn' }) + } + throw error + } } } diff --git a/src/main/claude/__fixtures__/catalog-signin.json b/src/main/claude/__fixtures__/catalog-signin.json new file mode 100644 index 000000000000..e9afdd725f5f --- /dev/null +++ b/src/main/claude/__fixtures__/catalog-signin.json @@ -0,0 +1,143 @@ +{ + "version": "2.1.310", + "accounts": { + "signedOut": { + "tokenSource": "none", + "apiProvider": "firstParty" + }, + "apiKey": { + "tokenSource": "none", + "apiKeySource": "ANTHROPIC_API_KEY", + "apiProvider": "firstParty" + }, + "bearer": { + "tokenSource": "ANTHROPIC_AUTH_TOKEN", + "apiProvider": "firstParty" + }, + "bedrock": { + "apiProvider": "bedrock" + } + }, + "models": { + "models": [ + { + "value": "default", + "resolvedModel": "claude-opus-5-5", + "displayName": "Default (recommended)", + "description": "Use the default model (currently Opus 5.5) \u00b7 $4/$20 per Mtok", + "supportsEffort": true, + "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], + "supportsAdaptiveThinking": true, + "supportsFastMode": true, + "supportsAutoMode": true, + "supportsThinkingOff": false + }, + { + "value": "opus", + "resolvedModel": "claude-opus-5-5", + "displayName": "Opus", + "description": "Opus 5.5 \u00b7 Best for everyday, complex tasks \u00b7 $4/$20 per Mtok", + "supportsEffort": true, + "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], + "supportsAdaptiveThinking": true, + "supportsFastMode": true, + "supportsAutoMode": true, + "supportsThinkingOff": false + }, + { + "value": "fable", + "resolvedModel": "claude-fable-5-1", + "displayName": "Fable", + "description": "Fable 5.1 \u00b7 Most capable for your hardest and longest-running tasks \u00b7 $10/$50 per Mtok", + "supportsEffort": true, + "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], + "supportsAdaptiveThinking": true, + "supportsAutoMode": true, + "supportsThinkingOff": false + }, + { + "value": "sonnet", + "resolvedModel": "claude-sonnet-5-5", + "displayName": "Sonnet", + "description": "Sonnet 5.5 \u00b7 Efficient for routine tasks \u00b7 $2/$10 per Mtok", + "supportsEffort": true, + "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], + "supportsAdaptiveThinking": true, + "supportsAutoMode": true, + "supportsThinkingOff": false + }, + { + "value": "haiku", + "resolvedModel": "claude-haiku-5-5", + "displayName": "Haiku", + "description": "Haiku 5.5 \u00b7 Fastest for quick answers \u00b7 $0.10/$0.50 per Mtok ($0.50/$2.50 for prompts over 100k)", + "supportsEffort": true, + "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], + "supportsAdaptiveThinking": true, + "supportsAutoMode": true, + "supportsThinkingOff": false + } + ] + }, + "settings": { + "effective": { + "disableAllHooks": true + }, + "sources": [ + { + "source": "flagSettings", + "settings": { + "disableAllHooks": true + } + } + ], + "applied": { + "model": "claude-opus-5-5", + "effort": "medium", + "advisor": null, + "ultracode": false, + "ultracodeRequested": false, + "ultracodeAvailable": true + } + }, + "endpointSettings": { + "effective": { + "env": { + "ANTHROPIC_BASE_URL": "https://gateway.example.invalid" + }, + "disableAllHooks": true + }, + "sources": [ + { + "source": "userSettings", + "settings": { + "env": { + "ANTHROPIC_BASE_URL": "https://gateway.example.invalid" + } + } + }, + { + "source": "projectSettings", + "settings": { + "env": { + "ANTHROPIC_BASE_URL": "https://gateway.example.invalid" + } + } + }, + { + "source": "flagSettings", + "settings": { + "disableAllHooks": true + } + } + ], + "applied": { + "model": "claude-opus-5-5", + "effort": "medium", + "advisor": null, + "ultracode": false, + "ultracodeRequested": false, + "ultracodeAvailable": true + } + } +} diff --git a/src/main/claude/claude-command-catalog-probe.ts b/src/main/claude/claude-command-catalog-probe.ts index 98bbdd892ec5..c13a4459e270 100644 --- a/src/main/claude/claude-command-catalog-probe.ts +++ b/src/main/claude/claude-command-catalog-probe.ts @@ -1,7 +1,7 @@ import type { AgentSessionSlashCommand } from '../../shared/agent-session/agent-session-wire' -import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' import { runClaudeListing, type ClaudeModelCatalogProbeDeps } from './claude-model-catalog-probe' import { ClaudeSlashCommandCatalog } from './claude-slash-command-catalog' +import { readClaudeControlResponses } from './claude-control-responses' const REQUEST_ID = 'orca-command-discovery' @@ -12,8 +12,6 @@ const CLAUDE_COMMAND_LIST_STDIN = `${JSON.stringify({ request: { subtype: 'initialize' } })}\n` -const INITIALIZE_JSON_LIMITS = { structuralTokens: 64 * 1024, nestingDepth: 16 } as const - export type ClaudeCommandCatalogProbe = (request: { cwd: string accountHomePath: string @@ -42,31 +40,8 @@ export function createClaudeCommandCatalogProbe( } function readInitializeCommands(stdout: string): AgentSessionSlashCommand[] | undefined { - for (const rawLine of stdout.split(/\r?\n/)) { - const line = rawLine.trim() - if (!line.startsWith('{') || !line.includes(REQUEST_ID)) { - continue - } - let parsed: unknown - try { - assertJsonTextStructureWithinLimits(line, INITIALIZE_JSON_LIMITS) - parsed = JSON.parse(line) - } catch { - continue - } - const response = - parsed && typeof parsed === 'object' && 'response' in parsed ? parsed.response : undefined - if ( - response && - typeof response === 'object' && - 'subtype' in response && - response.subtype === 'success' && - 'request_id' in response && - response.request_id === REQUEST_ID && - 'response' in response - ) { - return new ClaudeSlashCommandCatalog(undefined, response.response).commands - } - } - return undefined + const initialization = readClaudeControlResponses(stdout).get(REQUEST_ID) + return initialization === undefined + ? undefined + : new ClaudeSlashCommandCatalog(undefined, initialization).commands } diff --git a/src/main/claude/claude-control-responses.ts b/src/main/claude/claude-control-responses.ts new file mode 100644 index 000000000000..109315b22bf3 --- /dev/null +++ b/src/main/claude/claude-control-responses.ts @@ -0,0 +1,30 @@ +import { assertJsonTextStructureWithinLimits } from '../../shared/json-text-structure-limit' + +const CONTROL_JSON_LIMITS = { structuralTokens: 64 * 1024, nestingDepth: 16 } as const + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export function readClaudeControlResponses(stdout: string): Map { + const responses = new Map() + for (const line of stdout.split(/\r?\n/)) { + if (!line.trimStart().startsWith('{') || !line.includes('control_response')) { + continue + } + try { + assertJsonTextStructureWithinLimits(line, CONTROL_JSON_LIMITS) + const frame: unknown = JSON.parse(line) + if (!isRecord(frame) || frame.type !== 'control_response' || !isRecord(frame.response)) { + continue + } + const response = frame.response + if (response.subtype === 'success' && typeof response.request_id === 'string') { + responses.set(response.request_id, response.response) + } + } catch { + // Diagnostics and incomplete frames establish no control facts. + } + } + return responses +} diff --git a/src/main/claude/claude-model-catalog-probe.test.ts b/src/main/claude/claude-model-catalog-probe.test.ts index ab68544c4f4b..e231c33439d4 100644 --- a/src/main/claude/claude-model-catalog-probe.test.ts +++ b/src/main/claude/claude-model-catalog-probe.test.ts @@ -6,6 +6,7 @@ import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { PROVIDER_STDIN_END_GRACE_MS } from '../provider-process/provider-process-supervisor' import { createClaudeModelCatalogProbe } from './claude-model-catalog-probe' +import signinCapture from './__fixtures__/catalog-signin.json' import { discoveredModel } from '../../shared/agent-session/options/structured-agent-session-discovered-model' import { resolveClaudeStructuredInvocation } from './structured/launch/claude-structured-launch-resolution' import { resolveStructuredAgentCommand } from '../native-chat/structured-agent-command-resolution' @@ -328,7 +329,7 @@ it.each(['high', 'low'])( ?.trim() .split('\n') .map((line) => JSON.parse(line)) - ).toHaveLength(2) + ).toHaveLength(3) } ) @@ -345,6 +346,7 @@ it('qualifies only a correlated valid settings observation', async () => { runListing: async () => LISTED + JSON.stringify({ + type: 'control_response', response: { subtype: 'success', request_id: requestId, @@ -355,3 +357,71 @@ it('qualifies only a correlated valid settings observation', async () => { expect((await probe(HOME('/account'))).complete).toBe(complete) } }) + +function controlReply(requestId: string, response: unknown): string { + return `${JSON.stringify({ type: 'control_response', response: { subtype: 'success', request_id: requestId, response } })}\n` +} + +it.each(['system', 'managed'] as const)( + 'keeps recorded models with the signed-out %s account advisory', + async (account) => { + const runListing = vi.fn( + async () => + controlReply('orca-model-discovery', signinCapture.models) + + controlReply('orca-model-settings', signinCapture.settings) + + controlReply('orca-signin-discovery', { account: signinCapture.accounts.signedOut }) + ) + const probe = createClaudeModelCatalogProbe({ + ...probeDeps(), + resolveAuthPolicy: () => ({ account }), + runListing + }) + const success = await probe(HOME('/account')) + expect(success).toMatchObject({ unavailable: { reason: 'notSignedIn', account } }) + expect(success.models.length).toBeGreaterThan(0) + const [launch, options] = runListing.mock.calls[0] ?? [] + expect( + launch?.stdin + ?.trim() + .split('\n') + .map((line: string) => JSON.parse(line).request.subtype) + ).toEqual(['list_models', 'get_settings', 'initialize']) + expect( + options?.partialOutput?.ready(controlReply('orca-model-discovery', signinCapture.models)) + ).toBe(false) + expect( + options?.partialOutput?.ready( + controlReply('orca-model-discovery', signinCapture.models) + + controlReply('orca-model-settings', signinCapture.settings) + ) + ).toBe(true) + } +) + +it('requires definitive first-party account facts, preserving inventory when auth is unknown', async () => { + const cases = [ + { account: signinCapture.accounts.apiKey }, + { account: signinCapture.accounts.bearer }, + { account: signinCapture.accounts.bedrock }, + { account: { tokenSource: 'none', email: 'user@example.test' } }, + { account: { tokenSource: 'none', subscriptionType: 'max' } }, + { account: {} }, + { account: undefined }, + { account: signinCapture.accounts.signedOut, settings: signinCapture.endpointSettings }, + { account: signinCapture.accounts.signedOut, endpoint: 'https://gateway.example.test' }, + { account: signinCapture.accounts.signedOut, settings: {} } + ] + for (const { account, settings = signinCapture.settings, endpoint } of cases) { + const probe = createClaudeModelCatalogProbe({ + ...probeDeps(), + resolveEnv: () => (endpoint ? { ANTHROPIC_BASE_URL: endpoint } : {}), + runListing: async () => + controlReply('orca-model-discovery', signinCapture.models) + + controlReply('orca-model-settings', settings) + + (account === undefined ? '' : controlReply('orca-signin-discovery', { account })) + }) + const success = await probe(HOME('/account')) + expect(success.unavailable).toBeUndefined() + expect(success.models.length).toBeGreaterThan(0) + } +}) diff --git a/src/main/claude/claude-model-catalog-probe.ts b/src/main/claude/claude-model-catalog-probe.ts index b53165a45ac0..eda5b39ccec6 100644 --- a/src/main/claude/claude-model-catalog-probe.ts +++ b/src/main/claude/claude-model-catalog-probe.ts @@ -12,14 +12,14 @@ import { runClaudeBackgroundAttempts } from '../../shared/claude-background-run- import { isCompleteClaudeModelListing, listedModels, - record, - type ListedModel + record } from './structured/launch/claude-structured-model-catalog' import { claudeFastModeSupport, wireClaudeModels } from './structured/launch/claude-structured-catalog-listing' import { readClaudeSettingsApplied } from './structured/launch/claude-structured-session-options' +import { readClaudeControlResponses } from './claude-control-responses' import { claudeStructuredLaunchArgs } from './structured/launch/claude-structured-launch-args' import { AgentModelCatalogUnavailableError } from '../native-chat/agent-model-catalog/agent-model-catalog-unavailable' import { claudeConfigDirEnvPatch } from './claude-config-dir-pin' @@ -142,7 +142,7 @@ export function createClaudeModelCatalogProbe( } const sources = await resolveClaudeChildEnvSources(launchDeps) assertStructuredAgentResolutionCurrent(operation) - const { command, prefixArgs, env } = await resolveClaudeStructuredInvocation( + const { command, prefixArgs, env, account } = await resolveClaudeStructuredInvocation( launchDeps, (base) => ({ ...base, @@ -174,9 +174,23 @@ export function createClaudeModelCatalogProbe( command, args: [...prefixArgs, ...argsFor(strictMcp)], ...(cwd ? { cwd } : {}), - stdin: `${CLAUDE_MODEL_LIST_STDIN}${JSON.stringify({ type: 'control_request', request_id: 'orca-model-settings', request: { subtype: 'get_settings' } })}\n` + stdin: `${CLAUDE_MODEL_LIST_STDIN}${JSON.stringify({ type: 'control_request', request_id: 'orca-model-settings', request: { subtype: 'get_settings' } })}\n${JSON.stringify({ type: 'control_request', request_id: 'orca-signin-discovery', request: { subtype: 'initialize' } })}\n` }, - { site: 'claude-model-catalog-probe', inheritedEnv: env, signal } + { + site: 'claude-model-catalog-probe', + inheritedEnv: env, + signal, + partialOutput: { + timeoutMs: 2_000, + ready: (stdout) => { + const replies = readClaudeControlResponses(stdout) + return ( + listedModels(replies.get('orca-model-discovery')).length > 0 && + replies.has('orca-model-settings') + ) + } + } + } ).catch((error: unknown) => { if (error instanceof AgentModelCatalogProbeError) { return error @@ -196,37 +210,18 @@ export function createClaudeModelCatalogProbe( throw outcome } const stdout = outcome - let models: ListedModel[] = [] - let complete = false - let configured: AgentModelCatalogSuccess['configured'] - for (const line of stdout.split(/\r?\n/)) { - try { - const response = record(record(JSON.parse(line))?.response) - if (response?.subtype !== 'success') { - continue - } - const body = response.response - const listed = response.request_id === 'orca-model-discovery' ? listedModels(body) : [] - if (listed.length > 0) { - models = listed - complete = isCompleteClaudeModelListing(body) - } - if (response.request_id === 'orca-model-settings') { - const applied = record(record(body)?.applied) - if ( - applied && - !Array.isArray(applied) && - ['model', 'effort'].every( - (key) => applied[key] == null || typeof applied[key] === 'string' - ) - ) { - configured = readClaudeSettingsApplied(body) ?? undefined - } - } - } catch { - /* Non-JSON diagnostic output carries no catalog facts. */ - } - } + const replies = readClaudeControlResponses(stdout) + const listing = replies.get('orca-model-discovery') + const models = listedModels(listing) + const settings = replies.get('orca-model-settings') + const applied = record(record(settings)?.applied) + const configured = + applied && + ['model', 'effort'].every( + (key) => applied[key] == null || typeof applied[key] === 'string' + ) + ? (readClaudeSettingsApplied(settings) ?? undefined) + : undefined if (models.length === 0) { throw new Error('claude listed no models') } @@ -234,9 +229,12 @@ export function createClaudeModelCatalogProbe( return { models: wireClaudeModels(models), origin: 'probe', - complete: complete && configured !== undefined, + complete: isCompleteClaudeModelListing(listing) && configured !== undefined, ...(fastModeSupport ? { fastModeSupport } : {}), - ...(configured ? { configured } : {}) + ...(configured ? { configured } : {}), + ...(claudeCatalogSignedOut(replies.get('orca-signin-discovery'), settings, env) + ? { unavailable: { reason: 'notSignedIn', account } } + : {}) } } } @@ -257,3 +255,34 @@ export function createClaudeModelCatalogProbe( { resolve, resolveLegacy: (home, operation) => resolve(home, undefined, 'recover', operation) } ) } + +function claudeCatalogSignedOut( + initialization: unknown, + settings: unknown, + env: Readonly> +): boolean { + const account = record(record(initialization)?.account) + const effective = record(record(settings)?.effective) + if ( + !account || + !effective || + account.tokenSource !== 'none' || + (account.apiProvider !== undefined && account.apiProvider !== 'firstParty') || + (account.apiKeySource && account.apiKeySource !== 'none') || + account.email || + account.subscriptionType + ) { + return false + } + const configuredEnv = record(effective.env) + const endpoint = configuredEnv?.ANTHROPIC_BASE_URL ?? env.ANTHROPIC_BASE_URL + if (typeof endpoint === 'string' && endpoint.trim()) { + try { + const url = new URL(endpoint) + return url.protocol === 'https:' && url.hostname === 'api.anthropic.com' + } catch { + return false + } + } + return true +} diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts index 79b78ee3e9e6..a7b51036dd93 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts @@ -37,6 +37,26 @@ function isAlive(pid: number): boolean { const NODE_ENV = { ELECTRON_RUN_AS_NODE: '1' } describe('the shared catalog probe runner', () => { + it('returns required replies and stops the child when an optional response stalls', async () => { + const file = pidFile() + const stdout = await runAgentModelCatalogListing( + { + ...node( + `require('fs').writeFileSync(${JSON.stringify(file)}, String(process.pid));process.stdout.write('models\\nsettings\\n');setInterval(()=>{},1000)` + ), + env: NODE_ENV + }, + { + site: 'test-partial-listing', + timeoutMs: 5_000, + partialOutput: { ready: (output) => output.includes('settings\n'), timeoutMs: 50 } + } + ) + expect(stdout).toBe('models\nsettings\n') + const pid = Number(readFileSync(file, 'utf8')) + await vi.waitFor(() => expect(isAlive(pid)).toBe(false), { timeout: 5_000 }) + }, 20_000) + it('answers a listing’s stdout once it exits 0, with its request on stdin', async () => { const stdout = await runAgentModelCatalogListing( { diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.ts index be9f488cd58e..26677b02a6d1 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.ts @@ -58,6 +58,8 @@ export type AgentModelCatalogRunnerOptions = { spawnImpl?: PipedProcessSpawner platform?: NodeJS.Platform inheritedEnv?: NodeJS.ProcessEnv + /** Finish required replies even when an optional control request never completes. */ + partialOutput?: { ready: (stdout: string) => boolean; timeoutMs: number } } /** Bound the listing and await child teardown before success, failure or cancellation settles. */ @@ -83,6 +85,7 @@ export async function runAgentModelCatalogListing( const stderrDetail = (): string => providerStderrForDisplay(managed.stderrTail()).trim().slice(0, STDERR_DETAIL_MAX_CHARS) let timer: ReturnType | undefined + let partialTimer: ReturnType | undefined let onAbort = (): void => {} try { return await new Promise((resolve, reject) => { @@ -109,6 +112,8 @@ export async function runAgentModelCatalogListing( output.write(chunk) if (output.truncated()) { fail(`${launch.command} listed more than the model catalog reads`, 'output-overflow') + } else if (partialTimer === undefined && options.partialOutput?.ready(output.text())) { + partialTimer = setTimeout(() => resolve(output.text()), options.partialOutput.timeoutMs) } }) child.stdout.on('error', () => {}) @@ -142,6 +147,7 @@ export async function runAgentModelCatalogListing( }) } finally { clearTimeout(timer) + clearTimeout(partialTimer) options.signal?.removeEventListener('abort', onAbort) await managed.close() } diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts index 151e3fe0905b..b028ed4c8e50 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts @@ -3,6 +3,87 @@ import { createAgentModelCatalogService } from './agent-model-catalog-service' import { AgentModelCatalogStore, type AgentModelCatalogSuccess } from './agent-model-catalog-store' import type { AgentModelCatalogProbe } from './agent-model-catalog-discovery' import type { AgentModelCatalogReadParams } from './agent-model-catalog-service-contract' +import { createClaudeModelCatalogProbe } from '../../claude/claude-model-catalog-probe' +import { createAcpModelCatalogProbe } from '../../acp/acp-model-catalog-probe' +import { acpLaunchSpecFor } from '../../acp/acp-launch-specs' +import type { runAgentModelCatalogListing } from './agent-model-catalog-probe-runner' +import claudeCapture from '../../claude/__fixtures__/catalog-signin.json' +import grokCapture from '../../acp/__fixtures__/grok-catalog-signin.json' + +it('shares one bounded sign-in observation across N and 10N catalog demands', async () => { + for (const callers of [10, 100]) { + for (const agent of ['claude', 'grok'] as const) { + const runListing = vi.fn(async () => { + if (agent === 'grok') { + return grokCapture.models + } + return [ + ['orca-model-discovery', claudeCapture.models], + ['orca-model-settings', claudeCapture.settings], + ['orca-signin-discovery', { account: claudeCapture.accounts.signedOut }] + ] + .map(([request_id, response]) => + JSON.stringify({ + type: 'control_response', + response: { subtype: 'success', request_id, response } + }) + ) + .join('\n') + }) + const initialize = vi.fn(async () => grokCapture.initialize) + const close = vi.fn(async () => undefined) + const extension = vi.fn(async () => ({})) + const probe = + agent === 'claude' + ? createClaudeModelCatalogProbe({ + resolveInheritedEnv: async () => ({ PATH: '/usr/bin' }), + resolveEnv: () => ({}), + resolveCommand: () => ({ + command: process.execPath, + prefixArgs: ['run', '--', 'claude'] + }), + resolveAuthPolicy: () => ({ account: 'system' }), + runListing + }) + : createAcpModelCatalogProbe(acpLaunchSpecFor('grok')!, { + resolveEnvironment: async () => ({ PATH: '/usr/bin' }), + resolveCommand: () => process.execPath, + dataDirectory: '/orca-data', + homePath: '/home/test', + runListing, + connect: () => ({ initialize, close, requestSessionFreeExtension: extension }) + }) + const service = createAgentModelCatalogService({ + store: new AgentModelCatalogStore(), + getConversation: () => undefined, + getRecord: () => undefined, + drivesRecord: () => true, + resolveAccountHome: async () => ({ + variable: agent === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'GROK_HOME', + path: '/account' + }), + probes: { [agent]: probe } + }) + const request = { agent, workspacePath: '/project' } + try { + await Promise.all(Array.from({ length: callers }, () => service.demand(request))) + for (let index = 0; index < callers; index++) { + service.snapshot(request) + } + expect(runListing).toHaveBeenCalledTimes(1) + expect(initialize).toHaveBeenCalledTimes(agent === 'grok' ? 1 : 0) + expect(extension).not.toHaveBeenCalled() + expect(close).toHaveBeenCalledTimes(agent === 'grok' ? 1 : 0) + expect(service.snapshot(request)).toMatchObject({ unavailable: { reason: 'notSignedIn' } }) + if (agent === 'claude') { + expect(runListing.mock.calls[0]?.[0].stdin?.trim().split('\n')).toHaveLength(3) + } + } finally { + service.stop() + } + } + } +}) it('catalog-scale retires a many-scope frontier and releases actual listing waiters before recovery', async () => { for (const scopes of [5, 50]) { diff --git a/src/main/pi/rpc-model-catalog-probe.test.ts b/src/main/pi/rpc-model-catalog-probe.test.ts index 78d984bacb60..bc3e66a72fa2 100644 --- a/src/main/pi/rpc-model-catalog-probe.test.ts +++ b/src/main/pi/rpc-model-catalog-probe.test.ts @@ -19,6 +19,8 @@ import { AgentModelCatalogStore } from '../native-chat/agent-model-catalog/agent-model-catalog-store' import * as probeRunner from '../native-chat/agent-model-catalog/agent-model-catalog-probe-runner' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' const AVAILABLE_MODELS = { models: [ @@ -77,6 +79,34 @@ function catalog(request: PiRpcRequester['request'], now = Date.now) { } describe('Pi model catalog', () => { + it('projects the recorded signed-out sentinel independently of model inventory', async () => { + const frames = readFileSync(join(import.meta.dirname, '__fixtures__/signed-out.jsonl'), 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .filter((frame) => frame.dir === 'out') + .map((frame) => JSON.parse(frame.raw)) + const state = frames.find((frame) => frame.command === 'get_state').data + const empty = frames.find((frame) => frame.command === 'get_available_models').data + for (const inventory of [empty, AVAILABLE_MODELS]) { + const { probe, record } = catalog(async (command) => + command === 'get_state' ? state : inventory + ) + if (inventory === empty) { + await expect(probe(record.accountHome)).rejects.toMatchObject({ + unavailable: { reason: 'notSignedIn' } + }) + } else { + const result = await probe(record.accountHome) + expect(result.unavailable).toEqual({ reason: 'notSignedIn' }) + expect(result.models.length).toBeGreaterThan(0) + expect(result.configured).toBeUndefined() + } + } + const { probe, record } = catalog(async (command) => (command === 'get_state' ? {} : empty)) + await expect(probe(record.accountHome)).rejects.toThrow('pi listed no models') + }) + it.each(['', `"${process.execPath}" run -- pi`])( 'lists the native menu through sessionless RPC for Command %j', async (command) => { diff --git a/src/main/pi/rpc-model-catalog-probe.ts b/src/main/pi/rpc-model-catalog-probe.ts index 50ad681edc88..8b641cf29d6b 100644 --- a/src/main/pi/rpc-model-catalog-probe.ts +++ b/src/main/pi/rpc-model-catalog-probe.ts @@ -13,6 +13,7 @@ import type { AgentModelCatalogSuccess } from '../native-chat/agent-model-catalog/agent-model-catalog-store' import { runAgentModelCatalogSession } from '../native-chat/agent-model-catalog/agent-model-catalog-probe-runner' +import { AgentModelCatalogUnavailableError } from '../native-chat/agent-model-catalog/agent-model-catalog-unavailable' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' import { buildPiRpcLaunch, piRpcSessionlessEnvironment } from './rpc-launch' import { checkPiRpcVersion, resolvePiRpcInvocation } from './rpc-launch-resolution' @@ -89,9 +90,11 @@ export function createPiModelCatalogProbe(deps: PiModelCatalogProbeDeps): AgentM }), async (connection): Promise => { let configured: AgentModelCatalogSuccess['configured'] + let signedOut = false try { const state = discoveryStateSchema.safeParse(await connection.request('get_state')) - if (state.success && state.data.model) { + signedOut = state.success && state.data.model?.provider === 'unknown' + if (state.success && state.data.model && !signedOut) { configured = { model: piModelOptionId(state.data.model.provider, state.data.model.id), ...(typeof state.data.thinkingLevel === 'string' && state.data.thinkingLevel @@ -104,8 +107,21 @@ export function createPiModelCatalogProbe(deps: PiModelCatalogProbeDeps): AgentM } assertStructuredAgentResolutionCurrent(operation) options?.signal?.throwIfAborted() - const models = await readPiRpcModelOptions(connection) - return { models, origin: 'probe', ...(configured ? { configured } : {}) } + const models = await readPiRpcModelOptions(connection).catch((error: unknown) => { + if (signedOut) { + throw new AgentModelCatalogUnavailableError({ reason: 'notSignedIn' }) + } + throw error + }) + if (signedOut && models.length === 0) { + throw new AgentModelCatalogUnavailableError({ reason: 'notSignedIn' }) + } + return { + models, + origin: 'probe', + ...(configured ? { configured } : {}), + ...(signedOut ? { unavailable: { reason: 'notSignedIn' } as const } : {}) + } }, { label: 'pi', signal: options?.signal } ) diff --git a/src/shared/grok-model-list-probe.ts b/src/shared/grok-model-list-probe.ts index f1392fa79d5e..fe8b6f767f90 100644 --- a/src/shared/grok-model-list-probe.ts +++ b/src/shared/grok-model-list-probe.ts @@ -10,6 +10,10 @@ const MODEL_BULLET = /^\s*[*-]\s+([^\s(]+)(.*)$/ // Read markers from listed rows so preamble defaults cannot name unavailable IDs. const DEFAULT_MARKER = /\(default\)/ +export function grokModelListSignedOut(stdout: string): boolean { + return stdout.split(/\r?\n/).some((line) => line.trim() === 'You are not authenticated.') +} + export function parseGrokModelList(stdout: string): CommitMessageModel[] { const lines = stdout.split(/\r?\n/) // Require the header to exclude default/login preambles that resemble model IDs. From 28479935adad65b2e2b5a1e1571a0b930ed7716c Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:02:17 -0800 Subject: [PATCH 2/7] Drop the Grok status process; keep listed models when Pi or Claude replies arrive late Grok goes back to one listing process with no Grok-specific sign-out guess; an agent-authored auth-required reply still reports notSignedIn. Pi reports notSignedIn only when it has no provider and nothing listable. Once Claude's required replies are in, the listing deadline returns them instead of failing. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../acp/__fixtures__/grok-catalog-signin.json | 134 ------------------ .../session-new-auth-required.json | 14 ++ src/main/acp/acp-launch-specs.ts | 4 - src/main/acp/acp-model-catalog-probe.test.ts | 56 ++------ src/main/acp/acp-model-catalog-probe.ts | 36 +---- .../claude/__fixtures__/catalog-signin.json | 43 +++++- .../claude/claude-model-catalog-probe.test.ts | 21 +++ .../agent-model-catalog-probe-runner.test.ts | 15 ++ .../agent-model-catalog-probe-runner.ts | 11 +- .../agent-model-catalog-scaling.test.ts | 113 ++++++--------- src/main/pi/rpc-model-catalog-probe.test.ts | 27 ++-- src/main/pi/rpc-model-catalog-probe.ts | 18 +-- src/shared/grok-model-list-probe.ts | 4 - 13 files changed, 176 insertions(+), 320 deletions(-) delete mode 100644 src/main/acp/__fixtures__/grok-catalog-signin.json create mode 100644 src/main/acp/__fixtures__/session-new-auth-required.json diff --git a/src/main/acp/__fixtures__/grok-catalog-signin.json b/src/main/acp/__fixtures__/grok-catalog-signin.json deleted file mode 100644 index bcad83d1d35b..000000000000 --- a/src/main/acp/__fixtures__/grok-catalog-signin.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "version": "1.0.70", - "initialize": { - "protocolVersion": 1, - "agentCapabilities": { - "loadSession": true, - "promptCapabilities": { - "image": true, - "audio": false, - "embeddedContext": true - }, - "mcpCapabilities": { - "http": true, - "sse": true - }, - "sessionCapabilities": { - "list": {}, - "resume": {}, - "close": {} - }, - "auth": {}, - "_meta": { - "x.ai/fs_notify": true, - "x.ai/hooks": { - "blockingEvents": ["pre_tool_use", "stop", "subagent_stop"], - "decisions": ["deny", "block"], - "stopSignals": ["continue", "stopReason", "additionalContext"] - }, - "x.ai/capabilities": { - "toolOverrides": { - "x_keyword_search": true, - "x_semantic_search": true, - "x_user_search": false, - "x_thread_fetch": false - } - } - } - }, - "authMethods": [ - { - "id": "grok.com", - "name": "Grok", - "description": "Sign in with Grok" - } - ], - "_meta": { - "modelState": { - "currentModelId": "grok-4.6", - "availableModels": [ - { - "modelId": "grok-4.6", - "name": "Grok 4.6", - "description": "SpaceXAI's latest frontier model", - "_meta": { - "totalContextTokens": 500000, - "agentType": "grok-build-plan", - "supportsReasoningEffort": true, - "reasoningEffort": "high", - "reasoningEfforts": [ - { - "id": "xhigh", - "value": "xhigh", - "label": "Extra High Effort", - "description": "Highest effort and reasoning level", - "default": false - }, - { - "id": "high", - "value": "high", - "label": "High Effort", - "description": "Higher implementation quality with extensive reasoning", - "default": true - }, - { - "id": "medium", - "value": "medium", - "label": "Medium Effort", - "description": "Balanced effort with standard implementation and testing", - "default": false - }, - { - "id": "low", - "value": "low", - "label": "Low Effort", - "description": "Quick, fast implementations", - "default": false - } - ] - } - }, - { - "modelId": "grok-4.5", - "name": "Grok 4.5", - "_meta": { - "totalContextTokens": 500000, - "agentType": "grok-build-plan", - "supportsReasoningEffort": true, - "reasoningEffort": "high", - "reasoningEfforts": [ - { - "id": "high", - "value": "high", - "label": "High Effort", - "description": "Highest implementation quality with extensive reasoning", - "default": true - }, - { - "id": "medium", - "value": "medium", - "label": "Medium Effort", - "description": "Balanced effort with standard implementation and testing", - "default": false - }, - { - "id": "low", - "value": "low", - "label": "Low Effort", - "description": "Quick, fast implementations", - "default": false - } - ] - } - } - ] - } - } - }, - "authError": { - "code": -32000, - "message": "Authentication required", - "data": "no auth method id provided" - }, - "models": "You are not authenticated.\n\nDefault model: grok-4.6\n\nAvailable models:\n * grok-4.6 (default)\n - grok-4.5\n" -} diff --git a/src/main/acp/__fixtures__/session-new-auth-required.json b/src/main/acp/__fixtures__/session-new-auth-required.json new file mode 100644 index 000000000000..a3d956b0da64 --- /dev/null +++ b/src/main/acp/__fixtures__/session-new-auth-required.json @@ -0,0 +1,14 @@ +{ + "agent": "grok", + "version": "1.0.70", + "method": "session/new", + "reply": { + "jsonrpc": "2.0", + "id": 2, + "error": { + "code": -32000, + "message": "Authentication required", + "data": "no auth method id provided" + } + } +} diff --git a/src/main/acp/acp-launch-specs.ts b/src/main/acp/acp-launch-specs.ts index ee0151166635..5c3e4a861764 100644 --- a/src/main/acp/acp-launch-specs.ts +++ b/src/main/acp/acp-launch-specs.ts @@ -15,7 +15,6 @@ import type { InitializeResponse } from './generated/acp-protocol.generated' import { compareAppVersions } from '../../shared/app-version' import type { ManagedRuntime } from '../node-runtime/managed-runtime-release' import { readGrokModelCatalog } from './acp-dialects/grok-model-catalog' -import { GROK_MODEL_LIST_ARGS, grokModelListSignedOut } from '../../shared/grok-model-list-probe' import { OPENCODE_MODEL_LISTING_ARGS, parseOpenCodeModelListing @@ -66,8 +65,6 @@ export type AcpModelDiscovery = } ): Promise listingNamesConfiguredModel: boolean - /** Read after initialize, which may silently refresh credentials. */ - signInStatus?: { args: readonly string[]; signedOut(stdout: string): boolean } } | { kind: 'command' @@ -169,7 +166,6 @@ const GROK_LAUNCH_SPEC: AcpLaunchSpec = { modelDiscovery: { kind: 'initialize', read: readGrokModelCatalog, - signInStatus: { args: GROK_MODEL_LIST_ARGS, signedOut: grokModelListSignedOut }, listingNamesConfiguredModel: false }, visualsSkill: loadGrokVisualsSkill, diff --git a/src/main/acp/acp-model-catalog-probe.test.ts b/src/main/acp/acp-model-catalog-probe.test.ts index 89d5902e7e32..af24de5452b9 100644 --- a/src/main/acp/acp-model-catalog-probe.test.ts +++ b/src/main/acp/acp-model-catalog-probe.test.ts @@ -12,7 +12,7 @@ import { AcpScriptedAgent } from './acp-scripted-agent.test-support' import { AgentModelCatalogListingStoppedError } from '../native-chat/agent-model-catalog/agent-model-catalog-failures' import type { ProviderProcessLaunch } from '../provider-process/provider-process-launch' import { openCodeAcpAccountBinding } from '../opencode/opencode-structured-account-home' -import signinCapture from './__fixtures__/grok-catalog-signin.json' +import authRequired from './__fixtures__/session-new-auth-required.json' const PREFIX = ['run', '--', 'agent'] const launcherSettings = (agent: string) => ({ @@ -83,7 +83,6 @@ function scriptedGrok(initializeMeta: Record): { resolveLaunchEnv: () => ({ XAI_API_KEY: 'from-settings' }), resolveCommand: () => '/opt/grok/bin/grok', homePath: '/home/user', - runListing: vi.fn(async () => ''), connect: (launch) => { launches.push(launch) const runtime = new AcpSessionRuntime(agent.stdout, agent.stdin) @@ -169,47 +168,18 @@ function scriptedOpenCode2(canDelete: boolean): { } describe('ACP model catalog probes', () => { - it('retains Grok models with its recorded sign-out banner after initialize, without creating a session', async () => { - const { deps, agent, launches, closed } = scriptedGrok({}) - agent.on('initialize', (frame) => agent.reply(frame, signinCapture.initialize)) - deps.resolveCommandSettings = () => launcherSettings('grok') - const runListing = vi.fn>(async () => { - expect(agent.frames.map((frame) => frame.method)).toEqual(['initialize']) - return signinCapture.models - }) - deps.runListing = runListing - const success = await createAcpModelCatalogProbe(GROK, deps)(GROK_HOME) - expect(success.unavailable).toEqual({ reason: 'notSignedIn' }) - expect(success.models.map((model) => model.id)).toContain('grok-4.6') - expect(runListing).toHaveBeenCalledExactlyOnceWith( - { ...launches[0], args: [...PREFIX, 'models'] }, - expect.objectContaining({ timeoutMs: 2_000 }) - ) - expect(agent.frames.map((frame) => frame.method)).toEqual(['initialize']) - expect(closed()).toBe(1) - }) - - it('keeps usable Grok inventory when the optional status command fails', async () => { - const { deps } = scriptedGrok({ modelState: MODEL_STATE }) - deps.runListing = async () => { - throw new Error('unsupported command') - } - const result = await createAcpModelCatalogProbe(GROK, deps)(GROK_HOME) - expect(result.models.length).toBeGreaterThan(0) - expect(result.unavailable).toBeUndefined() - }) - - it('projects an agent-authored auth-required error, retaining unknown for other failures', async () => { - const { deps, agent } = scriptedGrok({}) - const { code, message, data } = signinCapture.authError - agent.on('x.ai/models/list', (frame) => agent.fail(frame, code, message, data)) - await expect(createAcpModelCatalogProbe(GROK, deps)(GROK_HOME)).rejects.toMatchObject({ - unavailable: { reason: 'notSignedIn' } - }) - agent.on('x.ai/models/list', (frame) => agent.fail(frame, -32603, 'Internal error')) - await expect(createAcpModelCatalogProbe(GROK, deps)(GROK_HOME)).rejects.toMatchObject({ - code: -32603 - }) + it('projects an agent-authored auth-required session/new error, retaining unknown for other failures', async () => { + const { agent, deps } = scriptedOpenCode2(true) + const { code, message, data } = authRequired.reply.error + agent.on('session/new', (frame) => agent.fail(frame, code, message, data)) + const list = () => + createAcpModelCatalogProbe(OPENCODE, deps)( + { kind: 'opencode', locator: { kind: 'unmanaged' } }, + { workspacePath: '/work/project' } + ) + await expect(list()).rejects.toMatchObject({ unavailable: { reason: 'notSignedIn' } }) + agent.on('session/new', (frame) => agent.fail(frame, -32603, 'Internal error')) + await expect(list()).rejects.toMatchObject({ code: -32603 }) }) it('lists Grok from initialize alone, under its pinned account, and closes the child', async () => { diff --git a/src/main/acp/acp-model-catalog-probe.ts b/src/main/acp/acp-model-catalog-probe.ts index 02b8ed5b0234..9786870d84fc 100644 --- a/src/main/acp/acp-model-catalog-probe.ts +++ b/src/main/acp/acp-model-catalog-probe.ts @@ -5,10 +5,7 @@ import { import { homedir } from 'node:os' import { join } from 'node:path' import type { AgentCliVersionReader } from '../agent-cli-version-probe' -import type { - AgentModelCatalogProbe, - AgentModelCatalogSuccess -} from '../native-chat/agent-model-catalog/agent-model-catalog-store' +import type { AgentModelCatalogProbe } from '../native-chat/agent-model-catalog/agent-model-catalog-store' import { runAgentModelCatalogListing, runAgentModelCatalogSession @@ -109,7 +106,6 @@ export function createAcpModelCatalogProbe( } const run = { label: spec.agent, signal: options?.signal } let models: AgentSessionModelOption[] - let unavailable: AgentModelCatalogSuccess['unavailable'] if (listing.kind === 'command') { const stdout = await (deps.runListing ?? runAgentModelCatalogListing)( { @@ -149,41 +145,15 @@ export function createAcpModelCatalogProbe( async (connection) => { const initialized = await connection.initialize() assertStructuredAgentResolutionCurrent(operation) - const listed = listing.read(initialized, connection).then( - (models) => ({ models }), - (error: unknown) => ({ error }) - ) - const signedOut = listing.signInStatus - ? await (deps.runListing ?? runAgentModelCatalogListing)( - { ...launch, args: [...prefixArgs, ...listing.signInStatus.args] }, - { site: `${spec.agent}-signin-probe`, signal: options?.signal, timeoutMs: 2_000 } - ) - .then(listing.signInStatus.signedOut) - .catch(() => false) - : false - assertStructuredAgentResolutionCurrent(operation) - const result = await listed - if (signedOut) { - unavailable = { reason: 'notSignedIn' } - } - if ('error' in result) { - if (unavailable) { - throw new AgentModelCatalogUnavailableError(unavailable) - } - throw result.error - } - return result.models + return listing.read(initialized, connection) }, run ) } if (models.length === 0) { - if (unavailable) { - throw new AgentModelCatalogUnavailableError(unavailable) - } throw new Error(`${spec.agent} listed no models`) } - return { models, origin: 'probe', ...(unavailable ? { unavailable } : {}) } + return { models, origin: 'probe' } } return async (home, options) => { try { diff --git a/src/main/claude/__fixtures__/catalog-signin.json b/src/main/claude/__fixtures__/catalog-signin.json index e9afdd725f5f..2f2950710a0b 100644 --- a/src/main/claude/__fixtures__/catalog-signin.json +++ b/src/main/claude/__fixtures__/catalog-signin.json @@ -24,7 +24,7 @@ "value": "default", "resolvedModel": "claude-opus-5-5", "displayName": "Default (recommended)", - "description": "Use the default model (currently Opus 5.5) \u00b7 $4/$20 per Mtok", + "description": "Use the default model (currently Opus 5.5) · $4/$20 per Mtok", "supportsEffort": true, "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], "supportsAdaptiveThinking": true, @@ -36,7 +36,7 @@ "value": "opus", "resolvedModel": "claude-opus-5-5", "displayName": "Opus", - "description": "Opus 5.5 \u00b7 Best for everyday, complex tasks \u00b7 $4/$20 per Mtok", + "description": "Opus 5.5 · Best for everyday, complex tasks · $4/$20 per Mtok", "supportsEffort": true, "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], "supportsAdaptiveThinking": true, @@ -48,7 +48,7 @@ "value": "fable", "resolvedModel": "claude-fable-5-1", "displayName": "Fable", - "description": "Fable 5.1 \u00b7 Most capable for your hardest and longest-running tasks \u00b7 $10/$50 per Mtok", + "description": "Fable 5.1 · Most capable for your hardest and longest-running tasks · $10/$50 per Mtok", "supportsEffort": true, "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], "supportsAdaptiveThinking": true, @@ -59,7 +59,7 @@ "value": "sonnet", "resolvedModel": "claude-sonnet-5-5", "displayName": "Sonnet", - "description": "Sonnet 5.5 \u00b7 Efficient for routine tasks \u00b7 $2/$10 per Mtok", + "description": "Sonnet 5.5 · Efficient for routine tasks · $2/$10 per Mtok", "supportsEffort": true, "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], "supportsAdaptiveThinking": true, @@ -70,7 +70,7 @@ "value": "haiku", "resolvedModel": "claude-haiku-5-5", "displayName": "Haiku", - "description": "Haiku 5.5 \u00b7 Fastest for quick answers \u00b7 $0.10/$0.50 per Mtok ($0.50/$2.50 for prompts over 100k)", + "description": "Haiku 5.5 · Fastest for quick answers · $0.10/$0.50 per Mtok ($0.50/$2.50 for prompts over 100k)", "supportsEffort": true, "supportedEffortLevels": ["low", "medium", "high", "xhigh", "max"], "supportsAdaptiveThinking": true, @@ -139,5 +139,38 @@ "ultracodeRequested": false, "ultracodeAvailable": true } + }, + "commandFrames": { + "error": { + "type": "control_response", + "response": { + "subtype": "error", + "request_id": "orca-command-discovery", + "error": "Unsupported control request subtype: orca_unknown_subtype" + } + }, + "initialize": { + "type": "control_response", + "response": { + "subtype": "success", + "request_id": "orca-command-discovery", + "response": { + "commands": [ + { + "name": "deep-research", + "description": "Deep research harness — fan-out web searches, fetch sources, adversarially verify claims, synthesize a cited report. (dynamic workflow)", + "argumentHint": "", + "builtin": true + }, + { + "name": "design", + "description": "Grant or revoke Claude agent access to your Design projects", + "argumentHint": "consent | revoke", + "builtin": true + } + ] + } + } + } } } diff --git a/src/main/claude/claude-model-catalog-probe.test.ts b/src/main/claude/claude-model-catalog-probe.test.ts index e231c33439d4..463e1e569fd7 100644 --- a/src/main/claude/claude-model-catalog-probe.test.ts +++ b/src/main/claude/claude-model-catalog-probe.test.ts @@ -6,6 +6,7 @@ import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { PROVIDER_STDIN_END_GRACE_MS } from '../provider-process/provider-process-supervisor' import { createClaudeModelCatalogProbe } from './claude-model-catalog-probe' +import { createClaudeCommandCatalogProbe } from './claude-command-catalog-probe' import signinCapture from './__fixtures__/catalog-signin.json' import { discoveredModel } from '../../shared/agent-session/options/structured-agent-session-discovered-model' import { resolveClaudeStructuredInvocation } from './structured/launch/claude-structured-launch-resolution' @@ -425,3 +426,23 @@ it('requires definitive first-party account facts, preserving inventory when aut expect(success.models.length).toBeGreaterThan(0) } }) + +it('reads slash commands from the real initialize frame past error, diagnostic and oversized lines', async () => { + const { error, initialize } = signinCapture.commandFrames + const nested = (depth: number): unknown => + depth === 0 ? 'orca-command-discovery' : { response: nested(depth - 1) } + const oversized = JSON.stringify({ type: 'control_response', response: nested(20) }) + const list = (lines: string[]) => + createClaudeCommandCatalogProbe({ + ...probeDeps(), + runListing: async () => `${lines.join('\n')}\n` + })({ cwd: '/project', accountHomePath: '/homes/account-a' }) + const rejected = [ + JSON.stringify(error), + 'orca-command-discovery control_response not ready', + oversized + ] + await expect(list(rejected)).rejects.toThrow('claude listed no commands') + const commands = await list([...rejected, JSON.stringify(initialize)]) + expect(commands.map((command) => command.name)).toEqual(['deep-research', 'design']) +}) diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts index a7b51036dd93..ac7f6d094ebc 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts @@ -57,6 +57,21 @@ describe('the shared catalog probe runner', () => { await vi.waitFor(() => expect(isAlive(pid)).toBe(false), { timeout: 5_000 }) }, 20_000) + it('keeps required replies when the listing deadline lands inside the optional grace', async () => { + const stdout = await runAgentModelCatalogListing( + { + ...node(`process.stdout.write('models\\nsettings\\n');setInterval(()=>{},1000)`), + env: NODE_ENV + }, + { + site: 'test-partial-deadline', + timeoutMs: 3_000, + partialOutput: { ready: (output) => output.includes('settings\n'), timeoutMs: 60_000 } + } + ) + expect(stdout).toBe('models\nsettings\n') + }, 20_000) + it('answers a listing’s stdout once it exits 0, with its request on stdin', async () => { const stdout = await runAgentModelCatalogListing( { diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.ts index 26677b02a6d1..869eb3ca18cc 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.ts @@ -102,16 +102,23 @@ export async function runAgentModelCatalogListing( failedExit ) ) + // Required replies already in hand outlive an optional reply that runs out of time or room. + const failUnlessReady = (message: string, reason: AgentModelCatalogProbeFailure): void => + partialTimer === undefined ? fail(message, reason) : resolve(output.text()) onAbort = () => fail(`${launch.command} listing stopped`, 'stopped') options.signal?.addEventListener('abort', onAbort, { once: true }) timer = setTimeout( - () => fail(`${launch.command} did not list models within ${timeoutMs}ms`, 'timeout'), + () => + failUnlessReady(`${launch.command} did not list models within ${timeoutMs}ms`, 'timeout'), timeoutMs ) child.stdout.on('data', (chunk: Buffer | string) => { output.write(chunk) if (output.truncated()) { - fail(`${launch.command} listed more than the model catalog reads`, 'output-overflow') + failUnlessReady( + `${launch.command} listed more than the model catalog reads`, + 'output-overflow' + ) } else if (partialTimer === undefined && options.partialOutput?.ready(output.text())) { partialTimer = setTimeout(() => resolve(output.text()), options.partialOutput.timeoutMs) } diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts index b028ed4c8e50..ae848ed09b99 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts @@ -4,83 +4,54 @@ import { AgentModelCatalogStore, type AgentModelCatalogSuccess } from './agent-m import type { AgentModelCatalogProbe } from './agent-model-catalog-discovery' import type { AgentModelCatalogReadParams } from './agent-model-catalog-service-contract' import { createClaudeModelCatalogProbe } from '../../claude/claude-model-catalog-probe' -import { createAcpModelCatalogProbe } from '../../acp/acp-model-catalog-probe' -import { acpLaunchSpecFor } from '../../acp/acp-launch-specs' import type { runAgentModelCatalogListing } from './agent-model-catalog-probe-runner' import claudeCapture from '../../claude/__fixtures__/catalog-signin.json' -import grokCapture from '../../acp/__fixtures__/grok-catalog-signin.json' it('shares one bounded sign-in observation across N and 10N catalog demands', async () => { for (const callers of [10, 100]) { - for (const agent of ['claude', 'grok'] as const) { - const runListing = vi.fn(async () => { - if (agent === 'grok') { - return grokCapture.models - } - return [ - ['orca-model-discovery', claudeCapture.models], - ['orca-model-settings', claudeCapture.settings], - ['orca-signin-discovery', { account: claudeCapture.accounts.signedOut }] - ] - .map(([request_id, response]) => - JSON.stringify({ - type: 'control_response', - response: { subtype: 'success', request_id, response } - }) - ) - .join('\n') - }) - const initialize = vi.fn(async () => grokCapture.initialize) - const close = vi.fn(async () => undefined) - const extension = vi.fn(async () => ({})) - const probe = - agent === 'claude' - ? createClaudeModelCatalogProbe({ - resolveInheritedEnv: async () => ({ PATH: '/usr/bin' }), - resolveEnv: () => ({}), - resolveCommand: () => ({ - command: process.execPath, - prefixArgs: ['run', '--', 'claude'] - }), - resolveAuthPolicy: () => ({ account: 'system' }), - runListing - }) - : createAcpModelCatalogProbe(acpLaunchSpecFor('grok')!, { - resolveEnvironment: async () => ({ PATH: '/usr/bin' }), - resolveCommand: () => process.execPath, - dataDirectory: '/orca-data', - homePath: '/home/test', - runListing, - connect: () => ({ initialize, close, requestSessionFreeExtension: extension }) - }) - const service = createAgentModelCatalogService({ - store: new AgentModelCatalogStore(), - getConversation: () => undefined, - getRecord: () => undefined, - drivesRecord: () => true, - resolveAccountHome: async () => ({ - variable: agent === 'claude' ? 'CLAUDE_CONFIG_DIR' : 'GROK_HOME', - path: '/account' - }), - probes: { [agent]: probe } - }) - const request = { agent, workspacePath: '/project' } - try { - await Promise.all(Array.from({ length: callers }, () => service.demand(request))) - for (let index = 0; index < callers; index++) { - service.snapshot(request) - } - expect(runListing).toHaveBeenCalledTimes(1) - expect(initialize).toHaveBeenCalledTimes(agent === 'grok' ? 1 : 0) - expect(extension).not.toHaveBeenCalled() - expect(close).toHaveBeenCalledTimes(agent === 'grok' ? 1 : 0) - expect(service.snapshot(request)).toMatchObject({ unavailable: { reason: 'notSignedIn' } }) - if (agent === 'claude') { - expect(runListing.mock.calls[0]?.[0].stdin?.trim().split('\n')).toHaveLength(3) - } - } finally { - service.stop() + const runListing = vi.fn(async () => + [ + ['orca-model-discovery', claudeCapture.models], + ['orca-model-settings', claudeCapture.settings], + ['orca-signin-discovery', { account: claudeCapture.accounts.signedOut }] + ] + .map(([request_id, response]) => + JSON.stringify({ + type: 'control_response', + response: { subtype: 'success', request_id, response } + }) + ) + .join('\n') + ) + const probe = createClaudeModelCatalogProbe({ + resolveInheritedEnv: async () => ({ PATH: '/usr/bin' }), + resolveEnv: () => ({}), + resolveCommand: () => ({ + command: process.execPath, + prefixArgs: ['run', '--', 'claude'] + }), + resolveAuthPolicy: () => ({ account: 'system' }), + runListing + }) + const service = createAgentModelCatalogService({ + store: new AgentModelCatalogStore(), + getConversation: () => undefined, + getRecord: () => undefined, + drivesRecord: () => true, + resolveAccountHome: async () => ({ variable: 'CLAUDE_CONFIG_DIR', path: '/account' }), + probes: { claude: probe } + }) + const request = { agent: 'claude', workspacePath: '/project' } as const + try { + await Promise.all(Array.from({ length: callers }, () => service.demand(request))) + for (let index = 0; index < callers; index++) { + service.snapshot(request) } + expect(runListing).toHaveBeenCalledTimes(1) + expect(service.snapshot(request)).toMatchObject({ unavailable: { reason: 'notSignedIn' } }) + expect(runListing.mock.calls[0]?.[0].stdin?.trim().split('\n')).toHaveLength(3) + } finally { + service.stop() } } }) diff --git a/src/main/pi/rpc-model-catalog-probe.test.ts b/src/main/pi/rpc-model-catalog-probe.test.ts index bc3e66a72fa2..8aec026daaad 100644 --- a/src/main/pi/rpc-model-catalog-probe.test.ts +++ b/src/main/pi/rpc-model-catalog-probe.test.ts @@ -79,7 +79,7 @@ function catalog(request: PiRpcRequester['request'], now = Date.now) { } describe('Pi model catalog', () => { - it('projects the recorded signed-out sentinel independently of model inventory', async () => { + it('projects the recorded signed-out sentinel only when nothing is listable', async () => { const frames = readFileSync(join(import.meta.dirname, '__fixtures__/signed-out.jsonl'), 'utf8') .trim() .split('\n') @@ -88,21 +88,22 @@ describe('Pi model catalog', () => { .map((frame) => JSON.parse(frame.raw)) const state = frames.find((frame) => frame.command === 'get_state').data const empty = frames.find((frame) => frame.command === 'get_available_models').data - for (const inventory of [empty, AVAILABLE_MODELS]) { + const closed = async () => { + throw new Error('rpc closed') + } + for (const read of [async () => empty, closed]) { const { probe, record } = catalog(async (command) => - command === 'get_state' ? state : inventory + command === 'get_state' ? state : read() ) - if (inventory === empty) { - await expect(probe(record.accountHome)).rejects.toMatchObject({ - unavailable: { reason: 'notSignedIn' } - }) - } else { - const result = await probe(record.accountHome) - expect(result.unavailable).toEqual({ reason: 'notSignedIn' }) - expect(result.models.length).toBeGreaterThan(0) - expect(result.configured).toBeUndefined() - } + await expect(probe(record.accountHome)).rejects.toMatchObject({ + unavailable: { reason: 'notSignedIn' } + }) } + const listed = catalog(async (command) => (command === 'get_state' ? state : AVAILABLE_MODELS)) + const result = await listed.probe(listed.record.accountHome) + expect(result.unavailable).toBeUndefined() + expect(result.models.length).toBeGreaterThan(0) + expect(result.configured).toBeUndefined() const { probe, record } = catalog(async (command) => (command === 'get_state' ? {} : empty)) await expect(probe(record.accountHome)).rejects.toThrow('pi listed no models') }) diff --git a/src/main/pi/rpc-model-catalog-probe.ts b/src/main/pi/rpc-model-catalog-probe.ts index 8b641cf29d6b..497c400397c7 100644 --- a/src/main/pi/rpc-model-catalog-probe.ts +++ b/src/main/pi/rpc-model-catalog-probe.ts @@ -90,11 +90,11 @@ export function createPiModelCatalogProbe(deps: PiModelCatalogProbeDeps): AgentM }), async (connection): Promise => { let configured: AgentModelCatalogSuccess['configured'] - let signedOut = false + let unknownProvider = false try { const state = discoveryStateSchema.safeParse(await connection.request('get_state')) - signedOut = state.success && state.data.model?.provider === 'unknown' - if (state.success && state.data.model && !signedOut) { + unknownProvider = state.success && state.data.model?.provider === 'unknown' + if (state.success && state.data.model && !unknownProvider) { configured = { model: piModelOptionId(state.data.model.provider, state.data.model.id), ...(typeof state.data.thinkingLevel === 'string' && state.data.thinkingLevel @@ -107,21 +107,17 @@ export function createPiModelCatalogProbe(deps: PiModelCatalogProbeDeps): AgentM } assertStructuredAgentResolutionCurrent(operation) options?.signal?.throwIfAborted() + // Pi's own "No models available" condition: no selected provider and nothing listable. const models = await readPiRpcModelOptions(connection).catch((error: unknown) => { - if (signedOut) { + if (unknownProvider) { throw new AgentModelCatalogUnavailableError({ reason: 'notSignedIn' }) } throw error }) - if (signedOut && models.length === 0) { + if (unknownProvider && models.length === 0) { throw new AgentModelCatalogUnavailableError({ reason: 'notSignedIn' }) } - return { - models, - origin: 'probe', - ...(configured ? { configured } : {}), - ...(signedOut ? { unavailable: { reason: 'notSignedIn' } as const } : {}) - } + return { models, origin: 'probe', ...(configured ? { configured } : {}) } }, { label: 'pi', signal: options?.signal } ) diff --git a/src/shared/grok-model-list-probe.ts b/src/shared/grok-model-list-probe.ts index fe8b6f767f90..f1392fa79d5e 100644 --- a/src/shared/grok-model-list-probe.ts +++ b/src/shared/grok-model-list-probe.ts @@ -10,10 +10,6 @@ const MODEL_BULLET = /^\s*[*-]\s+([^\s(]+)(.*)$/ // Read markers from listed rows so preamble defaults cannot name unavailable IDs. const DEFAULT_MARKER = /\(default\)/ -export function grokModelListSignedOut(stdout: string): boolean { - return stdout.split(/\r?\n/).some((line) => line.trim() === 'You are not authenticated.') -} - export function parseGrokModelList(stdout: string): CommitMessageModel[] { const lines = stdout.split(/\r?\n/) // Require the header to exclude default/login preambles that resemble model IDs. From 8d8edabc5b9f7d849c4ad257408788b0dcd80b0f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:07:08 -0800 Subject: [PATCH 3/7] Drop the runner tests' timeout bumps, which sat below the default Co-Authored-By: Claude Opus 5.5 (1M context) --- .../agent-model-catalog-probe-runner.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts index ac7f6d094ebc..11c34b40566b 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-probe-runner.test.ts @@ -55,7 +55,7 @@ describe('the shared catalog probe runner', () => { expect(stdout).toBe('models\nsettings\n') const pid = Number(readFileSync(file, 'utf8')) await vi.waitFor(() => expect(isAlive(pid)).toBe(false), { timeout: 5_000 }) - }, 20_000) + }) it('keeps required replies when the listing deadline lands inside the optional grace', async () => { const stdout = await runAgentModelCatalogListing( @@ -70,7 +70,7 @@ describe('the shared catalog probe runner', () => { } ) expect(stdout).toBe('models\nsettings\n') - }, 20_000) + }) it('answers a listing’s stdout once it exits 0, with its request on stdin', async () => { const stdout = await runAgentModelCatalogListing( From 058db57a3453729d4187c096661d6e4a1ce42794 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:20:25 -0800 Subject: [PATCH 4/7] Type the endpoint test's launch env as a string map Co-Authored-By: Claude Opus 5.5 (1M context) --- src/main/claude/claude-model-catalog-probe.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/claude/claude-model-catalog-probe.test.ts b/src/main/claude/claude-model-catalog-probe.test.ts index 463e1e569fd7..101b5eaf1981 100644 --- a/src/main/claude/claude-model-catalog-probe.test.ts +++ b/src/main/claude/claude-model-catalog-probe.test.ts @@ -415,7 +415,7 @@ it('requires definitive first-party account facts, preserving inventory when aut for (const { account, settings = signinCapture.settings, endpoint } of cases) { const probe = createClaudeModelCatalogProbe({ ...probeDeps(), - resolveEnv: () => (endpoint ? { ANTHROPIC_BASE_URL: endpoint } : {}), + resolveEnv: (): Record => (endpoint ? { ANTHROPIC_BASE_URL: endpoint } : {}), runListing: async () => controlReply('orca-model-discovery', signinCapture.models) + controlReply('orca-model-settings', settings) + From 63d1ed5f78761229d09ed4a203f2b0badfb3ffe1 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:34:26 -0800 Subject: [PATCH 5/7] Drive the signed-out Claude re-probe trigger in the scaling contract N and 10N distinct chats (distinct read claims, one account-scoped Claude fingerprint) read with waitForListing against the real probe and store on an injected clock: one listing before the held-reason TTL, exactly one more once it is due (held until every read has decided), none within the next TTL, each result carrying notSignedIn and models. Checked locally that it bites: making heldReasonDue ignore the TTL fails it (2 listings inside the TTL). Removing the in-flight dedup (the probeDue isListing guard plus the lister's early return) does not: store refresh then runs once per chat, but a newer flight replaces each queued one behind the two admission slots, so still one process per TTL. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../agent-model-catalog-scaling.test.ts | 89 +++++++++++++------ 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts index ae848ed09b99..ec4b0a917d91 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts @@ -6,11 +6,26 @@ import type { AgentModelCatalogReadParams } from './agent-model-catalog-service- import { createClaudeModelCatalogProbe } from '../../claude/claude-model-catalog-probe' import type { runAgentModelCatalogListing } from './agent-model-catalog-probe-runner' import claudeCapture from '../../claude/__fixtures__/catalog-signin.json' +import { AGENT_MODEL_CATALOG_FAILURE_TTL_MS } from './agent-model-catalog-failures' +import { agentSessionRecordFixture } from '../agent-session-record-test-fixture' +import { agentSessionConversationView } from '../../../shared/agent-session-conversation-view' -it('shares one bounded sign-in observation across N and 10N catalog demands', async () => { - for (const callers of [10, 100]) { - const runListing = vi.fn(async () => - [ +it('re-probes a signed-out Claude scope once per held-reason TTL for N and 10N chats', async () => { + const home = { variable: 'CLAUDE_CONFIG_DIR', path: '/account' } + // Distinct folders give distinct read claims; Claude's account-scoped catalog keeps one fingerprint. + const chat = (sessionId: string) => + agentSessionRecordFixture({ + sessionId, + provider: 'claude', + accountHome: home, + location: { workspaceId: `folder-${sessionId}`, workspaceKind: 'folder' } + }) + for (const chats of [10, 100]) { + let now = 1_000 + let listingGate: Promise | undefined + const runListing = vi.fn(async () => { + await listingGate + return [ ['orca-model-discovery', claudeCapture.models], ['orca-model-settings', claudeCapture.settings], ['orca-signin-discovery', { account: claudeCapture.accounts.signedOut }] @@ -22,33 +37,57 @@ it('shares one bounded sign-in observation across N and 10N catalog demands', as }) ) .join('\n') - ) - const probe = createClaudeModelCatalogProbe({ - resolveInheritedEnv: async () => ({ PATH: '/usr/bin' }), - resolveEnv: () => ({}), - resolveCommand: () => ({ - command: process.execPath, - prefixArgs: ['run', '--', 'claude'] - }), - resolveAuthPolicy: () => ({ account: 'system' }), - runListing }) + const store = new AgentModelCatalogStore({ now: () => now }) const service = createAgentModelCatalogService({ - store: new AgentModelCatalogStore(), - getConversation: () => undefined, - getRecord: () => undefined, + store, + getConversation: (id) => agentSessionConversationView(chat(id)), + getRecord: chat, drivesRecord: () => true, - resolveAccountHome: async () => ({ variable: 'CLAUDE_CONFIG_DIR', path: '/account' }), - probes: { claude: probe } + resolveAccountHome: async () => home, + probes: { + claude: createClaudeModelCatalogProbe({ + resolveInheritedEnv: async () => ({ PATH: '/usr/bin' }), + resolveEnv: () => ({}), + resolveCommand: () => ({ + command: process.execPath, + prefixArgs: ['run', '--', 'claude'] + }), + resolveAuthPolicy: () => ({ account: 'system' }), + runListing + }) + } }) - const request = { agent: 'claude', workspacePath: '/project' } as const + const readAll = () => + Promise.all( + Array.from({ length: chats }, (_, index) => + service.read({ agent: 'claude', sessionId: `chat-${index}`, waitForListing: true }) + ) + ) try { - await Promise.all(Array.from({ length: callers }, () => service.demand(request))) - for (let index = 0; index < callers; index++) { - service.snapshot(request) - } + await readAll() + now += AGENT_MODEL_CATALOG_FAILURE_TTL_MS - 1 + await readAll() expect(runListing).toHaveBeenCalledTimes(1) - expect(service.snapshot(request)).toMatchObject({ unavailable: { reason: 'notSignedIn' } }) + now += 1 + const opened = Promise.withResolvers() + listingGate = opened.promise + const decided = vi.spyOn(store, 'pendingListing') + const reading = readAll() + // Hold the re-probe until every chat's read has decided whether to start one. + await vi.waitFor(() => + expect(decided.mock.calls.filter(([, , terminal]) => terminal).length).toBe(chats) + ) + opened.resolve() + const due = await reading + expect(runListing).toHaveBeenCalledTimes(2) + for (const result of due) { + expect(result.unavailable).toMatchObject({ reason: 'notSignedIn' }) + expect(result.models?.length).toBeGreaterThan(0) + } + now += AGENT_MODEL_CATALOG_FAILURE_TTL_MS - 1 + await readAll() + expect(runListing).toHaveBeenCalledTimes(2) expect(runListing.mock.calls[0]?.[0].stdin?.trim().split('\n')).toHaveLength(3) } finally { service.stop() From d9215755904c8d8d23c7c8e062ae9bd4c8aa3f91 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:37:01 -0800 Subject: [PATCH 6/7] Accept Claude control replies as main did, without a type field The shared reader required type 'control_response' and prefiltered on that text, dropping success frames main's model and slash-command parsers accepted (the released-phone wire test's fake listing). It now prefilters on request_id and keeps the structure limit and success-only reading. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/main/claude/claude-control-responses.ts | 4 ++-- .../claude/claude-model-catalog-probe.test.ts | 23 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/claude/claude-control-responses.ts b/src/main/claude/claude-control-responses.ts index 109315b22bf3..498fde83c309 100644 --- a/src/main/claude/claude-control-responses.ts +++ b/src/main/claude/claude-control-responses.ts @@ -9,13 +9,13 @@ function isRecord(value: unknown): value is Record { export function readClaudeControlResponses(stdout: string): Map { const responses = new Map() for (const line of stdout.split(/\r?\n/)) { - if (!line.trimStart().startsWith('{') || !line.includes('control_response')) { + if (!line.trimStart().startsWith('{') || !line.includes('request_id')) { continue } try { assertJsonTextStructureWithinLimits(line, CONTROL_JSON_LIMITS) const frame: unknown = JSON.parse(line) - if (!isRecord(frame) || frame.type !== 'control_response' || !isRecord(frame.response)) { + if (!isRecord(frame) || !isRecord(frame.response)) { continue } const response = frame.response diff --git a/src/main/claude/claude-model-catalog-probe.test.ts b/src/main/claude/claude-model-catalog-probe.test.ts index 101b5eaf1981..3fe97c784b8d 100644 --- a/src/main/claude/claude-model-catalog-probe.test.ts +++ b/src/main/claude/claude-model-catalog-probe.test.ts @@ -427,22 +427,23 @@ it('requires definitive first-party account facts, preserving inventory when aut } }) -it('reads slash commands from the real initialize frame past error, diagnostic and oversized lines', async () => { +it('reads slash commands as main did: past error, diagnostic and oversized lines, with or without type', async () => { const { error, initialize } = signinCapture.commandFrames - const nested = (depth: number): unknown => - depth === 0 ? 'orca-command-discovery' : { response: nested(depth - 1) } - const oversized = JSON.stringify({ type: 'control_response', response: nested(20) }) + const nested = (depth: number): unknown => (depth === 0 ? {} : { response: nested(depth - 1) }) + const oversized = JSON.stringify({ + ...initialize, + response: { ...initialize.response, response: nested(20) } + }) const list = (lines: string[]) => createClaudeCommandCatalogProbe({ ...probeDeps(), runListing: async () => `${lines.join('\n')}\n` })({ cwd: '/project', accountHomePath: '/homes/account-a' }) - const rejected = [ - JSON.stringify(error), - 'orca-command-discovery control_response not ready', - oversized - ] + const rejected = [JSON.stringify(error), 'orca-command-discovery request_id not ready', oversized] await expect(list(rejected)).rejects.toThrow('claude listed no commands') - const commands = await list([...rejected, JSON.stringify(initialize)]) - expect(commands.map((command) => command.name)).toEqual(['deep-research', 'design']) + const { type: _type, ...untyped } = initialize + for (const frame of [initialize, untyped]) { + const commands = await list([...rejected, JSON.stringify(frame)]) + expect(commands.map((command) => command.name)).toEqual(['deep-research', 'design']) + } }) From 39bc807e50ec1fba1d22e0f1e02ba6494530e330 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:44:45 -0800 Subject: [PATCH 7/7] Narrow the scaling contract's catalog result before reading models Co-Authored-By: Claude Opus 5.5 (1M context) --- .../agent-model-catalog/agent-model-catalog-scaling.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts b/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts index ec4b0a917d91..905d9cb180e8 100644 --- a/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts +++ b/src/main/native-chat/agent-model-catalog/agent-model-catalog-scaling.test.ts @@ -83,7 +83,7 @@ it('re-probes a signed-out Claude scope once per held-reason TTL for N and 10N c expect(runListing).toHaveBeenCalledTimes(2) for (const result of due) { expect(result.unavailable).toMatchObject({ reason: 'notSignedIn' }) - expect(result.models?.length).toBeGreaterThan(0) + expect(result.origin === 'unknown' ? 0 : result.models.length).toBeGreaterThan(0) } now += AGENT_MODEL_CATALOG_FAILURE_TTL_MS - 1 await readAll()