Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions src/lib/__tests__/agent-interface.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { runAgent, createStopHook } from '@lib/agent/agent-interface';
import { AgentOutputSignals } from '@lib/agent/output-signals';
import type { WizardRunOptions } from '@utils/types';
import type { SpinnerHandle } from '@ui';
import {
Expand Down Expand Up @@ -393,26 +394,29 @@ describe('createStopHook', () => {
});

it('allows stop immediately on API error (401)', () => {
const collectedText = [
const signals = new AgentOutputSignals();
signals.push(
'Failed to authenticate. API Error: 401 {"detail":"Authentication required"}',
];
const hook = createStopHook([AdditionalFeature.LLM], collectedText);
);
const hook = createStopHook([AdditionalFeature.LLM], signals);

const result = hook(hookInput);
expect(result).toEqual({});
});

it('allows stop immediately on generic API error', () => {
const collectedText = ['API Error: 500 Internal Server Error'];
const hook = createStopHook([AdditionalFeature.LLM], collectedText);
const signals = new AgentOutputSignals();
signals.push('API Error: 500 Internal Server Error');
const hook = createStopHook([AdditionalFeature.LLM], signals);

const result = hook(hookInput);
expect(result).toEqual({});
});

it('proceeds normally when collectedText has no API error', () => {
const collectedText = ['Some normal agent output'];
const hook = createStopHook([], collectedText);
it('proceeds normally when output has no API error', () => {
const signals = new AgentOutputSignals();
signals.push('Some normal agent output'); // dropped: carries no signal
const hook = createStopHook([], signals);

// First call → remark prompt (normal behavior)
const first = hook(hookInput);
Expand Down
66 changes: 66 additions & 0 deletions src/lib/agent/__tests__/output-signals.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import { AgentOutputSignals } from '@lib/agent/output-signals';

describe('AgentOutputSignals', () => {
it('drops prose but detects each signal marker', () => {
const signals = new AgentOutputSignals();
signals.push('Thinking about the integration plan...'); // prose, dropped
signals.push('Reading files and editing config'); // prose, dropped
signals.push('Hit a wall. API Error: 401 unauthorized');
signals.push('[ERROR-MCP-MISSING] could not reach MCP');

expect(signals.hasApiError()).toBe(true);
expect(signals.hasApiErrorStatus(401)).toBe(true);
expect(signals.hasApiErrorStatus(429)).toBe(false);
expect(signals.has('MCP_MISSING')).toBe(true);
expect(signals.has('RESOURCE_MISSING')).toBe(false);
expect(signals.hasYaraViolation()).toBe(false);
expect(signals.remark()).toBeUndefined();
});

it('treats the API error status as a parameter, not a fixed marker', () => {
const signals = new AgentOutputSignals();
signals.push('API Error: 503 service unavailable');

expect(signals.hasApiError()).toBe(true); // generic match via the prefix
expect(signals.hasApiErrorStatus(503)).toBe(true);
expect(signals.hasApiErrorStatus(500)).toBe(false);
});

it('detects YARA violations from either marker', () => {
const critical = new AgentOutputSignals();
critical.push('[YARA CRITICAL] prompt injection detected');
expect(critical.hasYaraViolation()).toBe(true);

const scannerErr = new AgentOutputSignals();
scannerErr.push('[YARA] Scanner error: failed to load rules');
expect(scannerErr.hasYaraViolation()).toBe(true);
});

it('extracts only the API Error lines for the message', () => {
const signals = new AgentOutputSignals();
signals.push('Some prose before the error');
signals.push('Request failed: API Error: 429 rate limited, retry later');

expect(signals.hasApiErrorStatus(429)).toBe(true);
expect(signals.apiErrorMessage()).toBe(
'API Error: 429 rate limited, retry later',
);
});

it('extracts the trimmed remark after the marker', () => {
const signals = new AgentOutputSignals();
signals.push(
'Done. [WIZARD-REMARK] The MCP schema was larger than expected.',
);

expect(signals.remark()).toBe('The MCP schema was larger than expected.');
});

it('returns undefined extractions when no matching lines were retained', () => {
const signals = new AgentOutputSignals();
signals.push('Nothing interesting here');

expect(signals.apiErrorMessage()).toBeUndefined();
expect(signals.remark()).toBeUndefined();
});
});
Loading
Loading