Skip to content

fix(sep-2322): stop two scored scenarios going green on a server that rejects everything #537

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AmirK-S wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(sep-2322): stop two scored scenarios going green on a server that rejects everything #537

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions src/scenarios/server/input-required-result-reject-all.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import { createServer, Server } from 'http';
import { testContext } from '../../connection/testing';
import {
InputRequiredResultUnsupportedMethodsScenario,
InputRequiredResultValidateInputScenario
} from './input-required-result';

// #451: a server that rejects every request with the same unrelated error
// used to score two scored 2026-07-28 scenarios fully green:
// input-required-result-validate-input, whose checks accept any JSON-RPC
// error as proof of input validation, and
// input-required-result-unsupported-methods, whose MUST NOT is never
// exercised when every probe is rejected.
describe('SEP-2322 scenarios against a server that rejects everything', () => {
let server: Server;
let SERVER_URL: string;

beforeAll(async () => {
server = createServer((req, res) => {
let body = '';
req.on('data', (d) => (body += d));
req.on('end', () => {
let id: unknown = null;
try {
id = JSON.parse(body).id ?? null;
} catch {
// keep id null
}
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(
JSON.stringify({
jsonrpc: '2.0',
id,
error: {
code: -32000,
message: 'Bad Request: Server not initialized'
}
})
);
});
});
await new Promise<void>((resolve) => server.listen(0, resolve));
const port = (server.address() as { port: number }).port;
SERVER_URL = `http://localhost:${port}/mcp`;
});

afterAll(async () => {
await new Promise<void>((resolve) => server.close(() => resolve()));
});

it('validate-input reports both checks as not testable instead of SUCCESS', async () => {
const scenario = new InputRequiredResultValidateInputScenario();
const checks = await scenario.run(testContext(SERVER_URL));

expect(checks.map((c) => c.id).sort()).toEqual([
'sep-2322-error-on-protocol-error',
'sep-2322-validate-input-responses'
]);
for (const check of checks) {
expect(check.status).toBe('WARNING');
expect(check.errorMessage).toContain('Not testable:');
expect(check.errorMessage).toContain('-32000');
expect(check.details?.untestable).toBe(true);
}
}, 10000);

it('unsupported-methods reports its check as not testable instead of SUCCESS', async () => {
const scenario = new InputRequiredResultUnsupportedMethodsScenario();
const checks = await scenario.run(testContext(SERVER_URL));

expect(checks).toHaveLength(1);
expect(checks[0].id).toBe('sep-2322-not-on-unsupported-requests');
expect(checks[0].status).toBe('FAILURE');
expect(checks[0].errorMessage).toContain('Not testable:');
expect(checks[0].errorMessage).toContain('tools/list');
expect(checks[0].errorMessage).toContain('prompts/list');
expect(checks[0].details?.untestable).toBe(true);
}, 10000);
});
116 changes: 101 additions & 15 deletions src/scenarios/server/input-required-result.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import {
mockListRootsResponse,
MRTR_SPEC_REFERENCES
} from './input-required-result-helpers';
import { notTestable } from '../untestable';
import { notTestable, untestableCheck } from '../untestable';

// ─── A1: Basic Elicitation ────────────────────────────────────────────────────

Expand Down Expand Up @@ -1225,7 +1225,10 @@ export class InputRequiredResultUnsupportedMethodsScenario implements ClientScen
specVersions: SpecVersion[] = [DRAFT_PROTOCOL_VERSION];
description = `Test that server does NOT return InputRequiredResult on unsupported methods (SEP-2322).

Servers MUST NOT send InputRequiredResult responses on any client requests other than the supported ones (prompts/get, resources/read, tools/call, tasks/result).`;
Servers MUST NOT send InputRequiredResult responses on any client requests other than the supported ones (prompts/get, resources/read, tools/call, tasks/result).

The scenario probes \`tools/list\` and \`prompts/list\`. A rejected probe does not exercise the requirement, so if
neither returns a result the check reports \`Not testable:\` instead of SUCCESS.`;

async run(ctx: RunContext): Promise<ConformanceCheck[]> {
const { serverUrl } = ctx;
Expand All @@ -1235,19 +1238,43 @@ Servers MUST NOT send InputRequiredResult responses on any client requests other
const unsupportedMethods = ['tools/list', 'prompts/list'];

try {
// A method the server rejects never had the chance to return an
// InputRequiredResult, so it does not exercise the MUST NOT. A server
// without prompts may reject prompts/list; at least one probe has to
// come back with a result, or a server that rejects every request
// would score SUCCESS here (#451).
const rejected: string[] = [];
for (const method of unsupportedMethods) {
const resp = await sendRpc(serverUrl, method, {});
if (
resp.result &&
if (!resp.result) {
rejected.push(
resp.error
? `${method} (JSON-RPC error ${resp.error.code}: ${resp.error.message})`
: `${method} (no result)`
);
} else if (
(resp.result as Record<string, unknown>).resultType ===
'input_required'
'input_required'
) {
errors.push(
`${method} returned InputRequiredResult, but it is not a supported method for MRTR`
);
}
}

if (rejected.length === unsupportedMethods.length) {
checks.push(
untestableCheck(
'sep-2322-not-on-unsupported-requests',
'NotOnUnsupportedRequests',
'Server does not return InputRequiredResult on unsupported methods',
`no probe returned a result: ${rejected.join('; ')}`,
MRTR_SPEC_REFERENCES
)
);
return checks;
}

checks.push({
id: 'sep-2322-not-on-unsupported-requests',
name: 'NotOnUnsupportedRequests',
Expand Down Expand Up @@ -1575,13 +1602,55 @@ export class InputRequiredResultValidateInputScenario implements ClientScenario
Uses the same tool as A1: \`test_input_required_result_elicitation\`.

This scenario sends completely invalid inputResponses structures. The server SHOULD validate them
and return a JSON-RPC error or a new InputRequiredResult.`;
and return a JSON-RPC error or a new InputRequiredResult.

Both checks first require the plain call (no \`inputResponses\`) to return an InputRequiredResult,
as in A1. Otherwise a rejection of the invalid input cannot be told apart from a server that rejects
every request, and both checks report \`Not testable:\` instead of SUCCESS.`;

async run(ctx: RunContext): Promise<ConformanceCheck[]> {
const { serverUrl } = ctx;
const checks: ConformanceCheck[] = [];

try {
// Control: the same tool called without inputResponses must answer with
// an InputRequiredResult. The checks below pass on any JSON-RPC error, so
// without this a server that rejects every request (e.g. -32000 "Server
// not initialized") would score both as SUCCESS for an unrelated reason.
const control = await sendRpc(serverUrl, 'tools/call', {
name: 'test_input_required_result_elicitation',
arguments: {}
});
if (control.error || !isInputRequiredResult(control.result)) {
const got = control.error
? `JSON-RPC error ${control.error.code}: ${control.error.message}`
: control.result
? 'a complete result'
: 'no result';
const reason =
'test_input_required_result_elicitation called without inputResponses ' +
`returned ${got} instead of an InputRequiredResult, so a rejection of ` +
'invalid inputResponses cannot be attributed to input validation';
return [
untestableCheck(
'sep-2322-validate-input-responses',
'ValidateInputResponses',
'Server validates InputResponses structure',
reason,
MRTR_SPEC_REFERENCES,
'WARNING'
),
untestableCheck(
'sep-2322-error-on-protocol-error',
'ErrorOnProtocolError',
'Server returns JSON-RPC error for protocol-level input errors',
reason,
MRTR_SPEC_REFERENCES,
'WARNING'
)
];
}

// Send inputResponses with invalid structure (number instead of object
// for the response). Deliberately schema-invalid, so wire-schema
// validation is skipped for this call.
Expand Down Expand Up @@ -1656,15 +1725,32 @@ and return a JSON-RPC error or a new InputRequiredResult.`;
details: { result: resp2.result, error: resp2.error }
});
} catch (error) {
checks.push({
id: 'sep-2322-validate-input-responses',
name: 'ValidateInputResponses',
description: 'Server validates InputResponses structure',
status: 'WARNING',
timestamp: new Date().toISOString(),
errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}`,
specReferences: MRTR_SPEC_REFERENCES
});
// Report every check that did not get to run, so a throw does not
// shrink the emitted set.
const errorMessage = `Failed: ${error instanceof Error ? error.message : String(error)}`;
for (const [id, name, description] of [
[
'sep-2322-validate-input-responses',
'ValidateInputResponses',
'Server validates InputResponses structure'
],
[
'sep-2322-error-on-protocol-error',
'ErrorOnProtocolError',
'Server returns JSON-RPC error for protocol-level input errors'
]
]) {
if (checks.some((c) => c.id === id)) continue;
checks.push({
id,
name,
description,
status: 'WARNING',
timestamp: new Date().toISOString(),
errorMessage,
specReferences: MRTR_SPEC_REFERENCES
});
}
}

return checks;
Expand Down
Loading