test(claude-cli): add offline schema fixtures for Issue #6469

tests/providers/claude-cli-adapter.test.ts

@@ -12,6 +12,7 @@ import {
   type SpawnFn,
 } from "../../src/adapters/claude-cli/adapter";
 import { CLAUDE_REPLAY_SYSTEM_PROMPT } from "../../src/adapters/claude-cli/stable-replay";
+import { satisfiesOpenAiStrictSchema } from "../../src/adapters/anthropic-output-schema";
 import { baseScopedEnv } from "../../src/adapters/coding-agent/turn";
 import { CLAUDE_CLI_PROFILE, clearClaudeCliBinaryCache } from "../../src/adapters/claude-cli/profiles";
 import { effectiveAdapterContract, getAdapterDefinition } from "../../src/adapters/registry";
@@ -1047,6 +1048,11 @@ const textDelta = (text: string): string =>
 
 describe("Issue #6469 — offline fixtures", () => {
   const systemInit = enc.encode('{"type":"system","subtype":"init","mcp_servers":[{"name":"opencodex","status":"connected"}]}\n');
+  const schema = {
+    type: "object",
+    properties: { ocx_schema_probe_marker: { type: "string" } },
+    required: ["ocx_schema_probe_marker"],
+  };
 
   test("1. tool-use round trip: tool request, matching result, plain JSON final output", async () => {
     const firstChild = fakeChild([
@@ -1106,11 +1112,11 @@ describe("Issue #6469 — offline fixtures", () => {
 
   test("3. refusal: model refusal surfaces as an error, not fabricated output", async () => {
     const events = await run(adapterWith([
-      enc.encode('{"type":"result","subtype":"success","is_error":true,"message":"refusal","usage":{"input_tokens":2,"output_tokens":0}}\n'),
+      enc.encode('{"type":"result","subtype":"success","is_error":true,"result":"refusal","usage":{"input_tokens":2,"output_tokens":0}}\n'),
     ]), parsed());
     const err = events.find(e => e.type === "error");
     expect(err).toBeDefined();
-    expect(err).toMatchObject({ type: "error" });
+    expect(err).toMatchObject({ type: "error", message: "refusal" });
     expect(events.filter(e => e.type === "text_delta")).toEqual([]);
   });
 
@@ -1293,6 +1299,224 @@ describe("Issue #6469 — offline fixtures", () => {
     expect(promptFile).not.toBe("");
     expect(existsSync(promptFile)).toBe(false);
   });
+
+  test("8a. valid Core JSON instance surfaced as plain text, not validated against schema", async () => {
+    const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema } } });
+    const validInstance = JSON.stringify({ ocx_schema_probe_marker: "present" });
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta(validInstance)),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), schemaRequest);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe(validInstance);
+    expect(JSON.parse(text)).toEqual({ ocx_schema_probe_marker: "present" });
+    expect(events.some(e => e.type === "error")).toBe(false);
+    expect(events.at(-1)).toMatchObject({ type: "done", stopReason: "stop" });
+  });
+
+  test("9. enum/const constraints ignored — output violating them still passed as plain text", async () => {
+    const enumSchema = {
+      type: "object",
+      properties: {
+        status: { type: "string", enum: ["ok", "fail"] },
+        code: { type: "string", const: "exact" },
+      },
+      required: ["status", "code"],
+    };
+    const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: enumSchema } } });
+    const violating = JSON.stringify({ status: "not-allowed", code: "wrong" });
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta(violating)),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), schemaRequest);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe(violating);
+    expect(events.some(e => e.type === "error")).toBe(false);
+  });
+
+  test("10. additionalProperties:false ignored — extra properties still passed as plain text", async () => {
+    const strictSchema = {
+      type: "object",
+      properties: { answer: { type: "string" } },
+      required: ["answer"],
+      additionalProperties: false,
+    };
+    const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: strictSchema } } });
+    const withExtra = JSON.stringify({ answer: "42", extra: "should-not-matter" });
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta(withExtra)),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), schemaRequest);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe(withExtra);
+    expect(events.some(e => e.type === "error")).toBe(false);
+  });
+
+  test("11. missing schema field — textFormat.json_schema without schema treated as plain text", async () => {
+    const noSchema = parsed({ options: { textFormat: { type: "json_schema", name: "probe" } } });
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta("plain output")),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), noSchema);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe("plain output");
+    expect(events.some(e => e.type === "error")).toBe(false);
+    expect(events.at(-1)).toMatchObject({ type: "done", stopReason: "stop" });
+  });
+
+  test("12. metadata/title-only schema does not certify unconstrained shape — still plain text", async () => {
+    const titleOnly = {
+      type: "object",
+      title: "Probe",
+      description: "Only metadata, no constraints",
+    };
+    const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: titleOnly } } });
+    const unconstrained = JSON.stringify("goes");
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta(unconstrained)),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), schemaRequest);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe(unconstrained);
+    expect(events.some(e => e.type === "error")).toBe(false);
+  });
+
+  test("13. unsupported/out-of-subset keywords ignored — schema not applied", async () => {
+    const weirdSchema = {
+      type: "object",
+      properties: { answer: { type: "string" } },
+      required: ["answer"],
+      if: { properties: { answer: { const: "a" } } },
+      then: { properties: { answer: { const: "b" } } },
+    };
+    const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: weirdSchema } } });
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta('{"answer":"a"}')),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), schemaRequest);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe('{"answer":"a"}');
+    expect(events.some(e => e.type === "error")).toBe(false);
+  });
+
+  test("14. strict:true on uncertifiable schema ignored — no enforcement", async () => {
+    const uncertifiable = {
+      type: "object",
+      properties: { answer: { type: "string" } },
+      required: ["answer"],
+    };
+    expect(satisfiesOpenAiStrictSchema(uncertifiable)).toBe(false);
+    const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: uncertifiable, strict: true } } });
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta('{"answer":"42"}')),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), schemaRequest);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe('{"answer":"42"}');
+    expect(events.some(e => e.type === "error")).toBe(false);
+  });
+
+  test("15. oversized output surfaced as plain text without truncation", async () => {
+    const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema } } });
+    const big = "x".repeat(50_000);
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta(big)),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), schemaRequest);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe(big);
+    expect(text.length).toBe(50_000);
+    expect(events.some(e => e.type === "error")).toBe(false);
+  });
+
+  test("16. deeply nested JSON output surfaced as plain text without depth limits", async () => {
+    const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema } } });
+    let nested: any = { value: "leaf" };
+    for (let i = 0; i < 200; i++) nested = { wrapper: nested };
+    const deep = JSON.stringify(nested);
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta(deep)),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), schemaRequest);
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe(deep);
+    expect(events.some(e => e.type === "error")).toBe(false);
+  });
+
+  test("17. tool turn with no final text — done with stopReason tool_use, no text_delta", async () => {
+    const stdout = [
+      systemInit,
+      enc.encode(claudeCliToolFrame(0, "tu_1", CLAUDE_CLI_MCP_TOOL, '{"path":"a"}')),
+      enc.encode('{"type":"stream_event","event":{"type":"message_stop"}}\n'),
+      enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":0}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), claudeCliToolRequest());
+    const starts = events.filter(e => e.type === "tool_call_start");
+    expect(starts).toHaveLength(1);
+    expect(starts[0]).toMatchObject({ type: "tool_call_start", id: "tu_1", name: "read_file" });
+    expect(events.filter(e => e.type === "text_delta")).toEqual([]);
+    expect(events.some(e => e.type === "error")).toBe(false);
+    expect(events.at(-1)).toMatchObject({ type: "done", stopReason: "tool_use" });
+  });
+
+  test("18. terminal failure before complete output — is_error result ends turn with error", async () => {
+    const stdout = [
+      systemInit,
+      enc.encode(textDelta("partial")),
+      enc.encode('{"type":"result","subtype":"success","is_error":true,"result":"model refused","usage":{"input_tokens":4,"output_tokens":1}}\n'),
+    ];
+    const events = await run(adapterWith(stdout), parsed());
+    const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join("");
+    expect(text).toBe("partial");
+    const err = events.find(e => e.type === "error");
+    expect(err).toBeDefined();
+    expect(err).toMatchObject({ message: "model refused" });
+    expect(events.some(e => e.type === "done")).toBe(false);
+  });
+
+  test("19. failure order — protocol error after tool calls does not replay or erase exposed calls", async () => {
+    let spawnCount = 0;
+    const stdout = [
+      systemInit,
+      enc.encode(claudeCliToolFrame(0, "tu_1", CLAUDE_CLI_MCP_TOOL, '{"path":"a"}')),
+      enc.encode(claudeCliToolFrame(1, "tu_2", CLAUDE_CLI_MCP_TOOL, '{"path":"b"}')),
+      enc.encode("not-json\n"),
+    ];
+    const events = await run(createClaudeCliAdapter(provider(), {
+      spawn: () => { spawnCount++; return fakeChild(stdout, { exitCode: 0 }) as unknown as ChildProcess; },
+      which: () => "/opt/homebrew/bin/claude",
+      killGraceMs: 20,
+    }), claudeCliToolRequest());
+    const starts = events.filter(e => e.type === "tool_call_start");
+    expect(starts).toHaveLength(2);
+    expect(starts.map(e => (e as { id: string }).id)).toEqual(["tu_1", "tu_2"]);
+    expect(events.filter(e => e.type === "tool_call_delta").map(e => (e as { arguments: string }).arguments))
+      .toEqual(['{"path":"a"}', '{"path":"b"}']);
+    expect(events.filter(e => e.type === "tool_call_end")).toHaveLength(2);
+    const err = events.find(e => e.type === "error");
+    expect(err).toBeDefined();
+    expect(err).toMatchObject({ code: "protocol_error" });
+    expect(events.some(e => e.type === "done")).toBe(false);
+    expect(spawnCount).toBe(1);
+  });
 });
 
 function adapterWith(stdout: Uint8Array[], exitCode = 0): ReturnType<typeof createClaudeCliAdapter> {