diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index 23f43b98b6..8245dfda39 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -286,6 +286,14 @@ fn build_multi_format_client( are retained and opaque `encrypted_content` is removed. Switchyard's translated Anthropic reasoning payload is always removed. Set `Drop` for a backend that cannot consume provider reasoning. Messages and tool history are retained. +- `ModelConfig::with_responses_compatibility` selects Responses tool adaptations. + The default `Native` leaves namespaces, item order, and output fields unchanged. + `FlatTools` joins function namespaces and names with `__` in history, tool + definitions, and tool choice. `Kimi` also pairs outputs with matching calls + and adds the call name when the output has no name. Pairing stops at an + intervening message; outputs without a local call remain in place. These + adaptations run after translation and body overrides. The backend's required + mode requires an explicit setting; Switchyard does not infer it from model names. - `HttpBackendConfig::failure_cooldown` skips completion calls to a backend briefly after an exhausted transient failure. Zero disables it. State is shared per model within the client. Calls resume when the cooldown expires. Deployment TOML defaults diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index 4fa4db687f..7fe804576a 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -68,6 +68,7 @@ pub struct ModelConfig { default_backend: Backend, other_backends: Option>, responses_reasoning: crate::ResponsesReasoningPolicy, + responses_compatibility: crate::ResponsesCompatibility, } impl ModelConfig { @@ -84,9 +85,17 @@ impl ModelConfig { default_backend, other_backends, responses_reasoning: crate::ResponsesReasoningPolicy::default(), + responses_compatibility: crate::ResponsesCompatibility::default(), } } + /// This method selects the Responses request changes required by the model's provider. + #[must_use] + pub fn with_responses_compatibility(mut self, policy: crate::ResponsesCompatibility) -> Self { + self.responses_compatibility = policy; + self + } + /// Sets how Responses reasoning items are replayed to this model. #[must_use] pub fn with_responses_reasoning(mut self, policy: crate::ResponsesReasoningPolicy) -> Self { @@ -362,9 +371,6 @@ impl TranslatingLlmClient { // the upstream always sees the provider model. let upstream_model = self.provider_model(model); set_json_model(&mut body, upstream_model); - if matches!(backend, Backend::OpenAiResponses(_)) { - sanitize_openai_responses_provider_body(&mut body); - } // Strip before `merge_extra_body` so a target can reinstate either field // deliberately via `extra_body`. if matches!(backend, Backend::Anthropic(_)) { @@ -374,6 +380,11 @@ impl TranslatingLlmClient { omit_configured_body_fields(&mut body, backend.omit_body_fields()); merge_extra_body(&mut body, backend.extra_body()); if matches!(backend, Backend::OpenAiResponses(_)) { + self.model_to_config + .get(model) + .map(|config| config.responses_compatibility) + .unwrap_or_default() + .normalize(&mut body); self.model_to_config .get(model) .map(|config| config.responses_reasoning) @@ -1174,126 +1185,6 @@ fn rejects_template_options(error: &LlmClientError) -> bool { .any(|phrase| message.contains(phrase))) } -const CODEX_NAMESPACE_SEPARATOR: &str = "__"; - -/// Rewrites a Responses body into the shape OpenAI-compatible providers accept. -/// -/// Codex extends Responses with namespace containers and namespaced function calls, and -/// OpenAI-compatible providers expect a flat Responses tool namespace. -fn sanitize_openai_responses_provider_body(body: &mut Value) { - let Value::Object(object) = body else { - return; - }; - sanitize_openai_responses_input_for_provider(object.get_mut("input")); - sanitize_openai_responses_tools_for_provider(object.get_mut("tools")); - sanitize_openai_responses_tool_choice_for_provider(object.get_mut("tool_choice")); -} - -fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) { - let Some(Value::Array(items)) = input else { - return; - }; - for item in items { - let Some(object) = item.as_object_mut() else { - continue; - }; - if object.get("type").and_then(Value::as_str) == Some("function_call") { - qualify_responses_function_name(object); - } - } -} - -fn sanitize_openai_responses_tools_for_provider(tools: Option<&mut Value>) { - let Some(Value::Array(tools)) = tools else { - return; - }; - let mut flat_tools = Vec::with_capacity(tools.len()); - for tool in std::mem::take(tools) { - push_sanitized_openai_responses_tool(&mut flat_tools, tool); - } - *tools = flat_tools; -} - -fn push_sanitized_openai_responses_tool(out: &mut Vec, tool: Value) { - let Value::Object(mut object) = tool else { - out.push(tool); - return; - }; - if object.get("type").and_then(Value::as_str) == Some("namespace") { - let namespace = object - .get("name") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - let Some(Value::Array(children)) = object.remove("tools") else { - out.push(Value::Object(object)); - return; - }; - for child in children { - push_sanitized_namespaced_tool(out, &namespace, child); - } - return; - } - ensure_responses_function_tool_description(&mut object); - out.push(Value::Object(object)); -} - -fn push_sanitized_namespaced_tool(out: &mut Vec, namespace: &str, tool: Value) { - let Value::Object(mut object) = tool else { - out.push(tool); - return; - }; - if object.get("type").and_then(Value::as_str) == Some("function") { - qualify_responses_function_name_with_namespace(&mut object, namespace); - ensure_responses_function_tool_description(&mut object); - } - out.push(Value::Object(object)); -} - -fn sanitize_openai_responses_tool_choice_for_provider(tool_choice: Option<&mut Value>) { - let Some(Value::Object(object)) = tool_choice else { - return; - }; - if object.get("type").and_then(Value::as_str) == Some("function") { - qualify_responses_function_name(object); - } -} - -fn qualify_responses_function_name(object: &mut Map) { - let namespace = object - .remove("namespace") - .and_then(|value| value.as_str().map(ToOwned::to_owned)); - let Some(namespace) = namespace.as_deref() else { - return; - }; - qualify_responses_function_name_with_namespace(object, namespace); -} - -fn qualify_responses_function_name_with_namespace( - object: &mut Map, - namespace: &str, -) { - if namespace.is_empty() { - return; - } - let Some(name) = object.get("name").and_then(Value::as_str) else { - return; - }; - let prefix = format!("{namespace}{CODEX_NAMESPACE_SEPARATOR}"); - if name.starts_with(&prefix) { - return; - } - object.insert("name".to_string(), Value::String(format!("{prefix}{name}"))); -} - -fn ensure_responses_function_tool_description(object: &mut Map) { - if object.get("type").and_then(Value::as_str) == Some("function") - && !matches!(object.get("description"), Some(Value::String(_))) - { - object.insert("description".to_string(), Value::String(String::new())); - } -} - // Drops fields accepted by OpenAI-like APIs but rejected by Anthropic Messages. // // A router can serve earlier turns of a session from an OpenAI-format target and @@ -4063,7 +3954,13 @@ mod tests { .mount(&server) .await; - let client = TranslatingLlmClient::new(&responses_map(&format!("{}/v1", server.uri())))?; + let models = responses_map(&format!("{}/v1", server.uri())) + .into_iter() + .map(|model| { + model.with_responses_compatibility(crate::ResponsesCompatibility::FlatTools) + }) + .collect::>(); + let client = TranslatingLlmClient::new(&models)?; let parameters = json!({"type": "object", "properties": {"q": {"type": "string"}}}); let raw = json!({ "model": "client-facing", diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs index 3710ad150d..17154a21c9 100644 --- a/crates/libsy-llm-client/src/lib.rs +++ b/crates/libsy-llm-client/src/lib.rs @@ -30,6 +30,7 @@ mod observed_translation; mod openai_decisions; mod payload_capture; pub mod raw; +mod responses_compatibility; mod responses_reasoning; pub mod run; mod system_one; @@ -41,6 +42,7 @@ pub use observation::{ModelCallObservation, RunObservation, RunObserver}; pub use openai_decisions::OpenAiDecisionClient; pub use payload_capture::with_payload_capture; pub use raw::RawResponse; +pub use responses_compatibility::ResponsesCompatibility; pub use responses_reasoning::ResponsesReasoningPolicy; pub use run::{ClientRouter, decide, run}; pub use switchyard_translation::RawEventStream; diff --git a/crates/libsy-llm-client/src/responses_compatibility.rs b/crates/libsy-llm-client/src/responses_compatibility.rs new file mode 100644 index 0000000000..133e12008d --- /dev/null +++ b/crates/libsy-llm-client/src/responses_compatibility.rs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! This module applies explicitly configured Responses tool adaptations. + +use serde::Deserialize; +use serde_json::{Map, Value}; +use std::collections::HashMap; + +/// This policy selects the Responses request changes required by a configured provider. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResponsesCompatibility { + /// This mode leaves native item order, IDs, fields, and tool namespaces unchanged. + #[default] + Native, + /// This mode joins function namespaces and names with `__`. + FlatTools, + /// This mode flattens function namespaces, pairs outputs with calls, and adds missing names. + Kimi, +} + +impl ResponsesCompatibility { + pub(crate) fn normalize(self, body: &mut Value) { + if self == Self::Native { + return; + } + sanitize_openai_responses_provider_body(body); + if self == Self::Kimi { + pair_kimi_tool_outputs(body); + } + } +} + +// Kimi's translator needs output adjacency and names in addition to `call_id`. +// Pairing stops at intervening messages, and existing output names remain unchanged. +fn pair_kimi_tool_outputs(body: &mut Value) { + let Some(Value::Array(items)) = body.get_mut("input") else { + return; + }; + let names: HashMap = items + .iter() + .filter(|item| { + matches!( + item.get("type").and_then(Value::as_str), + Some("function_call" | "custom_tool_call") + ) + }) + .filter_map(|item| { + Some(( + item.get("call_id")?.as_str()?.to_string(), + item.get("name")?.clone(), + )) + }) + .collect(); + for item in items.iter_mut() { + if matches!( + item.get("type").and_then(Value::as_str), + Some("function_call_output" | "custom_tool_call_output") + ) && let Some(name) = item + .get("call_id") + .and_then(Value::as_str) + .and_then(|id| names.get(id)) + && let Some(item) = item.as_object_mut() + { + item.entry("name".to_string()) + .or_insert_with(|| name.clone()); + } + } + let mut index = 0; + while index < items.len() { + let output_kind = match items[index].get("type").and_then(Value::as_str) { + Some("function_call") => "function_call_output", + Some("custom_tool_call") => "custom_tool_call_output", + _ => { + index += 1; + continue; + } + }; + let Some(call_id) = items[index].get("call_id").and_then(Value::as_str) else { + index += 1; + continue; + }; + let output = items + .iter() + .enumerate() + .skip(index + 1) + .take_while(|(_, item)| { + item.get("role").is_none() + && item.get("type").and_then(Value::as_str) != Some("message") + }) + .find(|(_, item)| { + item.get("type").and_then(Value::as_str) == Some(output_kind) + && item.get("call_id").and_then(Value::as_str) == Some(call_id) + }) + .map(|(position, _)| position); + if let Some(output) = output { + let item = items.remove(output); + items.insert(index + 1, item); + } + index += 1; + } +} + +const CODEX_NAMESPACE_SEPARATOR: &str = "__"; + +// The `flat_tools` and `kimi` modes flatten function namespaces in history, tools, and tool choice. +// These modes also supply empty descriptions when function definitions lack string descriptions. +fn sanitize_openai_responses_provider_body(body: &mut Value) { + let Value::Object(object) = body else { + return; + }; + sanitize_openai_responses_input_for_provider(object.get_mut("input")); + sanitize_openai_responses_tools_for_provider(object.get_mut("tools")); + sanitize_openai_responses_tool_choice_for_provider(object.get_mut("tool_choice")); +} + +fn sanitize_openai_responses_input_for_provider(input: Option<&mut Value>) { + let Some(Value::Array(items)) = input else { + return; + }; + for item in items { + let Some(object) = item.as_object_mut() else { + continue; + }; + if object.get("type").and_then(Value::as_str) == Some("function_call") { + qualify_responses_function_name(object); + } + } +} + +fn sanitize_openai_responses_tools_for_provider(tools: Option<&mut Value>) { + let Some(Value::Array(tools)) = tools else { + return; + }; + let mut flat_tools = Vec::with_capacity(tools.len()); + for tool in std::mem::take(tools) { + push_sanitized_openai_responses_tool(&mut flat_tools, tool); + } + *tools = flat_tools; +} + +fn push_sanitized_openai_responses_tool(out: &mut Vec, tool: Value) { + let Value::Object(mut object) = tool else { + out.push(tool); + return; + }; + if object.get("type").and_then(Value::as_str) == Some("namespace") { + let namespace = object + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + let Some(Value::Array(children)) = object.remove("tools") else { + out.push(Value::Object(object)); + return; + }; + for child in children { + push_sanitized_namespaced_tool(out, &namespace, child); + } + return; + } + ensure_responses_function_tool_description(&mut object); + out.push(Value::Object(object)); +} + +fn push_sanitized_namespaced_tool(out: &mut Vec, namespace: &str, tool: Value) { + let Value::Object(mut object) = tool else { + out.push(tool); + return; + }; + if object.get("type").and_then(Value::as_str) == Some("function") { + qualify_responses_function_name_with_namespace(&mut object, namespace); + ensure_responses_function_tool_description(&mut object); + } + out.push(Value::Object(object)); +} + +fn sanitize_openai_responses_tool_choice_for_provider(tool_choice: Option<&mut Value>) { + let Some(Value::Object(object)) = tool_choice else { + return; + }; + if object.get("type").and_then(Value::as_str) == Some("function") { + qualify_responses_function_name(object); + } +} + +fn qualify_responses_function_name(object: &mut Map) { + let namespace = object + .remove("namespace") + .and_then(|value| value.as_str().map(ToOwned::to_owned)); + let Some(namespace) = namespace.as_deref() else { + return; + }; + qualify_responses_function_name_with_namespace(object, namespace); +} + +fn qualify_responses_function_name_with_namespace( + object: &mut Map, + namespace: &str, +) { + if namespace.is_empty() { + return; + } + let Some(name) = object.get("name").and_then(Value::as_str) else { + return; + }; + let prefix = format!("{namespace}{CODEX_NAMESPACE_SEPARATOR}"); + if name.starts_with(&prefix) { + return; + } + object.insert("name".to_string(), Value::String(format!("{prefix}{name}"))); +} + +fn ensure_responses_function_tool_description(object: &mut Map) { + if object.get("type").and_then(Value::as_str) == Some("function") + && !matches!(object.get("description"), Some(Value::String(_))) + { + object.insert("description".to_string(), Value::String(String::new())); + } +} diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 5646344aa1..c4b1e55306 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -932,6 +932,11 @@ impl ClientRouter { .preservation .requests .contains_key(&responses_format) + || request + .llm_request + .messages + .iter() + .any(|message| message.responses.is_some()) || fields.contains_key("previous_response_id") || fields.contains_key("conversation"); if !responses_request && !self.inner.track_provider_state { @@ -1076,6 +1081,7 @@ impl ClientRouter { } let mut segment = input.messages.to_vec(); segment.extend(response.outputs.iter().map(|output| Message { + responses: None, role: output.role, content: output.content.clone(), })); @@ -1543,7 +1549,7 @@ mod tests { .iter() .flat_map(|instruction| &instruction.content) .filter_map(|block| match block { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), _ => None, }) .collect() @@ -1966,7 +1972,7 @@ mod tests { let clients = ClientRouter::new(HashMap::from([(ModelId::from("weak"), client)])); let models = to_category_map(&["weak"]); - let seed = Request { + let mut seed = Request { llm_request: switchyard_translation::decode_request( WireFormat::OpenAiResponses, &json!({ @@ -1983,6 +1989,8 @@ mod tests { raw_request: None, metadata: None, }; + // Native item metadata must identify the Responses source after saved bodies are cleared. + seed.llm_request.preservation.requests.clear(); let (_, response) = run( Arc::new(switchyard_libsy::Passthrough), clients.clone(), @@ -2135,10 +2143,12 @@ mod tests { .fields .insert("previous_response_id".to_string(), json!("msg_stream")); follow.llm_request.messages = vec![Message { + responses: None, role: Role::Tool, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "toolu_stream".to_string(), content: vec![ContentBlock::Text { + responses: None, text: "ready".to_string(), }], is_error: None, diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index 1177a25325..7bfa860b1c 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -725,6 +725,7 @@ async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard let request = Request { llm_request: LlmRequest { messages: vec![Message { + responses: None, role: Role::User, content: vec![ContentBlock::Reasoning { text: "provider reasoning".to_string(), @@ -1278,6 +1279,7 @@ async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Res messages: vec![ Message::text(Role::User, "fix the build"), Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: "call_1".to_string(), @@ -1286,10 +1288,12 @@ async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Res })], }, Message { + responses: None, role: Role::Tool, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "call_1".to_string(), content: vec![ContentBlock::Text { + responses: None, text: "fatal runtime error: out of memory".to_string(), }], is_error: Some(true), diff --git a/crates/libsy/src/algorithms/advisor_gate.rs b/crates/libsy/src/algorithms/advisor_gate.rs index d44711e8d7..8af68c4696 100644 --- a/crates/libsy/src/algorithms/advisor_gate.rs +++ b/crates/libsy/src/algorithms/advisor_gate.rs @@ -381,15 +381,15 @@ impl AdvisorGate { review_tail: Option<&str>, trigger: &'static str, ) -> Result { - // The advisor reviews the FULL transcript: system/developer content is - // normalized out of `messages` into `instructions`, so prepend it back - // as leading messages (identical {role, content} shape) — the task - // constraints the verdict must check against usually live there. + // The advisor needs instructions as well as conversation content to check the task. + // Top-level instructions precede the messages in this transcript. + // Native Responses inline instructions retain their positions within the messages. let transcript_messages: Vec = base .llm_request .instructions .iter() .map(|block| Message { + responses: None, role: block.role, content: block.content.clone(), }) @@ -519,6 +519,7 @@ impl AdvisorGate { instructions: vec![InstructionBlock { role: Role::System, content: vec![ContentBlock::Text { + responses: None, text: self.config.reviewer_system_prompt.clone(), }], }], diff --git a/crates/libsy/src/algorithms/advisor_gate/signals.rs b/crates/libsy/src/algorithms/advisor_gate/signals.rs index 8209a81258..8ae7ea2131 100644 --- a/crates/libsy/src/algorithms/advisor_gate/signals.rs +++ b/crates/libsy/src/algorithms/advisor_gate/signals.rs @@ -66,10 +66,12 @@ mod tests { Message::text(Role::User, "build X"), Message::text(Role::Assistant, "working"), Message { + responses: None, role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "t1".to_string(), content: vec![ContentBlock::Text { + responses: None, text: "ok".to_string(), }], is_error: None, @@ -90,6 +92,7 @@ mod tests { role: Role::Assistant, content: vec![ ContentBlock::Text { + responses: None, text: "running a tool".to_string(), }, ContentBlock::ToolCall(ToolCall { diff --git a/crates/libsy/src/algorithms/advisor_gate/tests.rs b/crates/libsy/src/algorithms/advisor_gate/tests.rs index b59e549e8f..fc8445a229 100644 --- a/crates/libsy/src/algorithms/advisor_gate/tests.rs +++ b/crates/libsy/src/algorithms/advisor_gate/tests.rs @@ -106,6 +106,7 @@ fn tool_use_stop_turn() -> Response { outputs: vec![ResponseOutput { role: Role::Assistant, content: vec![ContentBlock::Text { + responses: None, text: "calling a tool".to_string(), }], url_citations: Vec::new(), @@ -473,7 +474,7 @@ async fn over_cap_consult_flags_truncation_to_the_reviewer() { .content .iter() .filter_map(|block| match block { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), _ => None, }) .collect(); @@ -916,6 +917,7 @@ async fn pattern_trigger_matches_on_tool_call_turns() { role: Role::Assistant, content: vec![ ContentBlock::Text { + responses: None, text: "task_complete: true".to_string(), }, ContentBlock::ToolCall(ToolCall { @@ -959,10 +961,12 @@ async fn min_tool_results_defers_gate() { let with_result = request(vec![ Message::text(Role::User, "build X"), Message { + responses: None, role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "t1".to_string(), content: vec![ContentBlock::Text { + responses: None, text: "ok".to_string(), }], is_error: None, @@ -987,6 +991,7 @@ async fn min_tool_results_counts_batched_result_blocks() { ContentBlock::ToolResult(ToolResult { tool_call_id: id.to_string(), content: vec![ContentBlock::Text { + responses: None, text: "ok".to_string(), }], is_error: None, @@ -995,6 +1000,7 @@ async fn min_tool_results_counts_batched_result_blocks() { let batched = request(vec![ Message::text(Role::User, "build X"), Message { + responses: None, role: Role::User, content: vec![tool_result("t1"), tool_result("t2")], }, @@ -1271,6 +1277,7 @@ async fn consult_request_shape() { assert_eq!( consult.instructions[0].content, vec![ContentBlock::Text { + responses: None, text: REVIEWER_SYSTEM_PROMPT.to_string() }] ); @@ -1296,6 +1303,7 @@ async fn consult_transcript_includes_system_instructions() { gated.llm_request.instructions = vec![InstructionBlock { role: Role::System, content: vec![ContentBlock::Text { + responses: None, text: "the deliverable must be a CSV".to_string(), }], }]; diff --git a/crates/libsy/src/algorithms/advisor_gate/turn.rs b/crates/libsy/src/algorithms/advisor_gate/turn.rs index 48f41545e3..24efcf0f37 100644 --- a/crates/libsy/src/algorithms/advisor_gate/turn.rs +++ b/crates/libsy/src/algorithms/advisor_gate/turn.rs @@ -27,7 +27,7 @@ pub(super) fn visible_text(agg: &AggLlmResponse) -> Option { .iter() .flat_map(|output| output.content.iter()) .filter_map(|block| match block { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), _ => None, }) .collect(); diff --git a/crates/libsy/src/algorithms/composite.rs b/crates/libsy/src/algorithms/composite.rs index 8e259bc142..6fe0364c28 100644 --- a/crates/libsy/src/algorithms/composite.rs +++ b/crates/libsy/src/algorithms/composite.rs @@ -514,6 +514,7 @@ mod tests { assert_eq!( outcome.request.llm_request.instructions[0].content, vec![switchyard_protocol::ContentBlock::Text { + responses: None, text: prompt.into() }] ); diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs index 9279260dde..a0157df40f 100644 --- a/crates/libsy/src/algorithms/escalation.rs +++ b/crates/libsy/src/algorithms/escalation.rs @@ -60,6 +60,7 @@ fn clear_streak(state: &mut State) { fn assistant_message(response: &AggLlmResponse) -> Message { Message { + responses: None, role: Role::Assistant, content: response .first_output() @@ -674,7 +675,7 @@ mod tests { .first() .and_then(|instruction| { instruction.content.iter().find_map(|block| match block { - ContentBlock::Text { text } => Some(text.clone()), + ContentBlock::Text { text, .. } => Some(text.clone()), _ => None, }) }); diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index de0b06e955..90af6689f5 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -92,15 +92,15 @@ impl TaskClassifierVerdict { } } -/// Keeps the opening task and the last `recent_turn_window` turns after it. A -/// window of `0` keeps the task alone. +/// This helper keeps the opening task and the last `recent_turn_window` turns after it. +/// A window of `0` keeps no turns after the task. /// -/// Inbound decoders normalize client system and developer content into -/// `LlmRequest::instructions`, so it never reaches this list. +/// The classifier passes the grouped conversation view, which separates native +/// inline instructions from messages. Direct callers can still supply system +/// and developer messages; this helper retains them before the task. /// -/// Selects by reference and clones only what survives — a coding-agent -/// conversation carries every tool result, so cloning it whole to keep a window -/// would copy the transcript on each judged turn. +/// This helper selects by reference and clones only retained messages. The +/// conversation view may already have cloned native Responses messages to group them. fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec { let is_instruction = |message: &Message| matches!(message.role, Role::System | Role::Developer); let mut kept: Vec<&Message> = messages.iter().filter(|m| is_instruction(m)).collect(); @@ -179,6 +179,7 @@ fn task_messages(messages: &[Message]) -> Vec { .into_iter() .flatten() .map(|message| Message { + responses: None, role: Role::User, content: message .content @@ -198,11 +199,12 @@ struct TaskInput { impl TaskInput { /// Selects conversation content without adding judge instructions. fn messages(&self, request: &Request) -> Vec { + let conversation = switchyard_translation::conversation(&request.llm_request); // The default preserves the whole-task anchor and latest user update. A // configured window widens that to the surrounding conversation. let mut messages = match self.recent_turn_window { - Some(window) => trim_messages(&request.llm_request.messages, window), - None => task_messages(&request.llm_request.messages), + Some(window) => trim_messages(&conversation.messages, window), + None => task_messages(&conversation.messages), }; // Reasoning is provider-private and not required to classify the task. Some // upstreams also reject an unsigned reasoning item replayed without the @@ -1053,7 +1055,7 @@ mod tests { .first() .and_then(|instruction| { instruction.content.iter().find_map(|b| { - if let ContentBlock::Text { text } = b { + if let ContentBlock::Text { text, .. } = b { Some(text.clone()) } else { None @@ -1561,6 +1563,7 @@ mod tests { }; let mut request = classify_session_request(); request.llm_request.messages.push(Message { + responses: None, role: Role::Assistant, content: vec![ ContentBlock::Reasoning { @@ -1569,6 +1572,7 @@ mod tests { details: vec![], }, ContentBlock::Text { + responses: None, text: "visible progress".into(), }, ], @@ -2156,6 +2160,7 @@ mod tests { fn tool_call(id: &str) -> Message { Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: id.to_string(), @@ -2167,10 +2172,12 @@ mod tests { fn tool_result(id: &str) -> Message { Message { + responses: None, role: Role::Tool, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: id.to_string(), content: vec![ContentBlock::Text { + responses: None, text: "tool output".to_string(), }], is_error: None, @@ -2188,6 +2195,7 @@ mod tests { result.role = Role::User; let mut mixed = result.clone(); mixed.content.push(ContentBlock::Text { + responses: None, text: "latest follow-up".to_string(), }); let input = TaskInput { @@ -2335,6 +2343,7 @@ mod tests { Message::text(Role::System, "client instructions"), Message::text(Role::User, "initial task"), Message { + responses: None, role: Role::Assistant, content: vec![ ContentBlock::Reasoning { @@ -2343,6 +2352,7 @@ mod tests { details: Vec::new(), }, ContentBlock::Text { + responses: None, text: "visible answer".to_string(), }, ], @@ -2350,6 +2360,7 @@ mod tests { tool_call("call-1"), tool_result("call-1"), Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::Reasoning { text: "reasoning-only turn".to_string(), diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs index 4963b5b194..549855541e 100644 --- a/crates/libsy/src/algorithms/noop.rs +++ b/crates/libsy/src/algorithms/noop.rs @@ -35,6 +35,7 @@ impl Algorithm for Noop { url_citations: Vec::new(), role: Role::Assistant, content: vec![ContentBlock::Text { + responses: None, text: "OK".to_string(), }], stop_reason: Some(StopReason::EndTurn), diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 55cc17ffb6..f1f0f2bae3 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -629,6 +629,7 @@ mod tests { &mut passing.llm_request.messages[2].content[0] { result.content = vec![switchyard_protocol::ContentBlock::Text { + responses: None, text: "test result: ok. 1 passed; 0 failed".into(), }]; } diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs index b236cf8765..c037cec33b 100644 --- a/crates/libsy/src/algorithms/util/affinity.rs +++ b/crates/libsy/src/algorithms/util/affinity.rs @@ -57,7 +57,7 @@ pub(crate) fn is_user_turn(message: &Message) -> bool { } fn is_environment_context(block: &ContentBlock) -> bool { - matches!(block, ContentBlock::Text { text } if text.trim() + matches!(block, ContentBlock::Text { text, .. } if text.trim() .strip_prefix("") .and_then(|context| context.split_once("")) .is_some_and(|(_, rest)| rest.trim().is_empty())) @@ -73,7 +73,10 @@ pub(crate) fn has_new_user_turn(messages: &[Message]) -> bool { messages .iter() .rev() - .find(|message| !is_environment_update(message)) + .find(|message| { + !matches!(message.role, Role::System | Role::Developer) + && !is_environment_update(message) + }) .is_some_and(is_user_turn) } @@ -358,6 +361,7 @@ mod tests { ))); let mut mixed = Message::text(Role::User, context); mixed.content.push(ContentBlock::Text { + responses: None, text: "fix the parser".into(), }); assert!(is_user_turn(&mixed)); @@ -370,6 +374,8 @@ mod tests { has_new_user_turn(&[ Message::text(role, "follow-up"), Message::text(Role::User, context), + Message::text(Role::System, "Keep the task focused."), + Message::text(Role::Developer, "Verify the result."), ]), expected ); @@ -670,9 +676,11 @@ mod tests { }; let text_only = request(Message::text(Role::User, "Implement the parser.")); let text_with_reasoning = request(Message { + responses: None, role: Role::User, content: vec![ ContentBlock::Text { + responses: None, text: "Implement the parser.".to_string(), }, ContentBlock::Reasoning { @@ -937,6 +945,7 @@ mod tests { // A tool continuation holds the assignment, so no judge call. let mut continued = opening.clone(); continued.llm_request.messages.push(Message { + responses: None, role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "call-1".to_string(), @@ -983,6 +992,7 @@ mod tests { // The new turn's tool continuation is not held on the earlier turn's model. let mut continued = spoke.clone(); continued.llm_request.messages.push(Message { + responses: None, role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "call-1".to_string(), diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index c65c598417..67171b21b7 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -202,10 +202,11 @@ pub(crate) struct EscalationInput { impl ClassifierInput for EscalationInput { fn build_messages(&self, _state: &State, request: &Request) -> Vec { + let conversation = switchyard_translation::conversation(&request.llm_request); let summary = summarize_for_judge( - &request.llm_request.instructions, - &request.llm_request.messages, - conversation_turn(request), + &conversation.instructions, + &conversation.messages, + conversation_turn(&conversation.messages), self.phase, &self.config, ); @@ -323,12 +324,9 @@ fn build_contract( /// message is this turn — no `+ 1`. Counting the caller's request instead would report the /// turn after the one under judgement. /// -/// Messages are already normalized by `switchyard-protocol`, so this needs no -/// per-format branching. -pub(crate) fn conversation_turn(request: &Request) -> usize { - request - .llm_request - .messages +/// Callers pass the conversation view so native Responses items share assistant turns. +pub(crate) fn conversation_turn(messages: &[Message]) -> usize { + messages .iter() .filter(|message| message.role == Role::Assistant) .count() @@ -387,7 +385,7 @@ fn summarize_for_judge( } for message in messages { - let text = message_text(message); + let text = message_text(message.role, &message.content); match message.role { Role::System | Role::Developer => instruction_anchors.push(format!( "[{}] {}", @@ -550,6 +548,7 @@ mod tests { assert_eq!( built.llm_request.instructions[0].content.as_slice(), &[ContentBlock::Text { + responses: None, text: PROMPT_TEMPLATE.to_string() }] ); @@ -611,7 +610,12 @@ mod tests { .llm_request .messages .push(Message::text(Role::Assistant, "this turn's reply")); - assert_eq!(conversation_turn(&judged), turn); + assert_eq!( + conversation_turn( + &switchyard_translation::conversation(&judged.llm_request).messages + ), + turn + ); } } @@ -671,7 +675,7 @@ mod tests { let built = judge.build_request(&State::default(), &request_at_turn(None, 1)); let system_prompt = built.llm_request.instructions[0].content.iter().find_map( |content| match content { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), _ => None, }, ); diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rs index 12b5692e34..588b10acf0 100644 --- a/crates/libsy/src/algorithms/util/subagent.rs +++ b/crates/libsy/src/algorithms/util/subagent.rs @@ -55,7 +55,7 @@ fn delegated_prompt_request(request: &Request) -> Option { .filter(|message| message.role == Role::User) .find_map(|message| { message.content.iter().rev().find_map(|block| match block { - ContentBlock::Text { text } if !text.trim().is_empty() => Some(text.clone()), + ContentBlock::Text { text, .. } if !text.trim().is_empty() => Some(text.clone()), _ => None, }) })?; diff --git a/crates/libsy/src/algorithms/util/tier_fixtures.rs b/crates/libsy/src/algorithms/util/tier_fixtures.rs index 1e126e3f1a..99817f0738 100644 --- a/crates/libsy/src/algorithms/util/tier_fixtures.rs +++ b/crates/libsy/src/algorithms/util/tier_fixtures.rs @@ -95,6 +95,7 @@ pub(crate) fn turn_request(failed: bool) -> Request { messages: vec![ Message::text(Role::User, "fix the build"), Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: "call_1".to_string(), @@ -103,10 +104,12 @@ pub(crate) fn turn_request(failed: bool) -> Request { })], }, Message { + responses: None, role: Role::Tool, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "call_1".to_string(), content: vec![ContentBlock::Text { + responses: None, text: content.to_string(), }], is_error: Some(failed), diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index f4d590e6d7..1f30f69b16 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -1055,8 +1055,8 @@ fn extract_tool_signals_with_window_and_semantics( semantics: &ToolSemantics, current_user_turn: bool, ) -> ToolSignals { - // Read the decoded conversation, including preserved built-in tool outputs. - let messages = &request.llm_request.messages; + let conversation = switchyard_translation::conversation(&request.llm_request); + let messages = &conversation.messages; let boundary = if current_user_turn { messages.iter().rposition(is_user_turn) } else { @@ -1297,7 +1297,7 @@ fn command_text(value: &Value) -> Option { /// Text carried by a content block, ignoring the non-textual kinds. fn text_of(block: &ContentBlock) -> Option<&str> { match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => Some(text.as_str()), _ => None, } } @@ -1767,6 +1767,7 @@ mod tests { // assistant message with a single named tool call fn tc(name: &str) -> Message { Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: String::new(), @@ -1779,6 +1780,7 @@ mod tests { // assistant Bash message carrying `command` fn bash(command: &str) -> Message { Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: String::new(), @@ -1791,10 +1793,12 @@ mod tests { // a tool result message (goes in a user-role message, as in Anthropic's normalised form) fn tr(text: &str) -> Message { Message { + responses: None, role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: String::new(), content: vec![ContentBlock::Text { + responses: None, text: text.to_string(), }], is_error: None, @@ -2100,6 +2104,7 @@ mod tests { #[test] fn retrieved_file_contents_are_ignored() { let call = |id: &str, name: &str, arguments: Value| Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: id.to_string(), @@ -2108,10 +2113,12 @@ mod tests { })], }; let result = |id: &str, text: &str| Message { + responses: None, role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: id.to_string(), content: vec![ContentBlock::Text { + responses: None, text: text.to_string(), }], is_error: None, @@ -2261,6 +2268,7 @@ mod tests { "error": null, }); let call = Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: String::new(), @@ -2415,6 +2423,7 @@ mod tests { let mut raw = raw.clone(); raw["call_id"] = json!(call_id); Message { + responses: None, role: Role::User, content: vec![ContentBlock::Unknown { provider: WireFormat::OpenAiResponses.into(), @@ -2453,6 +2462,7 @@ mod tests { for index in 0..3 { success["call_id"] = json!(format!("success_{index}")); request.llm_request.messages.push(Message { + responses: None, role: Role::User, content: vec![ContentBlock::Unknown { provider: WireFormat::OpenAiResponses.into(), @@ -2484,9 +2494,11 @@ mod tests { Message::text(Role::User, "do something"), Message::text(Role::Assistant, "working"), Message { + responses: None, role: Role::User, content: vec![ result(vec![ContentBlock::Text { + responses: None, text: "ok".to_string(), }]), result(Vec::new()), @@ -2535,6 +2547,7 @@ mod tests { fn exec_command(cmd: Value) -> Message { Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: String::new(), @@ -2860,6 +2873,7 @@ mod tests { ]; for arguments in arguments { let call = Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: String::new(), diff --git a/crates/libsy/src/algorithms/util/transcript.rs b/crates/libsy/src/algorithms/util/transcript.rs index 751fc4fd26..6e7ac31eae 100644 --- a/crates/libsy/src/algorithms/util/transcript.rs +++ b/crates/libsy/src/algorithms/util/transcript.rs @@ -3,18 +3,17 @@ //! Plain-text message rendering shared by routing judges. -use switchyard_protocol::{ContentBlock, Message, Role}; +use switchyard_protocol::{ContentBlock, Role}; -/// Flattens a message to plain text, tool calls and tool results included. +/// This helper renders content as plain text, including tool calls and results. /// -/// [`Message::text_content`] is deliberately not used here: it keeps only text and refusal -/// blocks, which would erase exactly the repeated-command signal the judge's loop detection -/// relies on. -pub(crate) fn message_text(message: &Message) -> String { +/// [`switchyard_protocol::Message::text_content`] omits the tool calls and results +/// that judges need for loop detection. For assistant content, this helper removes +/// Terminus command batches duplicated by structured tool calls. +pub(crate) fn message_text(role: Role, content: &[ContentBlock]) -> String { let mut parts = Vec::new(); - let terminus_commands = if message.role == Role::Assistant { - message - .content + let terminus_commands = if role == Role::Assistant { + content .iter() .filter_map(|block| match block { ContentBlock::ToolCall(call) if call.name == "bash_command" => call @@ -27,7 +26,7 @@ pub(crate) fn message_text(message: &Message) -> String { } else { Vec::new() }; - collect_text(&message.content, &mut parts, &terminus_commands); + collect_text(content, &mut parts, &terminus_commands); parts.join(" ") } @@ -105,7 +104,7 @@ pub(super) fn collect_text( ) { for block in content { match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => { + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => { parts.push(without_duplicated_terminus_commands(text, tool_commands)); } ContentBlock::ToolCall(call) => { @@ -126,9 +125,11 @@ mod tests { #[test] fn message_text_keeps_tool_calls_and_results() { let call = Message { + responses: None, role: Role::Assistant, content: vec![ ContentBlock::Text { + responses: None, text: "running it".to_string(), }, ContentBlock::ToolCall(ToolCall { @@ -138,21 +139,23 @@ mod tests { }), ], }; - let text = message_text(&call); + let text = message_text(call.role, &call.content); assert!(text.contains("running it"), "{text}"); assert!(text.contains(r#"tool_call bash({"cmd":"ls"})"#), "{text}"); let result = Message { + responses: None, role: Role::Tool, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "call-1".to_string(), content: vec![ContentBlock::Text { + responses: None, text: "no such file".to_string(), }], is_error: Some(true), })], }; - assert_eq!(message_text(&result), "no such file"); + assert_eq!(message_text(result.role, &result.content), "no such file"); } /// A raw command batch fully mirrored by structured tool calls is emptied in the judge view. @@ -161,9 +164,11 @@ mod tests { let first_command = "grep -n bug app.py\n"; let second_command = "sed -n '1,80p' app.py\n"; let message = Message { + responses: None, role: Role::Assistant, content: vec![ ContentBlock::Text { + responses: None, text: format!( "Before\n```json\n{}\n```\nAfter", json!({ @@ -188,7 +193,7 @@ mod tests { ], }; - let text = message_text(&message); + let text = message_text(message.role, &message.content); assert!(text.contains("inspect the reported file"), "{text}"); assert!(text.contains("Before"), "{text}"); @@ -212,9 +217,11 @@ mod tests { .to_string() }; let message = Message { + responses: None, role: Role::Assistant, content: vec![ ContentBlock::Text { + responses: None, text: format!( "First {} second {} repeated {}", batch(first_command), @@ -235,7 +242,7 @@ mod tests { ], }; - let text = message_text(&message); + let text = message_text(message.role, &message.content); assert_eq!(text.matches(r#""commands":[]"#).count(), 2, "{text}"); assert_eq!(text.matches("grep -n bug app.py").count(), 2, "{text}"); @@ -252,13 +259,21 @@ mod tests { }) .to_string(); let without_tool_call = Message { + responses: None, role: Role::Assistant, - content: vec![ContentBlock::Text { text: text.clone() }], + content: vec![ContentBlock::Text { + responses: None, + text: text.clone(), + }], }; let mismatched_tool_call = Message { + responses: None, role: Role::Assistant, content: vec![ - ContentBlock::Text { text }, + ContentBlock::Text { + responses: None, + text, + }, ContentBlock::ToolCall(ToolCall { id: "call-1".to_string(), name: "bash_command".to_string(), @@ -268,13 +283,13 @@ mod tests { }; assert_eq!( - message_text(&without_tool_call) + message_text(without_tool_call.role, &without_tool_call.content) .matches("grep -n bug app.py") .count(), 1 ); assert_eq!( - message_text(&mismatched_tool_call) + message_text(mismatched_tool_call.role, &mismatched_tool_call.content) .matches("grep -n bug app.py") .count(), 1 @@ -287,9 +302,11 @@ mod tests { let first_command = "grep -n bug app.py\n"; let second_command = "sed -n '1,80p' app.py\n"; let message = Message { + responses: None, role: Role::Assistant, content: vec![ ContentBlock::Text { + responses: None, text: json!({ "commands": [ {"keystrokes": first_command}, @@ -306,7 +323,7 @@ mod tests { ], }; - let text = message_text(&message); + let text = message_text(message.role, &message.content); assert_eq!(text.matches("grep -n bug app.py").count(), 2, "{text}"); assert_eq!(text.matches("sed -n '1,80p' app.py").count(), 1, "{text}"); @@ -318,9 +335,11 @@ mod tests { fn message_text_keeps_duplicate_commands_without_one_tool_call_each() { let command = "grep -n bug app.py\n"; let message = Message { + responses: None, role: Role::Assistant, content: vec![ ContentBlock::Text { + responses: None, text: json!({ "commands": [ {"keystrokes": command}, @@ -337,7 +356,7 @@ mod tests { ], }; - let text = message_text(&message); + let text = message_text(message.role, &message.content); assert_eq!(text.matches("grep -n bug app.py").count(), 3, "{text}"); assert!(!text.contains(r#""commands":[]"#), "{text}"); diff --git a/crates/libsy/src/algorithms/vgr/compaction.rs b/crates/libsy/src/algorithms/vgr/compaction.rs index de2308aa1f..e5bf197777 100644 --- a/crates/libsy/src/algorithms/vgr/compaction.rs +++ b/crates/libsy/src/algorithms/vgr/compaction.rs @@ -41,11 +41,11 @@ const EARLY_SECTION_BUDGET: usize = 2_500; const NARRATION_BUDGET: usize = 200; const RECENT_USER_TEXTS: usize = 2; -/// Condenses everything after the task message and records how, so later -/// requests of this user turn can be rewritten identically. +/// This helper condenses conversation after the task message and retains inline instructions. +/// Later requests of this user turn reuse the same recorded summary. /// -/// Returns `false`, leaving the request untouched, when there is no task -/// message followed by work from the efficient tier. +/// This helper returns `false` and leaves the request unchanged when no task +/// message is followed by work from the efficient tier. pub(super) fn compact(request: &mut Request, state: &mut State, note: &str) -> bool { let messages = &request.llm_request.messages; let start = turn_start(request); @@ -54,7 +54,10 @@ pub(super) fn compact(request: &mut Request, state: &mut State, note: &str) -> b }; let head = start + task + 1; let cut = messages.len(); - if cut <= head { + if !messages[head..cut] + .iter() + .any(|message| !matches!(message.role, Role::System | Role::Developer)) + { return false; } let mut summary = digest(&messages[head..cut]); @@ -121,10 +124,14 @@ fn recorded(state: &State, request: &Request) -> Option<(usize, usize, String)> fn rewrite(request: &mut Request, head: usize, cut: usize, summary: &str) { let messages = &mut request.llm_request.messages; - messages[head - 1].content.push(ContentBlock::Text { - text: summary.to_string(), - }); - messages.drain(head..cut); + messages[head - 1].content.push(ContentBlock::text(summary)); + // Inline instructions and their metadata must survive replacement of the surrounding history. + let instructions = messages[head..cut] + .iter() + .filter(|message| matches!(message.role, Role::System | Role::Developer)) + .cloned() + .collect::>(); + messages.splice(head..cut, instructions); drop_exact_replay(request); } @@ -156,7 +163,7 @@ fn result_text(result: &ToolResult) -> String { .content .iter() .map(|part| match part { - ContentBlock::Text { text } => text.as_str(), + ContentBlock::Text { text, .. } => text.as_str(), _ => "[non-text content]", }) .collect::>() @@ -170,10 +177,10 @@ fn digest(messages: &[Message]) -> String { for message in messages { for block in &message.content { match block { - ContentBlock::Text { text } if message.role == Role::Assistant => { + ContentBlock::Text { text, .. } if message.role == Role::Assistant => { narration.push(text.as_str()); } - ContentBlock::Text { text } if message.role == Role::User => { + ContentBlock::Text { text, .. } if message.role == Role::User => { user_text.push(clip_mid(text, RECENT_RESULT_BUDGET, 0.5)); } ContentBlock::ToolCall(call) => steps.push(Step { @@ -274,6 +281,7 @@ mod tests { fn call(id: &str, command: &str) -> Message { Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: id.into(), @@ -285,10 +293,11 @@ mod tests { fn result(id: &str, text: &str) -> Message { Message { + responses: None, role: Role::Tool, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: id.into(), - content: vec![ContentBlock::Text { text: text.into() }], + content: vec![ContentBlock::text(text)], is_error: None, })], } @@ -441,6 +450,7 @@ mod tests { #[test] fn batched_results_are_paired_by_tool_call_id() { let calls = Message { + responses: None, role: Role::Assistant, content: ["a", "b"] .into_iter() @@ -455,6 +465,7 @@ mod tests { }; for order in [["a", "b"], ["b", "a"]] { let results = Message { + responses: None, role: Role::Tool, content: order .into_iter() @@ -462,6 +473,7 @@ mod tests { ContentBlock::ToolResult(ToolResult { tool_call_id: id.into(), content: vec![ContentBlock::Text { + responses: None, text: format!("output_{id}"), }], is_error: Some(id == "b"), diff --git a/crates/libsy/src/algorithms/vgr/render.rs b/crates/libsy/src/algorithms/vgr/render.rs index 1e908cd986..4c3536506e 100644 --- a/crates/libsy/src/algorithms/vgr/render.rs +++ b/crates/libsy/src/algorithms/vgr/render.rs @@ -87,7 +87,7 @@ fn agentic_trajectory(request: &Request) -> Vec { for message in messages { for block in &message.content { let event = match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => { + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => { match message.role { Role::Assistant => Some(("assistant", text.clone())), Role::Tool => Some(("tool result", text.clone())), diff --git a/crates/libsy/src/algorithms/vgr/rungs.rs b/crates/libsy/src/algorithms/vgr/rungs.rs index ec677856f1..c676b6a192 100644 --- a/crates/libsy/src/algorithms/vgr/rungs.rs +++ b/crates/libsy/src/algorithms/vgr/rungs.rs @@ -90,9 +90,7 @@ pub(super) fn build_request( llm_request: LlmRequest { instructions: vec![InstructionBlock { role: Role::System, - content: vec![ContentBlock::Text { - text: question.system_prompt(), - }], + content: vec![ContentBlock::text(question.system_prompt())], }], messages: vec![Message::text(Role::User, judged_text)], output: OutputParams { @@ -145,7 +143,9 @@ pub(super) fn build_verdict_request( .into_iter() .flat_map(|output| &output.content) .filter_map(|block| match block { - ContentBlock::Reasoning { text, .. } | ContentBlock::Text { text } => Some(text.trim()), + ContentBlock::Reasoning { text, .. } | ContentBlock::Text { text, .. } => { + Some(text.trim()) + } _ => None, }) .collect::>() @@ -171,19 +171,39 @@ pub(super) fn build_verdict_request( /// Stable history messages let llama.cpp reuse its cache before the proposed turn. pub(super) fn turn_view(request: &Request, proposed: &AggLlmResponse) -> Vec { - let turns: Vec = request - .llm_request + let conversation = switchyard_translation::conversation(&request.llm_request); + let mut turns: Vec = conversation .instructions .iter() .map(|instruction| Turn { role: instruction.role, text: content_text(&instruction.content).0, }) - .chain(request.llm_request.messages.iter().map(|message| Turn { - role: message.role, - text: message_text(message), - })) .collect(); + for message in conversation.messages.iter() { + if message.content.is_empty() { + turns.push(Turn { + role: message.role, + text: String::new(), + }); + } + // Contiguous assistant blocks stay together so message_text can remove + // command batches duplicated by structured tool calls. + for content in message.content.chunk_by(|left, right| { + matches!(left, ContentBlock::ToolResult(_)) + == matches!(right, ContentBlock::ToolResult(_)) + }) { + let role = if matches!(content[0], ContentBlock::ToolResult(_)) { + Role::Tool + } else { + message.role + }; + turns.push(Turn { + role, + text: message_text(role, content), + }); + } + } let system_index = turns.iter().position(|turn| turn.role == Role::System); let first_user_index = turns.iter().position(|turn| turn.role == Role::User); @@ -249,7 +269,7 @@ fn proposed_line(proposed: &AggLlmResponse) -> String { { for block in &output.content { match block { - ContentBlock::Text { text } if !text.trim().is_empty() => { + ContentBlock::Text { text, .. } if !text.trim().is_empty() => { fragments.push(redact(text)); } ContentBlock::ToolCall(call) => fragments.push(format!( @@ -350,7 +370,7 @@ fn assistant_text(output: &ResponseOutput) -> Option { .content .iter() .filter_map(|block| match block { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), _ => None, }) .collect::>() @@ -366,7 +386,7 @@ mod tests { AggLlmResponse { outputs: vec![ResponseOutput { role: Role::Assistant, - content: vec![ContentBlock::Text { text: text.into() }], + content: vec![ContentBlock::text(text)], url_citations: Vec::new(), stop_reason, }], diff --git a/crates/libsy/src/algorithms/vgr/runtime/tests.rs b/crates/libsy/src/algorithms/vgr/runtime/tests.rs index fe9599559f..1717adb6e9 100644 --- a/crates/libsy/src/algorithms/vgr/runtime/tests.rs +++ b/crates/libsy/src/algorithms/vgr/runtime/tests.rs @@ -255,7 +255,7 @@ async fn a_typed_follow_up_answer_is_judged_on_the_answer_itself() -> Result<()> .first() .and_then(|block| block.content.first()) .and_then(|block| match block { - ContentBlock::Text { text } => Some(text.clone()), + ContentBlock::Text { text, .. } => Some(text.clone()), _ => None, }) .unwrap_or_default(); @@ -1137,7 +1137,7 @@ async fn default_verification_asks_capable_evidence_then_answer() -> Result<()> .iter() .flat_map(|block| &block.content) .any(|block| { - matches!(block, ContentBlock::Text { text } + matches!(block, ContentBlock::Text { text, .. } if text.starts_with("You are a strict answer verifier")) }); seen.lock().push(is_answer); diff --git a/crates/libsy/src/algorithms/vgr/tests.rs b/crates/libsy/src/algorithms/vgr/tests.rs index a0b43f4ac6..eb9e467072 100644 --- a/crates/libsy/src/algorithms/vgr/tests.rs +++ b/crates/libsy/src/algorithms/vgr/tests.rs @@ -20,6 +20,7 @@ pub(super) fn request(messages: Vec) -> Request { pub(super) fn call(id: &str) -> Message { Message { + responses: None, role: Role::Assistant, content: vec![ContentBlock::ToolCall(ToolCall { id: id.into(), @@ -31,10 +32,14 @@ pub(super) fn call(id: &str) -> Message { pub(super) fn result(id: &str, text: &str) -> Message { Message { + responses: None, role: Role::Tool, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: id.into(), - content: vec![ContentBlock::Text { text: text.into() }], + content: vec![ContentBlock::Text { + responses: None, + text: text.into(), + }], is_error: None, })], } @@ -137,6 +142,7 @@ fn missing_or_unrepresentable_evidence_fails_closed() { assert_eq!(branch(&request(Vec::new()), "done", None), Branch::Unknown); let image = request(vec![Message { + responses: None, role: Role::User, content: vec![ContentBlock::Image { source: ImageSource::Url { @@ -192,10 +198,12 @@ fn trajectory_labels_agree_with_tool_failure_counts() { Message::text(Role::User, "check the task"), call("c1"), Message { + responses: None, role: Role::Tool, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "c1".into(), content: vec![ContentBlock::Text { + responses: None, text: output.into(), }], is_error, diff --git a/crates/libsy/src/algorithms/vgr/text.rs b/crates/libsy/src/algorithms/vgr/text.rs index 6148a5b1a6..7d4f30cc15 100644 --- a/crates/libsy/src/algorithms/vgr/text.rs +++ b/crates/libsy/src/algorithms/vgr/text.rs @@ -57,7 +57,7 @@ pub(super) fn content_text(content: &[ContentBlock]) -> (String, bool) { let mut unsupported = false; for block in content { match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => parts.push(text), + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => parts.push(text), ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_) | ContentBlock::Reasoning { .. } => {} @@ -68,6 +68,7 @@ pub(super) fn content_text(content: &[ContentBlock]) -> (String, bool) { } pub(super) fn turns(request: &Request) -> (Vec, bool) { + let conversation = switchyard_translation::conversation(&request.llm_request); let mut flattened = Vec::new(); let mut unsupported = false; let mut push = |role: Role, content: &[ContentBlock]| { @@ -75,10 +76,10 @@ pub(super) fn turns(request: &Request) -> (Vec, bool) { unsupported |= block_unsupported; flattened.push(Turn { role, text }); }; - for InstructionBlock { role, content } in &request.llm_request.instructions { + for InstructionBlock { role, content } in conversation.instructions.iter() { push(*role, content); } - for Message { role, content } in &request.llm_request.messages { + for Message { role, content, .. } in conversation.messages.iter() { push(*role, content); } (flattened, unsupported) @@ -106,7 +107,7 @@ pub(super) fn newest_prompt_is_blank(request: &Request) -> bool { .find(|message| message.role == Role::User) .is_none_or(|message| { message.content.iter().all(|block| match block { - ContentBlock::Text { text } => text.chars().all(|c| { + ContentBlock::Text { text, .. } => text.chars().all(|c| { c.is_whitespace() || matches!(c, '\u{200B}'..='\u{200D}' | '\u{2060}' | '\u{FEFF}') }), @@ -129,10 +130,10 @@ pub(super) fn starts_user_turn(messages: &[Message], index: usize) -> bool { let (Some(message), Some(previous)) = ( messages.get(index), messages.get(..index).and_then(|previous| { - previous - .iter() - .rev() - .find(|message| !is_environment_update(message)) + previous.iter().rev().find(|message| { + !matches!(message.role, Role::System | Role::Developer) + && !is_environment_update(message) + }) }), ) else { return false; @@ -145,12 +146,20 @@ pub(super) fn starts_user_turn(messages: &[Message], index: usize) -> bool { .any(|block| matches!(block, ContentBlock::ToolCall(_))) } -/// The current turn's start, using the initiator header for the newest message. +/// This helper finds the current turn's start, using the initiator header for the newest message. +/// It returns an index into the original messages so compaction can edit native history. pub(super) fn turn_start(request: &Request) -> usize { let messages = &request.llm_request.messages; + let newest = messages + .iter() + .rposition(|message| { + !matches!(message.role, Role::System | Role::Developer) + && !is_environment_update(message) + }) + .unwrap_or(0); let end = match initiated_by_user(request) { - Some(true) => return messages.len().saturating_sub(1), - Some(false) => messages.len().saturating_sub(1), + Some(true) => return newest, + Some(false) => newest, None => messages.len(), }; (0..end) @@ -246,7 +255,7 @@ pub(super) fn tool_result_failed(result: &ToolResult) -> bool { .content .iter() .filter_map(|block| match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text), + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => Some(text), _ => None, }) .map(String::as_str) diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index deb7380b3a..3ec0b3c722 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -394,7 +394,7 @@ impl AnswerSummary { }) && response.outputs.iter().any(|output| { output.role == switchyard_protocol::Role::Assistant && output.content.iter().any( - |block| matches!(block, ContentBlock::Text { text } if !text.trim().is_empty()), + |block| matches!(block, ContentBlock::Text { text, .. } if !text.trim().is_empty()), ) }); Self { final_reply } @@ -976,7 +976,10 @@ mod tests { answer.outputs.push(tool); assert!(!AnswerSummary::of(&answer).final_reply); answer.outputs.pop(); - answer.outputs[0].content = vec![ContentBlock::Text { text: " ".into() }]; + answer.outputs[0].content = vec![ContentBlock::Text { + responses: None, + text: " ".into(), + }]; assert!(!AnswerSummary::of(&answer).final_reply); assert!(!AnswerSummary::of(&AggLlmResponse::default()).final_reply); } diff --git a/crates/prefill-router/tests/unit/algorithm.rs b/crates/prefill-router/tests/unit/algorithm.rs index 2aad5fc7cd..e4aef96b0b 100644 --- a/crates/prefill-router/tests/unit/algorithm.rs +++ b/crates/prefill-router/tests/unit/algorithm.rs @@ -183,10 +183,12 @@ async fn tool_result_after_the_person_is_walked_past_not_skipped() -> libsy::Res Message::text(Role::User, "follow-up task"), Message::text(Role::Assistant, "calling a tool"), Message { + responses: None, role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "call-1".to_string(), content: vec![ContentBlock::Text { + responses: None, text: "tool output".to_string(), }], is_error: None, @@ -236,10 +238,12 @@ async fn routes_on_latest_user_text_and_reuses_decision_for_tool_steps() -> libs Message::text(Role::User, "follow-up task"), Message::text(Role::Assistant, "calling a tool"), Message { + responses: None, role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: "call-1".to_string(), content: vec![ContentBlock::Text { + responses: None, text: "tool output".to_string(), }], is_error: None, diff --git a/crates/protocol/README.md b/crates/protocol/README.md index 83a1eff179..348e1720f4 100644 --- a/crates/protocol/README.md +++ b/crates/protocol/README.md @@ -27,16 +27,11 @@ serde_json = "1" ## Simple request ```rust -use switchyard_protocol::{ContentBlock, LlmRequest, Message, Role}; +use switchyard_protocol::{LlmRequest, Message, Role}; let request = LlmRequest { model: Some("provider/model".into()), - messages: vec![Message { - role: Role::User, - content: vec![ContentBlock::Text { - text: "Explain tail latency".into(), - }], - }], + messages: vec![Message::text(Role::User, "Explain tail latency")], ..LlmRequest::default() }; @@ -61,9 +56,7 @@ let request = Request { model: Some("provider/model".into()), instructions: vec![InstructionBlock { role: Role::System, - content: vec![ContentBlock::Text { - text: "Answer with concise operational guidance.".into(), - }], + content: vec![ContentBlock::text("Answer with concise operational guidance.")], }], messages: vec![Message::text(Role::User, "Why is p99 latency rising?")], tools: vec![ToolDefinition { @@ -99,6 +92,26 @@ let request = Request { assert_eq!(request.llm_request.tools[0].function().map(|tool| tool.name.as_str()), Some("lookup_metric")); ``` +## Responses request history + +Decoded Responses requests retain native input items as ordered messages, +including inline system and developer messages. Top-level `instructions` remain +separate. `Message.responses` retains item fields such as `id` and assistant +`phase`, including whether `phase` was missing, null, or supplied with a value. +The item `id` stays separate from the `call_id` that pairs a tool call with its +output. Text blocks retain their own metadata, so moving or editing a block +keeps that metadata with the block. +This metadata survives serialization and reconstruction after saved request +bodies are cleared. Reconstruction uses the current role and content for edited +fields. Edited reasoning does not reuse the original encrypted content or item ID. + +The translation crate provides `conversation(&LlmRequest)` for routing and APIs +that need grouped turns. The request owns the native history and its metadata. + +Use `Message::text` and `ContentBlock::text` to create plain text without native +metadata. Rust struct literals for `Message` and `ContentBlock::Text` must now +include `responses`; set it to `None` for content created without that metadata. + ## Response forms [`LlmResponse`] contains either a completed [`AggLlmResponse`] or a diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index 0123e55c97..7c0a5cf27b 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -63,9 +63,7 @@ pub fn text_response(model: Option, completion: impl Into) -> Ag outputs: vec![ResponseOutput { url_citations: Vec::new(), role: Role::Assistant, - content: vec![ContentBlock::Text { - text: completion.into(), - }], + content: vec![ContentBlock::text(completion)], stop_reason: None, }], ..AggLlmResponse::default() @@ -86,7 +84,7 @@ pub fn completion_text(response: &AggLlmResponse) -> String { .content .iter() .filter_map(|block| match block { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), _ => None, }) .collect::() diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index edcf87689c..2e06125c8f 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -48,6 +48,61 @@ pub struct Message { pub role: Role, /// Ordered message content. pub content: Vec, + /// This metadata retains native Responses fields outside `role` and `content`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub responses: Option, +} + +/// This enum records how a Responses item represented its content or tool output. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResponsesContentKind { + /// The item omitted its `content` or `output` field. + #[default] + Missing, + /// The item supplied an explicit null. + Null, + /// The item supplied one string. + String, + /// The item supplied an ordered array of content blocks. + Array, + /// The request supplied its entire `input` as one string. + InputString, +} + +/// Native item fields stay with their message when the conversation is edited. +#[derive(Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ResponsesItemMetadata { + /// These fields retain item data outside the decoded role, content, and tool call. + pub fields: Map, + /// This value records the original content or tool output representation. + pub content_kind: ResponsesContentKind, +} + +impl fmt::Debug for ResponsesItemMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ResponsesItemMetadata") + .field("fields", &self.fields.keys()) + .field("content_kind", &self.content_kind) + .finish() + } +} + +/// Native text-block fields stay attached when blocks are edited or reordered. +#[derive(Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ResponsesTextMetadata { + /// These fields retain block data outside the decoded text value. + pub fields: Map, +} + +impl fmt::Debug for ResponsesTextMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ResponsesTextMetadata") + .field("fields", &self.fields.keys()) + .finish() + } } impl Message { @@ -55,7 +110,8 @@ impl Message { pub fn text(role: Role, text: impl Into) -> Self { Self { role, - content: vec![ContentBlock::Text { text: text.into() }], + content: vec![ContentBlock::text(text)], + responses: None, } } @@ -65,7 +121,7 @@ impl Message { .content .iter() .filter_map(|block| match block { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), ContentBlock::Refusal { text } => Some(text.as_str()), _ => None, }) @@ -86,6 +142,9 @@ pub enum ContentBlock { Text { /// Text value. text: String, + /// This metadata retains native Responses fields, including the block type. + #[serde(default, skip_serializing_if = "Option::is_none")] + responses: Option, }, /// Model reasoning or thinking content. Reasoning { @@ -138,6 +197,16 @@ pub enum ContentBlock { }, } +impl ContentBlock { + /// This constructor creates plain text without provider metadata. + pub fn text(text: impl Into) -> Self { + Self::Text { + text: text.into(), + responses: None, + } + } +} + /// Image payload forms supported by the conversation model. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type", content = "data", rename_all = "snake_case")] diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs index 04dfbfe01a..13dcfec985 100644 --- a/crates/protocol/src/stream.rs +++ b/crates/protocol/src/stream.rs @@ -225,7 +225,7 @@ impl AggLlmResponse { for (output_index, output) in self.outputs.into_iter().enumerate() { for block in output.content { match block { - ContentBlock::Text { text } => { + ContentBlock::Text { text, .. } => { chunks.push(LlmResponseChunk::TextDelta { index: output_index, text, @@ -466,7 +466,10 @@ impl ResponseAccumulator { }); } if !self.text.is_empty() { - content.push(ContentBlock::Text { text: self.text }); + content.push(ContentBlock::Text { + responses: None, + text: self.text, + }); } for call in self.tool_calls.into_values() { content.push(ContentBlock::ToolCall(ToolCall { @@ -559,6 +562,7 @@ mod tests { assert_eq!( agg.outputs[0].content, vec![ContentBlock::Text { + responses: None, text: "Hello".to_string() }] ); @@ -611,6 +615,7 @@ mod tests { assert_eq!( aggregate.outputs[0].content, vec![ContentBlock::Text { + responses: None, text: "hello".to_string() }] ); @@ -715,6 +720,7 @@ mod tests { details: Vec::new(), }, ContentBlock::Text { + responses: None, text: "answer".to_string(), }, ] @@ -729,6 +735,7 @@ mod tests { outputs: vec![ResponseOutput { role: Role::Assistant, content: vec![ContentBlock::Text { + responses: None, text: "hello".to_string(), }], url_citations: Vec::new(), diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 39400e7026..9dc32bb0a1 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -15,7 +15,8 @@ use serde::{Deserialize, Deserializer}; use serde_json::Value; use switchyard_llm_client::{ AuxiliaryOperation, Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, - OpenAiDecisionClient, ResponsesReasoningPolicy, SystemOneClient, TranslatingLlmClient, + OpenAiDecisionClient, ResponsesCompatibility, ResponsesReasoningPolicy, SystemOneClient, + TranslatingLlmClient, }; use switchyard_protocol::{ LlmClientError, ModelId, RoutedDecisionClient, RoutedLlmClient, WireFormat, @@ -864,6 +865,13 @@ impl DeploymentConfig { for (name, client_config) in &self.llm_clients { validate_value("llm client name", name)?; + if client_config.responses_compatibility.is_some() + && !matches!(client_config.format, ClientFormat::OpenAiResponses) + { + return Err(RunnerError::configuration(format!( + "llm client {name} responses_compatibility is only valid for openai_responses" + ))); + } if client_config.responses_reasoning.is_some() && !matches!(client_config.format, ClientFormat::OpenAiResponses) { @@ -926,6 +934,9 @@ impl DeploymentConfig { None, ) .with_responses_reasoning(client_config.responses_reasoning.unwrap_or_default()) + .with_responses_compatibility( + client_config.responses_compatibility.unwrap_or_default(), + ) .with_provider_model(target.id.clone()), ); } @@ -1296,6 +1307,7 @@ struct LlmClientConfig { /// Deadline in milliseconds for all attempts and the complete response. Unset is unbounded. timeout_ms: Option, responses_reasoning: Option, + responses_compatibility: Option, } #[derive(Debug, Deserialize)] diff --git a/crates/switchyard-translation/README.md b/crates/switchyard-translation/README.md index ad32557089..6baf71e353 100644 --- a/crates/switchyard-translation/README.md +++ b/crates/switchyard-translation/README.md @@ -122,6 +122,15 @@ let policy = TranslationPolicy { }; ``` +## Read Responses conversation turns + +Use `switchyard_translation::conversation(&request)` to read grouped assistant +turns for routing or another API. The view combines reasoning and parallel tool +calls and returns inline instructions separately. It temporarily clones native +Responses messages; requests without native item metadata use borrowed slices. +Edit `LlmRequest.messages` directly when changing native history. Indexes in the +grouped view do not refer to the original items. + ## License Licensed under the Apache License, Version 2.0. See the diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index ca2f425877..d3ae4e960d 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -151,7 +151,11 @@ impl FormatCodec for AnthropicMessagesCodec { &mut diagnostics, policy, )?; - request.messages.push(Message { role, content }); + request.messages.push(Message { + role, + content, + responses: None, + }); } } request.tools = decode_anthropic_tools(body.get("tools")); @@ -202,15 +206,18 @@ impl FormatCodec for AnthropicMessagesCodec { request: &LlmRequest, policy: &TranslationPolicy, ) -> Result { + let mut diagnostics = Vec::new(); + crate::codecs::common::check_responses_metadata_loss( + request, + WireFormat::AnthropicMessages, + &mut diagnostics, + policy, + )?; if let Some(body) = exact_preserved_request(&request.preservation, WireFormat::AnthropicMessages, policy) { - return Ok(EncodedRequest { - body, - diagnostics: Vec::new(), - }); + return Ok(EncodedRequest { body, diagnostics }); } - let mut diagnostics = Vec::new(); validate_request_capabilities(request, &mut diagnostics, policy)?; let allowed = crate::codecs::common::allowed_function_tools(request)?; let (tools, tool_choice) = allowed.as_ref().map_or( @@ -221,12 +228,15 @@ impl FormatCodec for AnthropicMessagesCodec { if let Some(model) = &request.model { body.insert("model".to_string(), Value::String(model.clone())); } - let system_text = request + let conversation = crate::conversation(request); + let system_text = conversation .instructions .iter() .flat_map(|instruction| instruction.content.iter()) .filter_map(|block| match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => { + Some(text.as_str()) + } _ => None, }) .collect::>() @@ -238,7 +248,7 @@ impl FormatCodec for AnthropicMessagesCodec { body.insert( "messages".to_string(), Value::Array(encode_anthropic_messages( - &request.messages, + &conversation.messages, &mut diagnostics, policy, )?), @@ -369,9 +379,7 @@ impl FormatCodec for AnthropicMessagesCodec { } } if content.is_empty() { - content.push(ContentBlock::Text { - text: String::new(), - }); + content.push(ContentBlock::text(String::new())); } let response = AggLlmResponse { id: body @@ -589,9 +597,7 @@ fn strip_anthropic_unsupported_constraints(value: &mut Value) -> bool { // Decodes Anthropic's `system` field into instruction blocks. fn decode_anthropic_system(value: &Value) -> Result>> { match value { - Value::String(text) if !text.is_empty() => { - Ok(Some(vec![ContentBlock::Text { text: text.clone() }])) - } + Value::String(text) if !text.is_empty() => Ok(Some(vec![ContentBlock::text(text.clone())])), Value::String(_) | Value::Null => Ok(None), Value::Array(blocks) => { let mut content = Vec::new(); @@ -604,7 +610,7 @@ fn decode_anthropic_system(value: &Value) -> Result>> { .and_then(Value::as_str) .unwrap_or_default() .to_string(); - content.push(ContentBlock::Text { text }); + content.push(ContentBlock::text(text)); } } Ok((!content.is_empty()).then_some(content)) @@ -625,10 +631,8 @@ fn decode_anthropic_content( policy: &TranslationPolicy, ) -> Result> { match value { - Value::String(text) => Ok(vec![ContentBlock::Text { text: text.clone() }]), - Value::Null => Ok(vec![ContentBlock::Text { - text: String::new(), - }]), + Value::String(text) => Ok(vec![ContentBlock::text(text.clone())]), + Value::Null => Ok(vec![ContentBlock::text(String::new())]), Value::Array(blocks) => { let mut content = Vec::new(); for (index, block) in blocks.iter().enumerate() { @@ -649,15 +653,13 @@ fn decode_anthropic_content( )?); } if content.is_empty() { - content.push(ContentBlock::Text { - text: String::new(), - }); + content.push(ContentBlock::text(String::new())); } Ok(content) } - other => Ok(vec![ContentBlock::Text { - text: string_value(other).unwrap_or_default(), - }]), + other => Ok(vec![ContentBlock::text( + string_value(other).unwrap_or_default(), + )]), } } @@ -670,13 +672,13 @@ fn decode_anthropic_content_block( policy: &TranslationPolicy, ) -> Result> { Ok(match block.get("type").and_then(Value::as_str) { - Some("text") => vec![ContentBlock::Text { - text: block + Some("text") => vec![ContentBlock::text( + block .get("text") .and_then(Value::as_str) .unwrap_or_default() .to_string(), - }], + )], Some("thinking") => vec![ContentBlock::Reasoning { text: block .get("thinking") @@ -740,19 +742,19 @@ fn decode_anthropic_content_block( // Preserves supported Anthropic tool-result blocks in the neutral IR. fn decode_tool_result_content(value: &Value) -> Vec { match value { - Value::String(text) => vec![ContentBlock::Text { text: text.clone() }], + Value::String(text) => vec![ContentBlock::text(text.clone())], Value::Array(blocks) => { let mut content = Vec::new(); for block in blocks { if let Some(block) = block.as_object() { match block.get("type").and_then(Value::as_str) { - Some("text") => content.push(ContentBlock::Text { - text: block + Some("text") => content.push(ContentBlock::text( + block .get("text") .and_then(Value::as_str) .unwrap_or_default() .to_string(), - }), + )), Some("image") => content.push(ContentBlock::Image { source: ImageSource::Raw(Value::Object(block.clone())), }), @@ -768,12 +770,8 @@ fn decode_tool_result_content(value: &Value) -> Vec { } content } - Value::Null => vec![ContentBlock::Text { - text: String::new(), - }], - other => vec![ContentBlock::Text { - text: json_string(other), - }], + Value::Null => vec![ContentBlock::text(String::new())], + other => vec![ContentBlock::text(json_string(other))], } } @@ -1039,7 +1037,7 @@ fn encode_one_anthropic_response_block(block: &ContentBlock) -> Result Result> { Ok(match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => { + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => { vec![json!({"type": "text", "text": text})] } ContentBlock::Reasoning { diff --git a/crates/switchyard-translation/src/codecs/common.rs b/crates/switchyard-translation/src/codecs/common.rs index 303c90321b..de686905a2 100644 --- a/crates/switchyard-translation/src/codecs/common.rs +++ b/crates/switchyard-translation/src/codecs/common.rs @@ -5,8 +5,72 @@ use serde_json::{Map, Value}; +use crate::diagnostic::TranslationDiagnostic; use crate::error::{Result, TranslationError}; -use crate::llm::{ContentBlock, LlmRequest, Tool, ToolChoice}; +use crate::format::WireFormat; +use crate::llm::{ContentBlock, LlmRequest, Role, Tool, ToolChoice}; +use crate::policy::TranslationPolicy; +use crate::util::push_lossy; + +/// This check reports metadata loss or rejects conversion before grouping can discard the fields. +pub(crate) fn check_responses_metadata_loss( + request: &LlmRequest, + target: WireFormat, + diagnostics: &mut Vec, + policy: &TranslationPolicy, +) -> Result<()> { + let mut has_conversation = false; + for message in &request.messages { + let Some(metadata) = &message.responses else { + has_conversation = true; + continue; + }; + if metadata.fields.get("type").and_then(Value::as_str) == Some("additional_tools") { + continue; + } + if metadata.fields.contains_key("phase") { + push_lossy( + diagnostics, + policy, + format!("Responses assistant phase is not supported by {target}"), + )?; + } + // Reasoning item IDs live in the block's details and can survive Chat conversion. + let loses_reasoning_id = target != WireFormat::OpenAiChat + && message.content.iter().any(|block| { + matches!(block, ContentBlock::Reasoning { details, .. } + if details.iter().any(|item| item.get("id").is_some())) + }); + if metadata.fields.contains_key("id") || loses_reasoning_id { + push_lossy( + diagnostics, + policy, + format!( + "Responses item id is not supported by {target}; tool call_id remains separate" + ), + )?; + } + if matches!(message.role, Role::System | Role::Developer) { + if has_conversation { + push_lossy( + diagnostics, + policy, + format!("Responses inline instruction position changes when encoding {target}"), + )?; + } + if message.role == Role::Developer && target == WireFormat::AnthropicMessages { + push_lossy( + diagnostics, + policy, + "Responses developer instructions become Anthropic system instructions", + )?; + } + } else { + has_conversation = true; + } + } + Ok(()) +} // Internal provenance survives mutations that invalidate exact request replay. pub(crate) const ANTHROPIC_REQUEST_KEY: &str = "switchyard_anthropic_request"; @@ -100,7 +164,7 @@ pub(crate) fn text_from_blocks(content: &[ContentBlock], separator: &str) -> Str content .iter() .filter_map(|block| match block { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), ContentBlock::Refusal { text } => Some(text.as_str()), ContentBlock::Unknown { raw, .. } => raw.as_str(), _ => None, diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 71877834ea..9addbbe8ac 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -125,7 +125,7 @@ impl FormatCodec for OpenAiChatCodec { let text = content .iter() .filter_map(|block| match block { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), _ => None, }) .collect::>() @@ -139,9 +139,10 @@ impl FormatCodec for OpenAiChatCodec { role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id, - content: vec![ContentBlock::Text { text }], + content: vec![ContentBlock::text(text)], is_error: message.get("is_error").and_then(Value::as_bool), })], + responses: None, }); continue; } @@ -152,7 +153,11 @@ impl FormatCodec for OpenAiChatCodec { .push(InstructionBlock { role, content }); } Role::User | Role::Assistant | Role::Tool => { - request.messages.push(Message { role, content }); + request.messages.push(Message { + role, + content, + responses: None, + }); } } } @@ -191,15 +196,18 @@ impl FormatCodec for OpenAiChatCodec { request: &LlmRequest, policy: &TranslationPolicy, ) -> Result { + let mut diagnostics = Vec::new(); + crate::codecs::common::check_responses_metadata_loss( + request, + WireFormat::OpenAiChat, + &mut diagnostics, + policy, + )?; if let Some(body) = exact_preserved_request(&request.preservation, WireFormat::OpenAiChat, policy) { - return Ok(EncodedRequest { - body, - diagnostics: Vec::new(), - }); + return Ok(EncodedRequest { body, diagnostics }); } - let mut diagnostics = Vec::new(); validate_request_capabilities(request, &mut diagnostics, policy)?; let allowed = match &request.tool_choice { Some(ToolChoice::Raw(choice)) @@ -220,17 +228,22 @@ impl FormatCodec for OpenAiChatCodec { } let mut messages = Vec::new(); - for instruction in &request.instructions { + let conversation = crate::conversation(request); + for instruction in conversation.instructions.iter() { + let content = text_from_blocks(&instruction.content, "\n\n"); + if content.is_empty() { + continue; + } let role = match instruction.role { Role::Developer => "developer", _ => "system", }; messages.push(json!({ "role": role, - "content": text_from_blocks(&instruction.content, "\n\n"), + "content": content, })); } - for message in &request.messages { + for message in conversation.messages.iter() { messages.extend(encode_message_to_openai(message, &mut diagnostics, policy)?); } body.insert("messages".to_string(), Value::Array(messages)); @@ -555,10 +568,8 @@ pub(crate) fn decode_openai_content( ) -> Result> { let path = path.into(); match content { - Value::Null => Ok(vec![ContentBlock::Text { - text: String::new(), - }]), - Value::String(text) => Ok(vec![ContentBlock::Text { text: text.clone() }]), + Value::Null => Ok(vec![ContentBlock::text(String::new())]), + Value::String(text) => Ok(vec![ContentBlock::text(text.clone())]), Value::Array(blocks) => { let mut content = Vec::new(); for (index, block) in blocks.iter().enumerate() { @@ -572,13 +583,13 @@ pub(crate) fn decode_openai_content( }; match block.get("type").and_then(Value::as_str) { Some("text") | Some("input_text") | Some("output_text") => { - content.push(ContentBlock::Text { - text: block + content.push(ContentBlock::text( + block .get("text") .and_then(Value::as_str) .unwrap_or_default() .to_string(), - }); + )); } Some("refusal") => { content.push(ContentBlock::Refusal { @@ -609,16 +620,14 @@ pub(crate) fn decode_openai_content( } } if content.is_empty() { - Ok(vec![ContentBlock::Text { - text: String::new(), - }]) + Ok(vec![ContentBlock::text(String::new())]) } else { Ok(content) } } - other => Ok(vec![ContentBlock::Text { - text: string_value(other).unwrap_or_default(), - }]), + other => Ok(vec![ContentBlock::text( + string_value(other).unwrap_or_default(), + )]), } } @@ -902,6 +911,7 @@ fn push_pending_openai_message( let message = Message { role, content: std::mem::take(pending_content), + responses: None, }; out.push(encode_message_without_tool_results_to_openai( &message, @@ -1058,7 +1068,7 @@ fn copy_openai_chat_request_extensions( // Detects placeholder empty text generated while decoding assistant tool calls. fn is_empty_text_only(content: &[ContentBlock]) -> bool { - matches!(content, [ContentBlock::Text { text }] if text.is_empty()) + matches!(content, [ContentBlock::Text { text, .. }] if text.is_empty()) } /// Encodes normalized content into OpenAI Chat content JSON. @@ -1106,7 +1116,7 @@ pub(crate) fn encode_openai_content( for block in content { crate::codecs::openai_media::validate_media(block, WireFormat::OpenAiChat)?; match block { - ContentBlock::Text { text } => blocks.push(json!({"type": "text", "text": text})), + ContentBlock::Text { text, .. } => blocks.push(json!({"type": "text", "text": text})), ContentBlock::Refusal { text } => blocks.push(json!({"type": "text", "text": text})), ContentBlock::Image { source } => match openai_image_part(source) { Some(part) => blocks.push(part), diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 18b173891b..e85780bcbe 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -3,8 +3,6 @@ //! Buffered codec for OpenAI Responses request and response JSON. -use std::collections::HashMap; - use serde_json::{Map, Value, json}; use crate::codecs::common::{ @@ -24,9 +22,10 @@ use crate::diagnostic::TranslationDiagnostic; use crate::error::{Result, TranslationError}; use crate::format::{FormatId, WireFormat}; use crate::llm::{ - AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, - MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, - SamplingParams, StopReason, Tool, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, + AggLlmResponse, ContentBlock, FileSource, ImageSource, LlmRequest, MediaSource, Message, + OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, ResponsesContentKind, + ResponsesItemMetadata, ResponsesTextMetadata, Role, SamplingParams, StopReason, Tool, ToolCall, + ToolChoice, ToolDefinition, ToolResult, Usage, }; use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; use crate::util::{ @@ -80,32 +79,22 @@ impl FormatCodec for OpenAiResponsesCodec { ), ..LlmRequest::default() }; - if let Some(instructions) = body.get("instructions").and_then(Value::as_str) - && !instructions.is_empty() - { + if let Some(instructions) = body.get("instructions").and_then(Value::as_str) { request.instructions.push(crate::llm::InstructionBlock { role: Role::System, content: vec![ContentBlock::Text { + responses: None, text: instructions.to_string(), }], }); } - // With `previous_response_id`, the provider holds the earlier turns, so a tool output - // may answer a call that is not in this body. - let stored_state = body - .get("previous_response_id") - .and_then(Value::as_str) - .is_some_and(|id| !id.is_empty()); let mut custom_call_outputs = Vec::new(); - let (messages, instructions) = decode_responses_input( + request.messages = decode_responses_input( body.get("input").unwrap_or(&Value::String(String::new())), - stored_state, &mut custom_call_outputs, &mut diagnostics, policy, )?; - request.messages = messages; - request.instructions.extend(instructions); let mut tool_namespaces = Map::new(); let mut custom_tools = Map::new(); request.tools = @@ -196,13 +185,17 @@ impl FormatCodec for OpenAiResponsesCodec { .iter() .flat_map(|instruction| instruction.content.iter()) .filter_map(|block| match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => { + Some(text.as_str()) + } _ => None, }) - .collect::>() - .join("\n\n"); + .collect::>(); if !instructions.is_empty() { - body.insert("instructions".to_string(), Value::String(instructions)); + body.insert( + "instructions".to_string(), + Value::String(instructions.join("\n\n")), + ); } body.insert( "input".to_string(), @@ -216,18 +209,25 @@ impl FormatCodec for OpenAiResponsesCodec { )?, ); if let Some(additional) = crate::codex_custom_tools::additional_tools(&request.extensions) { - // A Responses-lite request carried its tools inside `input`; give them back the same - // way, verbatim, and leave top-level `tools` absent as the client did. - let item = json!({ - "type": "additional_tools", - "role": "developer", - "tools": additional, + let has_native_item = request.messages.iter().any(|message| { + message.responses.as_ref().is_some_and(|source| { + source.fields.get("type").and_then(Value::as_str) == Some("additional_tools") + }) }); - match body.get_mut("input") { - Some(Value::Array(input)) => input.insert(0, item), - _ => body - .insert("input".to_string(), Value::Array(vec![item])) - .map_or((), |_| ()), + if !has_native_item { + // Saved inline tools remain in `input` when no native item supplies them. + // The top-level `tools` field remains absent, as in the caller's request. + let item = json!({ + "type": "additional_tools", + "role": "developer", + "tools": additional, + }); + match body.get_mut("input") { + Some(Value::Array(input)) => input.insert(0, item), + _ => body + .insert("input".to_string(), Value::Array(vec![item])) + .map_or((), |_| ()), + } } } else if !request.tools.is_empty() { body.insert( @@ -340,7 +340,7 @@ impl FormatCodec for OpenAiResponsesCodec { .content .iter() .map(|block| match block { - ContentBlock::Text { text } | ContentBlock::Refusal { text } => { + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => { text.chars().count() } _ => 0, @@ -375,6 +375,7 @@ impl FormatCodec for OpenAiResponsesCodec { url_citations: Vec::new(), role: Role::Assistant, content: vec![ContentBlock::Text { + responses: None, text: String::new(), }], stop_reason: Some(StopReason::EndTurn), @@ -457,29 +458,24 @@ impl FormatCodec for OpenAiResponsesCodec { } } -/// Decodes Responses `input` into ordered normalized messages and inline -/// instruction blocks. -/// -/// Inline `system` and `developer` message items are returned as instruction -/// blocks rather than conversation turns. Classifying them here, before any -/// reasoning or tool-call state-machine transitions, prevents an instruction -/// item from flushing pending reasoning or disturbing tool-call grouping. +/// This decoder retains native Responses input items in their supplied order. fn decode_responses_input( value: &Value, - stored_state: bool, custom_call_outputs: &mut Vec, diagnostics: &mut Vec, policy: &TranslationPolicy, -) -> Result<(Vec, Vec)> { +) -> Result> { match value { - Value::String(text) => Ok((vec![Message::text(Role::User, text)], Vec::new())), + Value::String(text) => { + let mut message = Message::text(Role::User, text); + message.responses = Some(ResponsesItemMetadata { + fields: Map::new(), + content_kind: ResponsesContentKind::InputString, + }); + Ok(vec![message]) + } Value::Array(items) => { let mut messages = Vec::new(); - let mut instructions = Vec::new(); - let mut pending_tool_calls = Vec::new(); - let mut pending_tool_outputs = Vec::new(); - let mut deferred_messages = Vec::new(); - let mut pending_reasoning: Vec = Vec::new(); for (index, item) in items.iter().enumerate() { let Some(item) = item.as_object() else { push_lossy( @@ -489,8 +485,8 @@ fn decode_responses_input( )?; continue; }; - let item_type = match item.get("type") { - Some(Value::String(item_type)) => Some(item_type.as_str()), + let kind = match item.get("type") { + Some(Value::String(kind)) => Some(kind.as_str()), Some(_) => { return Err(TranslationError::InvalidType { path: format!("$.input[{index}].type"), @@ -499,232 +495,124 @@ fn decode_responses_input( } None => None, }; - let is_message = item_type == Some("message") - || (item_type.is_none() + let is_message = kind == Some("message") + || (kind.is_none() && item.contains_key("role") && item.contains_key("content")); - match item_type { - _ if is_message => { - let role = request_role_from_responses( - item.get("role").and_then(Value::as_str), - &format!("$.input[{index}].role"), - )?; - let content_value = item.get("content").unwrap_or(&Value::Null); - if is_responses_native_input_item(content_value) { - return Err(TranslationError::InvalidValue { - path: format!("$.input[{index}].content.type"), - message: "native Responses items must be top-level input items" - .to_string(), - }); - } - if let Some(content_index) = content_value.as_array().and_then(|blocks| { - blocks.iter().position(is_responses_native_input_item) - }) { - return Err(TranslationError::InvalidValue { - path: format!("$.input[{index}].content[{content_index}].type"), - message: "native Responses items must be top-level input items" - .to_string(), - }); - } - let content = decode_responses_content(content_value); - // Inline system and developer input items are instructions, not - // conversation turns. Classify them before any state-machine - // transitions so they do not flush pending reasoning or disturb - // tool-call grouping. - if matches!(role, Role::System | Role::Developer) { - instructions.push(InstructionBlock { role, content }); - continue; - } - let mut content = content; - // Reasoning that precedes an assistant message belongs to - // that turn; fold it in so it never surfaces as its own - // (empty-looking) assistant message downstream. - if role == Role::Assistant - && pending_tool_calls.is_empty() - && !pending_reasoning.is_empty() - { - let mut merged = std::mem::take(&mut pending_reasoning); - merged.append(&mut content); - content = merged; - } - if role != Role::Assistant { - flush_unattached_responses_reasoning( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - ); - } - push_responses_non_tool_message( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - Message { role, content }, - ); - } - Some("reasoning") => { - // A reasoning item after a completed tool block opens the - // next assistant turn: flush the finished block so this - // reasoning attaches to the turn that produced it. - if !pending_tool_outputs.is_empty() { - flush_responses_tool_block( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - ); - } - pending_reasoning.extend(decode_responses_reasoning_item(item)); - } - Some("function_call") => { - if !pending_tool_outputs.is_empty() { - flush_responses_tool_block( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - ); - } - let id = item - .get("call_id") - .and_then(Value::as_str) - .filter(|id| !id.is_empty()) - .map(ToOwned::to_owned) - .unwrap_or_else(|| match &policy.deterministic_ids { - DeterministicIdPolicy::GenerateStable { prefix } => { - stable_id(prefix, index + 1) - } - DeterministicIdPolicy::Preserve => String::new(), - }); - // History has to spell a tool the way its definition - // does, or the transcript teaches the model a name the - // upstream was never offered. - let name = item - .get("name") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - let name = match item - .get("namespace") - .and_then(Value::as_str) - .filter(|namespace| !namespace.is_empty()) - { - Some(namespace) => { - crate::codex_namespaces::qualified_tool_name(namespace, &name) - } - None => name, - }; - pending_tool_calls.push(ToolCall { - id, - name, - arguments: item.get("arguments").cloned().unwrap_or_else(|| json!({})), + let mut fields = item.clone(); + let mut content_kind = ResponsesContentKind::Missing; + let (role, content) = if is_message { + let role = request_role_from_responses( + item.get("role").and_then(Value::as_str), + &format!("$.input[{index}].role"), + )?; + let value = item.get("content").unwrap_or(&Value::Null); + if is_responses_native_input_item(value) { + return Err(TranslationError::InvalidValue { + path: format!("$.input[{index}].content.type"), + message: "native Responses items must be top-level input items".into(), }); } - kind @ (Some("function_call_output") | Some("custom_tool_call_output")) => { - let tool_call_id = item - .get("call_id") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - // A structured output keeps its typed text, image, and file parts; - // any other shape rides through as one text block. - let content = match item.get("output") { - Some(output @ Value::Array(_)) => decode_responses_content(output), - output => vec![ContentBlock::Text { - text: output.map(json_string).unwrap_or_default(), - }], - }; - // An output whose call is not in this body answers a call the provider - // holds behind `previous_response_id`; it stays a tool result so routing - // sees a tool continuation, not a new user turn. Without stored state - // the request is malformed, and the output becomes readable user text. - let answers_pending_call = pending_tool_calls - .iter() - .any(|call| call.id == tool_call_id); - if !answers_pending_call && !stored_state { - let output_text = text_from_blocks(&content, " "); - let message = Message::text( - Role::User, - format!("Tool result {tool_call_id}: {output_text}"), - ); - push_responses_non_tool_message( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - message, - ); - continue; - } - // The encoder types an output by the call it answers; a stored custom - // output has no call in this body, so its type is carried separately. - if !answers_pending_call && kind == Some("custom_tool_call_output") { - custom_call_outputs.push(tool_call_id.clone()); - } - pending_tool_outputs.push(ToolResult { - tool_call_id, - content, - is_error: None, + if let Some(content_index) = value + .as_array() + .and_then(|blocks| blocks.iter().position(is_responses_native_input_item)) + { + return Err(TranslationError::InvalidValue { + path: format!("$.input[{index}].content[{content_index}].type"), + message: "native Responses items must be top-level input items".into(), }); } - Some("custom_tool_call") => { - // A freeform call carries a raw `input` string; it rides through the IR - // as the single `input` argument of a function-style call. - if !pending_tool_outputs.is_empty() { - flush_responses_tool_block( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - ); + content_kind = responses_content_kind(item.get("content")); + fields.remove("role"); + fields.remove("content"); + (role, decode_responses_content(value, true)) + } else { + match kind { + Some("reasoning") => { + fields.retain(|key, _| key == "type"); + (Role::Assistant, decode_responses_reasoning_item(item, true)) } - let id = item - .get("call_id") - .and_then(Value::as_str) - .filter(|id| !id.is_empty()) - .map(ToOwned::to_owned) - .unwrap_or_else(|| match &policy.deterministic_ids { - DeterministicIdPolicy::GenerateStable { prefix } => { - stable_id(prefix, index + 1) + Some("function_call" | "custom_tool_call") => { + let id = item + .get("call_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| match &policy.deterministic_ids { + DeterministicIdPolicy::GenerateStable { prefix } => { + stable_id(prefix, index + 1) + } + DeterministicIdPolicy::Preserve => String::new(), + }); + let name = item.get("name").and_then(Value::as_str).unwrap_or_default(); + let name = item + .get("namespace") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map_or_else( + || name.to_string(), + |namespace| { + crate::codex_namespaces::qualified_tool_name( + namespace, name, + ) + }, + ); + let arguments = if kind == Some("custom_tool_call") { + json!({crate::codex_custom_tools::INPUT_ARGUMENT: item.get("input").and_then(Value::as_str).unwrap_or_default()}) + } else { + item.get("arguments").cloned().unwrap_or_else(|| json!({})) + }; + for key in ["call_id", "name", "arguments", "input"] { + fields.remove(key); + } + ( + Role::Assistant, + vec![ContentBlock::ToolCall(ToolCall { + id, + name, + arguments, + })], + ) + } + Some("function_call_output" | "custom_tool_call_output") => { + let tool_call_id = item + .get("call_id") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(); + if kind == Some("custom_tool_call_output") { + custom_call_outputs.push(tool_call_id.clone()); + } + content_kind = responses_content_kind(item.get("output")); + let content = match item.get("output") { + Some(value @ Value::Array(_)) => { + decode_responses_content(value, true) } - DeterministicIdPolicy::Preserve => String::new(), + value => vec![ContentBlock::text( + value.map(json_string).unwrap_or_default(), + )], + }; + fields.remove("call_id"); + fields.remove("output"); + ( + Role::User, + vec![ContentBlock::ToolResult(ToolResult { + tool_call_id, + content, + is_error: None, + })], + ) + } + None => { + return Err(TranslationError::InvalidValue { + path: format!("$.input[{index}].type"), + message: "missing type discriminator on a non-message input item" + .into(), }); - let name = item - .get("name") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - let input = item - .get("input") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(); - pending_tool_calls.push(ToolCall { - id, - name, - arguments: json!({crate::codex_custom_tools::INPUT_ARGUMENT: input}), - }); - } - None => { - return Err(TranslationError::InvalidValue { - path: format!("$.input[{index}].type"), - message: "missing type discriminator on a non-message input item" - .to_string(), - }); - } - // Tool definitions, not conversation; decoded separately by the request - // decoder and re-emitted in place by the request encoder. - Some("additional_tools") => {} - _ => { - let message = Message { - role: match item_type { + } + _ => { + fields.retain(|key, _| key == "type"); + let role = match kind { Some( "shell_call" | "apply_patch_call" @@ -736,144 +624,51 @@ fn decode_responses_input( | "apply_patch_call_output" | "computer_call_output", ) => Role::Tool, + Some("additional_tools") => Role::Developer, _ => Role::User, - }, - content: vec![ContentBlock::Unknown { - provider: WireFormat::OpenAiResponses.into(), - raw: Value::Object(item.clone()), - }], - }; - flush_unattached_responses_reasoning( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - ); - push_responses_non_tool_message( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - message, - ); + }; + ( + role, + vec![ContentBlock::Unknown { + provider: WireFormat::OpenAiResponses.into(), + raw: Value::Object(item.clone()), + }], + ) + } } - } - } - if !pending_tool_calls.is_empty() || !pending_tool_outputs.is_empty() { - flush_responses_tool_block( - &mut messages, - &mut pending_tool_calls, - &mut pending_tool_outputs, - &mut deferred_messages, - &mut pending_reasoning, - ); - } - // Trailing reasoning with no assistant turn to join keeps the - // historical standalone-message shape. - if !pending_reasoning.is_empty() { + }; messages.push(Message { - role: Role::Assistant, - content: pending_reasoning, + role, + content, + responses: Some(ResponsesItemMetadata { + fields, + content_kind, + }), }); } - Ok((messages, instructions)) + Ok(messages) } _ => Err(TranslationError::InvalidType { - path: "$.input".to_string(), + path: "$.input".into(), expected: "string or array", }), } } -/// Places non-tool input items without breaking Responses tool-call adjacency. -fn push_responses_non_tool_message( - messages: &mut Vec, - pending_tool_calls: &mut Vec, - pending_tool_outputs: &mut Vec, - deferred_messages: &mut Vec, - pending_reasoning: &mut Vec, - message: Message, -) { - if !pending_tool_calls.is_empty() && pending_tool_outputs.is_empty() { - deferred_messages.push(message); - return; - } - if !pending_tool_calls.is_empty() || !pending_tool_outputs.is_empty() { - flush_responses_tool_block( - messages, - pending_tool_calls, - pending_tool_outputs, - deferred_messages, - pending_reasoning, - ); - } - messages.push(message); -} - -/// Emits reasoning that cannot join an assistant turn as a standalone assistant -/// message at its original position. While tool calls are pending, the -/// reasoning belongs to the in-flight turn and stays pending for the flush. -fn flush_unattached_responses_reasoning( - messages: &mut Vec, - pending_tool_calls: &mut Vec, - pending_tool_outputs: &mut Vec, - deferred_messages: &mut Vec, - pending_reasoning: &mut Vec, -) { - if pending_reasoning.is_empty() || !pending_tool_calls.is_empty() { - return; - } - let message = Message { - role: Role::Assistant, - content: std::mem::take(pending_reasoning), - }; - push_responses_non_tool_message( - messages, - pending_tool_calls, - pending_tool_outputs, - deferred_messages, - pending_reasoning, - message, - ); -} - -/// Flushes pending function calls and matching outputs while preserving adjacency. -/// Pending reasoning rides in the same assistant message as the tool calls so a -/// Codex reasoning + function_call turn stays one assistant turn downstream. -fn flush_responses_tool_block( - messages: &mut Vec, - pending_tool_calls: &mut Vec, - pending_tool_outputs: &mut Vec, - deferred_messages: &mut Vec, - pending_reasoning: &mut Vec, -) { - if !pending_tool_calls.is_empty() { - let mut content = std::mem::take(pending_reasoning); - content.extend( - std::mem::take(pending_tool_calls) - .into_iter() - .map(ContentBlock::ToolCall), - ); - messages.push(Message { - role: Role::Assistant, - content, - }); - } - - for output in std::mem::take(pending_tool_outputs) { - messages.push(Message { - role: Role::User, - content: vec![ContentBlock::ToolResult(output)], - }); +fn responses_content_kind(value: Option<&Value>) -> ResponsesContentKind { + match value { + None => ResponsesContentKind::Missing, + Some(Value::Null) => ResponsesContentKind::Null, + Some(Value::Array(_)) => ResponsesContentKind::Array, + _ => ResponsesContentKind::String, } - - messages.append(deferred_messages); } -// Decodes a Responses reasoning item into private reasoning IR content. -fn decode_responses_reasoning_item(item: &Map) -> Vec { +// Request reconstruction needs the original reasoning fields as well as decoded text. +fn decode_responses_reasoning_item( + item: &Map, + preserve_metadata: bool, +) -> Vec { if let Some((text, signature)) = item .get("encrypted_content") .and_then(Value::as_str) @@ -891,9 +686,13 @@ fn decode_responses_reasoning_item(item: &Map) -> Vec) -> Vec Vec { +// Request decoding keeps metadata on each text block so it stays with edits and reordering. +fn decode_responses_content(value: &Value, preserve_metadata: bool) -> Vec { match value { - Value::String(text) => vec![ContentBlock::Text { text: text.clone() }], + Value::String(text) => vec![ContentBlock::text(text.clone())], Value::Array(blocks) => { let mut out = Vec::new(); for block in blocks { @@ -923,6 +722,13 @@ fn decode_responses_content(value: &Value) -> Vec { .and_then(Value::as_str) .unwrap_or_default() .to_string(), + responses: preserve_metadata.then(|| ResponsesTextMetadata { + fields: block + .iter() + .filter(|(key, _)| key.as_str() != "text") + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + }), }); } Some("refusal") => out.push(ContentBlock::Refusal { @@ -960,17 +766,17 @@ fn decode_responses_content(value: &Value) -> Vec { }), } } - if out.is_empty() { - out.push(ContentBlock::Text { - text: String::new(), - }); + if out.is_empty() && !preserve_metadata { + out.push(ContentBlock::text(String::new())); } out } Value::Null => vec![ContentBlock::Text { + responses: None, text: String::new(), }], other => vec![ContentBlock::Text { + responses: None, text: string_value(other).unwrap_or_default(), }], } @@ -1265,6 +1071,164 @@ fn encode_responses_text_format(response_format: &Value) -> Value { Value::Object(output) } +// Native metadata supplies fields outside the decoded role and content. +// This encoder reads edited values from the current role and content. +fn encode_native_responses_item( + message: &Message, + diagnostics: &mut Vec, + policy: &TranslationPolicy, +) -> Result> { + let Some(source) = &message.responses else { + return Ok(None); + }; + let kind = source.fields.get("type").and_then(Value::as_str); + let mut item = source.fields.clone(); + match (kind, message.content.as_slice()) { + (Some("function_call" | "custom_tool_call"), [ContentBlock::ToolCall(call)]) + if message.role == Role::Assistant => + { + item.insert("call_id".into(), Value::String(call.id.clone())); + let namespace = item + .get("namespace") + .and_then(Value::as_str) + .filter(|namespace| !namespace.is_empty()); + let name = if let Some(namespace) = namespace { + if let Some(name) = call.name.strip_prefix(&format!( + "{namespace}{}", + crate::codex_namespaces::NAMESPACE_SEPARATOR + )) { + name.to_owned() + } else { + item.remove("namespace"); + call.name.clone() + } + } else { + call.name.clone() + }; + item.insert("name".into(), Value::String(name)); + if kind == Some("custom_tool_call") { + item.insert( + "input".into(), + Value::String(crate::codex_custom_tools::input_from_arguments( + &call.arguments, + )), + ); + } else { + item.insert( + "arguments".into(), + Value::String(json_string(&call.arguments)), + ); + } + } + ( + Some("function_call_output" | "custom_tool_call_output"), + [ContentBlock::ToolResult(result)], + ) if matches!(message.role, Role::User | Role::Tool) => { + item.insert("call_id".into(), Value::String(result.tool_call_id.clone())); + if let Some(output) = encode_native_responses_content( + &result.content, + source.content_kind, + diagnostics, + policy, + )? { + item.insert("output".into(), output); + } + } + ( + Some("reasoning"), + [ + ContentBlock::Reasoning { + text, + signature: None, + details, + }, + ], + ) if message.role == Role::Assistant => { + let Some(original_item) = details.iter().find_map(|value| { + value + .as_object() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("reasoning")) + }) else { + return Ok(None); + }; + item = original_item.clone(); + let original = decode_responses_reasoning_item(original_item, false); + if !matches!(original.as_slice(), [ContentBlock::Reasoning { text: original_text, .. }] if original_text == text) + { + // Rewritten reasoning cannot retain an encrypted payload or ID for the old content. + for key in ["id", "content", "text", "encrypted_content"] { + item.remove(key); + } + item.insert( + "summary".into(), + json!([{"type":"summary_text", "text":text}]), + ); + } + } + (Some("message") | None, _) => { + if message.content.iter().any(|block| { + matches!( + block, + ContentBlock::ToolCall(_) + | ContentBlock::ToolResult(_) + | ContentBlock::Reasoning { .. } + ) + }) { + return Ok(None); + } + item.insert( + "role".into(), + Value::String(role_to_responses(message.role).into()), + ); + if let Some(content) = encode_native_responses_content( + &message.content, + source.content_kind, + diagnostics, + policy, + )? { + item.insert("content".into(), content); + } + } + (_, [ContentBlock::Unknown { provider, raw }]) + if provider.as_str() == WireFormat::OpenAiResponses.as_str() => + { + return Ok(Some(raw.clone())); + } + _ => return Ok(None), + } + Ok(Some(Value::Object(item))) +} + +// The saved representation keeps empty strings, empty arrays, null, and omitted fields distinct. +fn encode_native_responses_content( + content: &[ContentBlock], + kind: ResponsesContentKind, + diagnostics: &mut Vec, + policy: &TranslationPolicy, +) -> Result> { + let is_empty = content.is_empty() + || matches!(content, [ContentBlock::Text { text, .. }] if text.is_empty()); + match kind { + ResponsesContentKind::Missing if is_empty => return Ok(None), + ResponsesContentKind::Null if is_empty => return Ok(Some(Value::Null)), + ResponsesContentKind::String | ResponsesContentKind::InputString => { + if let [ContentBlock::Text { text, .. }] = content { + return Ok(Some(Value::String(text.clone()))); + } + } + _ => {} + } + let mut blocks = Vec::new(); + for block in content { + match encode_responses_content(std::slice::from_ref(block), diagnostics, policy)? { + Value::Array(items) => blocks.extend(items), + Value::String(text) => blocks.push(json!({"type":"input_text","text":text})), + _ => {} + } + } + Ok(Some(Value::Array(blocks))) +} + // Encodes normalized messages into the Responses `input` shape. fn encode_responses_input( messages: &[Message], @@ -1274,26 +1238,33 @@ fn encode_responses_input( custom_tools: &std::collections::HashSet, mut custom_call_ids: std::collections::HashSet, ) -> Result { - // Call ids of freeform tool calls emitted by this pass, seeded with the ids of stored-state - // custom outputs. A tool result is typed by the call it answers, and Responses history - // always lists the call before its output, so recording ids as calls are encoded is enough - // to type the outputs that follow. - // - // A lone user text is still sent as an item list: the Codex backend rejects a string `input`. + if let [message] = messages + && message + .responses + .as_ref() + .is_some_and(|source| source.content_kind == ResponsesContentKind::InputString) + && message.role == Role::User + && let [ContentBlock::Text { text, .. }] = message.content.as_slice() + { + return Ok(Value::String(text.clone())); + } + // `custom_call_ids` starts with IDs from stored-state custom outputs and gains local call IDs. + // These IDs determine the type of outputs that need reconstruction. + // New IR input uses an item list, even for one user text, for Codex compatibility. + // The early return above preserves string input decoded from a native Responses request. let mut encoded = Vec::new(); - // Some upstream translators (Kimi K3) resolve a tool output by an explicit - // `name`, so pair every output with the name of the call it answers. - let mut call_names: HashMap<&str, &str> = HashMap::new(); for message in messages { - for block in &message.content { - if let ContentBlock::ToolCall(call) = block { - call_names.insert(call.id.as_str(), call.name.as_str()); + if let Some(item) = encode_native_responses_item(message, diagnostics, policy)? { + if item.get("type").and_then(Value::as_str) == Some("custom_tool_call") + && let Some(id) = item.get("call_id").and_then(Value::as_str) + { + custom_call_ids.insert(id.to_owned()); } + encoded.push(item); + continue; } - } - for message in messages { - // The decoder carries each native top-level input item as one - // provider-qualified block, so replay that block in place. + + // Native items kept as opaque Responses blocks must retain their position in `input`. if let [ContentBlock::Unknown { provider, raw }] = message.content.as_slice() && provider.as_str() == WireFormat::OpenAiResponses.as_str() && is_responses_native_input_item(raw) @@ -1330,7 +1301,6 @@ fn encode_responses_input( if let Some(item) = encode_responses_special_input( block, namespaces, - &call_names, custom_tools, &mut custom_call_ids, diagnostics, @@ -1368,7 +1338,6 @@ fn encode_responses_input( if let Some(item) = encode_responses_special_input( block, namespaces, - &call_names, custom_tools, &mut custom_call_ids, diagnostics, @@ -1391,57 +1360,13 @@ fn encode_responses_input( })); } } - pair_tool_calls_with_outputs(&mut encoded); Ok(Value::Array(encoded)) } -// Moves each tool output directly behind the call it answers. A turn with -// parallel tool calls otherwise serializes as call,call,output,output, which -// upstreams that pair by adjacency (Kimi K3) resolve to the wrong call. -fn pair_tool_calls_with_outputs(items: &mut Vec) { - let call_id = |item: &Value, kind: &str| { - (item.get("type").and_then(Value::as_str) == Some(kind)) - .then(|| { - item.get("call_id") - .and_then(Value::as_str) - .map(str::to_owned) - }) - .flatten() - }; - let mut index = 0; - while index < items.len() { - // Freeform (`custom`) calls pair with `custom_tool_call_output` the same way. - let (id, output_kind) = match call_id(&items[index], "function_call") { - Some(id) => (id, "function_call_output"), - None => match call_id(&items[index], "custom_tool_call") { - Some(id) => (id, "custom_tool_call_output"), - None => { - index += 1; - continue; - } - }, - }; - let output = items - .iter() - .skip(index + 1) - .take_while(|item| item.get("type").and_then(Value::as_str) != Some("message")) - .position(|item| call_id(item, output_kind).as_deref() == Some(&id)) - .map(|offset| index + 1 + offset); - if let Some(output) = output - && output != index + 1 - { - let item = items.remove(output); - items.insert(index + 1, item); - } - index += 1; - } -} - // Encodes IR blocks that Responses represents as top-level input items. fn encode_responses_special_input( block: &ContentBlock, namespaces: Option<&Map>, - call_names: &HashMap<&str, &str>, custom_tools: &std::collections::HashSet, custom_call_ids: &mut std::collections::HashSet, diagnostics: &mut Vec, @@ -1487,7 +1412,7 @@ fn encode_responses_special_input( ContentBlock::ToolResult(result) => { // A result answering a freeform call replays as `custom_tool_call_output`; anything // else is a `function_call_output`. - let mut item = json!({ + let item = json!({ "type": if custom_call_ids.contains(&result.tool_call_id) { "custom_tool_call_output" } else { @@ -1496,24 +1421,14 @@ fn encode_responses_special_input( "call_id": result.tool_call_id, "output": encode_responses_tool_output(&result.content, diagnostics, policy)?, }); - // Carry the paired call's name, un-qualified to match the emitted - // function_call, for upstreams that resolve outputs by name. - if let Some(name) = call_names.get(result.tool_call_id.as_str()) { - let name = namespaces - .and_then(|namespaces| { - crate::codex_namespaces::split_qualified_name(namespaces, name) - }) - .map_or(*name, |(name, _)| name); - item["name"] = Value::String(name.to_string()); - } Some(item) } _ => None, }) } -// Encodes reasoning in the shape accepted for Responses input history. Response -// output items may carry `content`, but replayed input items must avoid it. +// This helper encodes reasoning for input history when native item reconstruction does not apply. +// Response output items may carry `content`, but this encoder omits it from input items. fn encode_responses_reasoning_input(text: &str, details: &[Value]) -> Option { for detail in details { let Some(detail) = detail.as_object() else { @@ -1522,6 +1437,12 @@ fn encode_responses_reasoning_input(text: &str, details: &[Value]) -> Option blocks.push(json!({"type": "input_text", "text": text})), + ContentBlock::Text { text, responses } => { + let mut fields = responses + .as_ref() + .map(|source| source.fields.clone()) + .unwrap_or_default(); + fields.entry("type").or_insert_with(|| json!("input_text")); + fields.insert("text".into(), Value::String(text.clone())); + blocks.push(Value::Object(fields)); + } ContentBlock::Refusal { text } => { blocks.push(json!({"type": "refusal", "refusal": text})); } @@ -1645,7 +1576,10 @@ fn encode_responses_tool_output( let text_only = content.iter().all(|block| { matches!( block, - ContentBlock::Text { .. } | ContentBlock::Refusal { .. } + ContentBlock::Text { + responses: None, + .. + } | ContentBlock::Refusal { .. } ) }); if text_only { @@ -1783,7 +1717,7 @@ fn decode_responses_output_item( Some("message") => Ok(Some(ResponseOutput { url_citations: decode_responses_url_citations(item.get("content")), role: role_from_responses(item.get("role").and_then(Value::as_str)), - content: decode_responses_content(item.get("content").unwrap_or(&Value::Null)), + content: decode_responses_content(item.get("content").unwrap_or(&Value::Null), false), stop_reason: Some(StopReason::EndTurn), })), Some("function_call") => Ok(Some(ResponseOutput { @@ -1828,7 +1762,7 @@ fn decode_responses_output_item( Some("reasoning") => Ok(Some(ResponseOutput { url_citations: Vec::new(), role: Role::Assistant, - content: decode_responses_reasoning_item(item), + content: decode_responses_reasoning_item(item, false), stop_reason: None, })), Some(kind) if super::is_native_output(kind) => Ok(Some(ResponseOutput { @@ -2143,6 +2077,7 @@ fn copy_responses_request_extensions( extensions: &Map, ) { for field in [ + "conversation", "include", "metadata", "parallel_tool_calls", diff --git a/crates/switchyard-translation/src/compaction.rs b/crates/switchyard-translation/src/compaction.rs index d45bf54e63..458f971a5f 100644 --- a/crates/switchyard-translation/src/compaction.rs +++ b/crates/switchyard-translation/src/compaction.rs @@ -4,27 +4,29 @@ use http::HeaderMap; use switchyard_protocol::{ContentBlock, LlmRequest, Message, Role}; -/// Normalizes current and historical compaction signals, preserving explicit IR annotations. +/// This function detects current and historical compaction while preserving explicit annotations. pub fn normalize_compaction(request: &mut LlmRequest, headers: Option<&HeaderMap>) { let is_compaction = headers .and_then(|headers| headers.get("x-codex-turn-metadata")) .and_then(|value| serde_json::from_slice::(value.as_bytes()).ok()) .is_some_and(|metadata| metadata["request_kind"] == "compaction"); - request.has_compaction |= request + // Both scans skip inline instructions so they cannot hide a trigger or count as a summary. + let messages = request .messages .iter() + .filter(|message| !matches!(message.role, Role::System | Role::Developer)); + request.has_compaction |= messages + .clone() .flat_map(|message| &message.content) .any(|block| match block { - ContentBlock::Text { text } => { + ContentBlock::Text { text, .. } => { let text = text.to_lowercase(); text.contains("session is being continued") || has_hermes_marker(&text) } _ => false, }); request.is_compaction |= is_compaction - || request - .messages - .iter() + || messages .rev() .find(|message| !is_environment_update(message)) .is_some_and(|message| message.role == Role::User && is_compaction_message(message)); @@ -34,7 +36,7 @@ fn is_environment_update(message: &Message) -> bool { message.role == Role::User && !message.content.is_empty() && message.content.iter().all(|block| { - matches!(block, ContentBlock::Text { text } if text.trim() + matches!(block, ContentBlock::Text { text, .. } if text.trim() .strip_prefix("") .and_then(|context| context.split_once("")) .is_some_and(|(_, rest)| rest.trim().is_empty())) @@ -59,7 +61,7 @@ fn is_compaction_block(block: &ContentBlock) -> bool { raw.get("type").and_then(serde_json::Value::as_str), Some("compaction" | "context_compaction" | "compaction_trigger") ), - ContentBlock::Text { text } => { + ContentBlock::Text { text, .. } => { let text = text.trim().to_ascii_lowercase(); text.starts_with("another language model started to solve this problem and produced a summary of its thinking process.") || text.starts_with("this session is being continued from a previous conversation that ran out of context.") diff --git a/crates/switchyard-translation/src/conversation.rs b/crates/switchyard-translation/src/conversation.rs new file mode 100644 index 0000000000..b8175bb596 --- /dev/null +++ b/crates/switchyard-translation/src/conversation.rs @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! This module groups native Responses input items into turns for routing and other APIs. + +use std::borrow::Cow; + +use crate::{ContentBlock, InstructionBlock, LlmRequest, Message, Role, ToolCall, ToolResult}; + +/// A temporary conversation view groups native items without changing the request. +pub struct Conversation<'a> { + /// Instructions include native inline system and developer messages. + pub instructions: Cow<'a, [InstructionBlock]>, + /// Reasoning and parallel tool calls share their original assistant turn. + pub messages: Cow<'a, [Message]>, +} + +/// This function groups the current content for routing and non-Responses APIs. +/// The request retains native item order and fields for Responses encoding. +/// Indexes in this view do not refer to the original request's messages. +pub fn conversation(request: &LlmRequest) -> Conversation<'_> { + if !request + .messages + .iter() + .any(|message| message.responses.is_some()) + { + return Conversation { + instructions: Cow::Borrowed(&request.instructions), + messages: Cow::Borrowed(&request.messages), + }; + } + // A state reference can identify provider-held calls that are absent from this request. + let fields = &request.extensions.fields; + let stored_state = fields + .get("previous_response_id") + .and_then(serde_json::Value::as_str) + .is_some_and(|id| !id.is_empty()) + || fields.get("conversation").is_some_and(|conversation| { + conversation + .as_str() + .or_else(|| conversation.get("id").and_then(serde_json::Value::as_str)) + .is_some_and(|id| !id.is_empty()) + }); + let mut instructions = request.instructions.clone(); + let mut turns = ConversationTurns::default(); + for message in &request.messages { + // Inline tool definitions are already present in the request's tool catalog. + if matches!(message.content.as_slice(), [ContentBlock::Unknown { raw, .. }] + if raw.get("type").and_then(serde_json::Value::as_str) == Some("additional_tools")) + { + continue; + } + // Inline instructions leave pending reasoning and tool calls in the same turn. + if matches!(message.role, Role::System | Role::Developer) { + instructions.push(InstructionBlock { + role: message.role, + content: message.content.clone(), + }); + continue; + } + match message.content.as_slice() { + [ContentBlock::Reasoning { .. }] if message.role == Role::Assistant => { + // Reasoning after tool outputs starts the next assistant turn. + if !turns.outputs.is_empty() { + turns.flush_tools(); + } + turns.reasoning.extend(message.content.clone()); + } + [ContentBlock::ToolCall(call)] => { + if !turns.outputs.is_empty() { + turns.flush_tools(); + } + turns.calls.push(call.clone()); + } + [ContentBlock::ToolResult(result)] => { + if stored_state + || turns + .calls + .iter() + .any(|call| call.id == result.tool_call_id) + { + turns.outputs.push(result.clone()); + } else { + let text = result + .content + .iter() + .filter_map(|block| match block { + ContentBlock::Text { text, .. } | ContentBlock::Refusal { text } => { + Some(text.as_str()) + } + ContentBlock::Unknown { raw, .. } => raw.as_str(), + _ => None, + }) + .collect::>() + .join(" "); + turns.push_message(Message::text( + Role::User, + format!("Tool result {}: {text}", result.tool_call_id), + )); + } + } + _ => { + let unknown_item = + matches!(message.content.as_slice(), [ContentBlock::Unknown { .. }]) + && message.responses.as_ref().is_some_and(|source| { + source + .fields + .get("type") + .and_then(serde_json::Value::as_str) + .is_some_and(|kind| kind != "message") + }); + let mut message = message.clone(); + message.responses = None; + if message.role == Role::Assistant && !unknown_item && turns.calls.is_empty() { + let mut content = std::mem::take(&mut turns.reasoning); + content.append(&mut message.content); + message.content = content; + } else if turns.calls.is_empty() { + turns.flush_reasoning(); + } + turns.push_message(message); + } + } + } + turns.flush_tools(); + turns.flush_reasoning(); + Conversation { + instructions: Cow::Owned(instructions), + messages: Cow::Owned(turns.messages), + } +} + +// These lists hold temporary clones while the view is built; the request owns the native history. +#[derive(Default)] +struct ConversationTurns { + messages: Vec, + calls: Vec, + outputs: Vec, + deferred: Vec, + reasoning: Vec, +} + +impl ConversationTurns { + fn push_message(&mut self, message: Message) { + if !self.calls.is_empty() && self.outputs.is_empty() { + self.deferred.push(message); + return; + } + self.flush_tools(); + self.messages.push(message); + } + + fn flush_tools(&mut self) { + if !self.calls.is_empty() { + let mut content = std::mem::take(&mut self.reasoning); + content.extend( + std::mem::take(&mut self.calls) + .into_iter() + .map(ContentBlock::ToolCall), + ); + self.messages.push(Message { + role: Role::Assistant, + content, + responses: None, + }); + } + for result in std::mem::take(&mut self.outputs) { + self.messages.push(Message { + role: Role::User, + content: vec![ContentBlock::ToolResult(result)], + responses: None, + }); + } + self.messages.append(&mut self.deferred); + } + + fn flush_reasoning(&mut self) { + if !self.reasoning.is_empty() { + let content = std::mem::take(&mut self.reasoning); + self.push_message(Message { + role: Role::Assistant, + content, + responses: None, + }); + } + } +} diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs index b221749cdf..fe5428050e 100644 --- a/crates/switchyard-translation/src/lib.rs +++ b/crates/switchyard-translation/src/lib.rs @@ -7,6 +7,7 @@ pub mod codecs; pub(crate) mod codex_custom_tools; pub(crate) mod codex_namespaces; mod compaction; +mod conversation; pub mod diagnostic; pub mod engine; pub mod error; @@ -25,6 +26,7 @@ pub use switchyard_protocol::stream::{ pub use switchyard_protocol::{format, llm}; pub use compaction::normalize_compaction; +pub use conversation::{Conversation, conversation}; pub use diagnostic::*; pub use engine::*; pub use error::*; diff --git a/crates/switchyard-translation/src/request_edit.rs b/crates/switchyard-translation/src/request_edit.rs index 9bc65ce25b..182bb607bf 100644 --- a/crates/switchyard-translation/src/request_edit.rs +++ b/crates/switchyard-translation/src/request_edit.rs @@ -60,11 +60,19 @@ pub fn append_note(request: &mut LlmRequest, note: &str) { /// This function prepends a system prompt and updates supported preserved provider bodies. /// It removes bodies it cannot edit so encoding rebuilds them from the updated request. pub fn prepend_system_prompt(request: &mut LlmRequest, prompt: &str) { + // An empty instruction preserves an explicit empty field until a prompt replaces it. + // Removing it here prevents reconstruction from adding an extra separator after the prompt. + if matches!(request.instructions.as_slice(), [InstructionBlock { content, .. }] + if matches!(content.as_slice(), [ContentBlock::Text { text, .. }] if text.is_empty())) + { + request.instructions.clear(); + } request.instructions.insert( 0, InstructionBlock { role: Role::System, content: vec![ContentBlock::Text { + responses: None, text: prompt.to_string(), }], }, diff --git a/crates/switchyard-translation/tests/capability_policy.rs b/crates/switchyard-translation/tests/capability_policy.rs index acd163e798..bd1fff340a 100644 --- a/crates/switchyard-translation/tests/capability_policy.rs +++ b/crates/switchyard-translation/tests/capability_policy.rs @@ -72,6 +72,7 @@ fn request_for(case: CapabilityCase) -> LlmRequest { }; if let Some(content) = content { request.messages.push(Message { + responses: None, role: Role::User, content: vec![content], }); diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rs index e3a9f497d4..9322a8d894 100644 --- a/crates/switchyard-translation/tests/extension_points.rs +++ b/crates/switchyard-translation/tests/extension_points.rs @@ -339,6 +339,7 @@ impl FormatCodec for MinimalCustomCodec { url_citations: Vec::new(), role: Role::Assistant, content: vec![switchyard_translation::ContentBlock::Text { + responses: None, text: body .get("text") .and_then(Value::as_str) @@ -375,7 +376,7 @@ impl FormatCodec for MinimalCustomCodec { .first_output() .and_then(|output| { output.content.iter().find_map(|block| match block { - switchyard_translation::ContentBlock::Text { text } => { + switchyard_translation::ContentBlock::Text { text, .. } => { Some(text.clone()) } _ => None, diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 07e3d3ec2a..d5d9e72aed 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -86,9 +86,7 @@ fn anthropic_server_tools_survive_request_reconstruction() -> TestResult { function.description = Some("Updated edit description".into()); request.instructions.push(InstructionBlock { role: Role::System, - content: vec![ContentBlock::Text { - text: "Plan first.".into(), - }], + content: vec![ContentBlock::text("Plan first.")], }); request.preservation.requests.clear(); let output = engine.encode_request(WireFormat::AnthropicMessages, &request, &policy)?; @@ -166,9 +164,7 @@ fn openai_chat_rejects_mcp_history_before_text_only_returns() { messages: vec![Message { role, content: vec![ - ContentBlock::Text { - text: "Keep this text".into(), - }, + ContentBlock::text("Keep this text"), ContentBlock::Mcp(McpContent::Listing { server_name: "inventory".into(), id: None, @@ -176,6 +172,7 @@ fn openai_chat_rejects_mcp_history_before_text_only_returns() { error: None, }), ], + responses: None, }], ..LlmRequest::default() }; @@ -1279,7 +1276,6 @@ fn anthropic_visible_text_survives_tool_history_to_responses() -> TestResult { { "type": "function_call_output", "call_id": "toolu_status", - "name": "lookup", "output": "ready" } ]) @@ -2580,7 +2576,8 @@ fn responses_reasoning_item_merges_into_next_assistant_message_for_openai_chat() { "role": "assistant", "content": "Let me check.", - "reasoning": "Reading." + "reasoning": "Reading.", + "reasoning_details": [body["input"][1]] } ]) ); @@ -2673,8 +2670,7 @@ fn openai_chat_encrypted_reasoning_details_retain_fallback() -> TestResult { Ok(()) } -// Verifies merged reasoning re-emerges as a Responses reasoning item ahead of -// the turn's function call when encoding back to the Responses format. +// Native reasoning and tool items retain their supplied fields during reconstruction. #[test] fn responses_reasoning_items_round_trip_through_decode_and_encode() -> TestResult { let engine = TranslationEngine::default(); @@ -2710,29 +2706,7 @@ fn responses_reasoning_items_round_trip_through_decode_and_encode() -> TestResul )? .body; - let input = output["input"].as_array().ok_or("input is not an array")?; - let item_types = input - .iter() - .map(|item| item["type"].as_str().unwrap_or_default()) - .collect::>(); - assert_eq!( - item_types, - vec![ - "message", - "reasoning", - "function_call", - "function_call_output" - ] - ); - assert_eq!( - input[1]["summary"], - json!([{"type": "summary_text", "text": "Simple ls."}]) - ); - assert!(input[1].get("content").is_none()); - assert_eq!(input[2]["call_id"], "call-ls"); - // Outputs carry the paired call's name for upstreams (Kimi K3) that - // resolve tool results by name rather than by call order. - assert_eq!(input[3]["name"], "shell"); + assert_eq!(output["input"], body["input"]); Ok(()) } @@ -2792,10 +2766,9 @@ fn responses_encrypted_reasoning_replays_without_input_content() -> TestResult { Ok(()) } -// Verifies an empty non-encrypted reasoning item is omitted instead of being -// replayed as an empty assistant message. +// An empty reasoning item still belongs to the caller's native history. #[test] -fn responses_empty_reasoning_without_encrypted_content_is_omitted() -> TestResult { +fn responses_empty_reasoning_without_encrypted_content_is_preserved() -> TestResult { let engine = TranslationEngine::default(); let policy = TranslationPolicy { preservation: switchyard_translation::PreservationPolicy::Disabled, @@ -2819,14 +2792,7 @@ fn responses_empty_reasoning_without_encrypted_content_is_omitted() -> TestResul )? .body; - let input = output["input"].as_array().ok_or("input is not an array")?; - let item_types = input - .iter() - .map(|item| item["type"].as_str().unwrap_or_default()) - .collect::>(); - assert_eq!(item_types, vec!["message", "message"]); - assert_eq!(input[0]["role"], "user"); - assert_eq!(input[1]["role"], "user"); + assert_eq!(output["input"], body["input"]); Ok(()) } @@ -4147,10 +4113,9 @@ fn responses_file_without_source_is_rejected_by_anthropic() { } } -// Verifies parallel tool calls serialize as adjacent call/output pairs so -// upstreams that resolve tool outputs by adjacency match the right call. +// Native Responses keeps parallel calls and outputs in the order supplied. #[test] -fn responses_parallel_tool_calls_pair_with_their_outputs() -> TestResult { +fn responses_parallel_tool_calls_keep_native_order() -> TestResult { let engine = TranslationEngine::default(); let policy = TranslationPolicy { preservation: switchyard_translation::PreservationPolicy::Disabled, @@ -4185,8 +4150,8 @@ fn responses_parallel_tool_calls_pair_with_their_outputs() -> TestResult { pairs, vec![ ("function_call", "call-a"), - ("function_call_output", "call-a"), ("function_call", "call-b"), + ("function_call_output", "call-a"), ("function_call_output", "call-b"), ] ); @@ -4341,12 +4306,15 @@ fn responses_lite_additional_tools_item_is_the_tool_list() -> TestResult { .collect::>(); assert_eq!(names, vec!["exec", "update_plan"]); assert!( - !decoded.request.messages.iter().any(|message| { - message - .content - .iter() - .any(|block| matches!(block, switchyard_protocol::ContentBlock::Unknown { .. })) - }), + !switchyard_translation::conversation(&decoded.request) + .messages + .iter() + .any(|message| { + message + .content + .iter() + .any(|block| matches!(block, switchyard_protocol::ContentBlock::Unknown { .. })) + }), "the additional_tools item must not become a conversation message" ); @@ -4443,15 +4411,13 @@ fn responses_lite_additional_tools_survive_a_single_text_input() -> TestResult { assert_eq!(input[0]["tools"], tools); assert_eq!(input.len(), 2, "{same}"); assert_eq!(input[1]["role"], "user"); - assert_eq!(input[1]["content"], "List files"); + assert_eq!(input[1]["content"], body["input"][1]["content"]); Ok(()) } -// Parallel freeform calls must pair with their `custom_tool_call_output` items the same way -// function calls do (#664), so upstreams that resolve outputs by adjacency see call, output, -// call, output. +// Native Responses keeps parallel custom calls and outputs in the order supplied. #[test] -fn responses_parallel_custom_tool_calls_pair_with_their_outputs() -> TestResult { +fn responses_parallel_custom_tool_calls_keep_native_order() -> TestResult { let engine = TranslationEngine::default(); let policy = TranslationPolicy { preservation: switchyard_translation::PreservationPolicy::Disabled, @@ -4487,8 +4453,8 @@ fn responses_parallel_custom_tool_calls_pair_with_their_outputs() -> TestResult pairs, vec![ ("custom_tool_call", "call-a"), - ("custom_tool_call_output", "call-a"), ("custom_tool_call", "call-b"), + ("custom_tool_call_output", "call-a"), ("custom_tool_call_output", "call-b"), ] ); diff --git a/docs/getting_started.md b/docs/getting_started.md index 01580f5845..f9e437e821 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -101,6 +101,7 @@ let mut request = request; request.llm_request.messages.extend([ Message { role: Role::Assistant, + responses: None, content: vec![ContentBlock::ToolCall(ToolCall { id: "c1".into(), name: "bash".into(), @@ -109,13 +110,12 @@ request.llm_request.messages.extend([ }, Message { role: Role::User, + responses: None, content: vec![ContentBlock::ToolResult(ToolResult { // The result refers to the tool call above by its ID. tool_call_id: "c1".into(), is_error: Some(true), - content: vec![ContentBlock::Text { - text: "MemoryError: cannot allocate memory\nKilled".into(), - }], + content: vec![ContentBlock::text("MemoryError: cannot allocate memory\nKilled")], })], }, ]); diff --git a/docs/protocols/llm.md b/docs/protocols/llm.md index 1008ff8dc7..d2c347239a 100644 --- a/docs/protocols/llm.md +++ b/docs/protocols/llm.md @@ -31,7 +31,7 @@ let request = Request { llm_request: LlmRequest { instructions: vec![InstructionBlock { role: Role::System, - content: vec![ContentBlock::Text { text: "Answer briefly.".into() }], + content: vec![ContentBlock::text("Answer briefly.")], }], messages: vec![Message::text(Role::User, "Explain tail latency.")], ..LlmRequest::default() diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index c7ec1db3ff..67024093da 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -55,6 +55,7 @@ route reaches no upstream. A file without a `[targets]` table is rejected with | `forward_auth` | No | `false` | Forward the caller's provider credential and application headers. A route's forwarding clients must use one credential family unless they all use the same scheme, host, and port, such as one LLM gateway. | | `extra_headers` | No | `{}` | Custom HTTP headers sent to the model server. Set credentials with `api_key_env` or `forward_auth`; the server rejects headers owned by the selected auth mode. Header names are case-insensitive. | | `responses_reasoning` | No | `"auto"` | Responses reasoning replay: `"auto"`, `"preserve_encrypted"`, `"preserve_plaintext"`, or `"drop"`. Only valid for `openai_responses`. | +| `responses_compatibility` | No | `"native"` | This setting selects `"native"`, `"flat_tools"`, or `"kimi"` tool handling for `openai_responses` clients. | | `fallback_on_rate_limit` | No | `true` | Try another completion candidate after an upstream HTTP 429 exhausts retries. Set to `false` to return the 429 instead. | | `max_retries` | No | `2` | Retry budget, `0`–`10`. | | `failure_cooldown_ms` | No | `5000` (5 seconds) | Skip a backend for this many milliseconds after an exhausted transient completion failure. Zero disables it. | @@ -86,6 +87,20 @@ All modes keep messages, tool calls and tool results. Set an explicit value when name behind a gateway. The setting is rejected on other client formats, even when no target uses the client. +`responses_compatibility = "native"` leaves tool namespaces, item order, and +output fields unchanged. Use `"flat_tools"` for a backend that requires flat +function names. Switchyard joins each function namespace and name with `__` in +history, tool definitions, and tool choice. Use `"kimi"` for a backend that also +requires each tool output immediately after its call and a `name` on the output. +Switchyard retains existing output names and adds missing names from matching +calls. Pairing stops at an intervening message. Outputs without a local call +remain in place, including outputs whose calls are stored upstream. + +These adaptations run after request translation and target body overrides. +Switchyard rejects unknown values and use with other client formats when building +the configuration, even when no target uses the client. Select the mode your +backend requires; Switchyard does not infer it from a model name or URL. + The TOML never contains the secret itself. `api_key_env` names a variable that must exist and be non-empty when the server loads. diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index b9b25077e4..ae8cd19961 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -402,6 +402,17 @@ request; they do not accumulate in the caller's conversation. The escalation note accompanies a signal-driven capable decision, and the optional de-escalation note accompanies a signal-driven efficient decision. +Switchyard appends notes directly to supported saved Chat Completions, +Responses, and Anthropic Messages request bodies. Unsupported bodies may still +require reconstruction. + +Switchyard places cache markers before the notes for Anthropic requests. For +Chat Completions and Responses, it does so only when it recognizes the upstream +model ID as part of the GPT-5.6 or GPT-6 family. Route aliases do not change +eligibility. OpenAI explicit-only cache mode skips these automatic markers. +Switchyard skips new markers when the request's existing marker limit is reached. +Cache reuse depends on the provider and request; measure it with your workload. + ```toml [routes.stage.handoff_notes] escalation_note = "the previous model was stalling; pick up the diagnosis" diff --git a/docs/tuning/examples.md b/docs/tuning/examples.md index bb1a635ff4..42aa175c32 100644 --- a/docs/tuning/examples.md +++ b/docs/tuning/examples.md @@ -48,11 +48,12 @@ cost, retries, and task success together. See the [stage-router guide](../routing_algorithms/stage_router_routing.md#pickers) for profile and threshold details. -**Handoff notes can break cache reuse across turns.** They are added only to the forwarded -request, so later requests can omit the note and break reuse of the cached -continuation. A fix is planned for a future release. Leave `handoff_notes` -unconfigured for this cost comparison; see -[handoff notes](../routing_algorithms/stage_router_routing.md#optional-handoff-notes). +**Handoff notes are an optional tuning control.** Use failures on calibration +tasks to choose the notes, then validate them on held-out tasks. +Keep the notes fixed while comparing profiles, thresholds, or hold turns. Measure +task success and total cost, including retries and cache charges. See +[handoff notes](../routing_algorithms/stage_router_routing.md#optional-handoff-notes) +for configuration and cache behavior. ### Split the benchmark and collect baselines diff --git a/examples/claude/src/main.rs b/examples/claude/src/main.rs index 827b4988f5..779d18147f 100644 --- a/examples/claude/src/main.rs +++ b/examples/claude/src/main.rs @@ -251,7 +251,7 @@ impl RoutedLlmClient for NativeClassifier { .iter() .flat_map(|b| &b.content) .filter_map(|b| match b { - ContentBlock::Text { text } => Some(text.as_str()), + ContentBlock::Text { text, .. } => Some(text.as_str()), _ => None, }) .collect::>()