diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 39400e702..3df0ddd50 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -137,12 +137,13 @@ pub(crate) fn load_runner(path: impl AsRef) -> RunnerResult { } pub(crate) fn runner_from_toml(source: &str) -> RunnerResult { - Arc::new(AlgorithmFactory::from_toml(source)?.config).build() + Runner::from_config(AlgorithmFactory::from_toml(source)?.config) } +/// Deployment configuration shared by typed Rust construction and TOML loading. #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -pub(crate) struct DeploymentConfig { +pub struct DeploymentConfig { schema_version: u32, fallback_client: Option, #[serde(default)] @@ -167,22 +168,30 @@ pub(crate) struct RouteConfig { eligibility: Option, } -#[derive(Debug, Deserialize)] +/// Requirements accepted by a route and ordered alternatives for its existing targets. +#[derive(Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] -struct TargetEligibilityConfig { +pub struct TargetEligibilityConfig { + /// Reject requests without an explicit requirement set; an empty collection counts. #[serde(default)] - require_explicit_requirements: bool, + pub require_explicit_requirements: bool, + /// Try the configured target first, then these alternatives in order. Mappings do not chain. #[serde(default)] - target_alternatives: BTreeMap>, + pub target_alternatives: BTreeMap>, + /// Whether requirements apply to this request or accumulate under its trusted task ID. #[serde(default)] - scope: RequirementScope, + pub scope: RequirementScope, } +/// How long the caller's requirements remain in effect. #[derive(Debug, Default, Deserialize)] #[serde(rename_all = "snake_case")] -enum RequirementScope { +pub enum RequirementScope { + /// Apply requirements only to the current request and its model calls. #[default] Request, + /// Retain the union of requirements for each task in this runner instance. + Task, } struct TargetPromptPolicy { @@ -391,7 +400,138 @@ impl RouteConfig { } } +impl Default for DeploymentConfig { + fn default() -> Self { + Self { + schema_version: SUPPORTED_SCHEMA_VERSION, + fallback_client: None, + llm_clients: BTreeMap::new(), + targets: BTreeMap::new(), + decision_clients: BTreeMap::new(), + decision_targets: BTreeMap::new(), + routes: BTreeMap::new(), + } + } +} + impl DeploymentConfig { + /// Starts an empty deployment. Build it with [`Runner::from_config`]. + pub fn new() -> Self { + Self::default() + } + + /// Adds a native HTTP client with TOML's transport defaults. + /// The credential is an environment variable name, not an API key. + /// Invalid URLs and duplicate names return configuration errors. + pub fn add_http_client( + &mut self, + name: impl Into, + format: WireFormat, + base_url: &str, + api_key_env: Option, + ) -> RunnerResult<()> { + let client = LlmClientConfig { + format: match format { + WireFormat::OpenAiChat => ClientFormat::OpenAiChat, + WireFormat::OpenAiResponses => ClientFormat::OpenAiResponses, + WireFormat::AnthropicMessages => ClientFormat::AnthropicMessages, + }, + base_url: HttpBaseUrl::parse(base_url).map_err(RunnerError::configuration)?, + context_discovery: None, + api_key_env, + forward_auth: false, + extra_headers: BTreeMap::new(), + max_retries: default_max_retries(), + fallback_on_rate_limit: default_fallback_on_rate_limit(), + failure_cooldown_ms: default_failure_cooldown_ms(), + timeout_ms: None, + responses_reasoning: None, + }; + insert_config(&mut self.llm_clients, "llm client", name.into(), client) + } + + /// Adds a model and its deployment-approved labels; duplicate names are rejected. + /// Model references and labels are validated when the runner is built. + pub fn add_target( + &mut self, + name: impl Into, + model: impl Into, + llm_client: impl Into, + labels: impl IntoIterator>, + ) -> RunnerResult<()> { + let target = TargetConfig { + id: model.into(), + llm_client: llm_client.into(), + labels: labels + .into_iter() + .map(|label| label.as_ref().to_owned()) + .collect(), + extra_body: BTreeMap::new(), + omit_body_fields: BTreeSet::new(), + system_prompt: None, + reasoning_effort: None, + }; + insert_config(&mut self.targets, "target", name.into(), target) + } + + /// Replaces approvals on an existing LLM or typed decision target. + /// Unknown names return an error; labels are validated when the runner is built. + pub fn set_target_labels( + &mut self, + name: &str, + labels: impl IntoIterator>, + ) -> RunnerResult<()> { + let target = if let Some(target) = self.targets.get_mut(name) { + &mut target.labels + } else if let Some(target) = self.decision_targets.get_mut(name) { + &mut target.labels + } else { + return Err(RunnerError::configuration(format!("unknown target {name}"))); + }; + *target = labels + .into_iter() + .map(|label| label.as_ref().to_owned()) + .collect(); + Ok(()) + } + + /// Adds an algorithm with optional eligibility; duplicate names are rejected. + /// Runner construction validates references and alternatives before execution. + pub fn add_route( + &mut self, + name: impl Into, + id: impl Into, + algorithm: AlgorithmSpec, + eligibility: Option, + ) -> RunnerResult<()> { + let route = RouteConfig { + id: id.into(), + context_window: None, + tool_calling: None, + reasoning: None, + vision: None, + eligibility, + algorithm, + infra: None, + }; + insert_config(&mut self.routes, "route", name.into(), route) + } + + /// Replaces eligibility before construction, using the TOML route name, not its model ID. + /// Unknown names return an error. + pub fn set_route_eligibility( + &mut self, + name: &str, + eligibility: Option, + ) -> RunnerResult<()> { + let route = self + .routes + .get_mut(name) + .ok_or_else(|| RunnerError::configuration(format!("unknown route {name}")))?; + route.eligibility = eligibility; + Ok(()) + } + pub(crate) fn requires_target_admission(&self) -> bool { self.routes .values() @@ -586,7 +726,7 @@ impl DeploymentConfig { }) } - fn build(self: Arc) -> RunnerResult { + pub(crate) fn build(self: Arc) -> RunnerResult { self.validate()?; let mut approvals = HashMap::new(); @@ -651,11 +791,10 @@ impl DeploymentConfig { let mut route = Route::from_lane(standard, caller_auth, capabilities, route_targets) .with_infra(config.infra.clone()); if let Some(eligibility) = &config.eligibility { - let policy = match eligibility.scope { - RequirementScope::Request => { - EligibilityPolicy::new(eligibility.require_explicit_requirements) - } - }; + let policy = EligibilityPolicy::new( + eligibility.require_explicit_requirements, + matches!(eligibility.scope, RequirementScope::Task), + ); route = route.with_eligibility( policy, EligibleLanes { @@ -1234,6 +1373,23 @@ impl DeploymentConfig { /// Values Anthropic accepts for `output_config.effort`. const ANTHROPIC_EFFORTS: [&str; 5] = ["low", "medium", "high", "xhigh", "max"]; +fn insert_config( + entries: &mut BTreeMap, + kind: &str, + name: String, + value: T, +) -> RunnerResult<()> { + match entries.entry(name) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(value); + Ok(()) + } + std::collections::btree_map::Entry::Occupied(entry) => Err(RunnerError::configuration( + format!("duplicate {kind} {}", entry.key()), + )), + } +} + fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize { let target_name = target_name.to_ascii_lowercase(); let model_id = model_id.to_ascii_lowercase(); @@ -1251,6 +1407,15 @@ fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize { struct HttpBaseUrl(reqwest::Url); impl HttpBaseUrl { + fn parse(raw: &str) -> Result { + let url = reqwest::Url::parse(raw.trim()) + .map_err(|error| format!("base_url must be an absolute HTTP(S) URL: {error}"))?; + if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { + return Err("base_url must be an absolute HTTP(S) URL".into()); + } + Ok(Self(url)) + } + fn as_str(&self) -> &str { self.0.as_str() } @@ -1262,15 +1427,7 @@ impl<'de> Deserialize<'de> for HttpBaseUrl { D: serde::Deserializer<'de>, { let raw = String::deserialize(deserializer)?; - let url = reqwest::Url::parse(raw.trim()).map_err(|error| { - serde::de::Error::custom(format!("base_url must be an absolute HTTP(S) URL: {error}")) - })?; - if !matches!(url.scheme(), "http" | "https") || url.host_str().is_none() { - return Err(serde::de::Error::custom( - "base_url must be an absolute HTTP(S) URL", - )); - } - Ok(Self(url)) + Self::parse(&raw).map_err(serde::de::Error::custom) } } @@ -2006,8 +2163,8 @@ secondary = ["approved_secondary"] "unknown field `require_explicit_requirement`", ), ( - format!("{configured}\nscope = 'task'\n"), - "unknown variant `task`", + format!("{configured}\nscope = 'tenant'\n"), + "unknown variant `tenant`", ), ] { let message = error_message(&invalid); diff --git a/crates/switchyard-runner/src/eligibility.rs b/crates/switchyard-runner/src/eligibility.rs index ee770d478..1a0f3aa90 100644 --- a/crates/switchyard-runner/src/eligibility.rs +++ b/crates/switchyard-runner/src/eligibility.rs @@ -1,10 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Trusted target requirements for one request. +//! Trusted target requirements and bounded task retention. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; +use parking_lot::Mutex; use serde_json::Value; use switchyard_protocol::{LlmClientError, Request, WireFormat}; @@ -12,25 +13,52 @@ const REQUIREMENTS_KEY: &str = "switchyard.internal.required_target_labels"; const MAX_LABELS: usize = 64; const MAX_LABEL_BYTES: usize = 128; const MAX_REQUIREMENTS_BYTES: usize = MAX_LABELS * (MAX_LABEL_BYTES * 6 + 4) + 2; +const MAX_TASK_ID_BYTES: usize = 512; +const MAX_TASKS: usize = 4_096; const RESPONSES_STATE_FIELDS: [&str; 2] = ["previous_response_id", "conversation"]; pub(crate) struct EligibilityPolicy { require_explicit_requirements: bool, + tasks: Option>>>, } impl EligibilityPolicy { - pub(crate) fn new(require_explicit_requirements: bool) -> Self { + pub(crate) fn new(require_explicit_requirements: bool, task_scope: bool) -> Self { Self { require_explicit_requirements, + tasks: task_scope.then(Mutex::default), } } + // Retain valid requirements before checking provider-held context for this turn. pub(crate) fn admit(&self, request: &Request) -> Result { let supplied = target_requirements(request)?; if self.require_explicit_requirements && supplied.is_none() { return Err(invalid("route requires explicit target requirements")); } - let required_labels = supplied.unwrap_or_default(); + let mut required_labels = supplied.unwrap_or_default(); + if let Some(tasks) = &self.tasks { + let task_id = request + .metadata + .as_ref() + .and_then(|metadata| metadata.task_id.as_deref()) + .filter(|id| !id.is_empty() && id.len() <= MAX_TASK_ID_BYTES && id.trim() == *id) + .ok_or_else(|| invalid("target eligibility needs a valid task id"))?; + // Validate and merge under one lock so concurrent turns cannot lose requirements. + let mut tasks = tasks.lock(); + if let Some(retained) = tasks.get_mut(task_id) { + if retained.len() + required_labels.difference(retained).count() > MAX_LABELS { + return Err(invalid("too many target requirement labels")); + } + retained.append(&mut required_labels); + required_labels.clone_from(retained); + } else if !required_labels.is_empty() { + if tasks.len() >= MAX_TASKS { + return Err(invalid("target requirement task capacity exhausted")); + } + tasks.insert(task_id.to_owned(), required_labels.clone()); + } + } validate_request_context(request)?; Ok(TargetRequirements { required_labels }) } @@ -109,7 +137,7 @@ fn target_requirements(request: &Request) -> Result>, Ll Ok(labels) } -/// Validated target requirements supplied by the caller. +/// Effective target requirements after combining caller requirements and retained task state. pub(crate) struct TargetRequirements { required_labels: BTreeSet, } @@ -163,8 +191,9 @@ mod tests { use super::*; - fn request(labels: &[&str]) -> Request { + fn task(id: &str, labels: &[&str]) -> Request { let mut request = Request::default(); + request.metadata.get_or_insert_default().task_id = Some(id.into()); require_target_labels(&mut request, labels).expect("trusted requirements"); request } @@ -190,16 +219,17 @@ mod tests { #[test] fn requirements_are_normalized_and_only_accumulate() { - let policy = EligibilityPolicy::new(false); - let mut request = request(&["customer_data", "private", "private"]); + let policy = EligibilityPolicy::new(false, false); + let mut request = task("one", &["customer_data", "private", "private"]); require_target_labels(&mut request, ["approved"]).expect("additional requirement"); require_target_labels(&mut request, [] as [&str; 0]).expect("cannot clear requirements"); assert_labels(&policy, &request, &["approved", "customer_data", "private"]); + assert_labels(&policy, &task("one", &[]), &[]); } #[test] fn invalid_requirements_do_not_replace_valid_requirements() { - let mut request = request(&["private"]); + let mut request = task("one", &["private"]); for invalid_label in [ "", " ", @@ -216,46 +246,98 @@ mod tests { for encoded in ["null", "{}", "[1]", "[\"\"]", "["] { metadata(&mut request).insert(REQUIREMENTS_KEY.into(), encoded.into()); assert!(has_target_requirements(&request)); - assert!(EligibilityPolicy::new(false).admit(&request).is_err()); + assert!( + EligibilityPolicy::new(false, false) + .admit(&request) + .is_err() + ); assert!(require_target_labels(&mut request, ["private"]).is_err()); } } #[test] fn required_labels_distinguish_missing_from_empty() { - let policy = EligibilityPolicy::new(true); + let policy = EligibilityPolicy::new(true, false); assert!(policy.admit(&Request::default()).is_err()); let decision = policy - .admit(&request(&[])) + .admit(&task("one", &[])) .expect("explicit empty requirements"); assert!(decision.required_labels().is_empty()); } + #[test] + fn task_requirements_accumulate_without_affecting_other_tasks() { + let policy = EligibilityPolicy::new(false, true); + policy + .admit(&task("one", &["private"])) + .expect("first requirement"); + let decision = policy + .admit(&task("one", &["customer_data"])) + .expect("union"); + assert_eq!( + decision.required_labels(), + &labels(&["private", "customer_data"]) + ); + assert_labels(&policy, &task("one", &[]), &["private", "customer_data"]); + assert_labels(&policy, &task("two", &[]), &[]); + let mut absent = Request::default(); + absent.metadata.get_or_insert_default().task_id = Some("one".into()); + assert_labels(&policy, &absent, &["private", "customer_data"]); + + let explicit = EligibilityPolicy::new(true, true); + assert_labels(&explicit, &task("one", &["private"]), &["private"]); + assert!(explicit.admit(&absent).is_err()); + assert_labels(&explicit, &task("one", &[]), &["private"]); + } + + // Invalid labels and task identities must not alter retained requirements. + #[test] + fn task_requirements_validate_signals_and_ids_before_retaining() { + let policy = EligibilityPolicy::new(false, true); + assert_labels(&policy, &task("one", &["private"]), &["private"]); + for encoded in ["null", "{}", "[1]", "[\"\"]", "["] { + let mut request = task("one", &[]); + metadata(&mut request).insert(REQUIREMENTS_KEY.into(), encoded.into()); + assert!(policy.admit(&request).is_err()); + assert_labels(&policy, &task("one", &[]), &["private"]); + } + assert!(policy.admit(&Request::default()).is_err()); + for id in ["", " ", " one", "one ", &"x".repeat(MAX_TASK_ID_BYTES + 1)] { + assert!(policy.admit(&task(id, &["customer_data"])).is_err()); + } + assert_eq!(policy.tasks.as_ref().unwrap().lock().len(), 1); + assert_labels( + &policy, + &task(&"x".repeat(MAX_TASK_ID_BYTES), &["private"]), + &["private"], + ); + } + #[test] fn rejected_provider_history_does_not_erase_valid_requirements() { - let policy = EligibilityPolicy::new(false); + let policy = EligibilityPolicy::new(false, true); for field in RESPONSES_STATE_FIELDS { - let mut request = request(&["private"]); + let mut request = task(field, &["private"]); request .llm_request .extensions .fields .insert(field.into(), Value::String("hidden".into())); assert!(policy.admit(&request).is_err()); - request.llm_request.extensions.fields.remove(field); - assert_labels(&policy, &request, &["private"]); - let mut preserved = self::request(&[]); + assert_labels(&policy, &task(field, &[]), &["private"]); + let mut preserved = task(field, &["customer_data"]); preserved.llm_request.preservation.requests.insert( WireFormat::OpenAiResponses.as_str().into(), serde_json::json!({field: "hidden"}), ); assert!(policy.admit(&preserved).is_err()); + assert_labels(&policy, &task(field, &[]), &["private", "customer_data"]); } } #[test] - fn label_limits_reject_without_replacing_valid_requirements() { - let mut request = request(&["private"]); + fn label_and_task_limits_reject_without_partially_retaining() { + let mut request = task("one", &["private"]); assert!( require_target_labels(&mut request, (0..MAX_LABELS).map(|index| index.to_string())) .is_err() @@ -264,5 +346,28 @@ mod tests { target_requirements(&request).unwrap(), Some(labels(&["private"])) ); + let policy = EligibilityPolicy::new(false, true); + let retained = (0..MAX_LABELS).map(|index| index.to_string()).collect(); + policy.tasks.as_ref().unwrap().lock().extend( + (0..MAX_TASKS).map(|index| (index.to_string(), BTreeSet::from(["private".into()]))), + ); + assert!(policy.admit(&task("new", &["private"])).is_err()); + assert_labels(&policy, &task("new", &[]), &[]); + assert_eq!(policy.tasks.as_ref().unwrap().lock().len(), MAX_TASKS); + policy + .tasks + .as_ref() + .unwrap() + .lock() + .insert("0".into(), retained); + assert!(policy.admit(&task("0", &["customer_data"])).is_err()); + assert_eq!( + policy + .admit(&task("0", &[])) + .unwrap() + .required_labels() + .len(), + MAX_LABELS + ); } } diff --git a/crates/switchyard-runner/src/lib.rs b/crates/switchyard-runner/src/lib.rs index 3e0ac98f6..7f0b64802 100644 --- a/crates/switchyard-runner/src/lib.rs +++ b/crates/switchyard-runner/src/lib.rs @@ -21,7 +21,10 @@ mod route; mod runner; #[cfg(not(target_os = "wasi"))] -pub use config::{ConfiguredTarget, RouteTargets, TargetModel, api_key_variables, route_targets}; +pub use config::{ + ConfiguredTarget, DeploymentConfig as RunnerConfig, RequirementScope, RouteTargets, + TargetEligibilityConfig, TargetModel, api_key_variables, route_targets, +}; #[cfg(not(target_os = "wasi"))] pub use eligibility::require_target_labels; diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs index 125508f35..4c8061eab 100644 --- a/crates/switchyard-runner/src/route.rs +++ b/crates/switchyard-runner/src/route.rs @@ -488,13 +488,13 @@ mod tests { } #[test] - fn infra_spillover_stays_within_request_requirements() { + fn infra_rechecks_task_requirements_and_spills_only_to_approved_models() { let algorithm = "type = 'llm_classifier'\nclassifier_target = 'judge'\n\ strong_target = 'capable'\nweak_target = 'fast'\nbase_threshold = 0.5"; let source = format!( "{}\n[routes.test.infra]\nmax_signal_age_ms = 1000\n\ min_waiting_requests = 4\nmax_capable_inflight = 2\n", - config(algorithm) + config(algorithm).replace("scope = 'request'", "scope = 'task'") ); let runner = Runner::from_toml(&source).expect("eligible queue policy"); let route = runner.route("route").expect("route"); @@ -521,8 +521,13 @@ mod tests { }) .collect(); - let mut stale = - RoutingOutcome::route_to("fast".into(), vec![], request("request", &["private"])); + let mut stale = RoutingOutcome::route_to("fast".into(), vec![], request("task", &[])); + route + .select_lane(&request("task", &["private"])) + .expect("retain private"); + route + .select_lane(&request("task", &["customer_data"])) + .expect("accumulate requirements"); let error = route .apply_infra(&mut stale, &signals) .expect_err("stale decision"); @@ -531,12 +536,11 @@ mod tests { "{error}" ); - for (labels, initial, expected) in [ - (&[] as &[&str], "fast", "capable"), - (&["private"], "approved_fast", "approved_capable"), + for (task, initial, expected) in [ + ("other", "fast", "capable"), + ("task", "approved_fast", "approved_capable"), ] { - let mut outcome = - RoutingOutcome::route_to(initial.into(), vec![], request("request", labels)); + let mut outcome = RoutingOutcome::route_to(initial.into(), vec![], request(task, &[])); assert!( route .apply_infra(&mut outcome, &signals) diff --git a/crates/switchyard-runner/src/runner.rs b/crates/switchyard-runner/src/runner.rs index ad6b1d60d..31d16d6eb 100644 --- a/crates/switchyard-runner/src/runner.rs +++ b/crates/switchyard-runner/src/runner.rs @@ -14,7 +14,7 @@ use switchyard_llm_client::TranslatingLlmClient; use switchyard_protocol::{ModelId, WireFormat}; use crate::config; -use crate::{ModelCapabilities, Route, RunnerError}; +use crate::{ModelCapabilities, Route, RunnerConfig, RunnerError}; /// Immutable named route table. pub struct Runner { @@ -70,6 +70,11 @@ impl Runner { config::runner_from_toml(source) } + /// Validates typed deployment configuration using the same construction as TOML. + pub fn from_config(config: RunnerConfig) -> Result { + Arc::new(config).build() + } + /// Builds a runner from named routes in caller-provided order. /// Pre-condition: There must be at least one route. pub fn new(routes: Vec<(ModelId, Route)>) -> Self { diff --git a/crates/switchyard-runner/tests/runner_config.rs b/crates/switchyard-runner/tests/runner_config.rs new file mode 100644 index 000000000..3b5d77737 --- /dev/null +++ b/crates/switchyard-runner/tests/runner_config.rs @@ -0,0 +1,281 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Typed deployments use the same role selection, task policy and validation as TOML. + +#![cfg(not(target_os = "wasi"))] + +use std::collections::BTreeMap; + +use libsy::PickerMode; +use switchyard_protocol::{Request, WireFormat, text_request}; +use switchyard_runner::{ + AlgorithmSpec, RequirementScope, Runner, RunnerConfig, RunnerError, StageSettingsConfig, + TargetEligibilityConfig, require_target_labels, +}; + +const CONFIG: &str = r#" +schema_version = 1 +[llm_clients.primary] +format = "openai_chat" +base_url = "http://127.0.0.1:9/v1" +[targets.fast] +id = "cloud/fast" +llm_client = "primary" +[targets.capable] +id = "cloud/capable" +llm_client = "primary" +[targets.private_fast] +id = "private/fast" +llm_client = "primary" +labels = ["private"] +[targets.private_capable] +id = "private/capable" +llm_client = "primary" +labels = ["private"] +[targets.approved_fast] +id = "approved/fast" +llm_client = "primary" +labels = ["private", "customer_data"] +[targets.approved_capable] +id = "approved/capable" +llm_client = "primary" +labels = ["private", "customer_data"] +[routes.coding] +id = "coding" +type = "stage_router" +efficient_target = "fast" +capable_target = "capable" +picker = "efficient_first" +confidence_threshold = 0.6 +[routes.coding.eligibility] +target_alternatives = {fast = ["private_fast", "approved_fast"], capable = ["private_capable", "approved_capable"]} +"#; + +fn typed_config( + scope: RequirementScope, + require_explicit_requirements: bool, +) -> Result { + let mut config = RunnerConfig::new(); + config.add_http_client( + "primary", + WireFormat::OpenAiChat, + "http://127.0.0.1:9/v1", + None, + )?; + for (name, model, labels) in [ + ("fast", "cloud/fast", &[] as &[&str]), + ("capable", "cloud/capable", &[]), + ("private_fast", "private/fast", &["private"]), + ("private_capable", "private/capable", &["private"]), + ( + "approved_fast", + "approved/fast", + &["private", "customer_data"], + ), + ( + "approved_capable", + "approved/capable", + &["private", "customer_data"], + ), + ] { + config.add_target(name, model, "primary", labels)?; + } + config.add_route( + "coding", + "coding", + AlgorithmSpec::StageRouter { + efficient_target: "fast".into(), + capable_target: "capable".into(), + capable_first_threshold: 0.0, + settings: StageSettingsConfig { + confidence_threshold: Some(0.6), + ..Default::default() + }, + picker: PickerMode::EfficientFirst, + classifier: None, + subagents: None, + }, + Some(eligibility(scope, require_explicit_requirements)), + )?; + Ok(config) +} + +fn eligibility( + scope: RequirementScope, + require_explicit_requirements: bool, +) -> TargetEligibilityConfig { + TargetEligibilityConfig { + scope, + require_explicit_requirements, + target_alternatives: BTreeMap::from([ + ( + "fast".into(), + vec!["private_fast".into(), "approved_fast".into()], + ), + ( + "capable".into(), + vec!["private_capable".into(), "approved_capable".into()], + ), + ]), + } +} + +fn request(labels: Option<&[&str]>) -> Request { + let mut request = Request { + llm_request: text_request(Some("coding".into()), "Summarize these synthetic notes."), + ..Default::default() + }; + request.metadata.get_or_insert_default().task_id = Some("renewal".into()); + if let Some(labels) = labels { + require_target_labels(&mut request, labels.iter().copied()).expect("requirements"); + } + request +} + +#[tokio::test] +async fn typed_and_toml_routes_select_the_same_ordered_roles() -> Result<(), RunnerError> { + let typed = Runner::from_config(typed_config(RequirementScope::Request, false)?)?; + let parsed = Runner::from_toml(CONFIG)?; + for labels in [ + None, + Some(&[][..]), + Some(&["private"][..]), + Some(&["customer_data", "private"][..]), + ] { + let expected = if labels.is_none_or(|labels| labels.is_empty()) { + ["fast", "capable"] + } else if labels.is_some_and(|labels| labels.len() == 1) { + ["private_fast", "private_capable"] + } else { + ["approved_fast", "approved_capable"] + }; + for runner in [&typed, &parsed] { + let result = runner + .route("coding") + .expect("route") + .decide(request(labels)) + .await?; + assert_eq!(result.selected_model_ids, expected); + } + } + for runner in [&typed, &parsed] { + assert!( + runner + .route("coding") + .expect("route") + .decide(request(Some(&["unknown"]))) + .await + .is_err() + ); + } + let mut edited: RunnerConfig = toml::from_str(CONFIG).expect("typed deployment schema"); + edited.set_target_labels("fast", ["private"])?; + let runner = Runner::from_config(edited)?; + let result = runner + .route("coding") + .expect("route") + .decide(request(Some(&["private"]))) + .await?; + assert_eq!(result.selected_model_ids, ["fast", "private_capable"]); + Ok(()) +} + +#[tokio::test] +async fn typed_and_toml_policies_require_labels_and_retain_task_union() -> Result<(), RunnerError> { + let typed = Runner::from_config(typed_config(RequirementScope::Task, true)?)?; + let parsed = Runner::from_toml(&format!( + "{CONFIG}\nscope = 'task'\nrequire_explicit_requirements = true" + ))?; + let mut edited: RunnerConfig = toml::from_str(CONFIG).expect("typed deployment schema"); + edited.set_route_eligibility("coding", Some(eligibility(RequirementScope::Task, true)))?; + let edited = Runner::from_config(edited)?; + for runner in [&typed, &parsed, &edited] { + let route = runner.route("coding").expect("route"); + assert!(route.decide(request(None)).await.is_err()); + for (labels, expected) in [ + (&[][..], "fast"), + (&["private"][..], "private_fast"), + (&[][..], "private_fast"), + (&["customer_data"][..], "approved_fast"), + (&[][..], "approved_fast"), + ] { + let result = route.decide(request(Some(labels))).await?; + assert_eq!( + result.selected_model_id().expect("selected model"), + expected + ); + } + assert!(route.decide(request(None)).await.is_err()); + } + Ok(()) +} + +fn build_error(config: RunnerConfig) -> String { + match Runner::from_config(config) { + Ok(_) => panic!("invalid typed configuration was accepted"), + Err(error) => error.to_string(), + } +} + +#[test] +fn typed_configuration_uses_shared_validation_without_overwriting_names() -> Result<(), RunnerError> +{ + let mut config = typed_config(RequirementScope::Request, false)?; + config.set_route_eligibility( + "coding", + Some(eligibility(RequirementScope::Request, false)), + )?; + assert!(config.set_target_labels("missing", ["private"]).is_err()); + assert!(config.set_route_eligibility("missing", None).is_err()); + assert!( + config + .add_target("fast", "replacement", "primary", ["private"]) + .is_err() + ); + assert!( + config + .add_http_client( + "primary", + WireFormat::OpenAiChat, + "http://127.0.0.1:9", + None + ) + .is_err() + ); + assert!( + config + .add_http_client("invalid", WireFormat::OpenAiChat, "file:///tmp/model", None) + .is_err() + ); + config.add_target("invalid", "unapproved", "primary", [" private"])?; + assert!(build_error(config).contains("target labels")); + + let mut config = typed_config(RequirementScope::Request, false)?; + config.add_route( + "bad", + "bad", + AlgorithmSpec::Passthrough { + target: "fast".into(), + subagents: None, + }, + Some(TargetEligibilityConfig { + target_alternatives: BTreeMap::from([("fast".into(), vec!["missing".into()])]), + ..Default::default() + }), + )?; + let typed = build_error(config); + let invalid = format!( + "{CONFIG}\n[routes.bad]\nid = 'bad'\ntype = 'passthrough'\ntarget = 'fast'\n[routes.bad.eligibility]\ntarget_alternatives = {{fast = ['missing']}}\n" + ); + let parsed = build_error(toml::from_str(&invalid).expect("configuration schema")); + assert_eq!(typed, parsed); + + let mut config: RunnerConfig = toml::from_str(&format!( + "{CONFIG}\n[decision_targets.audit]\nid = 'audit'\ndecision_client = 'missing'\n" + )) + .expect("typed decision target schema"); + config.set_target_labels("audit", [" private"])?; + assert!(build_error(config).contains("target labels")); + Ok(()) +} diff --git a/crates/switchyard-runner/tests/target_eligibility.rs b/crates/switchyard-runner/tests/target_eligibility.rs index ab1cdd714..566a4d03e 100644 --- a/crates/switchyard-runner/tests/target_eligibility.rs +++ b/crates/switchyard-runner/tests/target_eligibility.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Trusted request requirements constrain routing roles and provider calls. +//! Trusted requirements constrain routing roles, provider calls and retained tasks. use serde_json::{Value, json}; use switchyard_llm_client::AuxiliaryOperation; @@ -430,6 +430,59 @@ async fn explicit_requirements_apply_only_to_the_current_request() { assert!(models_called(&server).await.is_empty()); } +#[tokio::test] +async fn explicit_empty_differs_from_missing_and_task_requirements_only_accumulate() { + let server = MockServer::start().await; + let algorithm = STAGE + .replace("both_fast", "fast") + .replace("both_capable", "capable"); + let policy = format!("require_explicit_requirements = true\nscope = 'task'\n{ALTERNATIVES}"); + let source = config(&server.uri(), &algorithm, &policy); + let runner = Runner::from_toml(&source).expect("task-scoped route"); + let route = runner.route("chat").expect("route"); + assert!(route.decide(request("shared", None)).await.is_err()); + for (task, labels, expected) in [ + ("shared", &[] as &[&str], ["fast", "capable"]), + ("shared", &["private"], ["private_fast", "private_capable"]), + ("shared", &["customer_data"], ["both_fast", "both_capable"]), + ("shared", &[], ["both_fast", "both_capable"]), + ("other", &[], ["fast", "capable"]), + ] { + let outcome = route + .decide(request(task, Some(labels))) + .await + .expect("retained task requirements"); + assert_eq!(outcome.selected_model_ids, expected); + } + assert!(route.decide(request("shared", None)).await.is_err()); + + let unavailable = source.replace("[\"private\",\"customer_data\"]", "[\"private\"]"); + let runner = + Runner::from_toml(&unavailable).expect("valid config with unavailable intersection"); + let route = runner.route("chat").expect("route"); + route + .decide(request("blocked", Some(&["private"]))) + .await + .expect("initial private task"); + for labels in [&["customer_data"] as &[&str], &["private"], &[]] { + assert!( + route + .decide(request("blocked", Some(labels))) + .await + .is_err() + ); + } + let other = route + .decide(request("other", Some(&["private"]))) + .await + .expect("independent task requirements"); + assert_eq!( + other.selected_model_ids, + ["private_fast", "private_capable"] + ); + assert!(models_called(&server).await.is_empty()); +} + #[tokio::test] async fn unapproved_primary_fallback_or_judge_fails_before_provider_io() { let server = MockServer::start().await; diff --git a/docs/internal/target-eligibility.md b/docs/internal/target-eligibility.md index 07b13d4a8..fa2b487bc 100644 --- a/docs/internal/target-eligibility.md +++ b/docs/internal/target-eligibility.md @@ -51,8 +51,9 @@ capable = ["approved_capable"] supplied requirements, including an empty collection. - `require_explicit_requirements` defaults to `false`. When `true`, every request needs a collection; an explicitly empty one counts. -- `scope` is `"request"`, the default and only supported value. Requirements - cover this request and its internal model calls. +- `scope` defaults to `"request"`. Requirements cover this request and its + internal model calls. `"task"` also retains labels under the host-authorized + `request.metadata.task_id`. - `target_alternatives` maps configured target names to ordered alternatives; default `{}`. Keys name targets, not router roles. Alternatives must preserve target kind. @@ -85,16 +86,33 @@ let output = route.execute(request, None).await?; ``` An empty collection means the host assessed the request and found no requirements. -Omitting the helper supplies no assessment. With the default optional policy, -both allow normal routing. With `require_explicit_requirements = true`, omission -is rejected. Labels accumulate within one request but are not retained across -requests, even when they carry the same task ID. +Omitting the helper supplies no assessment. With request scope and the default +optional policy, both allow normal routing. With +`require_explicit_requirements = true`, omission is rejected. With request scope, +labels accumulate within one request but are not retained across requests, even +when they carry the same task ID. + +## Task requirements + +Set `scope = "task"` to retain requirements across requests. Every request needs +a trusted `request.metadata.task_id`: a nonempty string of at most 512 bytes, +without surrounding whitespace. The host must authorize task ownership. + +Labels only accumulate for the same task ID; different IDs are independent. +For example, a task requiring `private` that later adds `customer_data` requires +both from then on. Missing or empty requirements do not remove retained labels. +If the combined requirements cannot be served, the request fails and the labels +remain retained. + +State is in memory and belongs to one route instance. There is no reset, expiry, +replication, restart persistence or revocation. Overlapping in-flight requests +keep the requirements under which they were admitted. ## Execution ```mermaid flowchart TD - H["Trusted host: request + required labels"] --> A["Runner checks requirements"] + H["Trusted host: request + required labels"] --> A["Runner checks requirements and adds retained task labels"] A --> P["Resolve eligible roles; filter configured pools"] P -->|"no valid setup"| E["Error: no provider call"] P --> R["Existing routing algorithm"] @@ -107,8 +125,10 @@ targets. Each setup keeps its algorithm, clients, model groups and auxiliary targets together. Required judges, typed decisions, answer fallbacks and auxiliary calls must qualify. Provider codecs keep their existing translation role. -A route caches at most 128 additional model setups. Requirements contain at most -64 labels of 128 bytes each. Limits fail closed; routing state is not evicted. +A route retains at most 4,096 restricted tasks and caches at most 128 additional +model setups. Requirements contain at most 64 labels of 128 bytes each, including +the combined labels retained for a task. Limits fail closed; requirements are not +cleared and routing state is not evicted. Provider-held Responses continuation state and forwarded caller authentication are rejected on eligibility routes.