diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 780cf8b9c6..4eab2063f6 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -79,18 +79,28 @@ mise run helm:skaffold:run:sidecar mise run helm:skaffold:run:sidecar-mtls ``` -Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm +**Supervisor proxy-pod topology** (build once and leave running): +```bash +mise run helm:skaffold:run:proxy-pod +``` + +All Skaffold commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm chart. The sidecar profile renders an `openshell-network-init` init container for nftables setup and an `openshell-supervisor-network` runtime sidecar for proxying. Binary-aware policy mode runs that sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`; relaxed mode can run it as the configured proxy UID, which must be at least `1000` and distinct from the workload UID. The sidecar-mTLS profile reuses `ci/values-sidecar.yaml` and restores -`server.disableTls=false` inline for Skaffold. The `pkiInitJob` hook (a pre-install -Job that runs `openshell-gateway generate-certs`) generates mTLS secrets on first -install. The default Skaffold values export gateway and Kubernetes-driver traces to -the collector service installed by `helm:k3s:create`. Envoy Gateway opt-in; see the -Optional Add-ons section below. +`server.disableTls=false` inline for Skaffold. The proxy-pod profile renders +network supervision in a separate supervisor Deployment with one pod and relies +on Kubernetes NetworkPolicy enforcement so the agent pod can reach only its +paired supervisor plus DNS. The default local k3s/k3d cluster keeps k3s's +embedded NetworkPolicy controller enabled; if you replace the CNI, install a +policy-enforcing CNI before using proxy-pod. The `pkiInitJob` hook (a +pre-install Job that runs `openshell-gateway generate-certs`) generates mTLS +secrets on first install. The default Skaffold values export gateway and +Kubernetes-driver traces to the collector service installed by +`helm:k3s:create`. Envoy Gateway opt-in; see the Optional Add-ons section below. The gateway Service uses ClusterIP. Access is via Envoy Gateway (port `8080`) or the unified local forwarding task: @@ -131,6 +141,31 @@ export only while it is reachable. create the Secret named `openshell-ha-pg` with a `uri` key, then run `mise run helm:skaffold:run` or `mise run helm:skaffold:dev`. +### Kubernetes e2e profiles + +Run the default Kubernetes e2e environment: + +```bash +mise run e2e:kubernetes +``` + +Run the sidecar topology e2e environment: + +```bash +mise run e2e:kubernetes:sidecar +``` + +Run the proxy-pod topology e2e environment: + +```bash +mise run e2e:kubernetes:proxy-pod +``` + +The proxy-pod e2e task applies `ci/values-proxy-pod.yaml` through +`OPENSHELL_E2E_KUBE_EXTRA_VALUES`. Use an existing cluster with NetworkPolicy +enforcement, or let the wrapper create the default local k3d/k3s cluster with +k3s's embedded NetworkPolicy controller enabled. + ### TLS behaviour `ci/values-skaffold.yaml` sets `server.disableTls: true`, so Skaffold-based deploys run @@ -194,6 +229,12 @@ For a sidecar-profile deployment: mise run helm:skaffold:delete:sidecar ``` +For a proxy-pod-profile deployment: + +```bash +mise run helm:skaffold:delete:proxy-pod +``` + ### Delete the cluster entirely ```bash @@ -321,6 +362,7 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) | | `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay | | `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev | +| `deploy/helm/openshell/ci/values-proxy-pod.yaml` | Supervisor proxy-pod topology overlay for Kubernetes e2e/dev; requires NetworkPolicy enforcement | | `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay | | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev | | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) | diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 2e316f88bd..1a5d518bab 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -436,6 +436,24 @@ jobs: e2e-task: e2e:kubernetes:workspace-operator conformance-artifact-prefix: openshell-conformance + kubernetes-proxy-pod-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + # kind's default CNI does not enforce NetworkPolicies, so this exercises + # the proxy-pod control-plane contract (companions, readiness, sessionless + # relay rejection). The CNI-enforced egress isolation test is tracked + # separately and needs a policy-enforcing CNI. + job-name: Kubernetes E2E (proxy-pod topology) + e2e-task: e2e:kubernetes:proxy-pod + cli-artifact-prefix: rust-binary-cli + kubernetes-ha-e2e: needs: [pr_metadata, build-cli, build-conformance, build-gateway-image, build-supervisor-image] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' @@ -467,7 +485,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, docker-e2e, podman-e2e, vm-e2e, docker-external-driver-e2e, podman-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, docker-e2e, podman-e2e, vm-e2e, docker-external-driver-e2e, podman-external-driver-e2e, vm-external-driver-e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e, kubernetes-proxy-pod-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: diff --git a/Cargo.lock b/Cargo.lock index 72f615e934..670ac15e66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3952,6 +3952,7 @@ dependencies = [ "opentelemetry_sdk", "prost", "prost-types", + "rcgen", "serde", "serde_json", "temp-env", @@ -4524,6 +4525,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "socket2", + "temp-env", "tempfile", "tokio", "tokio-stream", @@ -5541,6 +5543,7 @@ dependencies = [ "ring", "rustls-pki-types", "time", + "x509-parser", "yasna", ] @@ -8627,6 +8630,7 @@ dependencies = [ "lazy_static", "nom", "oid-registry", + "ring", "rusticata-macros", "thiserror 1.0.69", "time", diff --git a/Cargo.toml b/Cargo.toml index 6936439a3b..16845307a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ http-body-util = "0.1" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12", "ring"] } rustls-pemfile = "2" -rcgen = { version = "0.13", features = ["crypto", "pem"] } +rcgen = { version = "0.13", features = ["crypto", "pem", "x509-parser"] } webpki-roots = "1" rustls-native-certs = "0.8" diff --git a/architecture/gateway.md b/architecture/gateway.md index f86411511b..e95a534816 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -240,10 +240,28 @@ to the selected compute driver's `AuthenticateSandbox` RPC. A capable driver is trusted to return the authenticated sandbox ID, while the gateway still requires a matching durable sandbox record before minting a JWT. The Kubernetes driver uses its own named configuration to run TokenReview and verify the live pod and -controlling Sandbox CR. The bootstrap path accepts +controlling Sandbox CR: agent pods must be directly controlled by the `Sandbox` +CR, while proxy-pod supervisor pods may be controlled through the Kubernetes +`Pod -> ReplicaSet -> Deployment -> Sandbox` chain. The bootstrap path accepts both `agents.x-k8s.io/v1beta1` ownerReferences from newer Agent Sandbox controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing -deployments. Supervisors renew gateway JWTs in memory before expiry only while +deployments. The proxy-pod gateway Role follows least privilege: the supervisor +Deployment, Service, CA Secret, and supervisor-ingress NetworkPolicy are +owner-referenced to the Sandbox CR and garbage-collected with it, so the gateway +holds no `delete` on them (Deployment create/get/patch, Service +create/get, Secret create only, plus get on the ReplicaSet for the owner-chain +check). In shared (single-namespace) mode the namespaced Role also grants +Deployment `list`/`watch`, backing a supervisor Deployment watch that pushes a +refreshed sandbox status within seconds of a supervisor availability change; +managed and operator modes deliberately omit those verbs to avoid cluster-wide +Deployment enumeration, folding readiness in through get/list instead. A periodic +reconcile (alongside the one at watch establishment) corrects supervisor replica +drift and reaps orphaned fences without waiting for the watch to drop. The agent +egress NetworkPolicy — the workload's egress fence — carries +no owner reference so it can outlive the workload pod during deletion; the gateway +manages its lifecycle directly and holds create/get/delete/list on NetworkPolicies +for ordered teardown and orphan reaping. +Supervisors renew gateway JWTs in memory before expiry only while the sandbox record still exists. Older tokens are not server-revoked; shared deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes. The config default is diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 4cd4e34a76..048d87827a 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -368,6 +368,94 @@ fn validate_memory_quantity(value: &str) -> Result { Ok(value.to_string()) } +/// True when the gateway reports that this sandbox's topology never opens a +/// supervisor session, so relay-backed operations cannot work. +/// +/// Checked before attempting a session rather than after: the failure would +/// otherwise surface through the `ssh` subprocess as `exit status 255`, which +/// tells the reader nothing. +fn sandbox_has_no_supervisor_session(sandbox: &Sandbox) -> bool { + sandbox.status.as_ref().is_some_and(|status| { + status.conditions.iter().any(|condition| { + condition.r#type == "SupervisorSession" + && condition.status.eq_ignore_ascii_case("false") + && condition.reason == "NotApplicable" + }) + }) +} + +/// True when an error is the gateway rejecting a relay-backed operation because +/// the sandbox's topology has no in-sandbox supervisor. +fn is_no_supervisor_session_error(err: &miette::Report) -> bool { + let marker = openshell_core::error::NO_SUPERVISOR_SESSION_MARKER; + // Check the whole chain: the marker may sit in a wrapped transport error + // rather than the outermost message. + format!("{err}").contains(marker) + || err + .chain() + .any(|source| source.to_string().contains(marker)) +} + +/// Explain a topology that cannot open sessions, instead of letting a raw gRPC +/// error imply the sandbox failed to start. +/// +/// The sandbox is running and its network policy is enforced; only the +/// interactive path is unavailable. The command still exits non-zero, because +/// a command passed to `sandbox create` did not run and callers must not read +/// success from the exit code. +fn report_no_supervisor_session(sandbox_name: &str, had_command: bool, persisted: bool) { + eprintln!(); + eprintln!( + "{} Sandbox '{}' is running, but this topology cannot open sessions.", + "!".yellow().bold(), + sandbox_name.bold() + ); + eprintln!(" SSH, exec, port forwarding, and file transfer need a supervisor inside"); + eprintln!(" the sandbox, which this topology does not run."); + eprintln!(); + if had_command { + eprintln!(" {} your command did not run.", "Note:".bold()); + eprintln!(" Set the workload entrypoint instead, so it starts with the container:"); + eprintln!( + " --driver-config-json '{{\"kubernetes\":{{\"containers\":{{\"agent\":{{\"command\":[...]}}}}}}}}'" + ); + } else { + eprintln!(" Policy-enforced network egress is unaffected."); + } + eprintln!(); + if persisted { + eprintln!(" Inspect it with:"); + eprintln!(" openshell logs {sandbox_name}"); + eprintln!(" openshell sandbox list"); + } + eprintln!(" Use the `combined`, `sidecar`, or `cni-sidecar` topology when you need"); + eprintln!(" interactive sessions."); +} + +/// Delete an ephemeral sandbox, explain the sessionless topology, and return +/// the error to surface. Shared by the pre-detach (command-bearing) and +/// post-detach (interactive) paths so both behave identically. +#[allow(clippy::too_many_arguments)] +async fn abort_sessionless_create( + server: &str, + sandbox_name: &str, + persist: bool, + workspace: &str, + tls: &TlsOptions, + gateway: &str, + had_command: bool, +) -> miette::Report { + if !persist { + let names = [sandbox_name.to_string()]; + if let Err(err) = sandbox_delete(server, &names, false, workspace, tls, gateway).await { + eprintln!("Failed to delete sandbox {sandbox_name}: {err}"); + } + } + report_no_supervisor_session(sandbox_name, had_command, persist); + miette::miette!("sandbox '{sandbox_name}' cannot open interactive sessions") +} + +#[allow(clippy::too_many_arguments)] async fn finalize_sandbox_create_session( server: &str, sandbox_name: &str, @@ -376,8 +464,20 @@ async fn finalize_sandbox_create_session( workspace: &str, tls: &TlsOptions, gateway: &str, + had_command: bool, ) -> Result { + let sessionless = session_result + .as_ref() + .err() + .is_some_and(is_no_supervisor_session_error); + if persist { + if sessionless { + report_no_supervisor_session(sandbox_name, had_command, true); + return Err(miette::miette!( + "sandbox '{sandbox_name}' is running but cannot open interactive sessions" + )); + } return session_result; } @@ -391,6 +491,15 @@ async fn finalize_sandbox_create_session( eprintln!("Failed to delete sandbox {sandbox_name}: {err}"); } + if sessionless { + // The sandbox has already been deleted per --no-keep, so do not point + // the reader at commands that would now fail. + report_no_supervisor_session(sandbox_name, had_command, false); + return Err(miette::miette!( + "sandbox '{sandbox_name}' could not open an interactive session" + )); + } + session_result } @@ -946,6 +1055,31 @@ pub async fn sandbox_create( drop(stream); drop(client); + // Detect a sessionless topology (e.g. proxy-pod) before any + // operation that needs a supervisor session. Uploads, port + // forwarding, the editor, an exec command, and interactive connect + // all require one. Handling this first means a discarded command is + // never reported as success (including with --output json) and an + // ephemeral (--no-keep) sandbox is cleaned up rather than leaked. + let sessionless = sandbox_has_no_supervisor_session(&last_sandbox); + if sessionless + && (!command.is_empty() + || !uploads.is_empty() + || forward.is_some() + || editor.is_some()) + { + return Err(abort_sessionless_create( + &effective_server, + &sandbox_name, + persist, + workspace, + &effective_tls, + gateway_name, + !command.is_empty(), + ) + .await); + } + let upload_count = uploads.len(); for (idx, (local_path, sandbox_path, git_ignore)) in uploads.iter().enumerate() { let dest = sandbox_path.as_deref(); @@ -1064,6 +1198,24 @@ pub async fn sandbox_create( return Ok(0); } + // An interactive bare create against a sessionless topology cannot + // attach (session-requiring operations were already rejected at the + // top of this arm). Skip the connect — which would spawn ssh, fail + // inside the subprocess, and surface as an opaque exit status — and + // explain instead. + if sessionless { + return Err(abort_sessionless_create( + &effective_server, + &sandbox_name, + persist, + workspace, + &effective_tls, + gateway_name, + false, + ) + .await); + } + let connect_result = if persist { sandbox_connect(&effective_server, &sandbox_name, &effective_tls, workspace).await } else { @@ -1084,6 +1236,7 @@ pub async fn sandbox_create( workspace, &effective_tls, gateway_name, + !command.is_empty(), ) .await } @@ -1110,6 +1263,7 @@ pub async fn sandbox_create( workspace, &effective_tls, gateway_name, + !command.is_empty(), ) .await } @@ -1134,6 +1288,7 @@ pub async fn sandbox_create( workspace, &effective_tls, gateway_name, + !command.is_empty(), ) .await } @@ -9012,6 +9167,78 @@ mod tests { assert!(sandbox_should_persist(true, None)); } + use crate::run::{is_no_supervisor_session_error, sandbox_has_no_supervisor_session}; + + #[test] + fn detects_the_sessionless_condition_on_a_sandbox() { + use openshell_core::proto::{Sandbox, SandboxCondition, SandboxStatus}; + + let sessionless = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: "SupervisorSession".to_string(), + status: "False".to_string(), + reason: "NotApplicable".to_string(), + message: openshell_core::error::no_supervisor_session_message(), + last_transition_time: String::new(), + }], + ..Default::default() + }), + ..Default::default() + }; + assert!(sandbox_has_no_supervisor_session(&sessionless)); + + // A supervisor that is merely not connected yet must not be mistaken + // for a topology that will never have one. + let still_settling = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "SupervisorNotConnected".to_string(), + message: "Backend ready; waiting for supervisor session".to_string(), + last_transition_time: String::new(), + }], + ..Default::default() + }), + ..Default::default() + }; + assert!(!sandbox_has_no_supervisor_session(&still_settling)); + + assert!(!sandbox_has_no_supervisor_session(&Sandbox::default())); + } + + #[test] + fn detects_the_no_supervisor_session_rejection() { + let err = miette::miette!("{}", openshell_core::error::no_supervisor_session_message()); + assert!(is_no_supervisor_session_error(&err)); + } + + #[test] + fn other_errors_are_not_mistaken_for_a_sessionless_topology() { + for message in [ + "supervisor session not connected", + "sandbox not found", + "timed out waiting for the sandbox to become ready", + ] { + let err = miette::miette!("{message}"); + assert!( + !is_no_supervisor_session_error(&err), + "{message} must not be treated as a sessionless topology" + ); + } + } + + /// The marker travels through gRPC as part of the status message, so + /// detection has to survive the wrapping the transport and CLI add. + #[test] + fn detection_survives_error_wrapping() { + let inner = + Status::failed_precondition(openshell_core::error::no_supervisor_session_message()); + let err = miette::miette!("failed to open session: {inner}"); + assert!(is_no_supervisor_session_error(&err)); + } + #[test] fn sandbox_should_not_persist_when_no_keep_is_set() { assert!(!sandbox_should_persist(false, None)); diff --git a/crates/openshell-core/src/error.rs b/crates/openshell-core/src/error.rs index 145106012d..aff43f48f4 100644 --- a/crates/openshell-core/src/error.rs +++ b/crates/openshell-core/src/error.rs @@ -138,3 +138,24 @@ impl From for tonic::Status { } } } + +/// Stable marker embedded in the gateway's rejection of relay-backed RPCs for +/// sandboxes whose topology has no in-sandbox process supervisor. +/// +/// SSH, `exec`, port forwarding, and file transfer all travel over the +/// supervisor session, which such a topology never opens. The CLI matches on +/// this marker to explain the situation rather than surfacing a raw gRPC +/// error, so callers must keep the two in sync. It is deliberately a distinct +/// token rather than prose so rewording the message cannot break detection. +pub const NO_SUPERVISOR_SESSION_MARKER: &str = "openshell:no-supervisor-session"; + +/// Full message returned for relay-backed RPCs against such a sandbox. +#[must_use] +pub fn no_supervisor_session_message() -> String { + format!( + "this sandbox's topology runs no supervisor inside the sandbox, so SSH, exec, port \ + forwarding, and file transfer are unavailable. Policy-enforced network egress is \ + unaffected. Use the `combined`, `sidecar`, or `cni-sidecar` topology when interactive \ + sessions are required. [{NO_SUPERVISOR_SESSION_MARKER}]" + ) +} diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 99ac55fe6e..1c0ee277a2 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -150,10 +150,26 @@ pub const SIDECAR_CONTROL_SOCKET: &str = "OPENSHELL_SIDECAR_CONTROL_SOCKET"; /// Optional TLS server name override used when connecting to the gateway. pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; +/// Explicit URL injected into sandbox child processes for proxy-mode egress. +/// +/// Kubernetes proxy-pod topology uses a headless Service DNS name, which +/// cannot be represented by the policy's `SocketAddr` proxy field. +pub const PROXY_URL: &str = "OPENSHELL_PROXY_URL"; + +/// Explicit listener address for the network supervisor's HTTP CONNECT proxy. +pub const PROXY_BIND_ADDR: &str = "OPENSHELL_PROXY_BIND_ADDR"; + /// Directory where the network supervisor writes the proxy CA files consumed /// by workload child processes. pub const PROXY_TLS_DIR: &str = "OPENSHELL_PROXY_TLS_DIR"; +/// Optional CA certificate PEM path used by the network supervisor instead of +/// generating an ephemeral CA. +pub const PROXY_CA_CERT_PATH: &str = "OPENSHELL_PROXY_CA_CERT_PATH"; + +/// Optional CA private key PEM path paired with [`PROXY_CA_CERT_PATH`]. +pub const PROXY_CA_KEY_PATH: &str = "OPENSHELL_PROXY_CA_KEY_PATH"; + /// Path to the CA certificate for mTLS communication with the gateway. pub const TLS_CA: &str = "OPENSHELL_TLS_CA"; diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d599cac169..f60528d172 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -45,11 +45,11 @@ use openshell_core::proto::compute::v1::{ GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, - StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, - watch_sandboxes_event, + StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, SupervisorSessionModel, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, + WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, + WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, + gateway_listener_requirement::Selector, watch_sandboxes_event, }; use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, @@ -2231,6 +2231,7 @@ fn pending_sandbox_snapshot( namespace: namespace.to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: sandbox.name.clone(), instance_id: String::new(), agent_fd: String::new(), @@ -3559,6 +3560,7 @@ fn driver_status_from_summary( let (ready, reason, message, deleting) = container_ready_condition(state); DriverSandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: summary_container_name(summary).unwrap_or_else(|| sandbox_name.to_string()), instance_id: summary.id.clone().unwrap_or_default(), agent_fd: String::new(), diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 07fb2b0ffb..79f452837f 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -15,7 +15,7 @@ use openshell_core::progress::{ use openshell_core::proto::compute::v1::{ DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, GetGatewayListenerRequirementsRequest, GpuResourceRequirements, ResourceRequirements, - gateway_listener_requirement::Selector, + SupervisorSessionModel, gateway_listener_requirement::Selector, }; use std::fs; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -3125,6 +3125,7 @@ fn exited_sandbox_with_ready_reason(reason: &str) -> DriverSandbox { last_transition_time: String::new(), }], deleting: false, + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, }), workspace: String::new(), } diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 3a3d843f1a..88b658796f 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -37,6 +37,7 @@ tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } notify = "8" +rcgen = { workspace = true } [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 02dcfe5e87..085962fea6 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -152,6 +152,20 @@ abstract socket whose peer PID must match that authenticated supervisor. Both supervisors exit if the control connection closes, coupling their container restart lifecycle before a new authoritative client can be established. +The `proxy-pod` supervisor topology runs network enforcement and gateway +forwarding in a separate supervisor Deployment with one pod. The agent pod runs +the sandbox image directly and reaches the supervisor through a per-sandbox +headless Service. The driver creates an owner-referenced supervisor +Deployment with one replica plus Service, proxy CA Secret, and NetworkPolicy +resources so agent egress is limited to its paired supervisor pod plus DNS. If +the supervisor pod is deleted, the Deployment recreates it. The workload pod +does not mount gateway credentials or the supervisor binary. Its proxy CA and +default workspace init containers run as the resolved sandbox UID/GID, disable +privilege escalation, and drop all capabilities. The workload mounts the +generated proxy CA bundle read-only. This topology +intentionally omits filesystem/process/binary enforcement, SSH/exec, +upload/download, sync, and provider environment injection. + The driver can request a Kubernetes AppArmor profile through `app_armor_profile`. diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 805c0314b0..1a05b56b05 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -22,7 +22,7 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; /// Default storage size for the workspace PVC. pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; -/// Default non-root UID for relaxed Kubernetes network supervisor sidecars. +/// Default UID for the long-running Kubernetes network proxy. pub const DEFAULT_PROXY_UID: u32 = 1337; /// How the supervisor binary is delivered into sandbox pods. @@ -72,6 +72,9 @@ pub enum SupervisorTopology { /// Run network supervision in a privileged sidecar and process supervision /// as a low-capability wrapper in the agent container. Sidecar, + /// Run network supervision in a separate supervisor pod and process + /// supervision as a low-capability wrapper in the agent pod. + ProxyPod, } impl std::fmt::Display for SupervisorTopology { @@ -79,6 +82,7 @@ impl std::fmt::Display for SupervisorTopology { match self { Self::Combined => f.write_str("combined"), Self::Sidecar => f.write_str("sidecar"), + Self::ProxyPod => f.write_str("proxy-pod"), } } } @@ -90,6 +94,7 @@ impl FromStr for SupervisorTopology { match s { "combined" => Ok(Self::Combined), "sidecar" => Ok(Self::Sidecar), + "proxy-pod" => Ok(Self::ProxyPod), other => Err(format!("unknown topology '{other}'")), } } @@ -177,6 +182,227 @@ impl KubernetesSidecarConfig { } } +/// Scheduling relationship between a proxy-pod workload and its paired +/// network-supervisor pod. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ProxyPodAffinity { + /// Do not add an OpenShell-managed pod-affinity term. + #[default] + Disabled, + /// Prefer same-node placement without making it a scheduling requirement. + Preferred, + /// Require the workload and network supervisor to run on the same node. + Required, +} + +impl std::fmt::Display for ProxyPodAffinity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Disabled => f.write_str("disabled"), + Self::Preferred => f.write_str("preferred"), + Self::Required => f.write_str("required"), + } + } +} + +impl FromStr for ProxyPodAffinity { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "disabled" => Ok(Self::Disabled), + "preferred" => Ok(Self::Preferred), + "required" => Ok(Self::Required), + other => Err(format!( + "unknown proxy-pod affinity '{other}'; expected 'disabled', 'preferred', or 'required'" + )), + } + } +} + +/// One cluster-DNS peer in the `proxy-pod` agent egress `NetworkPolicy`. +/// +/// Each peer renders as a single `to` entry combining a `namespaceSelector` +/// and a `podSelector`, so both selectors must match the same pod. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct ProxyPodDnsPeer { + /// Labels matched against the namespace hosting the DNS pods. + pub namespace_labels: BTreeMap, + /// Labels matched against the DNS pods themselves. + pub pod_labels: BTreeMap, + /// Port the DNS pods actually listen on. + /// + /// This is the **container** port, not the `Service` port. A + /// `NetworkPolicy` egress rule with a `podSelector` peer is evaluated + /// against the destination pod after `Service` address translation, so a + /// `Service` that maps 53 to a different container port needs that + /// container port here. Upstream `CoreDNS` listens on 53; `OpenShift`'s + /// `dns-default` listens on 5353 and maps 53 to it. + pub port: u16, +} + +impl Default for ProxyPodDnsPeer { + fn default() -> Self { + Self { + namespace_labels: BTreeMap::new(), + pod_labels: BTreeMap::new(), + port: DEFAULT_DNS_PORT, + } + } +} + +/// Default DNS container port, matching upstream `CoreDNS`/kube-dns. +pub const DEFAULT_DNS_PORT: u16 = 53; + +impl ProxyPodDnsPeer { + fn new(namespace_label: (&str, &str), pod_label: (&str, &str)) -> Self { + Self { + namespace_labels: std::iter::once(( + namespace_label.0.to_string(), + namespace_label.1.to_string(), + )) + .collect(), + pod_labels: std::iter::once((pod_label.0.to_string(), pod_label.1.to_string())) + .collect(), + port: DEFAULT_DNS_PORT, + } + } + + fn validate(&self, index: usize) -> Result<(), String> { + if self.port == 0 { + return Err(format!("proxy_pod.dns_peers[{index}].port must not be 0")); + } + if self.namespace_labels.is_empty() && self.pod_labels.is_empty() { + return Err(format!( + "proxy_pod.dns_peers[{index}] must set namespace_labels, pod_labels, or both; an \ + empty peer would allow DNS-port egress to every pod in the cluster" + )); + } + Ok(()) + } +} + +/// Upstream Kubernetes conventions for cluster DNS. +/// +/// These are conventions, not guarantees. `OpenShift`, `NodeLocal` `DNSCache`, and +/// custom DNS deployments all place cluster DNS elsewhere and require +/// `proxy_pod.dns_peers` to be set explicitly. +fn default_proxy_pod_dns_peers() -> Vec { + vec![ + ProxyPodDnsPeer::new( + ("kubernetes.io/metadata.name", "kube-system"), + ("k8s-app", "kube-dns"), + ), + ProxyPodDnsPeer::new( + ("kubernetes.io/metadata.name", "kube-system"), + ("k8s-app", "coredns"), + ), + ] +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesProxyPodConfig { + /// UID used by the network supervisor in `proxy-pod` topology. It must not + /// match the sandbox workload UID. + pub proxy_uid: u32, + /// Whether same-node placement with the paired supervisor is disabled, + /// preferred, or required. + pub affinity: ProxyPodAffinity, + /// Cluster DNS peers permitted by the agent egress `NetworkPolicy`. + /// + /// Defaults to the upstream `kube-system` conventions. Clusters that host + /// DNS elsewhere must override this or the agent pod cannot resolve any + /// name, including its own paired supervisor `Service`. + pub dns_peers: Vec, + /// Gateway peers the agent egress `NetworkPolicy` permits, so the in-pod + /// process supervisor can reach the `OpenShell` gateway for its session + /// (relays, policy, log push, token bootstrap). Same shape as `dns_peers` + /// (namespace/pod selectors + the gateway's TCP port). Empty by default; the + /// Helm chart renders it from the gateway's own pod labels and port. A + /// proxy-pod sandbox whose supervisor cannot reach the gateway never becomes + /// ready, so validation rejects an empty list for that topology. + pub gateway_peers: Vec, + /// Keep managing existing proxy-pod sandboxes after the configured topology + /// has been switched away from `proxy-pod`. + /// + /// The driver already manages each sandbox by its persisted creation-time + /// topology, but background upkeep — periodic companion reconciliation and + /// the shared-mode supervisor `Deployment` readiness watch — is gated on + /// whether this gateway manages proxy-pod sandboxes at all. When the + /// configured topology is not `proxy-pod`, that would otherwise be inferred + /// from a runtime sandbox list, which a transient discovery failure could + /// answer "none" and freeze for a whole watch session. Set this true during + /// a `retainCompanionRbac` migration (the Helm chart renders it from + /// `supervisor.proxyPod.retainCompanionRbac`) to keep that upkeep running + /// deterministically until every proxy-pod sandbox is deleted. + pub retain_companion_management: bool, +} + +impl Default for KubernetesProxyPodConfig { + fn default() -> Self { + Self { + proxy_uid: DEFAULT_PROXY_UID, + affinity: ProxyPodAffinity::Disabled, + dns_peers: default_proxy_pod_dns_peers(), + gateway_peers: Vec::new(), + retain_companion_management: false, + } + } +} + +impl KubernetesProxyPodConfig { + pub fn validate_proxy_uid(&self) -> Result<(), String> { + if self.proxy_uid < openshell_policy::MIN_SANDBOX_UID { + return Err(format!( + "proxy_pod.proxy_uid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) + } + + /// Validate the configured DNS peers. + /// + /// An empty list is rejected rather than silently denying DNS: a + /// `proxy-pod` sandbox with no DNS egress cannot resolve its own paired + /// supervisor `Service` and is inert. + pub fn validate_dns_peers(&self) -> Result<(), String> { + if self.dns_peers.is_empty() { + return Err( + "proxy_pod.dns_peers must not be empty; the agent pod needs cluster DNS to \ + resolve its paired supervisor Service" + .to_string(), + ); + } + for (index, peer) in self.dns_peers.iter().enumerate() { + peer.validate(index)?; + } + Ok(()) + } + + /// Validate the configured gateway peers. + /// + /// An empty list is rejected: the in-pod process supervisor must reach the + /// gateway for its session, and the agent egress `NetworkPolicy` otherwise + /// denies it, so the sandbox never becomes ready. + pub fn validate_gateway_peers(&self) -> Result<(), String> { + if self.gateway_peers.is_empty() { + return Err( + "proxy_pod.gateway_peers must not be empty; the in-pod process supervisor needs \ + egress to the gateway for its session" + .to_string(), + ); + } + for (index, peer) in self.gateway_peers.iter().enumerate() { + peer.validate(index)?; + } + Ok(()) + } +} + /// Kubernetes `AppArmor` profile requested for the sandbox agent container. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppArmorProfile { @@ -326,6 +552,8 @@ pub struct KubernetesComputeConfig { pub topology: SupervisorTopology, /// Sidecar-only settings used when `topology = "sidecar"`. pub sidecar: KubernetesSidecarConfig, + /// Proxy-pod-only settings used when `topology = "proxy-pod"`. + pub proxy_pod: KubernetesProxyPodConfig, /// Corporate HTTP forward proxy used by the network supervisor for /// policy-approved TLS CONNECT egress. pub https_proxy: Option, @@ -451,6 +679,7 @@ impl Default for KubernetesComputeConfig { supervisor_sideload_method: SupervisorSideloadMethod::default(), topology: SupervisorTopology::default(), sidecar: KubernetesSidecarConfig::default(), + proxy_pod: KubernetesProxyPodConfig::default(), https_proxy: None, no_proxy: None, proxy_auth_secret_name: None, @@ -503,7 +732,8 @@ impl KubernetesComputeConfig { } pub fn validate_proxy_uid(&self) -> Result<(), String> { - self.sidecar.validate_proxy_uid() + self.sidecar.validate_proxy_uid()?; + self.proxy_pod.validate_proxy_uid() } /// Validate the operator-owned corporate upstream proxy configuration. @@ -578,11 +808,11 @@ impl KubernetesComputeConfig { if self.proxy_auth_allow_insecure != Some(true) { return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); } - if self.topology == SupervisorTopology::Combined { - return Err( - "proxy credential Secrets require topology = \"sidecar\"; combined topology shares the credential mount with the workload and fsGroup can make it readable by the sandbox user" - .to_string(), - ); + if self.topology != SupervisorTopology::Sidecar { + return Err(format!( + "proxy credential Secrets require topology = \"sidecar\"; {} topology does not mount the credential into a supervisor container isolated from the workload", + self.topology + )); } } _ => { @@ -920,6 +1150,7 @@ mod tests { fn default_proxy_uid_is_dedicated_non_root_uid() { let cfg = KubernetesComputeConfig::default(); assert_eq!(cfg.sidecar.proxy_uid, DEFAULT_PROXY_UID); + assert_eq!(cfg.proxy_pod.affinity, ProxyPodAffinity::Disabled); } #[test] @@ -946,6 +1177,98 @@ mod tests { assert_eq!(cfg.topology, SupervisorTopology::Combined); } + #[test] + fn serde_override_topology_proxy_pod() { + let json = serde_json::json!({ + "topology": "proxy-pod" + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.topology, SupervisorTopology::ProxyPod); + assert_eq!(cfg.topology.to_string(), "proxy-pod"); + } + + #[test] + fn proxy_pod_dns_peers_default_to_kube_system() { + let cfg = KubernetesProxyPodConfig::default(); + assert_eq!(cfg.dns_peers.len(), 2); + cfg.validate_dns_peers().unwrap(); + } + + #[test] + fn proxy_pod_retain_companion_management_defaults_off_and_parses() { + // Absent from config → off (a non-migrating gateway). + assert!(!KubernetesProxyPodConfig::default().retain_companion_management); + // Present and unknown-field-strict: the field parses when rendered. + let cfg: KubernetesProxyPodConfig = + serde_json::from_value(serde_json::json!({"retain_companion_management": true})) + .unwrap(); + assert!(cfg.retain_companion_management); + } + + #[test] + fn proxy_pod_rejects_empty_dns_peers() { + let cfg = KubernetesProxyPodConfig { + dns_peers: Vec::new(), + ..KubernetesProxyPodConfig::default() + }; + let err = cfg.validate_dns_peers().unwrap_err(); + assert!(err.contains("must not be empty"), "{err}"); + } + + #[test] + fn proxy_pod_rejects_a_dns_peer_with_no_selectors() { + let cfg = KubernetesProxyPodConfig { + dns_peers: vec![ProxyPodDnsPeer::default()], + ..KubernetesProxyPodConfig::default() + }; + let err = cfg.validate_dns_peers().unwrap_err(); + assert!(err.contains("dns_peers[0]"), "{err}"); + } + + #[test] + fn serde_override_proxy_pod_dns_peers_nested() { + let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({ + "proxy_pod": { + "dns_peers": [{ + "namespace_labels": {"kubernetes.io/metadata.name": "openshift-dns"}, + "pod_labels": {"dns.operator.openshift.io/daemonset-dns": "default"} + }] + } + })) + .unwrap(); + assert_eq!(cfg.proxy_pod.dns_peers.len(), 1); + assert_eq!( + cfg.proxy_pod.dns_peers[0].pod_labels["dns.operator.openshift.io/daemonset-dns"], + "default" + ); + cfg.proxy_pod.validate_dns_peers().unwrap(); + } + + #[test] + fn serde_override_proxy_pod_proxy_uid_nested() { + let json = serde_json::json!({ + "proxy_pod": { + "proxy_uid": 2000, + "affinity": "preferred" + } + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.proxy_pod.proxy_uid, 2000); + assert_eq!(cfg.proxy_pod.affinity, ProxyPodAffinity::Preferred); + cfg.validate_proxy_uid().unwrap(); + } + + #[test] + fn serde_rejects_invalid_proxy_pod_affinity() { + let json = serde_json::json!({ + "proxy_pod": { + "affinity": "sometimes" + } + }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown variant")); + } + #[test] fn serde_rejects_sidecar_binary_identity_field() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index afe3f579e0..591b60c960 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -7,16 +7,18 @@ use super::AppArmorProfile; use crate::config::{ DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, - SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, is_dns_1123_label, - managed_namespace, managed_namespace_prefix, validate_managed_namespace_name, + ProxyPodAffinity, ProxyPodDnsPeer, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + is_dns_1123_label, managed_namespace, managed_namespace_prefix, + validate_managed_namespace_name, }; use futures::{Stream, StreamExt, TryStreamExt}; +use k8s_openapi::api::apps::v1::{Deployment, ReplicaSet}; use k8s_openapi::api::authentication::v1::{ TokenReview, TokenReviewSpec, TokenReviewStatus, UserInfo, }; use k8s_openapi::api::core::v1::{ Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Secret, - ServiceAccount, Volume, VolumeMount, + Service, ServiceAccount, Volume, VolumeMount, }; use k8s_openapi::api::networking::v1::{ NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, @@ -47,12 +49,14 @@ use openshell_core::proto::compute::v1::{ DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, DriverSandboxSpec as SandboxSpec, DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, - GetCapabilitiesResponse, GpuResourceRequirements, WatchSandboxesDeletedEvent, - WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesSandboxEvent, - watch_sandboxes_event, + GetCapabilitiesResponse, GpuResourceRequirements, SupervisorSessionModel, + WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, + WatchSandboxesSandboxEvent, watch_sandboxes_event, }; use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; +use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; use serde::Deserialize; +use serde::de::DeserializeOwned; use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; use std::pin::Pin; @@ -108,18 +112,52 @@ impl From for openshell_core::ComputeDriverError { /// API server is unreachable or slow. const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30); +/// Interval at which a proxy-pod gateway re-runs companion reconciliation while +/// its sandbox watch is established. `reconcile_proxy_pod_companions` otherwise +/// runs only at watch establishment, so a stop-time supervisor scale-down that +/// failed transiently (or an egress fence orphaned by a crash) would persist +/// until the next re-establishment. The periodic sweep bounds that window +/// without depending on the watch dropping. +const PROXY_POD_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); + /// Kubernetes defaults pod termination to 30 seconds when the pod template /// omits `terminationGracePeriodSeconds`. const DEFAULT_POD_TERMINATION_GRACE_PERIOD: Duration = Duration::from_secs(30); const STOP_INITIAL_POLL_INTERVAL: Duration = Duration::from_millis(250); const STOP_MAX_POLL_INTERVAL: Duration = Duration::from_secs(2); +/// After confirming the Sandbox CR is gone, the fence teardown waits out this +/// quiescence window (re-checking pod absence each interval) before deleting the +/// egress fence. It lets an in-flight controller reconciliation that read the CR +/// before deletion settle: if such a reconcile creates a dangling-ownerReference +/// workload Pod, the recheck sees it and retains the fence. A healthy controller +/// converges within its informer resync well inside this window. +const FENCE_QUIESCE_WINDOW: Duration = Duration::from_secs(6); +const FENCE_QUIESCE_INTERVAL: Duration = Duration::from_secs(2); + +/// Backoff before re-establishing a supervisor Deployment watch that ended, so a +/// rare stream end does not silently drop fast readiness for the remainder of a +/// sandbox watch's lifetime. +const DEPLOYMENT_WATCH_REESTABLISH_BACKOFF: Duration = Duration::from_secs(2); + const SANDBOX_GROUP: &str = "agents.x-k8s.io"; const SANDBOX_VERSION_V1BETA1: &str = "v1beta1"; const SANDBOX_VERSION_V1ALPHA1: &str = "v1alpha1"; const SANDBOX_VERSIONS: &[&str] = &[SANDBOX_VERSION_V1BETA1, SANDBOX_VERSION_V1ALPHA1]; pub const SANDBOX_KIND: &str = "Sandbox"; const SANDBOX_POD_NAME_ANNOTATION: &str = "agents.x-k8s.io/pod-name"; +/// Records the supervisor topology a Sandbox CR was created under. The gateway's +/// configured topology can change (e.g. a Helm value edit + restart), so +/// interpreting an existing CR with the current global topology would +/// misclassify its supervisor-session model and mis-target its companion +/// Deployment. Reading this annotation keeps status and lifecycle behavior tied +/// to the topology the sandbox was actually created with. +const ANNOTATION_SUPERVISOR_TOPOLOGY: &str = "openshell.ai/supervisor-topology"; +/// Records the name of the workload (agent) pod a proxy-pod egress fence guards. +/// The fence has no owner reference, so on delete/reap the gateway must confirm +/// that specific pod is gone before removing the fence — addressing it by name +/// (a scoped `get`) rather than enumerating pods cluster-wide. +const ANNOTATION_AGENT_POD_NAME: &str = "openshell.ai/agent-pod-name"; const SANDBOX_SUSPENDED_CONDITION: &str = "Suspended"; const SANDBOX_SUSPENDED_POD_NOT_OWNED_REASON: &str = "PodNotOwned"; const SANDBOX_TOKEN_AUDIENCE: &str = "openshell-gateway"; @@ -201,6 +239,16 @@ struct KubernetesDriverContainersConfig { struct KubernetesContainerDriverConfig { resources: KubernetesContainerResourceConfig, volume_mounts: Vec, + /// Entrypoint override for the workload container. + /// + /// Only meaningful in `proxy-pod` topology, where the sandbox image runs + /// directly. `combined` and `sidecar` replace the container command with + /// the supervisor binary, so an override there would be silently ignored + /// and is rejected instead. + command: Vec, + /// Arguments for `command`, or for the image entrypoint when `command` is + /// not set. + args: Vec, } #[derive(Debug, Clone, Default, Deserialize)] @@ -508,6 +556,22 @@ impl KubernetesComputeDriver { config .validate_proxy_uid() .map_err(KubernetesDriverError::Precondition)?; + // Validate DNS peers whenever this gateway may build proxy-pod + // companions: the configured topology is proxy-pod, or a migration is + // retaining companion management for pre-existing proxy-pod sandboxes + // (whose supervisor could be reconstructed with these peers). + if config.topology == SupervisorTopology::ProxyPod + || config.proxy_pod.retain_companion_management + { + config + .proxy_pod + .validate_dns_peers() + .map_err(KubernetesDriverError::Precondition)?; + config + .proxy_pod + .validate_gateway_peers() + .map_err(KubernetesDriverError::Precondition)?; + } config .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; @@ -580,7 +644,10 @@ impl KubernetesComputeDriver { } /// Authenticate the projected `ServiceAccount` token used by a sandbox pod. - pub async fn authenticate_sandbox(&self, credential: &str) -> Result { + pub async fn authenticate_sandbox( + &self, + credential: &str, + ) -> Result<(String, bool), tonic::Status> { let reviews: Api = Api::all(self.client.clone()); let review = TokenReview { metadata: ObjectMeta::default(), @@ -621,7 +688,24 @@ impl KubernetesComputeDriver { })?; validate_pod_uid(&pod, &identity.pod_uid)?; let sandbox_id = pod_sandbox_id(&pod)?; - let owner = sandbox_owner_reference(&pod)?; + // A proxy-pod agent pod carries the `agent` sandbox-role label. It is + // authenticated as a process-scoped caller so the gateway mints a + // narrowed `Process`-kind token that cannot read provider secrets or + // inference routing. Every other pod is a full-authority caller. + let scoped_process_caller = pod + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ROLE)) + .map(String::as_str) + == Some(SANDBOX_ROLE_AGENT); + // Resolve the controlling Sandbox. An agent pod is directly owned by the + // Sandbox CR; a proxy-pod supervisor pod is owned through the + // Pod -> ReplicaSet -> Deployment -> Sandbox chain, which is walked and + // UID-verified at each hop. + let owner = self + .resolve_sandbox_owner(&pod, &identity.namespace, &identity.pod_name) + .await?; let sandboxes = self .supported_agent_sandbox_api(self.client.clone(), &identity.namespace) .await @@ -632,8 +716,92 @@ impl KubernetesComputeDriver { warn!(sandbox = %owner.name, %error, "failed to read authenticated Sandbox resource"); tonic::Status::internal("failed to read authenticated Sandbox resource") })?.ok_or_else(|| tonic::Status::permission_denied("sandbox owner not found"))?; - validate_sandbox_owner_identity(owner, &sandbox_id, &sandbox)?; - Ok(sandbox_id) + validate_sandbox_owner_identity(&owner, &sandbox_id, &sandbox)?; + Ok((sandbox_id, scoped_process_caller)) + } + + /// Resolve the controlling `Sandbox` owner reference for an authenticated + /// pod. Agent pods are owned directly by the Sandbox CR. Proxy-pod + /// supervisor pods are owned through a `Pod -> ReplicaSet -> Deployment -> + /// Sandbox` chain; each hop is fetched and UID-verified before trusting the + /// next owner reference, so a pod cannot claim a Sandbox it does not belong + /// to. + #[allow(clippy::result_large_err)] + async fn resolve_sandbox_owner( + &self, + pod: &Pod, + namespace: &str, + pod_name: &str, + ) -> Result { + let pod_owners = pod.metadata.owner_references.as_deref().unwrap_or_default(); + if let Ok(owner) = sandbox_owner_reference(pod) { + return Ok(owner.clone()); + } + + // Proxy-pod supervisor pod: walk to the controlling Deployment. + let rs_owner = controller_owner_reference(pod_owners).ok_or_else(|| { + tonic::Status::permission_denied("pod is not controlled by a Sandbox") + })?; + if rs_owner.api_version != APPS_API_VERSION_V1 || rs_owner.kind != REPLICA_SET_KIND { + return Err(tonic::Status::permission_denied( + "pod is not controlled by a Sandbox", + )); + } + let replica_set = Api::::namespaced(self.client.clone(), namespace) + .get_opt(&rs_owner.name) + .await + .map_err(|error| { + warn!(pod = %pod_name, replica_set = %rs_owner.name, %error, "failed to read pod controller ReplicaSet"); + tonic::Status::internal(format!("replicaset GET failed: {error}")) + })? + .ok_or_else(|| { + tonic::Status::permission_denied("pod controller ReplicaSet not found") + })?; + validate_object_uid( + replica_set.metadata.uid.as_deref().unwrap_or_default(), + &rs_owner.uid, + "pod controller ReplicaSet UID mismatch", + )?; + + let deploy_owner = controller_owner_reference( + replica_set + .metadata + .owner_references + .as_deref() + .unwrap_or_default(), + ) + .ok_or_else(|| { + tonic::Status::permission_denied("ReplicaSet is not controlled by a Deployment") + })?; + if deploy_owner.api_version != APPS_API_VERSION_V1 || deploy_owner.kind != DEPLOYMENT_KIND { + return Err(tonic::Status::permission_denied( + "ReplicaSet is not controlled by a Deployment", + )); + } + let deployment = Api::::namespaced(self.client.clone(), namespace) + .get_opt(&deploy_owner.name) + .await + .map_err(|error| { + warn!(pod = %pod_name, deployment = %deploy_owner.name, %error, "failed to read controller Deployment"); + tonic::Status::internal(format!("deployment GET failed: {error}")) + })? + .ok_or_else(|| { + tonic::Status::permission_denied("ReplicaSet controller Deployment not found") + })?; + validate_object_uid( + deployment.metadata.uid.as_deref().unwrap_or_default(), + &deploy_owner.uid, + "ReplicaSet controller Deployment UID mismatch", + )?; + + let sandbox_owner = sandbox_owner_from_refs( + deployment + .metadata + .owner_references + .as_deref() + .unwrap_or_default(), + )?; + Ok(sandbox_owner.clone()) } fn accepts_auth_namespace(&self, namespace: &str) -> bool { @@ -1091,14 +1259,16 @@ impl KubernetesComputeDriver { &self, sandbox: &Sandbox, ) -> Result { - kubernetes_driver_config_for_spec( + let config = kubernetes_driver_config_for_spec( sandbox.spec.as_ref(), self.config.provider_spiffe_enabled().then_some( self.config .provider_spiffe_workload_api_socket_path .as_str(), ), - ) + )?; + validate_agent_command_for_topology(&config, self.config.topology)?; + Ok(config) } fn agent_sandbox_api( @@ -1152,10 +1322,46 @@ impl KubernetesComputeDriver { sandbox_lookup_selector_for(sandbox_id, &self.config.gateway_id) } + /// Live existence check for a Sandbox CR by sandbox id, scoped to this + /// gateway. `Some(true)`/`Some(false)` when the answer is known; `None` when + /// an API error or timeout makes it undeterminable (callers must treat that + /// as "cannot confirm absent" and retain, never delete). + async fn sandbox_cr_exists(&self, sandbox_id: &str) -> Option { + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .ok()?; + let lp = ListParams::default() + .labels(&self.sandbox_lookup_selector(sandbox_id)) + .limit(1); + match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { + Ok(Ok(list)) => Some(!list.items.is_empty()), + Ok(Err(err)) => { + warn!(sandbox_id = %sandbox_id, error = %err, "Could not confirm Sandbox CR existence"); + None + } + Err(_elapsed) => { + warn!(sandbox_id = %sandbox_id, "Timed out confirming Sandbox CR existence"); + None + } + } + } + fn openshell_sandbox_selector(&self) -> String { openshell_sandbox_selector_for(&self.config.gateway_id) } + /// Label selector matching this gateway's proxy-pod supervisor Deployments. + /// Scopes the supervisor Deployment watch to this gateway's supervisors so a + /// Deployment availability change can be turned into a sandbox readiness + /// refresh without observing unrelated Deployments. + fn proxy_pod_supervisor_selector(&self) -> String { + format!( + "{},{LABEL_SANDBOX_ROLE}={SANDBOX_ROLE_SUPERVISOR}", + self.openshell_sandbox_selector() + ) + } + async fn supported_sandbox_api_version(&self, client: Client) -> Result<&'static str, String> { self.sandbox_api_version .get_or_try_init( @@ -1313,6 +1519,19 @@ impl KubernetesComputeDriver { } pub async fn get_sandbox(&self, sandbox_id: &str) -> Result, String> { + // No override: the fold checks live supervisor availability itself. + self.lookup_sandbox_with_readiness(sandbox_id, None).await + } + + /// Look up a Sandbox CR by id and build its `Sandbox`, folding proxy-pod + /// supervisor readiness. `availability_override` supplies a supervisor + /// availability already known to the caller (e.g. a Deployment watch event), + /// avoiding a redundant Deployment GET that could otherwise fail open. + async fn lookup_sandbox_with_readiness( + &self, + sandbox_id: &str, + availability_override: Option, + ) -> Result, String> { info!( sandbox_id = %sandbox_id, workspace_mode = %self.config.workspace_mode, @@ -1325,20 +1544,35 @@ impl KubernetesComputeDriver { let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { - Ok(Ok(list)) => list.items.into_iter().next().map_or_else( - || { + Ok(Ok(list)) => { + let Some(obj) = list.items.into_iter().next() else { debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes"); - Ok(None) - }, - |obj| { - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - Ok(sandbox_from_object(&ns, obj).ok().map(|(_, s)| s)) - }, - ), + return Ok(None); + }; + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + // Capture identity before `sandbox_from_object` consumes the object. + let cr_topology = topology_from_object(&obj, self.config.topology); + let cr_name = obj.metadata.name.clone().unwrap_or_default(); + let cr_sandbox_id = sandbox_id_from_object(&obj).unwrap_or_default(); + let Ok((_, mut sandbox)) = sandbox_from_object(&ns, obj, self.config.topology) + else { + return Ok(None); + }; + self.apply_proxy_pod_supervisor_readiness( + &mut sandbox, + cr_topology, + &cr_name, + &cr_sandbox_id, + &ns, + availability_override, + ) + .await; + Ok(Some(sandbox)) + } Ok(Err(err)) => { warn!( sandbox_id = %sandbox_id, @@ -1380,25 +1614,42 @@ impl KubernetesComputeDriver { .await { Ok(Ok(list)) => { - let mut sandboxes: Vec = list - .items - .into_iter() - .filter_map(|obj| { - let name = obj.metadata.name.clone().unwrap_or_default(); - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - match sandbox_from_object(&ns, obj) { - Ok((_, s)) => Some(s), - Err(err) => { - warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); - None - } + let mut sandboxes: Vec = Vec::with_capacity(list.items.len()); + for obj in list.items { + let name = obj.metadata.name.clone().unwrap_or_default(); + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + // Capture identity before `sandbox_from_object` consumes the + // object so a proxy-pod sandbox's readiness can be checked + // against its live supervisor Deployment. + let cr_topology = topology_from_object(&obj, self.config.topology); + let cr_name = obj.metadata.name.clone().unwrap_or_default(); + let sandbox_id = sandbox_id_from_object(&obj).unwrap_or_default(); + match sandbox_from_object(&ns, obj, self.config.topology) { + Ok((_, mut sandbox)) => { + // The agent pod's Ready condition does not reflect the + // separate supervisor Deployment. Without this, a + // sandbox whose supervisor died after startup would + // stay Ready while policy-enforced egress is dead. + self.apply_proxy_pod_supervisor_readiness( + &mut sandbox, + cr_topology, + &cr_name, + &sandbox_id, + &ns, + None, + ) + .await; + sandboxes.push(sandbox); } - }) - .collect(); + Err(err) => { + warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); + } + } + } sandboxes.sort_by(|left, right| { left.name .cmp(&right.name) @@ -1503,50 +1754,20 @@ impl KubernetesComputeDriver { .resolve_sandbox_identity_in_namespace(&target_namespace) .await; - let params = SandboxPodParams { - default_image: &self.config.default_image, - image_pull_policy: &self.config.image_pull_policy, - image_pull_secrets: &self.config.image_pull_secrets, - supervisor_image: &self.config.supervisor_image, - supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, - supervisor_sideload_method: self.config.supervisor_sideload_method, - topology: self.config.topology, - proxy_uid: self.config.sidecar.proxy_uid, - process_binary_aware_network_policy: self - .config - .sidecar - .process_binary_aware_network_policy, - https_proxy: self.config.https_proxy.as_deref(), - no_proxy: self.config.no_proxy.as_deref(), - proxy_auth_secret_name: self.config.proxy_auth_secret_name.as_deref(), - proxy_auth_secret_key: self.config.proxy_auth_secret_key.as_deref(), - proxy_auth_allow_insecure: self.config.proxy_auth_allow_insecure == Some(true), - proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), - service_account_name: &self.config.service_account_name, - sandbox_id: &sandbox.id, - sandbox_name: &sandbox.name, - grpc_endpoint: &self.config.grpc_endpoint, - ssh_socket_path: self.ssh_socket_path(), - client_tls_secret_name: &self.config.client_tls_secret_name, - host_gateway_ip: &self.config.host_gateway_ip, - enable_user_namespaces: self.config.enable_user_namespaces, - app_armor_profile: self.config.app_armor_profile.as_ref(), - workspace_default_storage_size: &self.config.workspace_default_storage_size, - workspace_storage_class: &self.config.workspace_storage_class, - default_runtime_class_name: &self.config.default_runtime_class_name, - sa_token_ttl_secs: self.config.effective_sa_token_ttl_secs(), - provider_spiffe_enabled: self.config.provider_spiffe_enabled(), - provider_spiffe_workload_api_socket_path: &self - .config - .provider_spiffe_workload_api_socket_path, - sandbox_uid: resolved_user_id, - sandbox_gid: resolved_group_id, - }; - validate_sidecar_proxy_identity(¶ms)?; + let cr_name = self.config.kube_resource_name(workspace, name); + let params = self.build_sandbox_pod_params( + sandbox, + &target_namespace, + &cr_name, + resolved_user_id, + resolved_group_id, + self.config.topology, + ); + validate_proxy_identity(¶ms)?; let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) .map_err(KubernetesDriverError::InvalidArgument)?; - let kube_name = self.config.kube_resource_name(workspace, name); + let kube_name = cr_name.clone(); let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); let mut annotations = sandbox_annotations(sandbox); add_trace_context_annotation(&mut annotations); @@ -1558,28 +1779,37 @@ impl KubernetesComputeDriver { annotations.insert(key.to_string(), v.clone()); } } + // Persist the creation-time topology so watch/list and start/stop derive + // status and companion behavior from it rather than the gateway's + // current global config, which may have changed since creation. + annotations.insert( + ANNOTATION_SUPERVISOR_TOPOLOGY.to_string(), + self.config.topology.to_string(), + ); obj.metadata = ObjectMeta { name: Some(kube_name), - namespace: Some(target_namespace), + // Clone: `params` borrows `target_namespace` for later companion + // creation. + namespace: Some(target_namespace.clone()), labels: Some(sandbox_labels(sandbox, Some(&self.config.gateway_id))), annotations: Some(annotations), ..Default::default() }; obj.data = data; - match tokio::time::timeout( + let created = match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api.api.create(&PostParams::default(), &obj), ) .await { - Ok(Ok(_result)) => { + Ok(Ok(result)) => { info!( sandbox_id = %sandbox.id, sandbox_name = %name, "Sandbox created in Kubernetes successfully" ); - Ok(()) + result } Ok(Err(err)) => { warn!( @@ -1588,7 +1818,7 @@ impl KubernetesComputeDriver { error = %err, "Failed to create sandbox in Kubernetes" ); - Err(KubernetesDriverError::from_kube(err)) + return Err(KubernetesDriverError::from_kube(err)); } Err(_elapsed) => { warn!( @@ -1597,4908 +1827,8944 @@ impl KubernetesComputeDriver { timeout_secs = KUBE_API_TIMEOUT.as_secs(), "Timed out creating sandbox in Kubernetes" ); - Err(KubernetesDriverError::Message(format!( + return Err(KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - ))) - } - } - } - - #[tracing::instrument( - name = "kubernetes.stop_sandbox", - skip(self), - fields( - otel.name = "kubernetes.stop_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - ) - )] - pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let result = self.stop_sandbox_inner(sandbox_id).await; - span_status.finish(result) - } - - async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self - .patch_sandbox_operating_state(sandbox_id, false) - .await?; - let pod_api = Api::::namespaced(self.client.clone(), &namespace); - - let deadline = tokio::time::Instant::now() + stop_timeout; - let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; - loop { - let now = tokio::time::Instant::now(); - if now >= deadline { - return Err(KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes sandbox to stop", - stop_timeout.as_secs() ))); } - let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); - let object = tokio::time::timeout( - request_timeout, - agent_sandbox_api.api.get(&kube_name), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes API while checking sandbox stop", - request_timeout.as_secs() - )) - })? - .map_err(KubernetesDriverError::from_kube)?; - if let Some(error) = kubernetes_sandbox_stop_failure(&object) { - return Err(KubernetesDriverError::Message(error)); - } - let pod_is_gone = kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline) + }; + + if self.config.topology == SupervisorTopology::ProxyPod + && let Err(err) = self + .create_proxy_pod_resources( + sandbox, + sandbox.spec.as_ref(), + ¶ms, + &created, + &agent_sandbox_api.resource.api_version, + ) .await - .map_err(KubernetesDriverError::Message)?; - let stop_is_complete = kubernetes_sandbox_stop_is_complete( - &agent_sandbox_api.resource.version, - &object, - pod_is_gone, + { + warn!( + sandbox_id = %sandbox.id, + sandbox_name = %name, + error = %err, + "Failed to create proxy-pod resources; deleting Sandbox CR" ); - if stop_is_complete { - return Ok(()); + // Delete the CR we actually created, addressed by its returned name + // (the workspace-scoped CR name, not the bare sandbox name) and + // guarded by its UID so we never remove a same-named successor. + // Owner-referenced companions are garbage-collected with the CR. + let created_name = created.metadata.name.as_deref().unwrap_or(params.cr_name); + let mut delete_params = DeleteParams::default(); + if let Some(uid) = created.metadata.uid.clone() { + delete_params = delete_params.preconditions(Preconditions { + uid: Some(uid), + resource_version: None, + }); } - let now = tokio::time::Instant::now(); - if now >= deadline { - return Err(KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes sandbox to stop", - stop_timeout.as_secs() - ))); + let cr_deleted = match tokio::time::timeout( + KUBE_API_TIMEOUT, + agent_sandbox_api.api.delete(created_name, &delete_params), + ) + .await + { + Ok(Ok(_)) => true, + Ok(Err(KubeError::Api(err))) if err.code == 404 => true, + _ => false, + }; + // The agent egress NetworkPolicy carries no owner reference, so GC + // will not collect it. Only tear the fence down once the CR is + // confirmed gone AND the workload pod is gone: if the CR delete + // failed (never reached Kubernetes, timed out, precondition + // conflict), the surviving CR could still create an unfenced + // workload, so the fence must stay. A retained fence is reaped by + // reconciliation once its CR is truly absent. + if cr_deleted { + self.teardown_proxy_pod_fence( + params.namespace, + params.cr_name, + created.metadata.uid.as_deref(), + &sandbox.id, + DEFAULT_POD_TERMINATION_GRACE_PERIOD.saturating_add(KUBE_API_TIMEOUT), + ) + .await; + } else { + warn!( + sandbox_id = %sandbox.id, + "Sandbox CR deletion unconfirmed on rollback; retaining egress fence" + ); } - tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; - poll_interval = next_stop_poll_interval(poll_interval); + return Err(err); } + + Ok(()) } - #[tracing::instrument( - name = "kubernetes.start_sandbox", - skip(self), - fields( - otel.name = "kubernetes.start_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - ) - )] - pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let result = self - .patch_sandbox_operating_state(sandbox_id, true) - .await - .map(|_| ()); - span_status.finish(result) + /// Assemble the `SandboxPodParams` from gateway config for a sandbox. Shared + /// by the create path and the proxy-pod companion reconciliation path so both + /// render identical companions. + #[allow(clippy::similar_names)] + fn build_sandbox_pod_params<'a>( + &'a self, + sandbox: &'a Sandbox, + target_namespace: &'a str, + cr_name: &'a str, + sandbox_uid: u32, + sandbox_gid: u32, + // The topology to render for. Normally the gateway's configured topology, + // but companion reconstruction passes the CR's *persisted* topology so a + // proxy-pod sandbox is rebuilt with proxy-pod parameters (its supervisor + // UID, not the sidecar UID) even after the gateway's configured topology + // has been migrated away from proxy-pod. + topology: SupervisorTopology, + ) -> SandboxPodParams<'a> { + SandboxPodParams { + default_image: &self.config.default_image, + image_pull_policy: &self.config.image_pull_policy, + image_pull_secrets: &self.config.image_pull_secrets, + supervisor_image: &self.config.supervisor_image, + supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, + supervisor_sideload_method: self.config.supervisor_sideload_method, + topology, + proxy_uid: match topology { + SupervisorTopology::ProxyPod => self.config.proxy_pod.proxy_uid, + SupervisorTopology::Combined | SupervisorTopology::Sidecar => { + self.config.sidecar.proxy_uid + } + }, + process_binary_aware_network_policy: self + .config + .sidecar + .process_binary_aware_network_policy, + https_proxy: self.config.https_proxy.as_deref(), + no_proxy: self.config.no_proxy.as_deref(), + proxy_auth_secret_name: self.config.proxy_auth_secret_name.as_deref(), + proxy_auth_secret_key: self.config.proxy_auth_secret_key.as_deref(), + proxy_auth_allow_insecure: self.config.proxy_auth_allow_insecure == Some(true), + proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), + proxy_pod_affinity: self.config.proxy_pod.affinity, + proxy_pod_dns_peers: &self.config.proxy_pod.dns_peers, + proxy_pod_gateway_peers: &self.config.proxy_pod.gateway_peers, + namespace: target_namespace, + service_account_name: &self.config.service_account_name, + sandbox_id: &sandbox.id, + sandbox_name: &sandbox.name, + gateway_id: &self.config.gateway_id, + cr_name, + grpc_endpoint: &self.config.grpc_endpoint, + ssh_socket_path: self.ssh_socket_path(), + client_tls_secret_name: &self.config.client_tls_secret_name, + host_gateway_ip: &self.config.host_gateway_ip, + enable_user_namespaces: self.config.enable_user_namespaces, + app_armor_profile: self.config.app_armor_profile.as_ref(), + workspace_default_storage_size: &self.config.workspace_default_storage_size, + workspace_storage_class: &self.config.workspace_storage_class, + default_runtime_class_name: &self.config.default_runtime_class_name, + sa_token_ttl_secs: self.config.effective_sa_token_ttl_secs(), + provider_spiffe_enabled: self.config.provider_spiffe_enabled(), + provider_spiffe_workload_api_socket_path: &self + .config + .provider_spiffe_workload_api_socket_path, + sandbox_uid, + sandbox_gid, + } } - async fn patch_sandbox_operating_state( + async fn create_proxy_pod_resources( &self, - sandbox_id: &str, - running: bool, - ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { - let lookup_api = self - .supported_sandbox_api_for_lookup(self.client.clone()) - .await - .map_err(KubernetesDriverError::Message)?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let list = tokio::time::timeout( - KUBE_API_TIMEOUT, - lookup_api - .api - .list(&ListParams::default().labels(&selector)), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )) - })? - .map_err(KubernetesDriverError::from_kube)?; - let object = list - .items - .into_iter() - .next() - .ok_or(KubernetesDriverError::NotFound)?; - let namespace = object - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - let agent_sandbox_api = Self::agent_sandbox_api( - self.client.clone(), - &lookup_api.resource.version, - &namespace, - ); - let stop_timeout = kubernetes_sandbox_stop_timeout(&object); - let kube_name = object.metadata.name.ok_or_else(|| { - KubernetesDriverError::Message("sandbox resource has no name".to_string()) - })?; - let pod_name = object + sandbox: &Sandbox, + spec: Option<&SandboxSpec>, + params: &SandboxPodParams<'_>, + sandbox_cr: &DynamicObject, + sandbox_api_version: &str, + ) -> Result<(), KubernetesDriverError> { + // Companion names derive from the Sandbox CR name, which is unique per + // sandbox in every workspace mode. The bare sandbox name collides in + // shared mode, where `workspace-a/dev` and `workspace-b/dev` both have + // sandbox name `dev`. + let cr_name = sandbox_cr .metadata - .annotations - .as_ref() - .and_then(|annotations| annotations.get(SANDBOX_POD_NAME_ANNOTATION)) - .cloned() - .unwrap_or_else(|| kube_name.clone()); - let resource_version = object.metadata.resource_version.unwrap_or_default(); - let desired = sandbox_operating_state_patch( - &agent_sandbox_api.resource.version, - &resource_version, - running, - ); - tokio::time::timeout( - KUBE_API_TIMEOUT, - agent_sandbox_api.api.patch( - &kube_name, - &PatchParams::default(), - &Patch::Merge(&desired), - ), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )) - })? - .map_err(KubernetesDriverError::from_kube)?; + .name + .as_deref() + .unwrap_or(sandbox.name.as_str()); + let names = proxy_pod_resource_names(cr_name, &sandbox.id); + let template_environment = spec + .and_then(|spec| spec.template.as_ref()) + .map(|template| template.environment.clone()) + .unwrap_or_default(); + let spec_environment = spec_pod_env(spec); + let deployment_owner_ref = + proxy_pod_owner_reference(sandbox_cr, sandbox_api_version, true)?; + let dependent_owner_ref = + proxy_pod_owner_reference(sandbox_cr, sandbox_api_version, false)?; + let (ca_cert_pem, ca_key_pem) = generate_proxy_pod_ca()?; + + // Give the supervisor the workload's node placement so same-node + // affinity resolves to a node the workload can also use. Both the + // driver_config.pod placement and the public platform_config placement + // (runtime class, node selector, tolerations) the workload honors must + // be mirrored here. + let pod_driver_config = spec + .and_then(|spec| spec.template.as_ref()) + .and_then(|template| KubernetesSandboxDriverConfig::from_template(template).ok()) + .map(|config| config.pod) + .unwrap_or_default(); + let placement = + ProxyPodPlacement::from_template(spec.and_then(|spec| spec.template.as_ref())); + let companions = build_proxy_pod_companions( + &names, + params, + &template_environment, + &spec_environment, + &pod_driver_config, + &placement, + // A newly created sandbox starts running. + 1, + deployment_owner_ref, + dependent_owner_ref, + &ca_cert_pem, + &ca_key_pem, + ); + self.apply_proxy_pod_companions(params.namespace, &companions) + .await?; info!( - sandbox_id, - sandbox_api_version = %agent_sandbox_api.resource.version, - running, - "Updated Kubernetes sandbox operating state" + sandbox_id = %sandbox.id, + sandbox_name = %sandbox.name, + supervisor_deployment = %names.supervisor_deployment, + service = %names.service, + "Created proxy-pod supervisor resources" ); - Ok(( - agent_sandbox_api, - kube_name, - pod_name, - namespace, - stop_timeout, - )) + Ok(()) } - #[tracing::instrument( - name = "kubernetes.delete_sandbox", - skip(self), - fields( - otel.name = "kubernetes.delete_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, + /// Idempotently apply a proxy-pod companion set. Each object is created only + /// if absent (an `AlreadyExists` conflict is treated as success), so this is + /// safe to run from both the create path and the restart reconciliation + /// path. Purely additive: an existing CA Secret keeps its key material (no + /// rotation) and an existing supervisor Deployment keeps its replica count + /// (a stopped sandbox is not restarted). + async fn apply_proxy_pod_companions( + &self, + namespace: &str, + companions: &ProxyPodCompanions, + ) -> Result<(), KubernetesDriverError> { + let secrets: Api = Api::namespaced(self.client.clone(), namespace); + let services: Api = Api::namespaced(self.client.clone(), namespace); + let policies: Api = Api::namespaced(self.client.clone(), namespace); + let deployments: Api = Api::namespaced(self.client.clone(), namespace); + + // The CA Secret skips ownership verification: the gateway holds no + // Secret read permission, and the UUID-keyed name already implies the + // object is this sandbox's own. + create_companion_if_absent(&secrets, &companions.secret, "proxy-pod CA secret", false) + .await?; + create_companion_if_absent(&services, &companions.service, "proxy-pod service", true) + .await?; + // The egress fence carries no owner reference (it is gateway-managed), so + // owner-based verification cannot vouch for it. Validate its enforcement + // fields instead: an existing same-name policy must fence exactly this + // agent, or a stale/altered one would be silently accepted. + create_or_validate_egress_fence(&policies, &companions.agent_egress).await?; + create_companion_if_absent( + &policies, + &companions.supervisor_ingress, + "proxy-pod supervisor ingress NetworkPolicy", + true, ) - )] - pub async fn delete_sandbox(&self, sandbox_id: &str) -> Result { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let result = self.delete_sandbox_inner(sandbox_id).await; - span_status.finish(result) + .await?; + create_companion_if_absent( + &deployments, + &companions.supervisor_deployment, + "proxy-pod supervisor deployment", + true, + ) + .await?; + Ok(()) } - async fn delete_sandbox_inner(&self, sandbox_id: &str) -> Result { - info!( - sandbox_id = %sandbox_id, - workspace_mode = %self.config.workspace_mode, - "Deleting sandbox from Kubernetes" - ); + /// Downgrade a proxy-pod sandbox's readiness to `NotReady` when its + /// supervisor Deployment has no available replica. Shared by `get_sandbox` + /// and `list_sandboxes` so both the reconcile loop's status refresh and + /// direct queries reflect supervisor liveness. A no-op for other topologies. + async fn apply_proxy_pod_supervisor_readiness( + &self, + sandbox: &mut Sandbox, + topology: SupervisorTopology, + cr_name: &str, + sandbox_id: &str, + namespace: &str, + availability_override: Option, + ) { + if topology != SupervisorTopology::ProxyPod || sandbox_id.is_empty() { + return; + } + // Prefer a caller-supplied availability (e.g. taken directly from a + // Deployment watch event) over a fresh GET: it reflects the exact state + // that triggered the refresh and cannot fail open on a transient error. + let availability = if let Some(availability) = availability_override { + availability + } else { + let names = proxy_pod_resource_names(cr_name, sandbox_id); + self.proxy_pod_supervisor_availability(namespace, &names.supervisor_deployment) + .await + }; + // Fail closed: readiness is only `Ready` when the supervisor is + // *confirmed* available. The CR's own `Ready=True` reflects the agent + // pod, not the separate supervisor, so leaving it intact on `Unknown` + // (a GET error/timeout) would republish a possibly-dead-egress sandbox + // as Ready and could overwrite a prior `DependenciesNotReady`. Only a + // confirmed `Available` keeps `Ready`. + if availability != SupervisorAvailability::Available { + mark_supervisor_unavailable(sandbox); + } + } - let lookup_api = self + /// Tri-state availability of a proxy-pod sandbox's supervisor Deployment. A + /// missing Deployment is `Unavailable`; a transient API error is `Unknown`. + /// Callers fail closed (treat non-`Available` as not ready), so `Unknown` is + /// kept distinct only so a definite absence and an undeterminable check read + /// the same to readiness without conflating them in logs. + async fn proxy_pod_supervisor_availability( + &self, + namespace: &str, + deployment_name: &str, + ) -> SupervisorAvailability { + proxy_pod_supervisor_availability(&self.client, namespace, deployment_name).await + } + + /// Spawn a periodic proxy-pod companion reconciliation bound to a sandbox + /// watch's lifetime. `reconcile_proxy_pod_companions` otherwise runs only at + /// watch establishment, which leaves a transiently-failed supervisor + /// scale-down (or a crash-orphaned egress fence) uncorrected until the watch + /// re-establishes. The periodic sweep bounds that window to + /// `PROXY_POD_RECONCILE_INTERVAL`. + /// + /// Scheduled only when this gateway manages proxy-pod sandboxes (`enabled`): + /// either its configured topology is proxy-pod, or a `retainCompanionRbac` + /// migration left proxy-pod sandboxes it still owns. The task exits when the + /// watch stream consumer drops its receiver (observed through `tx.closed()`), + /// so each new watch establishment replaces the previous reconcile task + /// rather than accumulating one. + fn spawn_proxy_pod_periodic_reconcile( + &self, + tx: mpsc::Sender>, + enabled: bool, + ) { + if !enabled { + return; + } + let driver = self.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(PROXY_POD_RECONCILE_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // The first tick fires immediately; discard it because + // `watch_sandboxes` already reconciled before establishing the watch. + interval.tick().await; + loop { + tokio::select! { + _ = interval.tick() => { driver.reconcile_proxy_pod_companions().await; }, + () = tx.closed() => break, + } + } + }); + } + + /// Repair proxy-pod companions for every existing Sandbox CR. A gateway + /// crash between the CR create and its companion creates leaves a persisted + /// CR with a partial topology that ordinary reconciliation never repairs, + /// because the CR already exists. Runs on each `watch_sandboxes` call + /// (gateway start and watch re-establishment) and periodically thereafter + /// via `spawn_proxy_pod_periodic_reconcile`. Best-effort: failures are + /// logged, not fatal. + /// + /// Whether background upkeep (periodic reconcile, readiness watch) stays + /// scheduled is decided from configuration in `watch_sandboxes`, not from + /// this pass — a transient discovery failure here must never disable it. + async fn reconcile_proxy_pod_companions(&self) { + // Driven by each CR's persisted creation-time topology, not the + // gateway's current config: a gateway whose Helm topology was changed to + // `combined` must still reconcile sandboxes that were created as + // `proxy-pod` (whose companions and lifecycle it still owns). The + // per-CR filter below selects those. + let lookup_api = match self .supported_sandbox_api_for_lookup(self.client.clone()) - .await?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let lp = ListParams::default().labels(&selector); - let (kube_name, obj_namespace, _workspace, preconditions) = match tokio::time::timeout( - KUBE_API_TIMEOUT, - lookup_api.api.list(&lp), - ) - .await + .await { - Ok(Ok(list)) => { - if let Some(obj) = list.items.into_iter().next() { - match obj.metadata.name { - Some(name) => { - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - let ws = obj - .metadata - .labels - .as_ref() - .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) - .unwrap_or_default(); - let pc = Preconditions { - uid: obj.metadata.uid, - resource_version: obj.metadata.resource_version, - }; - (name, ns, ws, pc) - } - None => return Ok(false), - } - } else { - debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); - return Ok(false); - } + Ok(api) => api, + Err(err) => { + warn!(error = %err, "Skipping proxy-pod companion reconciliation: sandbox API unavailable"); + return; } + }; + let api_version = format!("{SANDBOX_GROUP}/{}", lookup_api.resource.version); + // Gateway-scoped selector: never touch another gateway's sandboxes, + // whose companions carry that gateway's image, endpoint, and config. + let lp = ListParams::default().labels(&self.openshell_sandbox_selector()); + let list = match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { + Ok(Ok(list)) => list, Ok(Err(err)) => { - warn!( - sandbox_id = %sandbox_id, - error = %err, - "Failed to list sandbox for deletion from Kubernetes" - ); - return Err(err.to_string()); + warn!(error = %err, "Skipping proxy-pod companion reconciliation: list failed"); + return; } Err(_elapsed) => { - warn!( - sandbox_id = %sandbox_id, - timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out listing sandbox for deletion from Kubernetes" - ); - return Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )); + warn!("Skipping proxy-pod companion reconciliation: list timed out"); + return; } }; - let delete_api = self - .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) - .await?; - let dp = DeleteParams::default().preconditions(preconditions); - match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { - Ok(Ok(_response)) => { - info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); - Ok(true) + let mut checked = 0usize; + let mut failed = 0usize; + let mut live_sandbox_ids = HashSet::new(); + for obj in list.items { + if !is_openshell_managed(&obj) + || topology_from_object(&obj, self.config.topology) != SupervisorTopology::ProxyPod + { + continue; } - Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { - debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted or replaced)"); - Ok(false) + if let Ok(id) = sandbox_id_from_object(&obj) { + live_sandbox_ids.insert(id); } - Ok(Err(err)) => { + checked += 1; + if let Err(err) = self + .ensure_proxy_pod_companions_for_cr(&obj, &api_version) + .await + { + failed += 1; warn!( - sandbox_id = %sandbox_id, + cr_name = ?obj.metadata.name, error = %err, - "Failed to delete sandbox from Kubernetes" - ); - Err(err.to_string()) - } - Err(_elapsed) => { - warn!( - sandbox_id = %sandbox_id, - timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out deleting sandbox from Kubernetes" + "Failed to reconcile proxy-pod companions" ); - Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )) } } - } - - pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { - let agent_sandbox_api = self - .supported_sandbox_api_for_lookup(self.client.clone()) - .await?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let lp = ListParams::default().labels(&selector); - match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { - Ok(Ok(list)) => Ok(!list.items.is_empty()), - Ok(Err(err)) => Err(err.to_string()), - Err(_elapsed) => Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )), + if checked > 0 { + info!( + checked, + failed, "Reconciled proxy-pod companions for existing sandboxes" + ); } - } - // Kept `async` to match the gRPC handler signature in `grpc.rs`, which awaits this method. - #[allow(clippy::unused_async)] - pub async fn watch_sandboxes(&self) -> Result { - if self.config.is_multi_namespace() { - self.watch_sandboxes_cluster_wide().await - } else { - self.watch_sandboxes_single_namespace().await - } + // Reap orphaned egress fences: the agent egress NetworkPolicy has no + // owner reference (so it can outlive its workload pod on delete), which + // means a gateway crash between CR deletion and fence teardown leaves it + // behind. Delete any whose sandbox CR no longer exists. + self.reap_orphaned_egress_fences(&live_sandbox_ids).await; } - async fn watch_sandboxes_single_namespace(&self) -> Result { - let namespace = self.config.namespace.clone(); - let agent_sandbox_api = self - .supported_agent_sandbox_api(self.watch_client.clone(), &self.config.namespace) - .await?; - let event_api: Api = Api::namespaced(self.watch_client.clone(), &namespace); - let watcher_config = watcher::Config::default().labels(&openshell_sandbox_label_selector()); - let mut sandbox_stream = recovering_watcher_stream( - watcher::watcher(agent_sandbox_api.api, watcher_config), - "sandbox-resource", - ) - .boxed(); - let mut event_stream = recovering_watcher_stream( - watcher::watcher(event_api, watcher::Config::default()), - "kubernetes-event", + /// Delete agent egress `NetworkPolicy` objects whose Sandbox CR is gone. + /// Their CRs having been reaped means the workload pods were torn down with + /// them, so removing the now-purposeless fence is safe. + async fn reap_orphaned_egress_fences(&self, live_sandbox_ids: &HashSet) { + let policies: Api = if self.config.is_multi_namespace() { + Api::all(self.client.clone()) + } else { + Api::namespaced(self.client.clone(), &self.config.namespace) + }; + // Gateway-scoped, agent-role egress policies only. + let selector = format!( + "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_GATEWAY_ID}={},{LABEL_SANDBOX_ROLE}={SANDBOX_ROLE_AGENT}", + self.config.gateway_id + ); + let list = match tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.list(&ListParams::default().labels(&selector)), ) - .boxed(); - let (tx, rx) = mpsc::channel(256); - - tokio::spawn(async move { - let mut sandbox_name_to_id = std::collections::HashMap::::new(); - let mut agent_pod_to_id = std::collections::HashMap::::new(); - - loop { - tokio::select! { - event = sandbox_stream.next() => match event { - Some(Event::Applied(obj)) => { - if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj) { - update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); - let event = WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } - )), - }; - if tx.send(Ok(event)).await.is_err() { - break; - } - } - } - Some(Event::Deleted(obj)) => { - if is_openshell_managed(&obj) - && let Ok(sandbox_id) = sandbox_id_from_object(&obj) - { - remove_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &sandbox_id); - let event = WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { sandbox_id } - )), - }; - if tx.send(Ok(event)).await.is_err() { - break; - } - } - } - Some(Event::Restarted(objs)) => { - for obj in objs { - if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj) { - update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); - let event = WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } - )), - }; - if tx.send(Ok(event)).await.is_err() { - return; - } - } - } - } - None => { - let _ = tx.send(Err(KubernetesDriverError::Message( - "sandbox watcher stream ended unexpectedly".to_string() - ))).await; - break; - } - }, - event = event_stream.next() => match event { - Some(Event::Applied(obj)) => { - if let Some((sandbox_id, event)) = map_kube_event_to_platform( - &sandbox_name_to_id, - &agent_pod_to_id, - &obj, - ) { - let event = WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::PlatformEvent( - WatchSandboxesPlatformEvent { sandbox_id, event: Some(event) } - )), - }; - if tx.send(Ok(event)).await.is_err() { - break; - } - } - } - Some(Event::Deleted(_)) => {} - Some(Event::Restarted(_)) => { - debug!(namespace = %namespace, "Kubernetes event watcher restarted"); - } - None => { - let _ = tx.send(Err(KubernetesDriverError::Message( - "kubernetes event watcher stream ended".to_string() - ))).await; - break; - } - }, - () = tx.closed() => break, + .await + { + Ok(Ok(list)) => list, + Ok(Err(err)) => { + warn!(error = %err, "Skipping orphaned egress fence reap: list failed"); + return; + } + Err(_elapsed) => { + warn!("Skipping orphaned egress fence reap: list timed out"); + return; + } + }; + for policy in list.items { + let sandbox_id = policy + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .cloned() + .unwrap_or_default(); + if sandbox_id.is_empty() || live_sandbox_ids.contains(&sandbox_id) { + continue; + } + let Some(name) = policy.metadata.name.as_deref() else { + continue; + }; + let ns = policy + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace) + .to_string(); + // The guarded workload pod's name, recorded on the fence at creation. + let Some(pod_name) = policy + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_AGENT_POD_NAME)) + .filter(|value| !value.is_empty()) + else { + debug!(policy = %name, "Retaining orphaned egress fence: no recorded workload pod name"); + continue; + }; + // `live_sandbox_ids` was snapshotted before this policy list, so a + // sandbox created in that window (CR then fence, in that order) is + // absent from it and would look orphaned. Re-confirm the CR is gone + // immediately before deleting; retain on "exists" or "unknown" so a + // freshly created sandbox never loses its egress fence to the reaper. + if self.sandbox_cr_exists(&sandbox_id).await != Some(false) { + debug!(sandbox_id = %sandbox_id, policy = %name, "Retaining egress fence: Sandbox CR not confirmed absent"); + continue; + } + // The CR is gone, but background garbage collection may still be + // terminating the workload pod. Only drop the fence once that pod is + // confirmed absent; retain it (this sweep or a later one reaps it) + // when absence cannot be confirmed, so a SIGTERM-ignoring workload + // never regains direct egress. + if self.workload_pod_absent(&ns, pod_name).await != Some(true) { + debug!(sandbox_id = %sandbox_id, policy = %name, "Retaining orphaned egress fence: workload pod not confirmed absent"); + continue; + } + let scoped: Api = Api::namespaced(self.client.clone(), &ns); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + scoped.delete(name, &DeleteParams::default()), + ) + .await + { + Ok(Ok(_)) => { + info!(sandbox_id = %sandbox_id, policy = %name, "Reaped orphaned proxy-pod egress fence"); + } + Ok(Err(KubeError::Api(_))) => {} + Ok(Err(err)) => { + warn!(policy = %name, error = %err, "Failed to reap orphaned egress fence"); + } + Err(_elapsed) => { + warn!(policy = %name, "Timed out reaping orphaned egress fence"); } } - }); - - Ok(Box::pin(ReceiverStream::new(rx))) + } } - async fn watch_sandboxes_cluster_wide(&self) -> Result { - let sandbox_api_version = self - .supported_sandbox_api_version(self.watch_client.clone()) + /// Ensure the companions for a single Sandbox CR exist, reconstructing the + /// render inputs from the CR itself. Placement (node selector, tolerations, + /// runtime class) and the log level are read back from the CR's rendered + /// agent pod so a repaired supervisor lands where the workload can pair with + /// it. Application is idempotent, so present companions are left untouched. + #[allow(clippy::similar_names)] + async fn ensure_proxy_pod_companions_for_cr( + &self, + obj: &DynamicObject, + sandbox_api_version: &str, + ) -> Result<(), KubernetesDriverError> { + let cr_name = + obj.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("sandbox CR has no name".to_string()) + })?; + let namespace = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let sandbox = Sandbox { + id: sandbox_id_from_object(obj).unwrap_or_default(), + name: annotation_or_label(obj, LABEL_SANDBOX_NAME).unwrap_or_default(), + namespace: namespace.clone(), + spec: None, + status: None, + workspace: annotation_or_label(obj, LABEL_SANDBOX_WORKSPACE).unwrap_or_default(), + }; + if sandbox.id.is_empty() { + return Err(KubernetesDriverError::Message(format!( + "sandbox CR {cr_name} has no sandbox id; cannot derive companion names" + ))); + } + let (sandbox_uid, sandbox_gid, _annotations) = + self.resolve_sandbox_identity_in_namespace(&namespace).await; + // This path only reconstructs proxy-pod companions (the caller filters to + // proxy-pod CRs), so render with proxy-pod parameters regardless of the + // gateway's currently-configured topology — otherwise a migrated + // combined/sidecar gateway would rebuild the supervisor with the sidecar + // UID instead of the proxy-pod supervisor UID. + let params = self.build_sandbox_pod_params( + &sandbox, + &namespace, + &cr_name, + sandbox_uid, + sandbox_gid, + SupervisorTopology::ProxyPod, + ); + let names = proxy_pod_resource_names(&cr_name, &sandbox.id); + let deployment_owner_ref = proxy_pod_owner_reference(obj, sandbox_api_version, true)?; + let dependent_owner_ref = proxy_pod_owner_reference(obj, sandbox_api_version, false)?; + // A fresh CA is generated but only used if the Secret is missing; + // create-if-absent keeps an existing CA rather than rotating it. + let (ca_cert_pem, ca_key_pem) = generate_proxy_pod_ca()?; + let placement = proxy_pod_placement_from_cr(obj); + let spec_environment = proxy_pod_log_level_env_from_cr(obj); + // Derive the desired supervisor replica count from the CR's operating + // state so a missing Deployment is recreated with the right count. + let replicas = desired_supervisor_replicas(obj); + let companions = build_proxy_pod_companions( + &names, + ¶ms, + &std::collections::HashMap::new(), + &spec_environment, + &KubernetesPodDriverConfig::default(), + &placement, + replicas, + deployment_owner_ref, + dependent_owner_ref, + &ca_cert_pem, + &ca_key_pem, + ); + self.apply_proxy_pod_companions(&namespace, &companions) .await?; - let cluster_api = - Self::cluster_wide_sandbox_api(self.watch_client.clone(), sandbox_api_version); - let selector = self.openshell_sandbox_selector(); - let watcher_config = watcher::Config::default().labels(&selector); - let sandbox_stream = recovering_watcher_stream( - watcher::watcher(cluster_api.api, watcher_config), - "sandbox-resource", + // Reconcile replica drift on an already-existing Deployment (create is + // idempotent and does not touch it): a crash between the CR operating- + // state patch and the scale could otherwise leave a running workload + // with zero supervisors, or a stopped sandbox with a live one. + self.scale_proxy_pod_supervisor( + &cr_name, + &sandbox.id, + &namespace, + SupervisorTopology::ProxyPod, + replicas, ) - .boxed(); + .await + } - Ok(cluster_wide_watch_stream( - sandbox_stream, - self.config.namespace.clone(), - )) + /// Scale a sandbox's paired supervisor `Deployment`. + /// + /// The supervisor runs in its own `Deployment`, so it does not stop when + /// the agent pod does. Without this, a stopped sandbox keeps consuming a + /// pod slot, CPU, and memory indefinitely. + /// + /// Failures are logged and swallowed. A supervisor that fails to scale down + /// wastes resources but does not break the stop; a supervisor that fails to + /// scale up is retried by the agent pod's connection attempts and surfaces + /// as a normal readiness failure. Neither should fail the caller's + /// start/stop RPC. + /// Scale a proxy-pod sandbox's supervisor Deployment. + /// + /// `cr_name` is the Sandbox CR resource name (cosmetic in the companion + /// names); `sandbox_id` is the immutable UUID the companion names are keyed + /// on. `namespace` is the sandbox's resolved namespace. + async fn scale_proxy_pod_supervisor( + &self, + cr_name: &str, + sandbox_id: &str, + namespace: &str, + topology: SupervisorTopology, + replicas: u32, + ) -> Result<(), KubernetesDriverError> { + if topology != SupervisorTopology::ProxyPod { + return Ok(()); + } + let names = proxy_pod_resource_names(cr_name, sandbox_id); + let deployments: Api = Api::namespaced(self.client.clone(), namespace); + let patch = serde_json::json!({"spec": {"replicas": replicas}}); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + deployments.patch( + &names.supervisor_deployment, + &PatchParams::default(), + &Patch::Merge(&patch), + ), + ) + .await + { + Ok(Ok(_)) => { + info!( + cr_name = %cr_name, + deployment = %names.supervisor_deployment, + replicas, + "Scaled proxy-pod supervisor Deployment" + ); + Ok(()) + } + Ok(Err(err)) => Err(KubernetesDriverError::from_kube(err)), + Err(_elapsed) => Err(KubernetesDriverError::Message(format!( + "timed out after {}s scaling proxy-pod supervisor Deployment {}", + KUBE_API_TIMEOUT.as_secs(), + names.supervisor_deployment + ))), + } } -} -fn cluster_wide_watch_stream(mut sandbox_stream: S, default_namespace: String) -> WatchStream -where - S: Stream> + Send + Unpin + 'static, -{ - let (tx, rx) = mpsc::channel(256); + #[tracing::instrument( + name = "kubernetes.stop_sandbox", + skip(self), + fields( + otel.name = "kubernetes.stop_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let result = self.stop_sandbox_inner(sandbox_id).await; + span_status.finish(result) + } - tokio::spawn(async move { - loop { - tokio::select! { - event = sandbox_stream.next() => match event { - Some(Event::Applied(obj)) => { - let ns = obj.metadata.namespace.clone() - .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { - let event = WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } - )), - }; - if tx.send(Ok(event)).await.is_err() { - break; - } - } - } - Some(Event::Deleted(obj)) => { - if is_openshell_managed(&obj) - && let Ok(sandbox_id) = sandbox_id_from_object(&obj) - { - let event = WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { sandbox_id } - )), - }; - if tx.send(Ok(event)).await.is_err() { - break; - } - } - } - Some(Event::Restarted(objs)) => { - for obj in objs { - let ns = obj.metadata.namespace.clone() - .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { - let event = WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } - )), - }; - if tx.send(Ok(event)).await.is_err() { - return; - } - } - } - } - None => { - let _ = tx.send(Err(KubernetesDriverError::Message( - "sandbox watcher stream ended unexpectedly".to_string() - ))).await; - break; - } - }, - () = tx.closed() => break, - } + async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout, topology) = self + .patch_sandbox_operating_state(sandbox_id, false) + .await?; + let stopped = self + .wait_for_sandbox_stopped( + &agent_sandbox_api, + &kube_name, + &pod_name, + &namespace, + stop_timeout, + ) + .await; + // Scale the paired supervisor down only once the workload has actually + // stopped, so a graceful shutdown that needs egress still has it. This + // is best-effort: the workload is already stopped, so a failed + // scale-down only wastes supervisor resources and must not fail the + // stop. A later start or delete reconciles the replica count. + if stopped.is_ok() + && let Err(err) = self + .scale_proxy_pod_supervisor(&kube_name, sandbox_id, &namespace, topology, 0) + .await + { + warn!( + sandbox_id = %sandbox_id, + cr_name = %kube_name, + error = %err, + "Failed to scale proxy-pod supervisor down on stop" + ); } - }); + stopped + } - Box::pin(ReceiverStream::new(rx)) -} + async fn wait_for_sandbox_stopped( + &self, + agent_sandbox_api: &AgentSandboxApi, + kube_name: &str, + pod_name: &str, + namespace: &str, + stop_timeout: Duration, + ) -> Result<(), KubernetesDriverError> { + let pod_api = Api::::namespaced(self.client.clone(), namespace); -fn recovering_watcher_stream( - stream: S, - watcher: &'static str, -) -> impl Stream> -where - S: Stream, E>>, - E: std::fmt::Display, -{ - continue_on_watcher_errors(stream.default_backoff(), watcher) -} - -/// Drop kube-runtime watcher errors after logging them so continued polling can -/// drive its built-in relist and recovery state machine. The production adapter -/// above applies backoff first to avoid hot-looping on persistent API failures. -fn continue_on_watcher_errors( - stream: S, - watcher: &'static str, -) -> impl Stream> -where - S: Stream, E>>, - E: std::fmt::Display, -{ - stream.filter_map(move |result| { - futures::future::ready(match result { - Ok(event) => Some(event), - Err(err) => { - warn!( - watcher, - error = %err, - "Kubernetes watcher stream error; waiting for kube-runtime recovery" - ); - None + let deadline = tokio::time::Instant::now() + stop_timeout; + let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes sandbox to stop", + stop_timeout.as_secs() + ))); } - }) - }) -} - -fn add_trace_context_annotation(annotations: &mut BTreeMap) { - let Some(carrier) = openshell_otel::current_trace_context_carrier() else { - return; - }; - if let Ok(value) = serde_json::to_string(&carrier) { - annotations.insert(AGENT_SANDBOX_TRACE_CONTEXT_ANNOTATION.to_string(), value); + let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); + let object = tokio::time::timeout( + request_timeout, + agent_sandbox_api.api.get(kube_name), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API while checking sandbox stop", + request_timeout.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + if let Some(error) = kubernetes_sandbox_stop_failure(&object) { + return Err(KubernetesDriverError::Message(error)); + } + let pod_is_gone = kubernetes_sandbox_pod_is_gone(&pod_api, pod_name, deadline) + .await + .map_err(KubernetesDriverError::Message)?; + if kubernetes_sandbox_stop_is_complete( + &agent_sandbox_api.resource.version, + &object, + pod_is_gone, + ) { + return Ok(()); + } + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes sandbox to stop", + stop_timeout.as_secs() + ))); + } + tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; + poll_interval = next_stop_poll_interval(poll_interval); + } } -} -fn should_try_next_sandbox_api_version(err: &KubeError) -> bool { - // Kubernetes returns a structured 404 for some missing API resources and a - // raw "404 page not found" body for others. Both mean the probed - // group/version is unavailable and the next supported Sandbox API version - // should be tried. - matches!(err, KubeError::Api(api) if api.code == 404) -} + #[tracing::instrument( + name = "kubernetes.start_sandbox", + skip(self), + fields( + otel.name = "kubernetes.start_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let result = self.start_sandbox_inner(sandbox_id).await; + span_status.finish(result) + } -fn validate_gpu_request( - gpu_requirements: Option<&GpuResourceRequirements>, -) -> Result<(), tonic::Status> { - let _ = - effective_driver_gpu_count(gpu_requirements).map_err(tonic::Status::invalid_argument)?; - Ok(()) -} + async fn start_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let (_api, kube_name, _pod_name, namespace, _timeout, topology) = + self.patch_sandbox_operating_state(sandbox_id, true).await?; + // Propagate scale-up failure: the agent pod cannot itself retry a + // Deployment scale, so a swallowed error would wedge the sandbox in + // Starting with a supervisor stuck at zero replicas. + self.scale_proxy_pod_supervisor(&kube_name, sandbox_id, &namespace, topology, 1) + .await?; + Ok(()) + } -const MAX_KUBE_NAME_LEN: usize = 63; + async fn patch_sandbox_operating_state( + &self, + sandbox_id: &str, + running: bool, + ) -> Result< + ( + AgentSandboxApi, + String, + String, + String, + Duration, + SupervisorTopology, + ), + KubernetesDriverError, + > { + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let list = tokio::time::timeout( + KUBE_API_TIMEOUT, + lookup_api + .api + .list(&ListParams::default().labels(&selector)), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + let object = list + .items + .into_iter() + .next() + .ok_or(KubernetesDriverError::NotFound)?; + // Resolve topology from the CR itself so start/stop scales the companion + // Deployment based on how the sandbox was created, not the gateway's + // current global topology. + let topology = topology_from_object(&object, self.config.topology); + let namespace = object + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let agent_sandbox_api = Self::agent_sandbox_api( + self.client.clone(), + &lookup_api.resource.version, + &namespace, + ); + let stop_timeout = kubernetes_sandbox_stop_timeout(&object); + let kube_name = object.metadata.name.ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no name".to_string()) + })?; + let pod_name = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(SANDBOX_POD_NAME_ANNOTATION)) + .cloned() + .unwrap_or_else(|| kube_name.clone()); + let resource_version = object.metadata.resource_version.unwrap_or_default(); + let desired = sandbox_operating_state_patch( + &agent_sandbox_api.resource.version, + &resource_version, + running, + ); + tokio::time::timeout( + KUBE_API_TIMEOUT, + agent_sandbox_api.api.patch( + &kube_name, + &PatchParams::default(), + &Patch::Merge(&desired), + ), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; -fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), tonic::Status> { - let combined = workspace.len() + 2 + name.len(); // "--" separator - if combined > MAX_KUBE_NAME_LEN { - return Err(tonic::Status::invalid_argument(format!( - "combined Kubernetes resource name '{workspace}--{name}' is {combined} characters, \ - exceeding the DNS-1123 limit of {MAX_KUBE_NAME_LEN}" - ))); + info!( + sandbox_id, + sandbox_api_version = %agent_sandbox_api.resource.version, + running, + "Updated Kubernetes sandbox operating state" + ); + Ok(( + agent_sandbox_api, + kube_name, + pod_name, + namespace, + stop_timeout, + topology, + )) } - Ok(()) -} - -fn is_namespace_owned_by_gateway( - labels: Option<&BTreeMap>, - gateway_id: &str, -) -> bool { - labels - .and_then(|l| l.get(LABEL_MANAGED_BY)) - .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) - && labels - .and_then(|l| l.get(LABEL_GATEWAY_ID)) - .is_some_and(|v| v == gateway_id) -} -fn gateway_id_label_needs_backfill( - labels: Option<&BTreeMap>, - gateway_id: &str, -) -> bool { - labels - .and_then(|labels| labels.get(LABEL_GATEWAY_ID)) - .is_none_or(|value| value != gateway_id) -} + #[tracing::instrument( + name = "kubernetes.delete_sandbox", + skip(self), + fields( + otel.name = "kubernetes.delete_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn delete_sandbox(&self, sandbox_id: &str) -> Result { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let result = self.delete_sandbox_inner(sandbox_id).await; + span_status.finish(result) + } -fn namespace_delete_params(uid: String) -> DeleteParams { - DeleteParams::default().preconditions(Preconditions { - uid: Some(uid), - resource_version: None, - }) -} + async fn delete_sandbox_inner(&self, sandbox_id: &str) -> Result { + info!( + sandbox_id = %sandbox_id, + workspace_mode = %self.config.workspace_mode, + "Deleting sandbox from Kubernetes" + ); -fn sandbox_lookup_selector_for(sandbox_id: &str, gateway_id: &str) -> String { - format!( - "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id},{LABEL_GATEWAY_ID}={gateway_id}" - ) -} + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let lp = ListParams::default().labels(&selector); + let (kube_name, obj_namespace, _workspace, preconditions, topology, stop_timeout) = + match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { + Ok(Ok(list)) => { + if let Some(obj) = list.items.into_iter().next() { + // Read fields that borrow the object before its `name` is moved. + let topology = topology_from_object(&obj, self.config.topology); + let stop_timeout = kubernetes_sandbox_stop_timeout(&obj); + match obj.metadata.name { + Some(name) => { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let ws = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) + .unwrap_or_default(); + let pc = Preconditions { + uid: obj.metadata.uid, + resource_version: obj.metadata.resource_version, + }; + (name, ns, ws, pc, topology, stop_timeout) + } + None => return Ok(false), + } + } else { + debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); + return Ok(false); + } + } + Ok(Err(err)) => { + warn!( + sandbox_id = %sandbox_id, + error = %err, + "Failed to list sandbox for deletion from Kubernetes" + ); + return Err(err.to_string()); + } + Err(_elapsed) => { + warn!( + sandbox_id = %sandbox_id, + timeout_secs = KUBE_API_TIMEOUT.as_secs(), + "Timed out listing sandbox for deletion from Kubernetes" + ); + return Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )); + } + }; -fn openshell_sandbox_selector_for(gateway_id: &str) -> String { - use std::fmt::Write; - let mut selector = openshell_sandbox_label_selector(); - write!(selector, ",{LABEL_GATEWAY_ID}={gateway_id}").unwrap(); - selector -} + let delete_api = self + .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) + .await?; + // Capture the UID before it is moved into the delete preconditions; the + // post-delete fence teardown re-checks that this exact CR is gone. + let cr_uid = preconditions.uid.clone(); + let dp = DeleteParams::default().preconditions(preconditions); + // Delete the Sandbox CR. Owner-referenced companions (supervisor + // Deployment, Service, CA Secret, supervisor-ingress NetworkPolicy) are + // reaped by garbage collection. The agent egress NetworkPolicy — the + // workload's fence — has no owner reference and is torn down explicitly + // below, only after the workload pod is gone, so a pod that ignores + // SIGTERM cannot regain direct egress during its termination grace + // period. The UID precondition means a 409 (replacement) or 404 leaves a + // successor untouched. + let deleted = match tokio::time::timeout( + KUBE_API_TIMEOUT, + delete_api.api.delete(&kube_name, &dp), + ) + .await + { + Ok(Ok(_response)) => { + info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); + Ok(true) + } + Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { + debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted or replaced)"); + Ok(false) + } + Ok(Err(err)) => { + warn!( + sandbox_id = %sandbox_id, + error = %err, + "Failed to delete sandbox from Kubernetes" + ); + Err(err.to_string()) + } + Err(_elapsed) => { + warn!( + sandbox_id = %sandbox_id, + timeout_secs = KUBE_API_TIMEOUT.as_secs(), + "Timed out deleting sandbox from Kubernetes" + ); + Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )) + } + }; -fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap { - let mut labels = BTreeMap::new(); - labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); - labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); - labels.insert( - LABEL_SANDBOX_WORKSPACE.to_string(), - sandbox.workspace.clone(), - ); - labels.insert( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ); - if let Some(gw_id) = gateway_id { - labels.insert(LABEL_GATEWAY_ID.to_string(), gw_id.to_string()); + // Ordered fence teardown for proxy-pod: only after THIS CR was confirmed + // deleted, wait for the workload pod to disappear, then delete its egress + // NetworkPolicy. A 409/404 (`Ok(false)`) means a successor owns the name, + // so we must not touch its fence. + if matches!(deleted, Ok(true)) && topology == SupervisorTopology::ProxyPod { + self.teardown_proxy_pod_fence( + &obj_namespace, + &kube_name, + cr_uid.as_deref(), + sandbox_id, + stop_timeout, + ) + .await; + } + deleted } - labels -} -fn managed_ssh_network_policy(namespace: &str, config: &KubernetesComputeConfig) -> NetworkPolicy { - NetworkPolicy { - metadata: ObjectMeta { - name: Some(MANAGED_SSH_NETWORK_POLICY_NAME.to_string()), - namespace: Some(namespace.to_string()), - labels: Some(BTreeMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }, - spec: Some(NetworkPolicySpec { - pod_selector: LabelSelector { - match_labels: Some(BTreeMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }, - policy_types: Some(vec!["Ingress".to_string()]), - ingress: Some(vec![NetworkPolicyIngressRule { - from: Some(vec![NetworkPolicyPeer { - namespace_selector: Some(LabelSelector { - match_labels: Some(BTreeMap::from([( - "kubernetes.io/metadata.name".to_string(), - config.managed_ssh_ingress.gateway_namespace.clone(), - )])), - ..Default::default() - }), - pod_selector: Some(LabelSelector { - match_labels: Some(config.managed_ssh_ingress.gateway_pod_selector.clone()), - ..Default::default() - }), - ..Default::default() - }]), - ports: Some(vec![NetworkPolicyPort { - port: Some(IntOrString::Int(2222)), - protocol: Some("TCP".to_string()), - ..Default::default() - }]), - }]), - ..Default::default() - }), - status: None, + /// Report whether the named workload (agent) pod is absent. + /// + /// Uses a name-scoped `get` (not a label `list`) so it needs only `pods:get`, + /// never cluster-wide pod enumeration. The workload pod is named after its + /// Sandbox CR, and that name is stamped on the fence so every path (delete, + /// rollback, reaper) can address it exactly. `Some(true)` means the pod does + /// not exist, `Some(false)` that it is still present, and `None` that the + /// check could not be performed — callers must retain the fence on `None`. + async fn workload_pod_absent(&self, namespace: &str, pod_name: &str) -> Option { + if pod_name.is_empty() { + return None; + } + let pods: Api = Api::namespaced(self.client.clone(), namespace); + match tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(pod_name)).await { + Ok(Ok(existing)) => Some(existing.is_none()), + Ok(Err(err)) => { + warn!(pod = %pod_name, error = %err, "Could not get workload pod"); + None + } + Err(_elapsed) => { + warn!(pod = %pod_name, "Timed out getting workload pod"); + None + } + } } -} -fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> Secret { - Secret { - metadata: ObjectMeta { - name: Some(secret_name.to_string()), - namespace: Some(namespace.to_string()), - labels: Some(BTreeMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }, - data: source.data, - type_: source.type_, - ..Default::default() + /// Whether the specific Sandbox CR we deleted — addressed by its Kubernetes + /// UID — is actually gone, not merely marked for deletion. A DELETE the API + /// server accepts only sets `deletionTimestamp`; finalizers or an in-flight + /// controller can keep the CR (and recreate its workload pod) afterward, so + /// pod-absence alone is not proof the fence is safe to drop. + /// + /// `Some(true)`: the CR is absent, or a *different*-UID object now holds the + /// name (our CR is gone; a successor owns its own separately-named fence). + /// `Some(false)`: the same-UID CR is still present (still terminating) — the + /// workload can reappear, so keep the fence. `None`: undeterminable. + async fn deleted_cr_is_gone( + &self, + namespace: &str, + cr_name: &str, + cr_uid: Option<&str>, + ) -> Option { + let api = self + .supported_agent_sandbox_api(self.client.clone(), namespace) + .await + .ok()?; + match tokio::time::timeout(KUBE_API_TIMEOUT, api.api.get_opt(cr_name)).await { + Ok(Ok(None)) => Some(true), + Ok(Ok(Some(obj))) => Some(obj.metadata.uid.as_deref() != cr_uid), + Ok(Err(err)) => { + warn!(cr = %cr_name, error = %err, "Could not confirm Sandbox CR deletion"); + None + } + Err(_elapsed) => { + warn!(cr = %cr_name, "Timed out confirming Sandbox CR deletion"); + None + } + } } -} -fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { - let mut annotations = BTreeMap::new(); - annotations.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); - annotations.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); - annotations.insert( - LABEL_SANDBOX_WORKSPACE.to_string(), - sandbox.workspace.clone(), - ); - annotations -} + /// Delete a proxy-pod sandbox's egress `NetworkPolicy`, but only once its + /// workload pod is confirmed gone, so the fence outlives a pod that ignores + /// `SIGTERM`. Waits up to `stop_timeout` for the pod to disappear and RETAINS + /// the fence if absence cannot be confirmed (leaving it for the reconciler to + /// reap once the pod is truly gone). Best-effort deletion. + async fn teardown_proxy_pod_fence( + &self, + namespace: &str, + cr_name: &str, + cr_uid: Option<&str>, + sandbox_id: &str, + stop_timeout: Duration, + ) { + // The workload pod is named after its Sandbox CR (see the pod-name + // annotation the controller sets, which equals the CR name). + let pod_name = cr_name; + let deadline = tokio::time::Instant::now() + stop_timeout; + let mut poll = STOP_INITIAL_POLL_INTERVAL; + loop { + match self.workload_pod_absent(namespace, pod_name).await { + Some(true) => break, + // Could not confirm, or pod still present: never drop the fence on + // a guess. Retain it; reconciliation reaps it once the pod is gone. + None => { + warn!(sandbox_id = %sandbox_id, "Leaving egress fence: workload pod absence unconfirmed"); + return; + } + Some(false) => {} + } + let now = tokio::time::Instant::now(); + if now >= deadline { + warn!(sandbox_id = %sandbox_id, "Workload pod still present at deadline; leaving egress fence for reconciliation"); + return; + } + tokio::time::sleep(poll.min(deadline.saturating_duration_since(now))).await; + poll = next_stop_poll_interval(poll); + } -fn sandbox_id_from_object(obj: &DynamicObject) -> Result { - if let Some(annotations) = obj.metadata.annotations.as_ref() - && let Some(id) = annotations.get(LABEL_SANDBOX_ID) - { - return Ok(id.clone()); - } - if let Some(labels) = obj.metadata.labels.as_ref() - && let Some(id) = labels.get(LABEL_SANDBOX_ID) - { - return Ok(id.clone()); - } - Err("sandbox id not found on object".to_string()) -} + // The accepted DELETE only set `deletionTimestamp`; a finalizer or an + // in-flight controller reconciliation can still recreate the workload + // after the pod-absence check above. Confirm THIS CR (by UID) is actually + // gone immediately before removing the fence; retain it otherwise so a + // reappearing workload never gets default-allow egress (reconciliation + // reaps the fence once the CR is truly absent). + match self.deleted_cr_is_gone(namespace, cr_name, cr_uid).await { + Some(true) => {} + Some(false) => { + warn!(sandbox_id = %sandbox_id, "Leaving egress fence: Sandbox CR still terminating (deletionTimestamp set, finalizers pending)"); + return; + } + None => { + warn!(sandbox_id = %sandbox_id, "Leaving egress fence: Sandbox CR deletion unconfirmed"); + return; + } + } -#[derive(Debug)] -struct TokenReviewIdentity { - namespace: String, - pod_name: String, - pod_uid: String, -} + // Even with the CR gone, a controller reconcile that read the CR before + // its deletion could still create a dangling-ownerReference workload Pod + // (Kubernetes permits references to a missing owner; GC orphan-collects it + // later). Wait out a short quiescence window, re-confirming pod absence, + // so such a stale Pod is observed and the fence retained rather than + // deleted out from under it. A healthy controller converges well within + // this window; anything still lingering is left for the reconciler. + let quiesce_deadline = tokio::time::Instant::now() + FENCE_QUIESCE_WINDOW; + while tokio::time::Instant::now() < quiesce_deadline { + tokio::time::sleep(FENCE_QUIESCE_INTERVAL).await; + if self.workload_pod_absent(namespace, pod_name).await != Some(true) { + warn!(sandbox_id = %sandbox_id, "Leaving egress fence: workload pod reappeared during quiescence (stale controller reconcile)"); + return; + } + } -#[allow(clippy::result_large_err)] -fn token_review_identity( - status: &TokenReviewStatus, - expected_service_account: &str, -) -> Result, tonic::Status> { - if status.authenticated != Some(true) { - return Ok(None); + let names = proxy_pod_resource_names(cr_name, sandbox_id); + let policies: Api = Api::namespaced(self.client.clone(), namespace); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.delete(&names.agent_egress_network_policy, &DeleteParams::default()), + ) + .await + { + Ok(Ok(_) | Err(KubeError::Api(_))) => {} + Ok(Err(err)) => { + warn!(sandbox_id = %sandbox_id, error = %err, "Failed to delete proxy-pod egress fence"); + } + Err(_elapsed) => { + warn!(sandbox_id = %sandbox_id, "Timed out deleting proxy-pod egress fence"); + } + } } - if !status - .audiences - .as_deref() - .unwrap_or_default() - .iter() - .any(|audience| audience == SANDBOX_TOKEN_AUDIENCE) - { - return Err(tonic::Status::unauthenticated( - "sandbox credential audience not accepted", - )); + + pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { + let agent_sandbox_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let lp = ListParams::default().labels(&selector); + match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { + Ok(Ok(list)) => Ok(!list.items.is_empty()), + Ok(Err(err)) => Err(err.to_string()), + Err(_elapsed) => Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )), + } } - let user = status - .user - .as_ref() - .ok_or_else(|| tonic::Status::permission_denied("TokenReview response missing user"))?; - let rest = user - .username - .as_deref() - .unwrap_or_default() - .strip_prefix("system:serviceaccount:") - .ok_or_else(|| tonic::Status::permission_denied("credential is not a service account"))?; - let (namespace, service_account) = rest - .split_once(':') - .filter(|(namespace, service_account)| !namespace.is_empty() && !service_account.is_empty()) - .ok_or_else(|| tonic::Status::permission_denied("invalid service account identity"))?; - if service_account != expected_service_account { - return Err(tonic::Status::permission_denied( - "credential is not from the configured sandbox service account", - )); - } - Ok(Some(TokenReviewIdentity { - namespace: namespace.to_string(), - pod_name: user_extra_one(user, POD_NAME_EXTRA)?, - pod_uid: user_extra_one(user, POD_UID_EXTRA)?, - })) -} -#[allow(clippy::result_large_err)] -fn user_extra_one(user: &UserInfo, key: &str) -> Result { - let values = user - .extra - .as_ref() - .and_then(|extra| extra.get(key)) - .ok_or_else(|| tonic::Status::permission_denied("sandbox credential is not pod-bound"))?; - if values.len() != 1 || values[0].is_empty() { - return Err(tonic::Status::permission_denied( - "sandbox credential has invalid pod binding", - )); + pub async fn watch_sandboxes(&self) -> Result { + // Repair any proxy-pod companions left partial by a gateway crash + // between the CR create and its companion creates. Runs on gateway start + // and on every watch re-establishment. + self.reconcile_proxy_pod_companions().await; + // Whether to keep periodic repair and the shared-mode supervisor + // Deployment readiness watch running. Determined from configuration — + // never from a runtime sandbox list — so a transient discovery failure + // cannot disable upkeep for a whole watch session. During a migration + // away from proxy-pod, `retain_companion_management` (rendered from Helm + // `retainCompanionRbac`) keeps it on until the last proxy-pod sandbox is + // deleted. + let manages_proxy_pod = self.config.topology == SupervisorTopology::ProxyPod + || self.config.proxy_pod.retain_companion_management; + if self.config.is_multi_namespace() { + self.watch_sandboxes_cluster_wide(manages_proxy_pod).await + } else { + self.watch_sandboxes_single_namespace(manages_proxy_pod) + .await + } } - Ok(values[0].clone()) -} -#[allow(clippy::result_large_err)] -fn pod_sandbox_id(pod: &Pod) -> Result { - pod.metadata - .annotations - .as_ref() - .and_then(|annotations| annotations.get(LABEL_SANDBOX_ID)) - .filter(|value| !value.is_empty()) - .cloned() - .ok_or_else(|| tonic::Status::permission_denied("pod is not bound to a sandbox identity")) -} + async fn watch_sandboxes_single_namespace( + &self, + manages_proxy_pod: bool, + ) -> Result { + let namespace = self.config.namespace.clone(); + let topology = self.config.topology; + // Plain client for supervisor Deployment readiness checks inside the task. + let client = self.client.clone(); + // Owned driver handle for CR lookups triggered by supervisor Deployment + // changes (proxy-pod readiness refresh). + let driver = self.clone(); + let agent_sandbox_api = self + .supported_agent_sandbox_api(self.watch_client.clone(), &self.config.namespace) + .await?; + let event_api: Api = Api::namespaced(self.watch_client.clone(), &namespace); + let watcher_config = watcher::Config::default().labels(&openshell_sandbox_label_selector()); + let mut sandbox_stream = recovering_watcher_stream( + watcher::watcher(agent_sandbox_api.api, watcher_config), + "sandbox-resource", + ) + .boxed(); + let mut event_stream = recovering_watcher_stream( + watcher::watcher(event_api, watcher::Config::default()), + "kubernetes-event", + ) + .boxed(); + // Watch supervisor Deployments so proxy-pod readiness reflects supervisor + // availability within seconds. Enabled whenever this gateway manages + // proxy-pod sandboxes (config topology or a retainCompanionRbac + // migration); otherwise it holds a stream that never yields. This is the + // single-namespace (shared workspace) path, so the watch is namespaced + // and needs no cluster-wide Deployment enumeration. + // Keep the watch inputs so the task can re-establish the Deployment watch + // after a recoverable end, rather than losing fast readiness for the rest + // of the sandbox watch's lifetime. + let deployment_watch: Option<(Api, String)> = manages_proxy_pod.then(|| { + ( + Api::namespaced(self.watch_client.clone(), &namespace), + self.proxy_pod_supervisor_selector(), + ) + }); + let mut deployment_stream = match &deployment_watch { + Some((api, selector)) => { + proxy_pod_supervisor_deployment_stream(true, api.clone(), selector.clone()) + } + None => futures::stream::pending().boxed(), + }; + let (tx, rx) = mpsc::channel(256); + self.spawn_proxy_pod_periodic_reconcile(tx.clone(), manages_proxy_pod); -#[allow(clippy::result_large_err)] -fn validate_pod_uid(pod: &Pod, expected_uid: &str) -> Result<(), tonic::Status> { - if pod.metadata.uid.as_deref() == Some(expected_uid) { - return Ok(()); - } - Err(tonic::Status::permission_denied( - "sandbox credential pod UID mismatch", - )) -} + tokio::spawn(async move { + let mut sandbox_name_to_id = std::collections::HashMap::::new(); + let mut agent_pod_to_id = std::collections::HashMap::::new(); -#[allow(clippy::result_large_err)] -fn sandbox_owner_reference(pod: &Pod) -> Result<&OwnerReference, tonic::Status> { - let mut owners = pod - .metadata - .owner_references - .as_deref() - .unwrap_or_default() - .iter() - .filter(|owner| { - owner.kind == SANDBOX_KIND - && matches!( - owner.api_version.as_str(), - "agents.x-k8s.io/v1beta1" | "agents.x-k8s.io/v1alpha1" - ) + loop { + tokio::select! { + event = sandbox_stream.next() => match event { + Some(Event::Applied(obj)) => { + if let Ok((kube_name, sandbox)) = sandbox_from_object_with_supervisor_readiness(&client, &namespace, obj, topology).await { + update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Some(Event::Deleted(obj)) => { + if is_openshell_managed(&obj) + && let Ok(sandbox_id) = sandbox_id_from_object(&obj) + { + remove_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &sandbox_id); + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Some(Event::Restarted(objs)) => { + for obj in objs { + if let Ok((kube_name, sandbox)) = sandbox_from_object_with_supervisor_readiness(&client, &namespace, obj, topology).await { + update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + return; + } + } + } + } + None => { + let _ = tx.send(Err(KubernetesDriverError::Message( + "sandbox watcher stream ended unexpectedly".to_string() + ))).await; + break; + } + }, + event = event_stream.next() => match event { + Some(Event::Applied(obj)) => { + if let Some((sandbox_id, event)) = map_kube_event_to_platform( + &sandbox_name_to_id, + &agent_pod_to_id, + &obj, + ) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::PlatformEvent( + WatchSandboxesPlatformEvent { sandbox_id, event: Some(event) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Some(Event::Deleted(_)) => {} + Some(Event::Restarted(_)) => { + debug!(namespace = %namespace, "Kubernetes event watcher restarted"); + } + None => { + let _ = tx.send(Err(KubernetesDriverError::Message( + "kubernetes event watcher stream ended".to_string() + ))).await; + break; + } + }, + result = deployment_stream.try_next() => match result { + Ok(Some(event)) => { + if !handle_supervisor_deployment_event(&driver, &tx, event).await { + break; + } + } + // kube's watcher self-heals transient errors and keeps + // yielding (it re-lists and re-watches with backoff), so + // keep polling the same stream instead of disabling the + // watch. The watch is only an optimization anyway — + // get/list and the periodic reconcile still fold in + // supervisor availability — so a persistent error (e.g. a + // migration without the Deployment RBAC) just backs off and + // logs; it never tears down the sandbox watch. + Err(err) => { + warn!(error = %err, "Supervisor Deployment watch error; retrying"); + } + // A watcher stream should not end; if it does, re-establish + // it after a short backoff so fast readiness is not lost for + // the rest of this sandbox watch's lifetime. + Ok(None) => { + if let Some((api, selector)) = &deployment_watch { + warn!("Supervisor Deployment watch ended; re-establishing"); + tokio::time::sleep(DEPLOYMENT_WATCH_REESTABLISH_BACKOFF).await; + deployment_stream = proxy_pod_supervisor_deployment_stream( + true, api.clone(), selector.clone(), + ); + } else { + deployment_stream = futures::stream::pending().boxed(); + } + } + }, + () = tx.closed() => break, + } + } }); - let owner = owners - .next() - .ok_or_else(|| tonic::Status::permission_denied("pod is not controlled by a Sandbox"))?; - if owners.next().is_some() - || owner.controller != Some(true) - || owner.name.is_empty() - || owner.uid.is_empty() - { - return Err(tonic::Status::permission_denied( - "pod has an invalid Sandbox owner", - )); - } - Ok(owner) -} -#[allow(clippy::result_large_err)] -fn validate_sandbox_owner_identity( - owner: &OwnerReference, - sandbox_id: &str, - sandbox: &DynamicObject, -) -> Result<(), tonic::Status> { - let uid_matches = sandbox.metadata.uid.as_deref() == Some(owner.uid.as_str()); - let sandbox_id_matches = sandbox - .metadata - .labels - .as_ref() - .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) - .is_some_and(|actual| actual == sandbox_id); - if uid_matches && sandbox_id_matches { - return Ok(()); + Ok(Box::pin(ReceiverStream::new(rx))) } - Err(tonic::Status::permission_denied( - "pod identity does not match its Sandbox owner", - )) -} -fn accepts_auth_namespace( - config: &KubernetesComputeConfig, - operator_allowlist: Option<&OperatorNamespaceAllowlist>, - namespace: &str, -) -> bool { - match config.workspace_mode { - WorkspaceMode::Shared => namespace == config.namespace, - WorkspaceMode::Managed => { - namespace.starts_with(&managed_namespace_prefix(&config.gateway_id)) - } - WorkspaceMode::Operator => { - operator_allowlist.is_some_and(|allowlist| allowlist.contains(namespace)) - } - } -} + async fn watch_sandboxes_cluster_wide( + &self, + manages_proxy_pod: bool, + ) -> Result { + let sandbox_api_version = self + .supported_sandbox_api_version(self.watch_client.clone()) + .await?; + let cluster_api = + Self::cluster_wide_sandbox_api(self.watch_client.clone(), sandbox_api_version); + let selector = self.openshell_sandbox_selector(); + let watcher_config = watcher::Config::default().labels(&selector); + let sandbox_stream = recovering_watcher_stream( + watcher::watcher(cluster_api.api, watcher_config), + "sandbox-resource", + ) + .boxed(); -fn annotation_or_label(obj: &DynamicObject, key: &str) -> Option { - obj.metadata - .annotations - .as_ref() - .and_then(|a| a.get(key)) - .or_else(|| obj.metadata.labels.as_ref().and_then(|l| l.get(key))) - .cloned() + // Multi-namespace (managed/operator) workspace modes deliberately do NOT + // watch supervisor Deployments: a cluster-wide Deployment informer would + // require cluster-scoped list/watch on Deployments, which is broad + // enumeration a compromised gateway could abuse. Proxy-pod supervisor + // readiness folds in via get/list and the periodic reconcile instead. + let (tx, rx) = mpsc::channel(256); + self.spawn_proxy_pod_periodic_reconcile(tx.clone(), manages_proxy_pod); + Ok(cluster_wide_watch_stream( + sandbox_stream, + self.config.namespace.clone(), + self.config.topology, + tx, + rx, + )) + } } -fn is_openshell_managed(obj: &DynamicObject) -> bool { - annotation_or_label(obj, LABEL_MANAGED_BY).as_deref() == Some(LABEL_MANAGED_BY_VALUE) +fn cluster_wide_watch_stream( + mut sandbox_stream: S, + default_namespace: String, + topology: SupervisorTopology, + tx: mpsc::Sender>, + rx: mpsc::Receiver>, +) -> WatchStream +where + S: Stream> + Send + Unpin + 'static, +{ + tokio::spawn(async move { + loop { + tokio::select! { + event = sandbox_stream.next() => match event { + Some(Event::Applied(obj)) => { + let ns = obj.metadata.namespace.clone() + .unwrap_or_else(|| default_namespace.clone()); + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj, topology) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Some(Event::Deleted(obj)) => { + if is_openshell_managed(&obj) + && let Ok(sandbox_id) = sandbox_id_from_object(&obj) + { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id } + )), + }; + if tx.send(Ok(event)).await.is_err() { + break; + } + } + } + Some(Event::Restarted(objs)) => { + for obj in objs { + let ns = obj.metadata.namespace.clone() + .unwrap_or_else(|| default_namespace.clone()); + if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj, topology) { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } + )), + }; + if tx.send(Ok(event)).await.is_err() { + return; + } + } + } + } + None => { + let _ = tx.send(Err(KubernetesDriverError::Message( + "sandbox watcher stream ended unexpectedly".to_string() + ))).await; + break; + } + }, + () = tx.closed() => break, + } + } + }); + + Box::pin(ReceiverStream::new(rx)) } -/// Returns `(kube_resource_name, DriverSandbox)`. -/// -/// Returns `Err` in two cases (callers should skip, not fail): -/// - The object is not managed by `OpenShell` (missing/wrong `managed-by` label). -/// - The object is managed by `OpenShell` but missing required fields (orphan). -fn sandbox_from_object(namespace: &str, obj: DynamicObject) -> Result<(String, Sandbox), String> { - let kube_name = obj.metadata.name.clone().unwrap_or_default(); +fn recovering_watcher_stream( + stream: S, + watcher: &'static str, +) -> impl Stream> +where + S: Stream, E>>, + E: std::fmt::Display, +{ + continue_on_watcher_errors(stream.default_backoff(), watcher) +} - if !is_openshell_managed(&obj) { - debug!(object = %kube_name, "skipping sandbox CR not managed by openshell"); - return Err(format!("object {kube_name} not managed by openshell")); - } +/// Drop kube-runtime watcher errors after logging them so continued polling can +/// drive its built-in relist and recovery state machine. The production adapter +/// above applies backoff first to avoid hot-looping on persistent API failures. +fn continue_on_watcher_errors( + stream: S, + watcher: &'static str, +) -> impl Stream> +where + S: Stream, E>>, + E: std::fmt::Display, +{ + stream.filter_map(move |result| { + futures::future::ready(match result { + Ok(event) => Some(event), + Err(err) => { + warn!( + watcher, + error = %err, + "Kubernetes watcher stream error; waiting for kube-runtime recovery" + ); + None + } + }) + }) +} - let Ok(id) = sandbox_id_from_object(&obj) else { - warn!(object = %kube_name, "openshell-managed sandbox CR missing id"); - return Err(format!("object {kube_name} missing sandbox id")); - }; - let Some(name) = annotation_or_label(&obj, LABEL_SANDBOX_NAME) else { - warn!(object = %kube_name, "openshell-managed sandbox CR missing name"); - return Err(format!("object {kube_name} missing sandbox name")); - }; - let Some(workspace) = annotation_or_label(&obj, LABEL_SANDBOX_WORKSPACE) else { - warn!(object = %kube_name, "openshell-managed sandbox CR missing workspace"); - return Err(format!("object {kube_name} missing sandbox workspace")); +fn add_trace_context_annotation(annotations: &mut BTreeMap) { + let Some(carrier) = openshell_otel::current_trace_context_carrier() else { + return; }; - - let namespace = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| namespace.to_string()); - let status = status_from_object(&obj); - - Ok(( - kube_name, - Sandbox { - id, - name, - namespace, - spec: None, - status, - workspace, - }, - )) + if let Ok(value) = serde_json::to_string(&carrier) { + annotations.insert(AGENT_SANDBOX_TRACE_CONTEXT_ANNOTATION.to_string(), value); + } } -fn update_indexes( - sandbox_name_to_id: &mut std::collections::HashMap, - agent_pod_to_id: &mut std::collections::HashMap, - kube_name: &str, - sandbox: &Sandbox, -) { - if !kube_name.is_empty() { - sandbox_name_to_id.insert(kube_name.to_string(), sandbox.id.clone()); - } - if let Some(status) = sandbox.status.as_ref() - && !status.instance_id.is_empty() - { - agent_pod_to_id.insert(status.instance_id.clone(), sandbox.id.clone()); +/// A supervisor Deployment watch scoped to `selector` when `enabled`, or a +/// stream that never yields otherwise (gateways that manage no supervisor +/// Deployments, or multi-namespace modes that avoid cluster-wide enumeration). +/// Boxing both arms to one type lets the watch loop poll a single branch +/// unconditionally. +fn proxy_pod_supervisor_deployment_stream( + enabled: bool, + deployments: Api, + selector: String, +) -> Pin, watcher::Error>> + Send>> { + if enabled { + let config = watcher::Config::default().labels(&selector); + watcher::watcher(deployments, config).boxed() + } else { + futures::stream::pending().boxed() } } -fn remove_indexes( - sandbox_name_to_id: &mut std::collections::HashMap, - agent_pod_to_id: &mut std::collections::HashMap, - sandbox_id: &str, -) { - sandbox_name_to_id.retain(|_, value| value != sandbox_id); - agent_pod_to_id.retain(|_, value| value != sandbox_id); +fn should_try_next_sandbox_api_version(err: &KubeError) -> bool { + // Kubernetes returns a structured 404 for some missing API resources and a + // raw "404 page not found" body for others. Both mean the probed + // group/version is unavailable and the next supported Sandbox API version + // should be tried. + matches!(err, KubeError::Api(api) if api.code == 404) } -fn map_kube_event_to_platform( - sandbox_name_to_id: &std::collections::HashMap, - agent_pod_to_id: &std::collections::HashMap, - obj: &KubeEventObj, -) -> Option<(String, PlatformEvent)> { - let involved = obj.involved_object.clone(); - let involved_kind = involved.kind.unwrap_or_default(); - let involved_name = involved.name.unwrap_or_default(); - - let sandbox_id = match involved_kind.as_str() { - "Sandbox" => sandbox_name_to_id.get(&involved_name).cloned()?, - "Pod" => sandbox_name_to_id - .get(&involved_name) - .cloned() - .or_else(|| agent_pod_to_id.get(&involved_name).cloned())?, - _ => return None, - }; +fn validate_gpu_request( + gpu_requirements: Option<&GpuResourceRequirements>, +) -> Result<(), tonic::Status> { + let _ = + effective_driver_gpu_count(gpu_requirements).map_err(tonic::Status::invalid_argument)?; + Ok(()) +} - let ts = obj - .last_timestamp - .as_ref() - .or(obj.first_timestamp.as_ref()) - .map_or(0, |t| t.0.timestamp_millis()); +const MAX_KUBE_NAME_LEN: usize = 63; - let mut metadata = std::collections::HashMap::new(); - metadata.insert("involved_kind".to_string(), involved_kind); - metadata.insert("involved_name".to_string(), involved_name); - if let Some(ns) = &obj.involved_object.namespace { - metadata.insert("namespace".to_string(), ns.clone()); - } - if let Some(count) = obj.count { - metadata.insert("count".to_string(), count.to_string()); +fn validate_kube_resource_name_length(workspace: &str, name: &str) -> Result<(), tonic::Status> { + let combined = workspace.len() + 2 + name.len(); // "--" separator + if combined > MAX_KUBE_NAME_LEN { + return Err(tonic::Status::invalid_argument(format!( + "combined Kubernetes resource name '{workspace}--{name}' is {combined} characters, \ + exceeding the DNS-1123 limit of {MAX_KUBE_NAME_LEN}" + ))); } - attach_kube_progress_metadata( - &mut metadata, - obj.reason.as_deref().unwrap_or_default(), - obj.message.as_deref().unwrap_or_default(), - ); + Ok(()) +} - Some(( - sandbox_id, - PlatformEvent { - timestamp_ms: ts, - source: "kubernetes".to_string(), - r#type: obj.type_.clone().unwrap_or_default(), - reason: obj.reason.clone().unwrap_or_default(), - message: obj.message.clone().unwrap_or_default(), - metadata, - }, - )) +fn is_namespace_owned_by_gateway( + labels: Option<&BTreeMap>, + gateway_id: &str, +) -> bool { + labels + .and_then(|l| l.get(LABEL_MANAGED_BY)) + .is_some_and(|v| v == LABEL_MANAGED_BY_VALUE) + && labels + .and_then(|l| l.get(LABEL_GATEWAY_ID)) + .is_some_and(|v| v == gateway_id) } -fn attach_kube_progress_metadata( - metadata: &mut std::collections::HashMap, - reason: &str, - message: &str, -) { - match reason { - "Scheduled" => { - mark_progress_complete( - metadata, - PROGRESS_STEP_REQUESTING_SANDBOX, - "Sandbox allocated", - ); - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - } - "Pulling" => { - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(image) = pulling_image_from_kube_message(message) { - mark_progress_detail(metadata, image); - } - } - "Pulled" => { - let label = pulled_image_label(message); - mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, label); - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - } - _ => {} - } +fn gateway_id_label_needs_backfill( + labels: Option<&BTreeMap>, + gateway_id: &str, +) -> bool { + labels + .and_then(|labels| labels.get(LABEL_GATEWAY_ID)) + .is_none_or(|value| value != gateway_id) } -fn pulling_image_from_kube_message(message: &str) -> Option { - let image = message - .strip_prefix("Pulling image ") - .map(str::trim) - .map(|value| value.trim_matches('"'))?; - (!image.is_empty()).then(|| image.to_string()) +fn namespace_delete_params(uid: String) -> DeleteParams { + DeleteParams::default().preconditions(Preconditions { + uid: Some(uid), + resource_version: None, + }) } -fn pulled_image_label(message: &str) -> String { - extract_image_size(message).map_or_else( - || "Image pulled".to_string(), - |bytes| format!("Image pulled ({})", format_bytes(bytes)), +fn sandbox_lookup_selector_for(sandbox_id: &str, gateway_id: &str) -> String { + format!( + "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_SANDBOX_ID}={sandbox_id},{LABEL_GATEWAY_ID}={gateway_id}" ) } -fn extract_image_size(message: &str) -> Option { - let size_prefix = "Image size: "; - let start = message.find(size_prefix)? + size_prefix.len(); - let rest = &message[start..]; - let end = rest.find(' ')?; - rest[..end].parse().ok() +fn openshell_sandbox_selector_for(gateway_id: &str) -> String { + use std::fmt::Write; + let mut selector = openshell_sandbox_label_selector(); + write!(selector, ",{LABEL_GATEWAY_ID}={gateway_id}").unwrap(); + selector } -/// Path where the supervisor binary is mounted inside the agent container. -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR; - -/// Name of the volume used to side-load the supervisor binary. -const SUPERVISOR_VOLUME_NAME: &str = "openshell-supervisor-bin"; - -/// Name of the init container that installs the supervisor binary. -const SUPERVISOR_INIT_CONTAINER_NAME: &str = "openshell-supervisor-install"; - -/// Name of the init container that prepares pod-level sidecar networking. -const SUPERVISOR_NETWORK_INIT_CONTAINER_NAME: &str = "openshell-network-init"; - -/// Container name for the network-only supervisor sidecar. -const SUPERVISOR_NETWORK_SIDECAR_NAME: &str = "openshell-supervisor-network"; - -/// UID used by strict process/binary-aware sidecars so Kubernetes grants the -/// requested capability set into the effective set without privilege escalation. -const BINARY_AWARE_SIDECAR_PROXY_UID: u32 = 0; - -/// Shared volume used by the network sidecar and process-only supervisor for -/// local coordination in sidecar topology. -const SIDECAR_STATE_VOLUME_NAME: &str = "openshell-sidecar-state"; -const SIDECAR_STATE_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; -const SIDECAR_CONTROL_SOCKET: &str = openshell_core::container_paths::SIDECAR_CONTROL_SOCKET; -// Linux abstract socket names are scoped to the pod's shared network namespace. -// Unlike a filesystem socket in the shared state volume, the workload cannot -// unlink and replace this relay endpoint after the trusted supervisor binds it. -const SIDECAR_SSH_SOCKET_FILE: &str = "@openshell-sidecar-ssh"; +fn sandbox_labels(sandbox: &Sandbox, gateway_id: Option<&str>) -> BTreeMap { + let mut labels = BTreeMap::new(); + labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); + labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); + labels.insert( + LABEL_SANDBOX_WORKSPACE.to_string(), + sandbox.workspace.clone(), + ); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + if let Some(gw_id) = gateway_id { + labels.insert(LABEL_GATEWAY_ID.to_string(), gw_id.to_string()); + } + labels +} -/// Shared TLS work directory. The network sidecar writes the proxy CA bundle -/// here, while the agent container consumes it after sidecar bootstrap. -const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; -const SIDECAR_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; -const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_CLIENT_TLS_DIR; +fn managed_ssh_network_policy(namespace: &str, config: &KubernetesComputeConfig) -> NetworkPolicy { + NetworkPolicy { + metadata: ObjectMeta { + name: Some(MANAGED_SSH_NETWORK_POLICY_NAME.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + policy_types: Some(vec!["Ingress".to_string()]), + ingress: Some(vec![NetworkPolicyIngressRule { + from: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + config.managed_ssh_ingress.gateway_namespace.clone(), + )])), + ..Default::default() + }), + pod_selector: Some(LabelSelector { + match_labels: Some(config.managed_ssh_ingress.gateway_pod_selector.clone()), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(2222)), + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + }]), + ..Default::default() + }), + status: None, + } +} -/// Build the emptyDir volume that holds the supervisor binary. -/// -/// The init container writes the binary here; the agent container reads it. -fn supervisor_volume() -> serde_json::Value { - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "emptyDir": {} - }) +fn image_pull_secret_copy(secret_name: &str, namespace: &str, source: Secret) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(secret_name.to_string()), + namespace: Some(namespace.to_string()), + labels: Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])), + ..Default::default() + }, + data: source.data, + type_: source.type_, + ..Default::default() + } } -/// Build the read-only volume mount for the supervisor binary in the agent container. -fn supervisor_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "mountPath": SUPERVISOR_MOUNT_PATH, - "readOnly": true - }) +fn sandbox_annotations(sandbox: &Sandbox) -> BTreeMap { + let mut annotations = BTreeMap::new(); + annotations.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); + annotations.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); + annotations.insert( + LABEL_SANDBOX_WORKSPACE.to_string(), + sandbox.workspace.clone(), + ); + annotations } -/// Build an image volume that mounts the supervisor OCI image directly. -/// -/// Requires Kubernetes >= v1.33 (`ImageVolume` beta) or >= v1.36 (GA). -/// The entire image filesystem is mounted read-only, making the binary -/// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. -fn supervisor_image_volume( - supervisor_image: &str, - supervisor_image_pull_policy: &str, -) -> serde_json::Value { - let mut image_spec = serde_json::json!({ - "reference": supervisor_image, - }); - if !supervisor_image_pull_policy.is_empty() { - image_spec["pullPolicy"] = serde_json::json!(supervisor_image_pull_policy); +fn sandbox_id_from_object(obj: &DynamicObject) -> Result { + if let Some(annotations) = obj.metadata.annotations.as_ref() + && let Some(id) = annotations.get(LABEL_SANDBOX_ID) + { + return Ok(id.clone()); } - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "image": image_spec - }) + if let Some(labels) = obj.metadata.labels.as_ref() + && let Some(id) = labels.get(LABEL_SANDBOX_ID) + { + return Ok(id.clone()); + } + Err("sandbox id not found on object".to_string()) } -/// Build the init container that copies the supervisor binary into the emptyDir. -/// -/// The supervisor image contains the supervisor binary at `/openshell-sandbox`. -/// We invoke that binary with the `copy-self` subcommand so it copies itself -/// into the shared emptyDir volume, where the agent container then executes it -/// from a fixed, writable path. This pattern (binary self-copy) avoids requiring -/// `sh`/`cp` in the supervisor image and mirrors the approach used by argoexec's -/// emissary executor. -fn supervisor_init_container( - supervisor_image: &str, - supervisor_image_pull_policy: &str, -) -> serde_json::Value { - let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); - let mut spec = serde_json::json!({ - "name": SUPERVISOR_INIT_CONTAINER_NAME, - "image": supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "copy-self", - installed_path, - ], - "securityContext": {"runAsUser": 0}, - "volumeMounts": [{ - "name": SUPERVISOR_VOLUME_NAME, - "mountPath": SUPERVISOR_MOUNT_PATH, - "readOnly": false - }] - }); - if !supervisor_image_pull_policy.is_empty() { - spec["imagePullPolicy"] = serde_json::json!(supervisor_image_pull_policy); - } - spec +#[derive(Debug)] +struct TokenReviewIdentity { + namespace: String, + pod_name: String, + pod_uid: String, } -fn apply_supervisor_binary_source( - spec: &mut serde_json::Map, - supervisor_image: &str, - supervisor_image_pull_policy: &str, - method: SupervisorSideloadMethod, -) { - let volumes = spec - .entry("volumes") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volumes) = volumes { - match method { - SupervisorSideloadMethod::ImageVolume => { - volumes.push(supervisor_image_volume( - supervisor_image, - supervisor_image_pull_policy, - )); - } - SupervisorSideloadMethod::InitContainer => { - volumes.push(supervisor_volume()); - } - } +#[allow(clippy::result_large_err)] +fn token_review_identity( + status: &TokenReviewStatus, + expected_service_account: &str, +) -> Result, tonic::Status> { + if status.authenticated != Some(true) { + return Ok(None); } - - if method == SupervisorSideloadMethod::InitContainer { - let init_containers = spec - .entry("initContainers") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(init_containers) = init_containers { - init_containers.push(supervisor_init_container( - supervisor_image, - supervisor_image_pull_policy, - )); - } + if !status + .audiences + .as_deref() + .unwrap_or_default() + .iter() + .any(|audience| audience == SANDBOX_TOKEN_AUDIENCE) + { + return Err(tonic::Status::unauthenticated( + "sandbox credential audience not accepted", + )); } -} - -/// Apply supervisor side-load transforms to an already-built pod template JSON. -/// -/// Depending on the sideload method: -/// - **`ImageVolume`**: mounts the supervisor OCI image directly as a read-only -/// volume (no init container needed, requires K8s >= v1.33). -/// - **`InitContainer`**: injects an emptyDir volume and an init container that -/// copies the supervisor binary from the supervisor image into that volume. -/// -/// In both cases, the agent container gets a command override to run the -/// side-loaded binary as root so it can create network namespaces, set up the -/// proxy, and configure Landlock/seccomp. -#[allow(clippy::similar_names)] -fn apply_supervisor_sideload_with_params( - pod_template: &mut serde_json::Value, - params: &SandboxPodParams<'_>, -) { - let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { - return; - }; - - apply_supervisor_binary_source( - spec, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, - ); - - // Find the agent container and add volume mount + command override - let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { - return; - }; - - let mut target_index = None; - for (i, c) in containers.iter().enumerate() { - if c.get("name").and_then(|v| v.as_str()) == Some("agent") { - target_index = Some(i); - break; - } + let user = status + .user + .as_ref() + .ok_or_else(|| tonic::Status::permission_denied("TokenReview response missing user"))?; + let rest = user + .username + .as_deref() + .unwrap_or_default() + .strip_prefix("system:serviceaccount:") + .ok_or_else(|| tonic::Status::permission_denied("credential is not a service account"))?; + let (namespace, service_account) = rest + .split_once(':') + .filter(|(namespace, service_account)| !namespace.is_empty() && !service_account.is_empty()) + .ok_or_else(|| tonic::Status::permission_denied("invalid service account identity"))?; + if service_account != expected_service_account { + return Err(tonic::Status::permission_denied( + "credential is not from the configured sandbox service account", + )); } - let index = target_index.unwrap_or(0); - - if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { - // Override command to use the side-loaded supervisor binary - let mut command = vec![ - format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--workdir".to_string(), - driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), - ]; - command.extend(upstream_proxy_cli_args(params)); - container.insert("command".to_string(), serde_json::json!(command)); - - // Force the supervisor to run as root (UID 0). Sandbox images may set - // a non-root USER directive (e.g. `USER sandbox`), but the supervisor - // needs root to create network namespaces, set up the proxy, and - // configure Landlock/seccomp. The supervisor itself drops privileges - // for child processes via the policy's `run_as_user`/`run_as_group`. - let security_context = container - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = security_context.as_object_mut() { - sc.insert("runAsUser".to_string(), serde_json::json!(0)); - } - - // Add volume mount - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(supervisor_volume_mount()); - } + Ok(Some(TokenReviewIdentity { + namespace: namespace.to_string(), + pod_name: user_extra_one(user, POD_NAME_EXTRA)?, + pod_uid: user_extra_one(user, POD_UID_EXTRA)?, + })) +} - // Inject the protected resolved identity contract. Clearing the OCI - // input prevents image or user environment from selecting a - // conflicting identity path. - let env = container - .entry("env") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(env) = env { - apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); - } - if has_upstream_proxy_credentials(params) { - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(upstream_proxy_auth_volume_mount()); - } - } +#[allow(clippy::result_large_err)] +fn user_extra_one(user: &UserInfo, key: &str) -> Result { + let values = user + .extra + .as_ref() + .and_then(|extra| extra.get(key)) + .ok_or_else(|| tonic::Status::permission_denied("sandbox credential is not pod-bound"))?; + if values.len() != 1 || values[0].is_empty() { + return Err(tonic::Status::permission_denied( + "sandbox credential has invalid pod binding", + )); } + Ok(values[0].clone()) } -#[cfg(test)] -#[allow(clippy::similar_names)] -fn apply_supervisor_sideload( - pod_template: &mut serde_json::Value, - supervisor_image: &str, - supervisor_image_pull_policy: &str, - method: SupervisorSideloadMethod, - sandbox_uid: u32, - sandbox_gid: u32, -) { - let params = SandboxPodParams { - supervisor_image, - supervisor_image_pull_policy, - supervisor_sideload_method: method, - sandbox_uid, - sandbox_gid, - ..SandboxPodParams::default() - }; - apply_supervisor_sideload_with_params(pod_template, ¶ms); +#[allow(clippy::result_large_err)] +fn pod_sandbox_id(pod: &Pod) -> Result { + pod.metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(LABEL_SANDBOX_ID)) + .filter(|value| !value.is_empty()) + .cloned() + .ok_or_else(|| tonic::Status::permission_denied("pod is not bound to a sandbox identity")) } -fn upstream_proxy_cli_args(params: &SandboxPodParams<'_>) -> Vec { - let mut args = Vec::new(); - if let Some(url) = params.https_proxy { - args.extend(["--upstream-proxy".to_string(), url.to_string()]); - } - if let Some(list) = params.no_proxy { - args.extend(["--upstream-no-proxy".to_string(), list.to_string()]); - } - if has_upstream_proxy_credentials(params) { - args.extend([ - "--upstream-proxy-auth-file".to_string(), - openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), - ]); - } - if params.proxy_auth_allow_insecure { - args.push("--upstream-proxy-auth-allow-insecure".to_string()); - } - if params.proxy_connect_by_hostname { - args.push("--upstream-proxy-connect-by-hostname".to_string()); +#[allow(clippy::result_large_err)] +fn validate_pod_uid(pod: &Pod, expected_uid: &str) -> Result<(), tonic::Status> { + if pod.metadata.uid.as_deref() == Some(expected_uid) { + return Ok(()); } - args + Err(tonic::Status::permission_denied( + "sandbox credential pod UID mismatch", + )) } -fn upstream_proxy_auth_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, - "mountPath": upstream_proxy_auth_volume_mount_path(), - "readOnly": true, - }) +#[allow(clippy::result_large_err)] +fn sandbox_owner_reference(pod: &Pod) -> Result<&OwnerReference, tonic::Status> { + sandbox_owner_from_refs(pod.metadata.owner_references.as_deref().unwrap_or_default()) +} + +/// Extract the single controlling `Sandbox` owner reference from an owner list. +/// Shared by direct-owner (agent pod) and controller-chain (proxy-pod supervisor +/// Deployment) resolution. +fn sandbox_owner_from_refs(owners: &[OwnerReference]) -> Result<&OwnerReference, tonic::Status> { + let mut matching = owners.iter().filter(|owner| { + owner.kind == SANDBOX_KIND + && matches!( + owner.api_version.as_str(), + "agents.x-k8s.io/v1beta1" | "agents.x-k8s.io/v1alpha1" + ) + }); + let owner = matching + .next() + .ok_or_else(|| tonic::Status::permission_denied("not controlled by a Sandbox"))?; + if matching.next().is_some() + || owner.controller != Some(true) + || owner.name.is_empty() + || owner.uid.is_empty() + { + return Err(tonic::Status::permission_denied("invalid Sandbox owner")); + } + Ok(owner) } -fn upstream_proxy_auth_volume_mount_path() -> &'static str { - Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) - .parent() - .and_then(Path::to_str) - .expect("upstream proxy auth path has a parent directory") +const REPLICA_SET_KIND: &str = "ReplicaSet"; +const DEPLOYMENT_KIND: &str = "Deployment"; +const APPS_API_VERSION_V1: &str = "apps/v1"; + +/// The single controlling (`controller: true`) owner reference, if any. +fn controller_owner_reference(owners: &[OwnerReference]) -> Option<&OwnerReference> { + owners.iter().find(|owner| owner.controller == Some(true)) } -fn upstream_proxy_auth_file_name() -> &'static str { - Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) - .file_name() - .and_then(|name| name.to_str()) - .expect("upstream proxy auth path has a UTF-8 file name") +#[allow(clippy::result_large_err)] +fn validate_object_uid(actual: &str, expected: &str, message: &str) -> Result<(), tonic::Status> { + if actual.is_empty() || actual != expected { + return Err(tonic::Status::permission_denied(message.to_string())); + } + Ok(()) } -fn has_upstream_proxy_credentials(params: &SandboxPodParams<'_>) -> bool { - params.proxy_auth_secret_name.is_some() && params.proxy_auth_secret_key.is_some() +#[allow(clippy::result_large_err)] +fn validate_sandbox_owner_identity( + owner: &OwnerReference, + sandbox_id: &str, + sandbox: &DynamicObject, +) -> Result<(), tonic::Status> { + let uid_matches = sandbox.metadata.uid.as_deref() == Some(owner.uid.as_str()); + let sandbox_id_matches = sandbox + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .is_some_and(|actual| actual == sandbox_id); + if uid_matches && sandbox_id_matches { + return Ok(()); + } + Err(tonic::Status::permission_denied( + "pod identity does not match its Sandbox owner", + )) } -fn sidecar_state_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SIDECAR_STATE_VOLUME_NAME, - "mountPath": SIDECAR_STATE_MOUNT_PATH, - }) +fn accepts_auth_namespace( + config: &KubernetesComputeConfig, + operator_allowlist: Option<&OperatorNamespaceAllowlist>, + namespace: &str, +) -> bool { + match config.workspace_mode { + WorkspaceMode::Shared => namespace == config.namespace, + WorkspaceMode::Managed => { + namespace.starts_with(&managed_namespace_prefix(&config.gateway_id)) + } + WorkspaceMode::Operator => { + operator_allowlist.is_some_and(|allowlist| allowlist.contains(namespace)) + } + } } -fn sidecar_tls_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SIDECAR_TLS_VOLUME_NAME, - "mountPath": SIDECAR_TLS_MOUNT_PATH, - }) +fn annotation_or_label(obj: &DynamicObject, key: &str) -> Option { + obj.metadata + .annotations + .as_ref() + .and_then(|a| a.get(key)) + .or_else(|| obj.metadata.labels.as_ref().and_then(|l| l.get(key))) + .cloned() } -fn copy_log_level_env( - env: &mut Vec, - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, -) { - if let Some(value) = spec_environment - .get(openshell_core::sandbox_env::LOG_LEVEL) - .or_else(|| template_environment.get(openshell_core::sandbox_env::LOG_LEVEL)) +fn is_openshell_managed(obj: &DynamicObject) -> bool { + annotation_or_label(obj, LABEL_MANAGED_BY).as_deref() == Some(LABEL_MANAGED_BY_VALUE) +} + +/// Resolve the supervisor topology a Sandbox CR was created under. +/// +/// Falls back to `fallback` (the gateway's current configured topology) for CRs +/// created before this annotation existed, preserving their prior behavior. +fn topology_from_object(obj: &DynamicObject, fallback: SupervisorTopology) -> SupervisorTopology { + if let Some(topology) = annotation_or_label(obj, ANNOTATION_SUPERVISOR_TOPOLOGY) + .and_then(|value| value.parse::().ok()) { - upsert_env(env, openshell_core::sandbox_env::LOG_LEVEL, value); + return topology; + } + // No annotation: this CR predates the topology annotation, which every + // sandbox created by this code stamps — including all proxy-pod sandboxes. + // Such a CR therefore was NEVER proxy-pod, so it must not be classified as + // proxy-pod even when the gateway is now configured that way (which would + // wrongly report it sessionless and hunt for companions it never had). + // Combined and sidecar share the session model and have no companions, so + // collapsing an unknown fallback to combined is safe. + if fallback == SupervisorTopology::ProxyPod { + SupervisorTopology::Combined + } else { + fallback } } -fn supervisor_sidecar_env( - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - params: &SandboxPodParams<'_>, -) -> Vec { - let mut env = Vec::new(); - apply_required_env( - &mut env, - params.sandbox_id, - params.sandbox_name, - params.grpc_endpoint, - "", - !params.client_tls_secret_name.is_empty(), - provider_spiffe_socket_path(params), - ); - if !params.client_tls_secret_name.is_empty() { - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_CA, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/ca.crt"), - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_CERT, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.crt"), - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_KEY, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.key"), - ); - } - copy_log_level_env(&mut env, template_environment, spec_environment); - upsert_env( - &mut env, - openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, - "sidecar", - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, - "sidecar-nftables", - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, - SIDECAR_CONTROL_SOCKET, - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - SIDECAR_SSH_SOCKET_FILE, - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::PROXY_TLS_DIR, - SIDECAR_TLS_MOUNT_PATH, - ); - apply_resolved_identity_env(&mut env, params.sandbox_uid, params.sandbox_gid); - if !params.process_binary_aware_network_policy { - upsert_env( - &mut env, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, - "relaxed", - ); +/// Returns `(kube_resource_name, DriverSandbox)`. +/// +/// Returns `Err` in two cases (callers should skip, not fail): +/// - The object is not managed by `OpenShell` (missing/wrong `managed-by` label). +/// - The object is managed by `OpenShell` but missing required fields (orphan). +fn sandbox_from_object( + namespace: &str, + obj: DynamicObject, + topology: SupervisorTopology, +) -> Result<(String, Sandbox), String> { + let kube_name = obj.metadata.name.clone().unwrap_or_default(); + + if !is_openshell_managed(&obj) { + debug!(object = %kube_name, "skipping sandbox CR not managed by openshell"); + return Err(format!("object {kube_name} not managed by openshell")); } - env -} -fn supervisor_sidecar_container( - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - params: &SandboxPodParams<'_>, -) -> serde_json::Value { - let proxy_uid = effective_sidecar_proxy_uid(params); - let capabilities = if params.process_binary_aware_network_policy { - serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) - } else { - serde_json::json!({ - "drop": ["ALL"] - }) + let Ok(id) = sandbox_id_from_object(&obj) else { + warn!(object = %kube_name, "openshell-managed sandbox CR missing id"); + return Err(format!("object {kube_name} missing sandbox id")); }; - let mut container = serde_json::json!({ - "name": SUPERVISOR_NETWORK_SIDECAR_NAME, - "image": params.supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network", - ], - "env": supervisor_sidecar_env(template_environment, spec_environment, params), - "securityContext": { - "runAsUser": proxy_uid, - "runAsGroup": params.sandbox_gid, - "runAsNonRoot": proxy_uid != 0, - "allowPrivilegeEscalation": false, - "capabilities": capabilities + let Some(name) = annotation_or_label(&obj, LABEL_SANDBOX_NAME) else { + warn!(object = %kube_name, "openshell-managed sandbox CR missing name"); + return Err(format!("object {kube_name} missing sandbox name")); + }; + let Some(workspace) = annotation_or_label(&obj, LABEL_SANDBOX_WORKSPACE) else { + warn!(object = %kube_name, "openshell-managed sandbox CR missing workspace"); + return Err(format!("object {kube_name} missing sandbox workspace")); + }; + + let namespace = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| namespace.to_string()); + // Derive the session model from the CR's creation-time topology, falling + // back to the gateway's current topology for CRs predating the annotation. + let resolved_topology = topology_from_object(&obj, topology); + let status = status_from_object(&obj, resolved_topology); + + Ok(( + kube_name, + Sandbox { + id, + name, + namespace, + spec: None, + status, + workspace, }, - "volumeMounts": [ - sidecar_state_volume_mount(), - sidecar_tls_volume_mount(), - { - "name": "openshell-sa-token", - "mountPath": "/var/run/secrets/openshell", - "readOnly": true - } - ] - }); - container["command"] - .as_array_mut() - .expect("network supervisor command is an array") - .extend( - upstream_proxy_cli_args(params) - .into_iter() - .map(serde_json::Value::String), - ); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); - } - if params.provider_spiffe_enabled { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), - "readOnly": true, - })); - } - if has_upstream_proxy_credentials(params) { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(upstream_proxy_auth_volume_mount()); + )) +} + +fn update_indexes( + sandbox_name_to_id: &mut std::collections::HashMap, + agent_pod_to_id: &mut std::collections::HashMap, + kube_name: &str, + sandbox: &Sandbox, +) { + if !kube_name.is_empty() { + sandbox_name_to_id.insert(kube_name.to_string(), sandbox.id.clone()); } - if let Some(profile) = params.app_armor_profile { - container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); + if let Some(status) = sandbox.status.as_ref() + && !status.instance_id.is_empty() + { + agent_pod_to_id.insert(status.instance_id.clone(), sandbox.id.clone()); } - container } -fn effective_sidecar_proxy_uid(params: &SandboxPodParams<'_>) -> u32 { - if params.process_binary_aware_network_policy { - BINARY_AWARE_SIDECAR_PROXY_UID - } else { - params.proxy_uid - } +fn remove_indexes( + sandbox_name_to_id: &mut std::collections::HashMap, + agent_pod_to_id: &mut std::collections::HashMap, + sandbox_id: &str, +) { + sandbox_name_to_id.retain(|_, value| value != sandbox_id); + agent_pod_to_id.retain(|_, value| value != sandbox_id); } -fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_json::Value { - let proxy_uid = effective_sidecar_proxy_uid(params); - let mut container = serde_json::json!({ - "name": SUPERVISOR_NETWORK_INIT_CONTAINER_NAME, - "image": params.supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network-init", - "--proxy-uid", - proxy_uid.to_string(), - "--proxy-gid", - params.sandbox_gid.to_string(), - "--sidecar-state-dir", - SIDECAR_STATE_MOUNT_PATH, - "--sidecar-tls-dir", - SIDECAR_TLS_MOUNT_PATH, - ], - "securityContext": { - "runAsUser": 0, - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"], - "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] - } - }, - "volumeMounts": [ - sidecar_state_volume_mount(), - sidecar_tls_volume_mount(), - ] - }); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); - } - if !params.client_tls_secret_name.is_empty() { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(serde_json::json!({ - "name": "openshell-client-tls", - "mountPath": openshell_core::container_paths::CLIENT_TLS_DIR, - "readOnly": true - })); +fn map_kube_event_to_platform( + sandbox_name_to_id: &std::collections::HashMap, + agent_pod_to_id: &std::collections::HashMap, + obj: &KubeEventObj, +) -> Option<(String, PlatformEvent)> { + let involved = obj.involved_object.clone(); + let involved_kind = involved.kind.unwrap_or_default(); + let involved_name = involved.name.unwrap_or_default(); + + let sandbox_id = match involved_kind.as_str() { + "Sandbox" => sandbox_name_to_id.get(&involved_name).cloned()?, + "Pod" => sandbox_name_to_id + .get(&involved_name) + .cloned() + .or_else(|| agent_pod_to_id.get(&involved_name).cloned())?, + _ => return None, + }; + + let ts = obj + .last_timestamp + .as_ref() + .or(obj.first_timestamp.as_ref()) + .map_or(0, |t| t.0.timestamp_millis()); + + let mut metadata = std::collections::HashMap::new(); + metadata.insert("involved_kind".to_string(), involved_kind); + metadata.insert("involved_name".to_string(), involved_name); + if let Some(ns) = &obj.involved_object.namespace { + metadata.insert("namespace".to_string(), ns.clone()); } - if let Some(profile) = params.app_armor_profile { - container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); + if let Some(count) = obj.count { + metadata.insert("count".to_string(), count.to_string()); } - container + attach_kube_progress_metadata( + &mut metadata, + obj.reason.as_deref().unwrap_or_default(), + obj.message.as_deref().unwrap_or_default(), + ); + + Some(( + sandbox_id, + PlatformEvent { + timestamp_ms: ts, + source: "kubernetes".to_string(), + r#type: obj.type_.clone().unwrap_or_default(), + reason: obj.reason.clone().unwrap_or_default(), + message: obj.message.clone().unwrap_or_default(), + metadata, + }, + )) } -fn apply_supervisor_sidecar_topology( - pod_template: &mut serde_json::Value, - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - params: &SandboxPodParams<'_>, +fn attach_kube_progress_metadata( + metadata: &mut std::collections::HashMap, + reason: &str, + message: &str, ) { - let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { - return; - }; - - let pod_security_context = spec - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = pod_security_context.as_object_mut() { - sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); + match reason { + "Scheduled" => { + mark_progress_complete( + metadata, + PROGRESS_STEP_REQUESTING_SANDBOX, + "Sandbox allocated", + ); + mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); + } + "Pulling" => { + mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); + if let Some(image) = pulling_image_from_kube_message(message) { + mark_progress_detail(metadata, image); + } + } + "Pulled" => { + let label = pulled_image_label(message); + mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, label); + mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); + } + _ => {} } +} - spec.insert("shareProcessNamespace".to_string(), serde_json::json!(true)); +fn pulling_image_from_kube_message(message: &str) -> Option { + let image = message + .strip_prefix("Pulling image ") + .map(str::trim) + .map(|value| value.trim_matches('"'))?; + (!image.is_empty()).then(|| image.to_string()) +} - apply_supervisor_binary_source( - spec, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, - ); +fn pulled_image_label(message: &str) -> String { + extract_image_size(message).map_or_else( + || "Image pulled".to_string(), + |bytes| format!("Image pulled ({})", format_bytes(bytes)), + ) +} - let volumes = spec - .entry("volumes") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volumes) = volumes { - volumes.push(serde_json::json!({ - "name": SIDECAR_STATE_VOLUME_NAME, - "emptyDir": {} - })); - volumes.push(serde_json::json!({ - "name": SIDECAR_TLS_VOLUME_NAME, - "emptyDir": {} - })); - } +fn extract_image_size(message: &str) -> Option { + let size_prefix = "Image size: "; + let start = message.find(size_prefix)? + size_prefix.len(); + let rest = &message[start..]; + let end = rest.find(' ')?; + rest[..end].parse().ok() +} - let init_containers = spec - .entry("initContainers") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(init_containers) = init_containers { - init_containers.push(supervisor_network_init_container(params)); - } +/// Path where the supervisor binary is mounted inside the agent container. +const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR; - let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { - return; - }; +/// Name of the volume used to side-load the supervisor binary. +const SUPERVISOR_VOLUME_NAME: &str = "openshell-supervisor-bin"; - let target_index = containers - .iter() - .position(|c| c.get("name").and_then(|v| v.as_str()) == Some("agent")) - .unwrap_or(0); +/// Name of the init container that installs the supervisor binary. +const SUPERVISOR_INIT_CONTAINER_NAME: &str = "openshell-supervisor-install"; - if let Some(container) = containers - .get_mut(target_index) - .and_then(|v| v.as_object_mut()) - { - container.insert( - "command".to_string(), - serde_json::json!([ - format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--mode=process", - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT - ]), - ); +/// Name of the init container that prepares pod-level sidecar networking. +const SUPERVISOR_NETWORK_INIT_CONTAINER_NAME: &str = "openshell-network-init"; - let security_context = container - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = security_context.as_object_mut() { - sc.insert( - "runAsUser".to_string(), - serde_json::json!(params.sandbox_uid), - ); - sc.insert( - "runAsGroup".to_string(), - serde_json::json!(params.sandbox_gid), - ); - sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); - sc.insert( - "allowPrivilegeEscalation".to_string(), - serde_json::json!(false), - ); - sc.insert( - "capabilities".to_string(), - serde_json::json!({ - "drop": ["ALL"] - }), - ); - } +/// Container name for the network-only supervisor sidecar. +const SUPERVISOR_NETWORK_SIDECAR_NAME: &str = "openshell-supervisor-network"; - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - remove_volume_mount(volume_mounts, "openshell-sa-token"); - remove_volume_mount(volume_mounts, "openshell-client-tls"); - remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); - volume_mounts.push(supervisor_volume_mount()); - volume_mounts.push(sidecar_state_volume_mount()); - volume_mounts.push(sidecar_tls_volume_mount()); +/// UID used by strict process/binary-aware sidecars so Kubernetes grants the +/// requested capability set into the effective set without privilege escalation. +const BINARY_AWARE_SIDECAR_PROXY_UID: u32 = 0; + +/// Shared volume used by the network sidecar and process-only supervisor for +/// local coordination in sidecar topology. +const SIDECAR_STATE_VOLUME_NAME: &str = "openshell-sidecar-state"; +const SIDECAR_STATE_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; +const SIDECAR_CONTROL_SOCKET: &str = openshell_core::container_paths::SIDECAR_CONTROL_SOCKET; +// Linux abstract socket names are scoped to the pod's shared network namespace. +// Unlike a filesystem socket in the shared state volume, the workload cannot +// unlink and replace this relay endpoint after the trusted supervisor binds it. +const SIDECAR_SSH_SOCKET_FILE: &str = "@openshell-sidecar-ssh"; + +// Abstract socket for the proxy-pod agent's in-pod process supervisor. The agent +// pod runs non-root, so it cannot create a filesystem socket under `/run`; an +// abstract socket needs no writable directory and is scoped to the agent pod's +// own network namespace. The gateway relay and SSH server share this process. +const PROXY_POD_SSH_SOCKET_FILE: &str = "@openshell-proxy-pod-ssh"; + +/// Shared TLS work directory. The network sidecar writes the proxy CA bundle +/// here, while the agent container consumes it after sidecar bootstrap. +const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; +const SIDECAR_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; +const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_CLIENT_TLS_DIR; + +const LABEL_SANDBOX_ROLE: &str = "openshell.ai/sandbox-role"; +const SANDBOX_ROLE_AGENT: &str = "agent"; +const SANDBOX_ROLE_SUPERVISOR: &str = "supervisor"; +const PROXY_POD_PROXY_PORT: u16 = 3128; +const PROXY_POD_WAIT_INIT_CONTAINER_NAME: &str = "openshell-wait-for-proxy"; +/// Upper bound on how long the agent pod waits for its paired supervisor. +/// Exceeding it fails the init container, which surfaces as a pod-level error +/// rather than a workload that silently has no egress. +const PROXY_POD_WAIT_TIMEOUT_SECS: u64 = 180; +const PROXY_POD_NETWORK_ENFORCEMENT_MODE: &str = "proxy-pod"; +const PROXY_POD_CA_SECRET_MOUNT_PATH: &str = "/var/run/openshell-proxy-ca"; +const PROXY_POD_CA_CERT_FILE: &str = "openshell-ca.pem"; +const PROXY_POD_CA_KEY_FILE: &str = "openshell-ca-key.pem"; + +/// Build the emptyDir volume that holds the supervisor binary. +/// +/// The init container writes the binary here; the agent container reads it. +fn supervisor_volume() -> serde_json::Value { + serde_json::json!({ + "name": SUPERVISOR_VOLUME_NAME, + "emptyDir": {} + }) +} + +/// Build the read-only volume mount for the supervisor binary in the agent container. +fn supervisor_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": SUPERVISOR_VOLUME_NAME, + "mountPath": SUPERVISOR_MOUNT_PATH, + "readOnly": true + }) +} + +/// Build an image volume that mounts the supervisor OCI image directly. +/// +/// Requires Kubernetes >= v1.33 (`ImageVolume` beta) or >= v1.36 (GA). +/// The entire image filesystem is mounted read-only, making the binary +/// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. +fn supervisor_image_volume( + supervisor_image: &str, + supervisor_image_pull_policy: &str, +) -> serde_json::Value { + let mut image_spec = serde_json::json!({ + "reference": supervisor_image, + }); + if !supervisor_image_pull_policy.is_empty() { + image_spec["pullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + } + serde_json::json!({ + "name": SUPERVISOR_VOLUME_NAME, + "image": image_spec + }) +} + +/// Build the init container that copies the supervisor binary into the emptyDir. +/// +/// The supervisor image contains the supervisor binary at `/openshell-sandbox`. +/// We invoke that binary with the `copy-self` subcommand so it copies itself +/// into the shared emptyDir volume, where the agent container then executes it +/// from a fixed, writable path. This pattern (binary self-copy) avoids requiring +/// `sh`/`cp` in the supervisor image and mirrors the approach used by argoexec's +/// emissary executor. +fn supervisor_init_container( + supervisor_image: &str, + supervisor_image_pull_policy: &str, + run_as: Option<(u32, u32)>, +) -> serde_json::Value { + let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); + // The copy-self init only writes the binary into the shared emptyDir. In + // root-capable topologies (combined/sidecar) it runs as UID 0. In the + // proxy-pod topology the pod runs under a non-root SCC (e.g. OpenShift + // nonroot-v2), so it must run as the sandbox uid/gid; the emptyDir is + // group-writable via the pod's fsGroup, so the non-root copy succeeds. + let security_context = match run_as { + None => serde_json::json!({ "runAsUser": 0 }), + Some((uid, gid)) => serde_json::json!({ + "runAsUser": uid, + "runAsGroup": gid, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "capabilities": { "drop": ["ALL"] } + }), + }; + let mut spec = serde_json::json!({ + "name": SUPERVISOR_INIT_CONTAINER_NAME, + "image": supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "copy-self", + installed_path, + ], + "securityContext": security_context, + "volumeMounts": [{ + "name": SUPERVISOR_VOLUME_NAME, + "mountPath": SUPERVISOR_MOUNT_PATH, + "readOnly": false + }] + }); + if !supervisor_image_pull_policy.is_empty() { + spec["imagePullPolicy"] = serde_json::json!(supervisor_image_pull_policy); + } + spec +} + +fn apply_supervisor_binary_source( + spec: &mut serde_json::Map, + supervisor_image: &str, + supervisor_image_pull_policy: &str, + method: SupervisorSideloadMethod, + init_run_as: Option<(u32, u32)>, +) { + let volumes = spec + .entry("volumes") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volumes) = volumes { + match method { + SupervisorSideloadMethod::ImageVolume => { + volumes.push(supervisor_image_volume( + supervisor_image, + supervisor_image_pull_policy, + )); + } + SupervisorSideloadMethod::InitContainer => { + volumes.push(supervisor_volume()); + } } + } - let env = container - .entry("env") + if method == SupervisorSideloadMethod::InitContainer { + let init_containers = spec + .entry("initContainers") .or_insert_with(|| serde_json::json!([])) .as_array_mut(); - if let Some(env) = env { - remove_env(env, openshell_core::sandbox_env::ENDPOINT); - remove_env(env, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - remove_env(env, openshell_core::sandbox_env::TLS_CA); - remove_env(env, openshell_core::sandbox_env::TLS_CERT); - remove_env(env, openshell_core::sandbox_env::TLS_KEY); - remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN); - remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - remove_env(env, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE); - remove_env( - env, - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - ); - upsert_env( - env, - openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, - "sidecar", - ); - upsert_env( - env, - openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, - "sidecar-nftables", - ); - upsert_env( - env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - SIDECAR_SSH_SOCKET_FILE, - ); - upsert_env( - env, - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, - SIDECAR_CONTROL_SOCKET, - ); - upsert_env( - env, - openshell_core::sandbox_env::PROXY_TLS_DIR, - SIDECAR_TLS_MOUNT_PATH, - ); - apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); + if let Some(init_containers) = init_containers { + init_containers.push(supervisor_init_container( + supervisor_image, + supervisor_image_pull_policy, + init_run_as, + )); } } - - containers.push(supervisor_sidecar_container( - template_environment, - spec_environment, - params, - )); } -/// Apply workspace persistence transforms to an already-built pod template. -/// -/// This injects: -/// 1. A volume mount on the agent container at `/sandbox`. -/// 2. An init container (same image) that seeds the PVC with the image's -/// original `/sandbox` contents on first use. +/// Apply supervisor side-load transforms to an already-built pod template JSON. /// -/// The PVC volume itself is **not** added here — the Sandbox CRD controller -/// automatically creates a volume for each entry in `volumeClaimTemplates` -/// (following the `StatefulSet` convention). Adding one here would create a -/// duplicate volume name and fail pod validation. +/// Depending on the sideload method: +/// - **`ImageVolume`**: mounts the supervisor OCI image directly as a read-only +/// volume (no init container needed, requires K8s >= v1.33). +/// - **`InitContainer`**: injects an emptyDir volume and an init container that +/// copies the supervisor binary from the supervisor image into that volume. /// -/// The init container mounts the PVC at a temporary path so it can still see -/// the image's `/sandbox` directory. It checks for a sentinel file and skips -/// the copy if the PVC was already initialised. +/// In both cases, the agent container gets a command override to run the +/// side-loaded binary as root so it can create network namespaces, set up the +/// proxy, and configure Landlock/seccomp. #[allow(clippy::similar_names)] -fn apply_workspace_persistence( +fn apply_supervisor_sideload_with_params( pod_template: &mut serde_json::Value, - image: &str, - image_pull_policy: &str, - sandbox_gid: u32, + params: &SandboxPodParams<'_>, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { return; }; - // fsGroup is a pod-level field — it instructs kubelet to chown mounted - // volumes to this GID. It is invalid at the container securityContext level. - let pod_sc = spec - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(pod_sc_obj) = pod_sc.as_object_mut() { - pod_sc_obj.insert("fsGroup".to_string(), serde_json::json!(sandbox_gid)); - } + apply_supervisor_binary_source( + spec, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, + None, + ); - // 1. Add workspace volume mount to the agent container - let containers = spec.get_mut("containers").and_then(|v| v.as_array_mut()); - if let Some(containers) = containers { - let mut target_index = None; - for (i, c) in containers.iter().enumerate() { - if c.get("name").and_then(|v| v.as_str()) == Some("agent") { - target_index = Some(i); - break; - } - } - let index = target_index.unwrap_or(0); + // Find the agent container and add volume mount + command override + let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { + return; + }; - if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(serde_json::json!({ - "name": WORKSPACE_VOLUME_NAME, - "mountPath": WORKSPACE_MOUNT_PATH - })); - } + let mut target_index = None; + for (i, c) in containers.iter().enumerate() { + if c.get("name").and_then(|v| v.as_str()) == Some("agent") { + target_index = Some(i); + break; } } + let index = target_index.unwrap_or(0); - // 3. Add the init container that seeds the PVC from the image - let init_containers = spec - .entry("initContainers") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(init_containers) = init_containers { - // The init container mounts the PVC at a temp path so it can still - // read the image's original /sandbox contents. It copies them into - // the PVC only when the sentinel file is absent. - // - // Prefer a tar stream over `cp -a`: some sandbox images contain - // self-referential symlinks under `/sandbox/.uv`, and GNU cp can - // fail while seeding the PVC even though preserving the symlink as-is - // is valid. `tar` copies the tree without dereferencing those links. - // Archive only the contents, not the `/sandbox` directory entry - // itself, so extraction never tries to chmod the PVC mount root. - // Extract without restoring owner, mode, or timestamps so the - // non-root init container can seed kubelet-owned PVCs. - // - // The inner `[ -d ... ]` guard handles custom images that don't have - // a /sandbox directory — the copy is skipped but the sentinel is - // still written so subsequent starts are instant. - let copy_cmd = format!( - "if [ ! -f {WORKSPACE_INIT_MOUNT_PATH}/{WORKSPACE_SENTINEL} ]; then \ - if [ -d {WORKSPACE_MOUNT_PATH} ]; then \ - tmp=$(mktemp) && rm -f \"$tmp\" && \ - (cd {WORKSPACE_MOUNT_PATH} && find . -mindepth 1 -maxdepth 1 -exec tar -cf \"$tmp\" {{}} +) && \ - if [ -f \"$tmp\" ]; then \ - tar -C {WORKSPACE_INIT_MOUNT_PATH} --no-same-owner --no-same-permissions --touch -xf \"$tmp\" && \ - rm -f \"$tmp\"; \ - fi; \ - fi && \ - touch {WORKSPACE_INIT_MOUNT_PATH}/{WORKSPACE_SENTINEL}; \ - fi" - ); + if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { + // Override command to use the side-loaded supervisor binary + let mut command = vec![ + format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), + "--workdir".to_string(), + driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), + ]; + command.extend(upstream_proxy_cli_args(params, true)); + container.insert("command".to_string(), serde_json::json!(command)); - let mut init_spec = serde_json::json!({ - "name": WORKSPACE_INIT_CONTAINER_NAME, - "image": image, - "command": ["sh", "-c", copy_cmd], - "securityContext": { - "runAsUser": 0, - }, - "volumeMounts": [{ - "name": WORKSPACE_VOLUME_NAME, - "mountPath": WORKSPACE_INIT_MOUNT_PATH - }] - }); - if !image_pull_policy.is_empty() { - init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + // Force the supervisor to run as root (UID 0). Sandbox images may set + // a non-root USER directive (e.g. `USER sandbox`), but the supervisor + // needs root to create network namespaces, set up the proxy, and + // configure Landlock/seccomp. The supervisor itself drops privileges + // for child processes via the policy's `run_as_user`/`run_as_group`. + let security_context = container + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(sc) = security_context.as_object_mut() { + sc.insert("runAsUser".to_string(), serde_json::json!(0)); } - init_containers.push(init_spec); - } -} -/// Build the default `volumeClaimTemplates` array for sandbox pods. -/// -/// Provides a single PVC named "workspace" that backs the `/sandbox` -/// directory. The init container seeds it from the image on first use. -/// -/// When `storage_class` is non-empty, it is written to the PVC's -/// `storageClassName`. An empty value omits the field so the cluster's -/// default `StorageClass` applies. Clusters with no default `StorageClass` -/// must set this to prevent the PVC from staying `Pending`. -fn default_workspace_volume_claim_templates( - storage_size: &str, - storage_class: &str, -) -> serde_json::Value { - let size = if storage_size.is_empty() { - DEFAULT_WORKSPACE_STORAGE_SIZE - } else { - storage_size - }; - let mut spec = serde_json::json!({ - "accessModes": ["ReadWriteOnce"], - "resources": { - "requests": { - "storage": size + // Add volume mount + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + volume_mounts.push(supervisor_volume_mount()); + } + + // Inject the protected resolved identity contract. Clearing the OCI + // input prevents image or user environment from selecting a + // conflicting identity path. + let env = container + .entry("env") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(env) = env { + apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); + } + if has_upstream_proxy_credentials(params) { + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + volume_mounts.push(upstream_proxy_auth_volume_mount()); } } - }); - if !storage_class.is_empty() { - spec["storageClassName"] = serde_json::json!(storage_class); } - serde_json::json!([{ - "metadata": { - "name": WORKSPACE_VOLUME_NAME - }, - "spec": spec - }]) } -/// Parameters shared by `sandbox_to_k8s_spec` and `sandbox_template_to_k8s`. -#[allow(clippy::struct_excessive_bools)] -struct SandboxPodParams<'a> { - default_image: &'a str, - image_pull_policy: &'a str, - image_pull_secrets: &'a [String], - supervisor_image: &'a str, - supervisor_image_pull_policy: &'a str, - supervisor_sideload_method: SupervisorSideloadMethod, - topology: SupervisorTopology, - proxy_uid: u32, - process_binary_aware_network_policy: bool, - https_proxy: Option<&'a str>, - no_proxy: Option<&'a str>, - proxy_auth_secret_name: Option<&'a str>, - proxy_auth_secret_key: Option<&'a str>, - proxy_auth_allow_insecure: bool, - proxy_connect_by_hostname: bool, - service_account_name: &'a str, - sandbox_id: &'a str, - sandbox_name: &'a str, - grpc_endpoint: &'a str, - ssh_socket_path: &'a str, - client_tls_secret_name: &'a str, - host_gateway_ip: &'a str, - enable_user_namespaces: bool, - app_armor_profile: Option<&'a AppArmorProfile>, - workspace_default_storage_size: &'a str, - workspace_storage_class: &'a str, - default_runtime_class_name: &'a str, - /// Lifetime (seconds) of the projected `ServiceAccount` token used - /// for the bootstrap `IssueSandboxToken` exchange. - sa_token_ttl_secs: i64, - provider_spiffe_enabled: bool, - provider_spiffe_workload_api_socket_path: &'a str, - /// Resolved sandbox UID for supervisor `runAsUser` and env var. +#[cfg(test)] +#[allow(clippy::similar_names)] +fn apply_supervisor_sideload( + pod_template: &mut serde_json::Value, + supervisor_image: &str, + supervisor_image_pull_policy: &str, + method: SupervisorSideloadMethod, sandbox_uid: u32, - /// Resolved sandbox GID for PVC init container operations. sandbox_gid: u32, +) { + let params = SandboxPodParams { + supervisor_image, + supervisor_image_pull_policy, + supervisor_sideload_method: method, + sandbox_uid, + sandbox_gid, + ..SandboxPodParams::default() + }; + apply_supervisor_sideload_with_params(pod_template, ¶ms); } -impl Default for SandboxPodParams<'_> { - fn default() -> Self { - Self { - default_image: "", - image_pull_policy: "", - image_pull_secrets: &[], - supervisor_image: "", - supervisor_image_pull_policy: "", - supervisor_sideload_method: SupervisorSideloadMethod::default(), - topology: SupervisorTopology::default(), - proxy_uid: DEFAULT_PROXY_UID, - process_binary_aware_network_policy: true, - https_proxy: None, - no_proxy: None, - proxy_auth_secret_name: None, - proxy_auth_secret_key: None, - proxy_auth_allow_insecure: false, - proxy_connect_by_hostname: false, - service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - sandbox_id: "", - sandbox_name: "", - grpc_endpoint: "", - ssh_socket_path: "", - client_tls_secret_name: "", - host_gateway_ip: "", - enable_user_namespaces: false, - app_armor_profile: None, - workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE, - workspace_storage_class: "", - default_runtime_class_name: "", - sa_token_ttl_secs: 3600, - provider_spiffe_enabled: false, - provider_spiffe_workload_api_socket_path: "", - sandbox_uid: DEFAULT_SANDBOX_UID, - sandbox_gid: DEFAULT_SANDBOX_UID, +/// Build the `--upstream-proxy*` CLI arguments for a network supervisor. +/// +/// `include_credentials` gates the arguments that depend on the mounted proxy +/// credential file. Sidecar topology mounts that Secret and passes `true`; +/// proxy-pod topology does not mount it (yet) and passes `false`, so it still +/// routes egress through the operator's corporate proxy (URL, `no_proxy`, CONNECT +/// mode) without referencing an auth file that would not exist in the +/// supervisor pod. +fn upstream_proxy_cli_args( + params: &SandboxPodParams<'_>, + include_credentials: bool, +) -> Vec { + let mut args = Vec::new(); + if let Some(url) = params.https_proxy { + args.extend(["--upstream-proxy".to_string(), url.to_string()]); + } + if let Some(list) = params.no_proxy { + args.extend(["--upstream-no-proxy".to_string(), list.to_string()]); + } + if include_credentials && has_upstream_proxy_credentials(params) { + args.extend([ + "--upstream-proxy-auth-file".to_string(), + openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + if params.proxy_auth_allow_insecure { + args.push("--upstream-proxy-auth-allow-insecure".to_string()); } } + if params.proxy_connect_by_hostname { + args.push("--upstream-proxy-connect-by-hostname".to_string()); + } + args } -fn validate_sidecar_proxy_identity( - params: &SandboxPodParams<'_>, -) -> Result<(), KubernetesDriverError> { - if params.topology == SupervisorTopology::Sidecar && params.proxy_uid == params.sandbox_uid { - return Err(KubernetesDriverError::Precondition(format!( - "proxy_uid ({}) must not match sandbox_uid ({}) in sidecar topology", - params.proxy_uid, params.sandbox_uid - ))); - } - Ok(()) +fn upstream_proxy_auth_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, + "mountPath": upstream_proxy_auth_volume_mount_path(), + "readOnly": true, + }) } -fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap { - let mut env = spec.map_or_else(Default::default, |s| s.environment.clone()); - if let Some(s) = spec.filter(|s| !s.log_level.is_empty()) { - env.insert( - openshell_core::sandbox_env::LOG_LEVEL.to_string(), - s.log_level.clone(), - ); - } - env +fn upstream_proxy_auth_volume_mount_path() -> &'static str { + Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) + .parent() + .and_then(Path::to_str) + .expect("upstream proxy auth path has a parent directory") } -fn kubernetes_driver_config_for_spec( - spec: Option<&SandboxSpec>, - provider_spiffe_workload_api_socket_path: Option<&str>, -) -> Result { - let config = spec - .and_then(|spec| spec.template.as_ref()) - .map(KubernetesSandboxDriverConfig::from_template) - .transpose()? - .unwrap_or_default(); - let mut protected_paths = KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS.to_vec(); - let provider_spiffe_mount_path; - if let Some(socket_path) = provider_spiffe_workload_api_socket_path { - provider_spiffe_mount_path = spiffe_socket_mount_path(socket_path); - protected_paths.push(&provider_spiffe_mount_path); - } - validate_kubernetes_protected_path_conflicts( - &config.containers.agent.volume_mounts, - &protected_paths, - )?; - Ok(config) +fn upstream_proxy_auth_file_name() -> &'static str { + Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) + .file_name() + .and_then(|name| name.to_str()) + .expect("upstream proxy auth path has a UTF-8 file name") } -fn sandbox_to_k8s_spec( - spec: Option<&SandboxSpec>, - params: &SandboxPodParams<'_>, -) -> Result { - let driver_config = - kubernetes_driver_config_for_spec(spec, provider_spiffe_socket_path(params))?; - let mut root = serde_json::Map::new(); +fn has_upstream_proxy_credentials(params: &SandboxPodParams<'_>) -> bool { + params.proxy_auth_secret_name.is_some() && params.proxy_auth_secret_key.is_some() +} - // Determine early whether OpenShell should inject its default workspace - // PVC. Explicit Kubernetes driver-config mounts under /sandbox/ take - // ownership of workspace persistence. - // We need this flag before building the podTemplate because the workspace - // persistence transforms are applied inside sandbox_template_to_k8s. - let user_has_explicit_workspace_mount = driver_config.has_explicit_sandbox_data_mount(); - let inject_workspace = !user_has_explicit_workspace_mount; +fn sidecar_state_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": SIDECAR_STATE_VOLUME_NAME, + "mountPath": SIDECAR_STATE_MOUNT_PATH, + }) +} - if let Some(spec) = spec { - let pod_env = spec_pod_env(Some(spec)); - if let Some(template) = spec.template.as_ref() { - root.insert( - "podTemplate".to_string(), - sandbox_template_to_k8s_with_validated_config( - template, - driver_gpu_requirements(spec.resource_requirements.as_ref()), - &pod_env, - Some(spec), - &driver_config, - inject_workspace, - params, - ), - ); - if !template.agent_socket_path.is_empty() { - root.insert( - "agentSocket".to_string(), - serde_json::json!(template.agent_socket_path), - ); +fn sidecar_tls_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": SIDECAR_TLS_VOLUME_NAME, + "mountPath": SIDECAR_TLS_MOUNT_PATH, + }) +} + +#[derive(Debug, Clone)] +struct ProxyPodResourceNames { + supervisor_deployment: String, + service: String, + proxy_ca_secret: String, + agent_egress_network_policy: String, + supervisor_ingress_network_policy: String, +} + +/// Derive the companion resource names for a sandbox. +/// +/// `cr_name` is cosmetic (readability in `kubectl get`); uniqueness comes from +/// `sandbox_id`, the immutable per-instance UUID. Keying the suffix on the UUID +/// (not the truncatable CR name) means two distinct sandbox instances never +/// collide, and a new sandbox that reuses a name while an old instance's +/// dependents are still being garbage-collected gets its own distinct names +/// rather than binding to the stale objects. +fn proxy_pod_resource_names(cr_name: &str, sandbox_id: &str) -> ProxyPodResourceNames { + ProxyPodResourceNames { + supervisor_deployment: dns_label_name("os-sup", cr_name, sandbox_id), + service: dns_label_name("os-svc", cr_name, sandbox_id), + proxy_ca_secret: dns_label_name("os-ca", cr_name, sandbox_id), + agent_egress_network_policy: dns_label_name("os-eg", cr_name, sandbox_id), + supervisor_ingress_network_policy: dns_label_name("os-ing", cr_name, sandbox_id), + } +} + +fn dns_label_name(prefix: &str, readable: &str, unique_key: &str) -> String { + // FNV-1a over the immutable unique key (sandbox UUID) rather than the + // readable name. A 64-bit suffix makes collisions between distinct sandbox + // instances negligible, where the previous 32-bit hash of a truncatable + // name had a deterministic collision path. Fall back to the readable name + // only when no unique key is available. + let key = if unique_key.is_empty() { + readable + } else { + unique_key + }; + let mut hash = 0xcbf2_9ce4_8422_2325_u64; + for byte in key.as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + let suffix = format!("{hash:016x}"); + let mut sanitized = readable + .chars() + .map(|c| { + let c = c.to_ascii_lowercase(); + if c.is_ascii_alphanumeric() || c == '-' { + c + } else { + '-' } - } + }) + .collect::(); + sanitized = sanitized + .trim_matches('-') + .split('-') + .filter(|part| !part.is_empty()) + .collect::>() + .join("-"); + if sanitized.is_empty() { + sanitized = "sandbox".to_string(); } - - if inject_workspace { - root.insert( - "volumeClaimTemplates".to_string(), - default_workspace_volume_claim_templates( - params.workspace_default_storage_size, - params.workspace_storage_class, - ), - ); + let max_base_len = 63usize.saturating_sub(prefix.len() + suffix.len() + 2); + if sanitized.len() > max_base_len { + sanitized.truncate(max_base_len); + sanitized = sanitized.trim_matches('-').to_string(); } + format!("{prefix}-{sanitized}-{suffix}") +} - // podTemplate is required by the Kubernetes CRD - ensure it's always present - if !root.contains_key("podTemplate") { - let pod_env = spec_pod_env(spec); - root.insert( - "podTemplate".to_string(), - sandbox_template_to_k8s_with_validated_config( - &SandboxTemplate::default(), - driver_gpu_requirements(spec.and_then(|s| s.resource_requirements.as_ref())), - &pod_env, - spec, - &driver_config, - inject_workspace, - params, - ), - ); - } +fn proxy_pod_service_dns(service_name: &str, namespace: &str) -> String { + // Search-domain-relative rather than a hardcoded `.svc.cluster.local` FQDN: + // clusters can run a custom cluster domain, and the pod resolver's search + // list (`.svc.`, `svc.`, ``) resolves this form + // on any domain. Hardcoding `cluster.local` would leave the workload's + // wait-for-proxy init container unable to resolve its supervisor there. + format!("{service_name}.{namespace}.svc") +} - Ok(serde_json::Value::Object( - std::iter::once(("spec".to_string(), serde_json::Value::Object(root))).collect(), - )) +fn proxy_pod_proxy_url(service_dns: &str) -> String { + format!("http://{service_dns}:{PROXY_POD_PROXY_PORT}") } -#[cfg(test)] -fn sandbox_template_to_k8s( - template: &SandboxTemplate, - gpu: bool, - spec_environment: &std::collections::HashMap, - inject_workspace: bool, - params: &SandboxPodParams<'_>, -) -> serde_json::Value { - let gpu_requirements = gpu.then_some(GpuResourceRequirements { count: None }); - let driver_config = KubernetesSandboxDriverConfig::from_template(template) - .expect("test Kubernetes driver_config should be valid"); - sandbox_template_to_k8s_with_validated_config( - template, - gpu_requirements.as_ref(), - spec_environment, - None, - &driver_config, - inject_workspace, - params, - ) +fn apply_host_gateway_aliases( + spec: &mut serde_json::Map, + host_gateway_ip: &str, +) { + if host_gateway_ip.is_empty() { + return; + } + spec.insert( + "hostAliases".to_string(), + serde_json::json!([{ + "ip": host_gateway_ip, + "hostnames": ["host.docker.internal", "host.openshell.internal"] + }]), + ); } -#[cfg(test)] -fn sandbox_template_to_k8s_with_gpu_requirements( - template: &SandboxTemplate, - gpu_requirements: Option<&GpuResourceRequirements>, +fn copy_log_level_env( + env: &mut Vec, + template_environment: &std::collections::HashMap, spec_environment: &std::collections::HashMap, - inject_workspace: bool, - params: &SandboxPodParams<'_>, -) -> serde_json::Value { - let driver_config = KubernetesSandboxDriverConfig::from_template(template) - .expect("test Kubernetes driver_config should be valid"); - sandbox_template_to_k8s_with_validated_config( - template, - gpu_requirements, - spec_environment, - None, - &driver_config, - inject_workspace, - params, - ) +) { + if let Some(value) = spec_environment + .get(openshell_core::sandbox_env::LOG_LEVEL) + .or_else(|| template_environment.get(openshell_core::sandbox_env::LOG_LEVEL)) + { + upsert_env(env, openshell_core::sandbox_env::LOG_LEVEL, value); + } } -fn sandbox_template_to_k8s_with_validated_config( - template: &SandboxTemplate, - gpu_requirements: Option<&GpuResourceRequirements>, +fn supervisor_sidecar_env( + template_environment: &std::collections::HashMap, spec_environment: &std::collections::HashMap, - sandbox_spec: Option<&openshell_core::proto::compute::v1::DriverSandboxSpec>, - driver_config: &KubernetesSandboxDriverConfig, - inject_workspace: bool, params: &SandboxPodParams<'_>, -) -> serde_json::Value { - let mut metadata = serde_json::Map::new(); - let mut pod_labels = template - .labels - .iter() - .map(|(key, value)| (key.clone(), serde_json::Value::String(value.clone()))) - .collect::>(); - if params.provider_spiffe_enabled { - pod_labels.insert( - LABEL_MANAGED_BY.to_string(), - serde_json::Value::String(LABEL_MANAGED_BY_VALUE.to_string()), +) -> Vec { + let mut env = Vec::new(); + apply_required_env( + &mut env, + params.sandbox_id, + params.sandbox_name, + params.grpc_endpoint, + "", + !params.client_tls_secret_name.is_empty(), + provider_spiffe_socket_path(params), + ); + if !params.client_tls_secret_name.is_empty() { + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CA, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/ca.crt"), ); - if !params.sandbox_id.is_empty() { - pod_labels.insert( - LABEL_SANDBOX_ID.to_string(), - serde_json::Value::String(params.sandbox_id.to_string()), - ); - } - } - if !pod_labels.is_empty() { - metadata.insert("labels".to_string(), serde_json::Value::Object(pod_labels)); - } - // Carry the sandbox UUID as a pod annotation so the gateway can resolve - // a projected SA token claim (pod name + uid) back to a sandbox identity - // when the supervisor calls `IssueSandboxToken` at startup. The gateway - // also verifies the pod's controlling Sandbox ownerReference against the - // live CR before accepting this annotation. Its K8s Role does NOT grant - // `patch pods`, so this annotation is effectively immutable post-create. - let mut pod_annotations = platform_config_struct(template, "annotations") - .and_then(|v| match v { - serde_json::Value::Object(map) => Some(map), - _ => None, - }) - .unwrap_or_default(); - if !params.sandbox_id.is_empty() { - pod_annotations.insert( - LABEL_SANDBOX_ID.to_string(), - serde_json::Value::String(params.sandbox_id.to_string()), + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CERT, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.crt"), ); - } - if !pod_annotations.is_empty() { - metadata.insert( - "annotations".to_string(), - serde_json::Value::Object(pod_annotations), + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_KEY, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.key"), ); } - - let mut spec = serde_json::Map::new(); - let runtime_class_name = platform_config_string(template, "runtime_class_name") - .or_else(|| { - (!driver_config.pod.runtime_class_name.is_empty()) - .then(|| driver_config.pod.runtime_class_name.clone()) - }) - .or_else(|| { - (!params.default_runtime_class_name.is_empty()) - .then(|| params.default_runtime_class_name.to_string()) - }); - if let Some(runtime_class) = runtime_class_name { - spec.insert( - "runtimeClassName".to_string(), - serde_json::json!(runtime_class), + copy_log_level_env(&mut env, template_environment, spec_environment); + upsert_env( + &mut env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "sidecar", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + "sidecar-nftables", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, + SIDECAR_CONTROL_SOCKET, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + SIDECAR_SSH_SOCKET_FILE, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + apply_resolved_identity_env(&mut env, params.sandbox_uid, params.sandbox_gid); + if !params.process_binary_aware_network_policy { + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + "relaxed", ); } - if let Some(node_selector) = platform_config_struct(template, "node_selector") { - spec.insert("nodeSelector".to_string(), node_selector); + env +} + +fn supervisor_sidecar_container( + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) -> serde_json::Value { + let proxy_uid = effective_sidecar_proxy_uid(params); + let capabilities = if params.process_binary_aware_network_policy { + serde_json::json!({ + "drop": ["ALL"], + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] + }) + } else { + serde_json::json!({ + "drop": ["ALL"] + }) + }; + let mut container = serde_json::json!({ + "name": SUPERVISOR_NETWORK_SIDECAR_NAME, + "image": params.supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network", + ], + "env": supervisor_sidecar_env(template_environment, spec_environment, params), + "securityContext": { + "runAsUser": proxy_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": proxy_uid != 0, + "allowPrivilegeEscalation": false, + "capabilities": capabilities + }, + "volumeMounts": [ + sidecar_state_volume_mount(), + sidecar_tls_volume_mount(), + { + "name": "openshell-sa-token", + "mountPath": "/var/run/secrets/openshell", + "readOnly": true + } + ] + }); + container["command"] + .as_array_mut() + .expect("network supervisor command is an array") + .extend( + upstream_proxy_cli_args(params, true) + .into_iter() + .map(serde_json::Value::String), + ); + if !params.supervisor_image_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); } - if let Some(tolerations) = platform_config_struct(template, "tolerations") { - spec.insert("tolerations".to_string(), tolerations); + if params.provider_spiffe_enabled { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), + "readOnly": true, + })); } - apply_pod_driver_config(&mut spec, &driver_config.pod); - - // Per-sandbox portable intent overrides the cluster-wide default. This - // driver owns the Kubernetes-specific `hostUsers` translation. Accept the - // former platform_config encoding during rolling upgrades from gateways - // that predate the typed field. - let use_user_namespaces = template - .user_namespaces - .or_else(|| platform_config_bool(template, "host_users").map(|host_users| !host_users)) - .unwrap_or(params.enable_user_namespaces); - - if use_user_namespaces { - spec.insert("hostUsers".to_string(), serde_json::json!(false)); - if gpu_requirements.is_some() { - warn!( - "GPU sandbox with user namespaces enabled — \ - NVIDIA device plugin compatibility is unverified" - ); - } + if has_upstream_proxy_credentials(params) { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(upstream_proxy_auth_volume_mount()); } - - if !params.service_account_name.is_empty() { - spec.insert( - "serviceAccountName".to_string(), - serde_json::json!(params.service_account_name), - ); + if let Some(profile) = params.app_armor_profile { + container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); } + container +} - let image_pull_secrets = image_pull_secret_refs(params.image_pull_secrets); - if !image_pull_secrets.is_empty() { - spec.insert( - "imagePullSecrets".to_string(), - serde_json::Value::Array(image_pull_secrets), - ); +fn effective_sidecar_proxy_uid(params: &SandboxPodParams<'_>) -> u32 { + if params.process_binary_aware_network_policy { + BINARY_AWARE_SIDECAR_PROXY_UID + } else { + params.proxy_uid } +} - // Disable service account token auto-mounting for security hardening. - // Sandbox pods should not have access to the Kubernetes API by default. - spec.insert( - "automountServiceAccountToken".to_string(), - serde_json::json!(false), - ); - // Do not let kubelet replace the canonical main-process generation after - // the supervisor exits. The gateway records that exit as terminal Error. - spec.insert("restartPolicy".to_string(), serde_json::json!("Never")); +fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_json::Value { + let proxy_uid = effective_sidecar_proxy_uid(params); + let mut container = serde_json::json!({ + "name": SUPERVISOR_NETWORK_INIT_CONTAINER_NAME, + "image": params.supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network-init", + "--proxy-uid", + proxy_uid.to_string(), + "--proxy-gid", + params.sandbox_gid.to_string(), + "--sidecar-state-dir", + SIDECAR_STATE_MOUNT_PATH, + "--sidecar-tls-dir", + SIDECAR_TLS_MOUNT_PATH, + ], + "securityContext": { + "runAsUser": 0, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"], + "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] + } + }, + "volumeMounts": [ + sidecar_state_volume_mount(), + sidecar_tls_volume_mount(), + ] + }); + if !params.supervisor_image_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + } + if !params.client_tls_secret_name.is_empty() { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": "openshell-client-tls", + "mountPath": openshell_core::container_paths::CLIENT_TLS_DIR, + "readOnly": true + })); + } + if let Some(profile) = params.app_armor_profile { + container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); + } + container +} - let mut container = serde_json::Map::new(); - container.insert("name".to_string(), serde_json::json!("agent")); - // Use template image if provided, otherwise fall back to default - let image = if template.image.is_empty() { - params.default_image - } else { - &template.image +fn apply_supervisor_sidecar_topology( + pod_template: &mut serde_json::Value, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) { + let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { + return; }; - if !image.is_empty() { - container.insert("image".to_string(), serde_json::json!(image)); - if !params.image_pull_policy.is_empty() { - container.insert( - "imagePullPolicy".to_string(), - serde_json::json!(params.image_pull_policy), - ); - } + + let pod_security_context = spec + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(sc) = pod_security_context.as_object_mut() { + sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); } - // Build environment variables - start with OpenShell-required vars - let env = build_env_list( + spec.insert("shareProcessNamespace".to_string(), serde_json::json!(true)); + + apply_supervisor_binary_source( + spec, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, None, - &template.environment, - spec_environment, - sandbox_spec, - params.sandbox_id, - params.sandbox_name, - params.grpc_endpoint, - params.ssh_socket_path, - !params.client_tls_secret_name.is_empty(), - provider_spiffe_socket_path(params), ); - container.insert("env".to_string(), serde_json::Value::Array(env)); - - let mut capabilities: Vec<&str> = vec!["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"]; - if use_user_namespaces { - // In a user namespace the bounding set is reset. SETUID/SETGID are - // needed for the supervisor to drop privileges to the sandbox user. - // DAC_READ_SEARCH is needed for cross-UID /proc//fd/ access - // for process identity resolution in network policy enforcement. - capabilities.extend(["SETUID", "SETGID", "DAC_READ_SEARCH"]); - } - let mut security_context = serde_json::json!({ - "capabilities": { - "add": capabilities - } - }); - if let Some(profile) = params.app_armor_profile { - security_context["appArmorProfile"] = app_armor_profile_to_k8s(profile); - } - container.insert("securityContext".to_string(), security_context); - - // Mount client TLS secret for mTLS to the server. Gateway identity uses - // the projected ServiceAccount bootstrap token. Provider token grants may - // additionally mount the SPIFFE Workload API socket. - let mut volume_mounts: Vec = Vec::new(); - if !params.client_tls_secret_name.is_empty() { - volume_mounts.push(serde_json::json!({ - "name": CLIENT_TLS_VOLUME_NAME, - "mountPath": openshell_core::container_paths::CLIENT_TLS_DIR, - "readOnly": true - })); - } - if params.provider_spiffe_enabled { - volume_mounts.push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), - "readOnly": true, - })); - } - volume_mounts.push(serde_json::json!({ - "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, - "mountPath": SERVICE_ACCOUNT_TOKEN_MOUNT_PATH, - "readOnly": true, - })); - volume_mounts.extend( - driver_config - .containers - .agent - .volume_mounts - .iter() - .map(kubernetes_driver_volume_mount_to_k8s), - ); - container.insert( - "volumeMounts".to_string(), - serde_json::Value::Array(volume_mounts), - ); - - if let Some(resources) = container_resources(template, gpu_requirements) { - container.insert("resources".to_string(), resources); - } - apply_agent_driver_resources(&mut container, &driver_config.containers.agent.resources); - spec.insert( - "containers".to_string(), - serde_json::Value::Array(vec![serde_json::Value::Object(container)]), - ); - - // Add TLS secret volume. Combined mode uses mode 0400 because the - // supervisor starts as root and drops privileges before running workload - // children. Sidecar mode keeps the process supervisor non-root, so it uses - // pod fsGroup + 0440 to preserve gateway session and SSH control behavior. - let mut volumes: Vec = Vec::new(); - if !params.client_tls_secret_name.is_empty() { - let client_tls_default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; - volumes.push(serde_json::json!({ - "name": CLIENT_TLS_VOLUME_NAME, - "secret": { - "secretName": params.client_tls_secret_name, - "defaultMode": client_tls_default_mode - } - })); - } - if has_upstream_proxy_credentials(params) { - let secret_name = params - .proxy_auth_secret_name - .expect("complete proxy credential reference has a Secret name"); - let secret_key = params - .proxy_auth_secret_key - .expect("complete proxy credential reference has a Secret key"); - // The credential volume is mounted only into the container that runs - // network supervision. Sidecar mode uses the pod fsGroup already - // required for its non-root network supervisor. - let default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; + let volumes = spec + .entry("volumes") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volumes) = volumes { volumes.push(serde_json::json!({ - "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, - "secret": { - "secretName": secret_name, - "defaultMode": default_mode, - "items": [{ - "key": secret_key, - "path": upstream_proxy_auth_file_name(), - }] - } + "name": SIDECAR_STATE_VOLUME_NAME, + "emptyDir": {} })); - } - if params.provider_spiffe_enabled { volumes.push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "csi": { - "driver": "csi.spiffe.io", - "readOnly": true - } + "name": SIDECAR_TLS_VOLUME_NAME, + "emptyDir": {} })); } - // Projected ServiceAccountToken volume — kubelet writes a short-lived - // audience-bound JWT into /var/run/secrets/openshell/token and rotates - // it automatically. The supervisor exchanges this for a gateway-minted - // JWT via `IssueSandboxToken` once at startup. In sidecar topology both - // supervisor containers run with the sandbox GID and need group-read access. - let sa_token_default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; - volumes.push(serde_json::json!({ - "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, - "projected": { - "sources": [{ - "serviceAccountToken": { - "audience": "openshell-gateway", - "expirationSeconds": params.sa_token_ttl_secs, - "path": "token" - } - }], - "defaultMode": sa_token_default_mode - } - })); - volumes.extend( - driver_config - .volumes - .iter() - .map(kubernetes_driver_volume_to_k8s), - ); - spec.insert("volumes".to_string(), serde_json::Value::Array(volumes)); - - // Add hostAliases so sandbox pods can reach the Docker host. - if !params.host_gateway_ip.is_empty() { - spec.insert( - "hostAliases".to_string(), - serde_json::json!([{ - "ip": params.host_gateway_ip, - "hostnames": ["host.docker.internal", "host.openshell.internal"] - }]), - ); - } - let mut template_value = serde_json::Map::new(); - if !metadata.is_empty() { - template_value.insert("metadata".to_string(), serde_json::Value::Object(metadata)); + let init_containers = spec + .entry("initContainers") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(init_containers) = init_containers { + init_containers.push(supervisor_network_init_container(params)); } - template_value.insert("spec".to_string(), serde_json::Value::Object(spec)); - let mut result = serde_json::Value::Object(template_value); + let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { + return; + }; - match params.topology { - SupervisorTopology::Combined => { - apply_supervisor_sideload_with_params(&mut result, params); - } - SupervisorTopology::Sidecar => { - apply_supervisor_sidecar_topology( - &mut result, - &template.environment, - spec_environment, - params, - ); - } - } + let target_index = containers + .iter() + .position(|c| c.get("name").and_then(|v| v.as_str()) == Some("agent")) + .unwrap_or(0); - // Inject workspace persistence (init container + PVC volume mount) so - // that /sandbox data survives pod rescheduling. Skipped when the user - // provides custom storage through driver_config. - if inject_workspace { - apply_workspace_persistence( - &mut result, - image, - params.image_pull_policy, - params.sandbox_gid, + if let Some(container) = containers + .get_mut(target_index) + .and_then(|v| v.as_object_mut()) + { + container.insert( + "command".to_string(), + serde_json::json!([ + format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), + "--mode=process", + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT + ]), ); - } - - result -} -fn apply_pod_driver_config( - spec: &mut serde_json::Map, - config: &KubernetesPodDriverConfig, -) { - if !config.node_selector.is_empty() { - let node_selector = spec - .entry("nodeSelector".to_string()) + let security_context = container + .entry("securityContext") .or_insert_with(|| serde_json::json!({})); - merge_string_map(node_selector, &config.node_selector); - } - - if !config.priority_class_name.is_empty() { - spec.entry("priorityClassName".to_string()) - .or_insert_with(|| serde_json::json!(config.priority_class_name)); - } - - if !config.tolerations.is_empty() { - let tolerations = spec - .entry("tolerations".to_string()) - .or_insert_with(|| serde_json::json!([])); - if let Some(existing) = tolerations.as_array_mut() { - existing.extend(config.tolerations.iter().cloned()); - } else { - *tolerations = serde_json::Value::Array(config.tolerations.clone()); + if let Some(sc) = security_context.as_object_mut() { + sc.insert( + "runAsUser".to_string(), + serde_json::json!(params.sandbox_uid), + ); + sc.insert( + "runAsGroup".to_string(), + serde_json::json!(params.sandbox_gid), + ); + sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); + sc.insert( + "allowPrivilegeEscalation".to_string(), + serde_json::json!(false), + ); + sc.insert( + "capabilities".to_string(), + serde_json::json!({ + "drop": ["ALL"] + }), + ); } - } -} -fn apply_agent_driver_resources( - container: &mut serde_json::Map, - resources: &KubernetesContainerResourceConfig, -) { - if resources.requests.is_empty() && resources.limits.is_empty() { - return; + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + remove_volume_mount(volume_mounts, "openshell-sa-token"); + remove_volume_mount(volume_mounts, "openshell-client-tls"); + remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); + volume_mounts.push(supervisor_volume_mount()); + volume_mounts.push(sidecar_state_volume_mount()); + volume_mounts.push(sidecar_tls_volume_mount()); + } + + let env = container + .entry("env") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(env) = env { + remove_env(env, openshell_core::sandbox_env::ENDPOINT); + remove_env(env, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); + remove_env(env, openshell_core::sandbox_env::TLS_CA); + remove_env(env, openshell_core::sandbox_env::TLS_CERT); + remove_env(env, openshell_core::sandbox_env::TLS_KEY); + remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN); + remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + remove_env(env, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE); + remove_env( + env, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ); + upsert_env( + env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "sidecar", + ); + upsert_env( + env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + "sidecar-nftables", + ); + upsert_env( + env, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + SIDECAR_SSH_SOCKET_FILE, + ); + upsert_env( + env, + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, + SIDECAR_CONTROL_SOCKET, + ); + upsert_env( + env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); + } } - let target = container - .entry("resources".to_string()) - .or_insert_with(|| serde_json::json!({})); - apply_resource_quantity_map(target, "requests", &resources.requests); - apply_resource_quantity_map(target, "limits", &resources.limits); + containers.push(supervisor_sidecar_container( + template_environment, + spec_environment, + params, + )); } -fn merge_string_map(target: &mut serde_json::Value, values: &BTreeMap) { - if !target.is_object() { - *target = serde_json::json!({}); - } - let target = target - .as_object_mut() - .expect("target was converted to object"); - for (key, value) in values { - target - .entry(key.clone()) - .or_insert_with(|| serde_json::json!(value)); - } +fn proxy_pod_ca_source_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": "openshell-proxy-pod-ca-source", + "mountPath": PROXY_POD_CA_SECRET_MOUNT_PATH, + "readOnly": true + }) } -fn apply_resource_quantity_map( - target: &mut serde_json::Value, - section: &str, - values: &BTreeMap, -) { - if values.is_empty() { - return; - } - if !target.is_object() { - *target = serde_json::json!({}); - } - let target = target - .as_object_mut() - .expect("target was converted to object"); - let section_value = target - .entry(section.to_string()) - .or_insert_with(|| serde_json::json!({})); - merge_string_map(section_value, values); +fn proxy_pod_ca_tls_volume_mount(read_only: bool) -> serde_json::Value { + serde_json::json!({ + "name": "openshell-proxy-pod-tls", + "mountPath": SIDECAR_TLS_MOUNT_PATH, + "readOnly": read_only, + }) } -fn image_pull_secret_refs(secrets: &[String]) -> Vec { - secrets - .iter() - .map(|secret| secret.trim()) - .filter(|secret| !secret.is_empty()) - .map(|secret| serde_json::json!({ "name": secret })) - .collect() +fn proxy_pod_ca_init_container( + image: &str, + image_pull_policy: &str, + run_as_user: u32, + run_as_group: u32, +) -> serde_json::Value { + let copy_cmd = format!( + "set -eu; \ + mkdir -p {SIDECAR_TLS_MOUNT_PATH}; \ + cp {PROXY_POD_CA_SECRET_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE} {SIDECAR_TLS_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE}; \ + bundle={SIDECAR_TLS_MOUNT_PATH}/ca-bundle.pem; \ + found=0; \ + for path in /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/ca-bundle.pem /etc/ssl/cert.pem; do \ + if [ -f \"$path\" ]; then cat \"$path\" > \"$bundle\"; found=1; break; fi; \ + done; \ + if [ \"$found\" = 0 ]; then : > \"$bundle\"; fi; \ + printf '\\n' >> \"$bundle\"; \ + cat {PROXY_POD_CA_SECRET_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE} >> \"$bundle\"" + ); + let mut init_spec = serde_json::json!({ + "name": "openshell-proxy-ca-install", + "image": image, + "command": ["sh", "-c", copy_cmd], + "securityContext": { + "runAsUser": run_as_user, + "runAsGroup": run_as_group, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { + "drop": ["ALL"] + } + }, + "volumeMounts": [ + proxy_pod_ca_source_volume_mount(), + proxy_pod_ca_tls_volume_mount(false), + ] + }); + if !image_pull_policy.is_empty() { + init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + } + init_spec } -fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { - let mut value = serde_json::json!({ - "type": profile.to_k8s_type() +fn proxy_pod_wait_for_proxy_init_container( + image: &str, + image_pull_policy: &str, + run_as_user: u32, + run_as_group: u32, + service_dns: &str, +) -> serde_json::Value { + let mut init_spec = serde_json::json!({ + "name": PROXY_POD_WAIT_INIT_CONTAINER_NAME, + "image": image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "wait-for-tcp", + format!("{service_dns}:{PROXY_POD_PROXY_PORT}"), + PROXY_POD_WAIT_TIMEOUT_SECS.to_string(), + ], + "securityContext": { + "runAsUser": run_as_user, + "runAsGroup": run_as_group, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { + "drop": ["ALL"] + } + } }); - if let Some(localhost_profile) = profile.localhost_profile() { - value["localhostProfile"] = serde_json::json!(localhost_profile); + if !image_pull_policy.is_empty() { + init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); } - value + init_spec } -fn container_resources( - template: &SandboxTemplate, - gpu_requirements: Option<&GpuResourceRequirements>, -) -> Option { - // Start from the raw resources passthrough in platform_config (preserves - // custom resource types like GPU limits that users set via the public API - // Struct), then overlay the typed DriverResourceRequirements on top. - let mut resources = - platform_config_struct(template, "resources_raw").unwrap_or_else(|| serde_json::json!({})); +fn apply_proxy_pod_affinity( + spec: &mut serde_json::Map, + sandbox_id: &str, + mode: ProxyPodAffinity, +) { + if sandbox_id.is_empty() || mode == ProxyPodAffinity::Disabled { + return; + } - // Overlay typed CPU/memory from DriverResourceRequirements. - if let Some(ref req) = template.resources { - let obj = resources.as_object_mut().unwrap(); - let mut apply = |section: &str, key: &str, value: &str| { - if !value.is_empty() { - let sec = obj.entry(section).or_insert_with(|| serde_json::json!({})); - sec[key] = serde_json::json!(value); - } - }; - apply("limits", "cpu", &req.cpu_limit); - apply("limits", "memory", &req.memory_limit); + let term = serde_json::json!({ + "labelSelector": { + "matchLabels": proxy_pod_match_labels(sandbox_id, SANDBOX_ROLE_SUPERVISOR) + }, + "topologyKey": "kubernetes.io/hostname" + }); - let cpu_request = if req.cpu_request.is_empty() { - &req.cpu_limit - } else { - &req.cpu_request - }; - let memory_request = if req.memory_request.is_empty() { - &req.memory_limit - } else { - &req.memory_request - }; - apply("requests", "cpu", cpu_request); - apply("requests", "memory", memory_request); + let affinity = spec + .entry("affinity".to_string()) + .or_insert_with(|| serde_json::json!({})); + if !affinity.is_object() { + *affinity = serde_json::json!({}); } - - if let Some(gpu) = gpu_requirements { - let quantity = gpu.count.unwrap_or(1).to_string(); - apply_gpu_limit(&mut resources, &quantity); + let affinity = affinity + .as_object_mut() + .expect("affinity was converted to object"); + let pod_affinity = affinity + .entry("podAffinity".to_string()) + .or_insert_with(|| serde_json::json!({})); + if !pod_affinity.is_object() { + *pod_affinity = serde_json::json!({}); } - if resources.as_object().is_some_and(serde_json::Map::is_empty) { - None - } else { - Some(resources) + let pod_affinity = pod_affinity + .as_object_mut() + .expect("podAffinity was converted to object"); + match mode { + ProxyPodAffinity::Disabled => {} + ProxyPodAffinity::Preferred => { + let preferred = pod_affinity + .entry("preferredDuringSchedulingIgnoredDuringExecution".to_string()) + .or_insert_with(|| serde_json::json!([])); + if !preferred.is_array() { + *preferred = serde_json::json!([]); + } + if let Some(preferred) = preferred.as_array_mut() { + preferred.push(serde_json::json!({ + "weight": 100, + "podAffinityTerm": term, + })); + } + } + ProxyPodAffinity::Required => { + let required = pod_affinity + .entry("requiredDuringSchedulingIgnoredDuringExecution".to_string()) + .or_insert_with(|| serde_json::json!([])); + if !required.is_array() { + *required = serde_json::json!([]); + } + if let Some(required) = required.as_array_mut() { + required.push(term); + } + } } } -fn apply_gpu_limit(resources: &mut serde_json::Value, quantity: &str) { - let Some(resources_obj) = resources.as_object_mut() else { - *resources = serde_json::json!({}); - return apply_gpu_limit(resources, quantity); +fn apply_supervisor_proxy_pod_topology( + pod_template: &mut serde_json::Value, + params: &SandboxPodParams<'_>, +) { + let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { + return; }; - let limits = resources_obj - .entry("limits") + let pod_security_context = spec + .entry("securityContext") .or_insert_with(|| serde_json::json!({})); - let Some(limits_obj) = limits.as_object_mut() else { - *limits = serde_json::json!({}); - return apply_gpu_limit(resources, quantity); - }; + if let Some(sc) = pod_security_context.as_object_mut() { + sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); + } - limits_obj.insert(GPU_RESOURCE_NAME.to_string(), serde_json::json!(quantity)); -} + apply_proxy_pod_affinity(spec, params.sandbox_id, params.proxy_pod_affinity); -#[allow(clippy::too_many_arguments)] -fn build_env_list( - existing_env: Option<&Vec>, - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - sandbox_spec: Option<&openshell_core::proto::compute::v1::DriverSandboxSpec>, - sandbox_id: &str, - sandbox_name: &str, - grpc_endpoint: &str, - ssh_socket_path: &str, - tls_enabled: bool, - provider_spiffe_socket_path: Option<&str>, -) -> Vec { - let mut env = existing_env.cloned().unwrap_or_default(); - apply_env_map(&mut env, template_environment); - apply_env_map(&mut env, spec_environment); - let mut user_env = template_environment.clone(); - user_env.extend(spec_environment.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - upsert_env( - &mut env, - openshell_core::sandbox_env::USER_ENVIRONMENT, - &json, - ); - } - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox_spec) - .expect("main process config serialization cannot fail"); - upsert_env( - &mut env, - openshell_core::sandbox_env::MAIN_PROCESS_SPEC, - &main_process, - ); - apply_required_env( - &mut env, - sandbox_id, - sandbox_name, - grpc_endpoint, - ssh_socket_path, - tls_enabled, - provider_spiffe_socket_path, - ); - env -} + let names = proxy_pod_resource_names(params.cr_name, params.sandbox_id); + let service_dns = proxy_pod_service_dns(&names.service, params.namespace); -fn apply_env_map( - env: &mut Vec, - values: &std::collections::HashMap, -) { - for (key, value) in values { - upsert_env(env, key, value); + let volumes = spec + .entry("volumes") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volumes) = volumes { + // The provider SPIFFE Workload API socket belongs only in the supervisor + // (proxy) pod, which mints provider credentials. The agent pod holds a + // scoped process-kind credential that cannot access provider secrets, so + // it must not carry an SVID: drop the CSI volume entirely (no container + // in the agent pod mounts it after the cleanup below). + volumes.retain(|v| { + v.get("name").and_then(serde_json::Value::as_str) + != Some(SPIFFE_WORKLOAD_API_VOLUME_NAME) + }); + volumes.push(serde_json::json!({ + "name": "openshell-proxy-pod-ca-source", + "secret": { + "secretName": names.proxy_ca_secret, + "defaultMode": 0o444, + "items": [{ + "key": PROXY_POD_CA_CERT_FILE, + "path": PROXY_POD_CA_CERT_FILE, + }] + } + })); + volumes.push(serde_json::json!({ + "name": "openshell-proxy-pod-tls", + "emptyDir": {} + })); } -} -// Required env vars are passed individually for clarity at call sites; grouping into a struct -// would not improve readability for this internal helper. -fn apply_required_env( - env: &mut Vec, - sandbox_id: &str, - sandbox_name: &str, - grpc_endpoint: &str, - ssh_socket_path: &str, - tls_enabled: bool, - provider_spiffe_socket_path: Option<&str>, -) { - upsert_env(env, openshell_core::sandbox_env::SANDBOX_ID, sandbox_id); - upsert_env(env, openshell_core::sandbox_env::SANDBOX, sandbox_name); - upsert_env(env, openshell_core::sandbox_env::ENDPOINT, grpc_endpoint); - upsert_env( - env, - openshell_core::sandbox_env::TELEMETRY_ENABLED, - openshell_core::telemetry::enabled_env_value(), - ); - // Runtime capabilities are driver-owned. Kubernetes topologies do not yet - // provide the complete policy DNS and transparent TCP substrate. - upsert_env( - env, - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - "", - ); - if !ssh_socket_path.is_empty() { - upsert_env( - env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - ssh_socket_path, - ); - } - // TLS cert paths for sandbox-to-server mTLS. Only set when TLS is enabled - // and the client TLS secret is mounted into the sandbox pod. - if tls_enabled { - upsert_env( - env, - openshell_core::sandbox_env::TLS_CA, - "/etc/openshell-tls/client/ca.crt", - ); - upsert_env( - env, - openshell_core::sandbox_env::TLS_CERT, - "/etc/openshell-tls/client/tls.crt", - ); - upsert_env( - env, - openshell_core::sandbox_env::TLS_KEY, - "/etc/openshell-tls/client/tls.key", - ); - } - // Projected ServiceAccount token written by kubelet (see the volume - // definition in `sandbox_template_to_k8s`). The supervisor reads this - // and exchanges it for a gateway-minted JWT via `IssueSandboxToken`. - upsert_env( - env, - openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, - "/var/run/secrets/openshell/token", + // The agent pod runs the process supervisor (`--mode=process`), so it needs + // the supervisor binary mounted just like combined/sidecar. Unlike those + // root-capable topologies, the proxy-pod agent pod runs under a non-root + // SCC, so the copy-self init container must run as the sandbox uid/gid. + apply_supervisor_binary_source( + spec, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, + Some((params.sandbox_uid, params.sandbox_gid)), ); - if let Some(socket_path) = provider_spiffe_socket_path { - upsert_env( - env, - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - socket_path, - ); + + let init_containers = spec + .entry("initContainers") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(init_containers) = init_containers { + init_containers.push(proxy_pod_ca_init_container( + params.supervisor_image, + params.supervisor_image_pull_policy, + params.sandbox_uid, + params.sandbox_gid, + )); + // Hold the workload until the paired supervisor is accepting proxy + // connections, so the process supervisor's children never race ahead of + // a working egress path. + init_containers.push(proxy_pod_wait_for_proxy_init_container( + params.supervisor_image, + params.supervisor_image_pull_policy, + params.sandbox_uid, + params.sandbox_gid, + &service_dns, + )); } -} -fn provider_spiffe_socket_path<'a>(params: &'a SandboxPodParams<'a>) -> Option<&'a str> { - params - .provider_spiffe_enabled - .then_some(params.provider_spiffe_workload_api_socket_path) -} + let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { + return; + }; + let target_index = containers + .iter() + .position(|c| c.get("name").and_then(|v| v.as_str()) == Some("agent")) + .unwrap_or(0); + if let Some(container) = containers + .get_mut(target_index) + .and_then(|v| v.as_object_mut()) + { + // Run the process supervisor as the agent container's entrypoint; it + // launches the workload (from MAIN_PROCESS_SPEC) and serves relays + // locally. Network enforcement is external (the paired proxy pod), so + // there is no `--mode=network` and no in-pod netns/nftables. + container.insert( + "command".to_string(), + serde_json::json!([ + format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), + "--mode=process", + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT + ]), + ); -fn spiffe_socket_mount_path(socket_path: &str) -> String { - Path::new(socket_path) - .parent() - .and_then(Path::to_str) - .filter(|path| !path.is_empty() && *path != "/") - .expect("provider SPIFFE socket path should be validated before pod rendering") - .to_string() -} + let security_context = container + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if !security_context.is_object() { + *security_context = serde_json::json!({}); + } + if let Some(sc) = security_context.as_object_mut() { + sc.insert( + "runAsUser".to_string(), + serde_json::json!(params.sandbox_uid), + ); + sc.insert( + "runAsGroup".to_string(), + serde_json::json!(params.sandbox_gid), + ); + sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); + sc.insert( + "allowPrivilegeEscalation".to_string(), + serde_json::json!(false), + ); + sc.insert( + "capabilities".to_string(), + serde_json::json!({ "drop": ["ALL"] }), + ); + } -fn upsert_env(env: &mut Vec, name: &str, value: &str) { - if let Some(existing) = env - .iter_mut() - .find(|item| item.get("name").and_then(|value| value.as_str()) == Some(name)) - { - *existing = serde_json::json!({"name": name, "value": value}); - return; - } + // Retain the gateway credentials: unlike the network-only design, the + // in-pod process supervisor holds its own (scoped, process-kind) gateway + // session for relays, policy, and log push. Add the supervisor binary + // mount and the proxy CA trust dir. + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + // Drop the provider SPIFFE Workload API mount from the agent + // container; it belongs only in the supervisor (proxy) pod. + remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); + volume_mounts.push(supervisor_volume_mount()); + volume_mounts.push(proxy_pod_ca_tls_volume_mount(true)); + } - env.push(serde_json::json!({"name": name, "value": value})); -} + let env = container + .entry("env") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(env) = env { + // The provider SPIFFE Workload API socket lives only in the proxy + // pod; strip its env so the agent never advertises a socket it no + // longer mounts. + remove_env( + env, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ); -fn apply_resolved_identity_env(env: &mut Vec, uid: u32, gid: u32) { - remove_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER); - remove_env(env, openshell_core::sandbox_env::SANDBOX_UID); - remove_env(env, openshell_core::sandbox_env::SANDBOX_GID); - upsert_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER, ""); - upsert_env( - env, - openshell_core::sandbox_env::SANDBOX_UID, - &uid.to_string(), - ); - upsert_env( - env, - openshell_core::sandbox_env::SANDBOX_GID, - &gid.to_string(), - ); -} + // Select the process-supervisor + remote-proxy runtime path: own + // gateway session (no sidecar control socket), NetworkOnly + // enforcement, and the proxy CA read from the mounted TLS dir. + upsert_env( + env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "proxy-pod", + ); + upsert_env( + env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + "proxy-pod", + ); + upsert_env( + env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); -fn remove_env(env: &mut Vec, name: &str) { - env.retain(|item| item.get("name").and_then(|value| value.as_str()) != Some(name)); -} + // The default SSH relay socket lives under `/run`, whose parent the + // non-root agent cannot create. Use an abstract socket instead: it + // needs no writable directory and is scoped to this pod's netns. + upsert_env( + env, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + PROXY_POD_SSH_SOCKET_FILE, + ); -fn remove_volume_mount(volume_mounts: &mut Vec, name: &str) { - volume_mounts.retain(|mount| mount.get("name").and_then(|value| value.as_str()) != Some(name)); -} + // The workload's children egress through the remote proxy via these + // proxy variables; the supervisor's own gateway dial is a direct + // gRPC channel that does not consult them. + let proxy_url = proxy_pod_proxy_url(&service_dns); + // The process supervisor clears relay-spawned child environments + // and reconstructs proxy variables from this canonical URL. + upsert_env(env, openshell_core::sandbox_env::PROXY_URL, &proxy_url); + for name in [ + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "http_proxy", + "https_proxy", + "grpc_proxy", + ] { + upsert_env(env, name, &proxy_url); + } + for name in ["NO_PROXY", "no_proxy"] { + upsert_env(env, name, "127.0.0.1,localhost,::1"); + } + upsert_env(env, "NODE_USE_ENV_PROXY", "1"); -/// Extract a string value from the template's `platform_config` Struct. -fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option { - let config = template.platform_config.as_ref()?; - let value = config.fields.get(key)?; - match value.kind.as_ref() { - Some(prost_types::value::Kind::StringValue(s)) if !s.is_empty() => Some(s.clone()), - _ => None, + let ca_cert = format!("{SIDECAR_TLS_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE}"); + let ca_bundle = format!("{SIDECAR_TLS_MOUNT_PATH}/ca-bundle.pem"); + for name in ["NODE_EXTRA_CA_CERTS", "DENO_CERT"] { + upsert_env(env, name, &ca_cert); + } + for name in [ + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CURL_CA_BUNDLE", + "GIT_SSL_CAINFO", + ] { + upsert_env(env, name, &ca_bundle); + } + } } } -fn platform_config_bool(template: &SandboxTemplate, key: &str) -> Option { - let config = template.platform_config.as_ref()?; - let value = config.fields.get(key)?; - match value.kind.as_ref() { - Some(prost_types::value::Kind::BoolValue(value)) => Some(*value), - _ => None, - } -} +/// Apply workspace persistence transforms to an already-built pod template. +/// +/// This injects: +/// 1. A volume mount on the agent container at `/sandbox`. +/// 2. An init container (same image) that seeds the PVC with the image's +/// original `/sandbox` contents on first use. +/// +/// The PVC volume itself is **not** added here — the Sandbox CRD controller +/// automatically creates a volume for each entry in `volumeClaimTemplates` +/// (following the `StatefulSet` convention). Adding one here would create a +/// duplicate volume name and fail pod validation. +/// +/// The init container mounts the PVC at a temporary path so it can still see +/// the image's `/sandbox` directory. It checks for a sentinel file and skips +/// the copy if the PVC was already initialised. +#[allow(clippy::similar_names)] +fn apply_workspace_persistence( + pod_template: &mut serde_json::Value, + image: &str, + image_pull_policy: &str, + sandbox_uid: u32, + sandbox_gid: u32, + topology: SupervisorTopology, +) { + let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { + return; + }; -/// Extract a nested Struct value from the template's `platform_config`, -/// converting it to `serde_json::Value`. -fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option { - let config = template.platform_config.as_ref()?; - let value = config.fields.get(key)?; - let json = value_to_json(value); - // Return None for null/empty objects so callers can distinguish - // "field absent" from "field present but empty". - match &json { - serde_json::Value::Null => None, - serde_json::Value::Object(m) if m.is_empty() => None, - _ => Some(json), + // fsGroup is a pod-level field — it instructs kubelet to chown mounted + // volumes to this GID. It is invalid at the container securityContext level. + let pod_sc = spec + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(pod_sc_obj) = pod_sc.as_object_mut() { + pod_sc_obj.insert("fsGroup".to_string(), serde_json::json!(sandbox_gid)); } -} -fn status_from_object(obj: &DynamicObject) -> Option { - let status = obj.data.get("status")?; - let status_obj = status.as_object()?; + // 1. Add workspace volume mount to the agent container + let containers = spec.get_mut("containers").and_then(|v| v.as_array_mut()); + if let Some(containers) = containers { + let mut target_index = None; + for (i, c) in containers.iter().enumerate() { + if c.get("name").and_then(|v| v.as_str()) == Some("agent") { + target_index = Some(i); + break; + } + } + let index = target_index.unwrap_or(0); - let conditions = status_obj - .get("conditions") - .and_then(|val| val.as_array()) - .map(|items| { - items - .iter() - .filter_map(condition_from_value) - .collect::>() - }) - .unwrap_or_default(); + if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + volume_mounts.push(serde_json::json!({ + "name": WORKSPACE_VOLUME_NAME, + "mountPath": WORKSPACE_MOUNT_PATH + })); + } + } + } - Some(SandboxStatus { - sandbox_name: status_obj - .get("sandboxName") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - instance_id: status_obj - .get("agentPod") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - agent_fd: status_obj - .get("agentFd") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - sandbox_fd: status_obj - .get("sandboxFd") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - conditions, - deleting: obj.metadata.deletion_timestamp.is_some(), - }) -} + // 3. Add the init container that seeds the PVC from the image + let init_containers = spec + .entry("initContainers") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(init_containers) = init_containers { + // The init container mounts the PVC at a temp path so it can still + // read the image's original /sandbox contents. It copies them into + // the PVC only when the sentinel file is absent. + // + // Prefer a tar stream over `cp -a`: some sandbox images contain + // self-referential symlinks under `/sandbox/.uv`, and GNU cp can + // fail while seeding the PVC even though preserving the symlink as-is + // is valid. `tar` copies the tree without dereferencing those links. + // Archive only the contents, not the `/sandbox` directory entry + // itself, so extraction never tries to chmod the PVC mount root. + // Extract without restoring owner, mode, or timestamps so the + // non-root init container can seed kubelet-owned PVCs. + // + // The inner `[ -d ... ]` guard handles custom images that don't have + // a /sandbox directory — the copy is skipped but the sentinel is + // still written so subsequent starts are instant. + let copy_cmd = format!( + "if [ ! -f {WORKSPACE_INIT_MOUNT_PATH}/{WORKSPACE_SENTINEL} ]; then \ + if [ -d {WORKSPACE_MOUNT_PATH} ]; then \ + tmp=$(mktemp) && rm -f \"$tmp\" && \ + (cd {WORKSPACE_MOUNT_PATH} && find . -mindepth 1 -maxdepth 1 -exec tar -cf \"$tmp\" {{}} +) && \ + if [ -f \"$tmp\" ]; then \ + tar -C {WORKSPACE_INIT_MOUNT_PATH} --no-same-owner --no-same-permissions --touch -xf \"$tmp\" && \ + rm -f \"$tmp\"; \ + fi; \ + fi && \ + touch {WORKSPACE_INIT_MOUNT_PATH}/{WORKSPACE_SENTINEL}; \ + fi" + ); -fn kubernetes_sandbox_has_stopped_condition(obj: &DynamicObject) -> bool { - obj.data - .get("status") - .and_then(|status| status.get("conditions")) - .and_then(serde_json::Value::as_array) - .is_some_and(|conditions| { - conditions.iter().any(|condition| { - condition.get("type").and_then(serde_json::Value::as_str) - == Some(SANDBOX_SUSPENDED_CONDITION) - && condition - .get("status") - .and_then(serde_json::Value::as_str) - .is_some_and(|status| status.eq_ignore_ascii_case("true")) + let security_context = if topology == SupervisorTopology::ProxyPod { + serde_json::json!({ + "runAsUser": sandbox_uid, + "runAsGroup": sandbox_gid, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + } }) - }) -} - -fn kubernetes_sandbox_stop_is_complete( - api_version: &str, - obj: &DynamicObject, - pod_is_gone: bool, -) -> bool { - if api_version == SANDBOX_VERSION_V1ALPHA1 { - // v1alpha1 omits a usable stopped condition. - pod_is_gone - } else { - kubernetes_sandbox_has_stopped_condition(obj) && pod_is_gone + } else { + serde_json::json!({ + "runAsUser": 0, + }) + }; + let mut init_spec = serde_json::json!({ + "name": WORKSPACE_INIT_CONTAINER_NAME, + "image": image, + "command": ["sh", "-c", copy_cmd], + "securityContext": security_context, + "volumeMounts": [{ + "name": WORKSPACE_VOLUME_NAME, + "mountPath": WORKSPACE_INIT_MOUNT_PATH + }] + }); + if !image_pull_policy.is_empty() { + init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); + } + init_containers.push(init_spec); } } -fn kubernetes_sandbox_stop_failure(obj: &DynamicObject) -> Option { - obj.data - .get("status")? - .get("conditions")? - .as_array()? - .iter() - .find_map(|condition| { - let is_terminal = condition.get("type").and_then(serde_json::Value::as_str) - == Some(SANDBOX_SUSPENDED_CONDITION) - && condition - .get("status") - .and_then(serde_json::Value::as_str) - .is_some_and(|status| status.eq_ignore_ascii_case("false")) - && condition.get("reason").and_then(serde_json::Value::as_str) - == Some(SANDBOX_SUSPENDED_POD_NOT_OWNED_REASON); - if !is_terminal { - return None; +/// Build the default `volumeClaimTemplates` array for sandbox pods. +/// +/// Provides a single PVC named "workspace" that backs the `/sandbox` +/// directory. The init container seeds it from the image on first use. +/// +/// When `storage_class` is non-empty, it is written to the PVC's +/// `storageClassName`. An empty value omits the field so the cluster's +/// default `StorageClass` applies. Clusters with no default `StorageClass` +/// must set this to prevent the PVC from staying `Pending`. +fn default_workspace_volume_claim_templates( + storage_size: &str, + storage_class: &str, +) -> serde_json::Value { + let size = if storage_size.is_empty() { + DEFAULT_WORKSPACE_STORAGE_SIZE + } else { + storage_size + }; + let mut spec = serde_json::json!({ + "accessModes": ["ReadWriteOnce"], + "resources": { + "requests": { + "storage": size } - - let message = condition - .get("message") - .and_then(serde_json::Value::as_str) - .filter(|message| !message.is_empty()) - .unwrap_or("backing pod is not owned by this sandbox"); - Some(format!("Kubernetes sandbox stop rejected: {message}")) - }) + } + }); + if !storage_class.is_empty() { + spec["storageClassName"] = serde_json::json!(storage_class); + } + serde_json::json!([{ + "metadata": { + "name": WORKSPACE_VOLUME_NAME + }, + "spec": spec + }]) } -async fn kubernetes_sandbox_pod_is_gone( - pod_api: &Api, - pod_name: &str, - deadline: tokio::time::Instant, -) -> Result { - let request_timeout = - KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(tokio::time::Instant::now())); - if request_timeout.is_zero() { - return Ok(false); - } +/// Parameters shared by `sandbox_to_k8s_spec` and `sandbox_template_to_k8s`. +#[allow(clippy::struct_excessive_bools)] +struct SandboxPodParams<'a> { + default_image: &'a str, + image_pull_policy: &'a str, + image_pull_secrets: &'a [String], + supervisor_image: &'a str, + supervisor_image_pull_policy: &'a str, + supervisor_sideload_method: SupervisorSideloadMethod, + topology: SupervisorTopology, + proxy_uid: u32, + process_binary_aware_network_policy: bool, + https_proxy: Option<&'a str>, + no_proxy: Option<&'a str>, + proxy_auth_secret_name: Option<&'a str>, + proxy_auth_secret_key: Option<&'a str>, + proxy_auth_allow_insecure: bool, + proxy_connect_by_hostname: bool, + proxy_pod_affinity: ProxyPodAffinity, + proxy_pod_dns_peers: &'a [ProxyPodDnsPeer], + proxy_pod_gateway_peers: &'a [ProxyPodDnsPeer], + namespace: &'a str, + service_account_name: &'a str, + sandbox_id: &'a str, + sandbox_name: &'a str, + /// Gateway that owns this sandbox. Stamped as a label on proxy-pod + /// companions so reconciliation can list and reap only this gateway's + /// resources, never another gateway's. + gateway_id: &'a str, + /// Sandbox CR resource name (`kube_resource_name`), unique per sandbox in + /// every workspace mode. Companion resource names derive from this so they + /// match across the workload pod template and the companion objects. + cr_name: &'a str, + grpc_endpoint: &'a str, + ssh_socket_path: &'a str, + client_tls_secret_name: &'a str, + host_gateway_ip: &'a str, + enable_user_namespaces: bool, + app_armor_profile: Option<&'a AppArmorProfile>, + workspace_default_storage_size: &'a str, + workspace_storage_class: &'a str, + default_runtime_class_name: &'a str, + /// Lifetime (seconds) of the projected `ServiceAccount` token used + /// for the bootstrap `IssueSandboxToken` exchange. + sa_token_ttl_secs: i64, + provider_spiffe_enabled: bool, + provider_spiffe_workload_api_socket_path: &'a str, + /// Resolved sandbox UID for supervisor `runAsUser` and env var. + sandbox_uid: u32, + /// Resolved sandbox GID for PVC init container operations. + sandbox_gid: u32, +} - match tokio::time::timeout(request_timeout, pod_api.get(pod_name)).await { - Ok(Ok(_)) => Ok(false), - Ok(Err(KubeError::Api(err))) if err.code == 404 => Ok(true), - Ok(Err(err)) => Err(err.to_string()), - Err(_) => Err(format!( - "timed out after {}s waiting for Kubernetes API while checking sandbox pod termination", - request_timeout.as_secs() - )), +impl Default for SandboxPodParams<'_> { + fn default() -> Self { + Self { + default_image: "", + image_pull_policy: "", + image_pull_secrets: &[], + supervisor_image: "", + supervisor_image_pull_policy: "", + supervisor_sideload_method: SupervisorSideloadMethod::default(), + topology: SupervisorTopology::default(), + proxy_uid: DEFAULT_PROXY_UID, + process_binary_aware_network_policy: true, + https_proxy: None, + no_proxy: None, + proxy_auth_secret_name: None, + proxy_auth_secret_key: None, + proxy_auth_allow_insecure: false, + proxy_connect_by_hostname: false, + proxy_pod_affinity: ProxyPodAffinity::Disabled, + proxy_pod_dns_peers: &[], + proxy_pod_gateway_peers: &[], + namespace: "default", + service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + sandbox_id: "", + sandbox_name: "", + gateway_id: "", + cr_name: "", + grpc_endpoint: "", + ssh_socket_path: "", + client_tls_secret_name: "", + host_gateway_ip: "", + enable_user_namespaces: false, + app_armor_profile: None, + workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE, + workspace_storage_class: "", + default_runtime_class_name: "", + sa_token_ttl_secs: 3600, + provider_spiffe_enabled: false, + provider_spiffe_workload_api_socket_path: "", + sandbox_uid: DEFAULT_SANDBOX_UID, + sandbox_gid: DEFAULT_SANDBOX_UID, + } } } -fn kubernetes_sandbox_stop_timeout(obj: &DynamicObject) -> Duration { - let termination_grace_period = obj - .data - .get("spec") - .and_then(|spec| spec.get("podTemplate")) - .and_then(|template| template.get("spec")) - .and_then(|spec| spec.get("terminationGracePeriodSeconds")) - .and_then(serde_json::Value::as_u64) - .map_or(DEFAULT_POD_TERMINATION_GRACE_PERIOD, Duration::from_secs); - - // The controller must observe the desired state, wait for the pod grace - // period and kubelet teardown, then reconcile the deleted pod into the - // Sandbox status. Keep one API timeout of headroom around that grace. - termination_grace_period.saturating_add(KUBE_API_TIMEOUT) +fn validate_proxy_identity(params: &SandboxPodParams<'_>) -> Result<(), KubernetesDriverError> { + if matches!( + params.topology, + SupervisorTopology::Sidecar | SupervisorTopology::ProxyPod + ) && params.proxy_uid == params.sandbox_uid + { + let topology = params.topology.to_string(); + return Err(KubernetesDriverError::Precondition(format!( + "proxy_uid ({}) must not match sandbox_uid ({}) in {topology} topology", + params.proxy_uid, params.sandbox_uid + ))); + } + Ok(()) } -fn next_stop_poll_interval(current: Duration) -> Duration { - current.saturating_mul(2).min(STOP_MAX_POLL_INTERVAL) +fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap { + let mut env = spec.map_or_else(Default::default, |s| s.environment.clone()); + if let Some(s) = spec.filter(|s| !s.log_level.is_empty()) { + env.insert( + openshell_core::sandbox_env::LOG_LEVEL.to_string(), + s.log_level.clone(), + ); + } + env } -fn sandbox_operating_state_patch( - api_version: &str, - resource_version: &str, - running: bool, -) -> serde_json::Value { - if api_version == SANDBOX_VERSION_V1BETA1 { - serde_json::json!({ - "metadata": {"resourceVersion": resource_version}, - "spec": {"operatingMode": if running { "Running" } else { "Suspended" }} - }) - } else { - serde_json::json!({ - "metadata": {"resourceVersion": resource_version}, - "spec": {"replicas": i32::from(running)} - }) +/// Reject workload entrypoint overrides in topologies that ignore them. +/// +/// `combined` and `sidecar` replace the agent container's command with the +/// supervisor binary, so an override there would be accepted and then silently +/// dropped. Failing at validation is better than a sandbox that quietly does +/// something other than what was asked. +fn validate_agent_command_for_topology( + config: &KubernetesSandboxDriverConfig, + topology: SupervisorTopology, +) -> Result<(), String> { + let agent = &config.containers.agent; + if agent.command.is_empty() && agent.args.is_empty() { + return Ok(()); } + // Every topology now runs the OpenShell supervisor as the agent container's + // entrypoint (proxy-pod included, since it keeps the process supervisor in + // the pod), so a container command/args override would be ignored. The + // initial workload command is delivered through the sandbox spec / session + // (`openshell sandbox create -- `). + Err(format!( + "containers.agent.command and containers.agent.args are not supported; {topology} \ + topology runs the OpenShell supervisor as the container entrypoint and would ignore \ + them — pass the workload command via `openshell sandbox create -- `" + )) } -fn condition_from_value(value: &serde_json::Value) -> Option { - let obj = value.as_object()?; - Some(SandboxCondition { - r#type: obj.get("type")?.as_str()?.to_string(), - status: obj.get("status")?.as_str()?.to_string(), - reason: obj - .get("reason") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - message: obj - .get("message") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - last_transition_time: obj - .get("lastTransitionTime") - .and_then(|val| val.as_str()) - .unwrap_or_default() - .to_string(), - }) +fn kubernetes_driver_config_for_spec( + spec: Option<&SandboxSpec>, + provider_spiffe_workload_api_socket_path: Option<&str>, +) -> Result { + let config = spec + .and_then(|spec| spec.template.as_ref()) + .map(KubernetesSandboxDriverConfig::from_template) + .transpose()? + .unwrap_or_default(); + let mut protected_paths = KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS.to_vec(); + let provider_spiffe_mount_path; + if let Some(socket_path) = provider_spiffe_workload_api_socket_path { + provider_spiffe_mount_path = spiffe_socket_mount_path(socket_path); + protected_paths.push(&provider_spiffe_mount_path); + } + validate_kubernetes_protected_path_conflicts( + &config.containers.agent.volume_mounts, + &protected_paths, + )?; + Ok(config) } -fn spawn_namespace_label_watcher( - client: Client, - label_selector: String, - allowlist: OperatorNamespaceAllowlist, - mut shutdown_rx: tokio::sync::watch::Receiver, -) { - let ns_api: Api = Api::all(client); - let watcher_config = watcher::Config::default().labels(&label_selector); - let jitter_seed = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |duration| { - duration.as_secs() ^ u64::from(duration.subsec_nanos()) - }); +fn sandbox_to_k8s_spec( + spec: Option<&SandboxSpec>, + params: &SandboxPodParams<'_>, +) -> Result { + let driver_config = + kubernetes_driver_config_for_spec(spec, provider_spiffe_socket_path(params))?; + let mut root = serde_json::Map::new(); - tokio::spawn(async move { - let mut retry_attempt = 0; - loop { - let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); + // Determine early whether OpenShell should inject its default workspace + // PVC. Explicit Kubernetes driver-config mounts under /sandbox/ take + // ownership of workspace persistence. + // We need this flag before building the podTemplate because the workspace + // persistence transforms are applied inside sandbox_template_to_k8s. + let user_has_explicit_workspace_mount = driver_config.has_explicit_sandbox_data_mount(); + let inject_workspace = !user_has_explicit_workspace_mount; - loop { - let event = tokio::select! { - result = stream.try_next() => result, - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - continue; - } - }; - match event { - Ok(Some(Event::Applied(ns))) => { - retry_attempt = 0; - if let Some(name) = ns.metadata.name.as_deref() - && allowlist.insert(name.to_string()) - { - info!(namespace = name, "operator namespace added to allowlist"); - } - } - Ok(Some(Event::Deleted(ns))) => { - retry_attempt = 0; - if let Some(name) = ns.metadata.name.as_deref() - && allowlist.remove(name) - { - info!( - namespace = name, - "operator namespace removed from allowlist" - ); - } - } - Ok(Some(Event::Restarted(namespaces))) => { - retry_attempt = 0; - let names: std::collections::BTreeSet = namespaces - .into_iter() - .filter_map(|ns| ns.metadata.name) - .collect(); - let count = names.len(); - allowlist.replace(names); - info!( - total = count, - "operator namespace allowlist replaced from full relist" - ); - } - Ok(None) => { - warn!("operator namespace watcher stream ended unexpectedly"); - break; - } - Err(err) => { - warn!(error = %err, "operator namespace watcher stream error"); - break; - } - } - } - - let retry_delay = namespace_watcher_retry_delay(retry_attempt, jitter_seed); - warn!(?retry_delay, "operator namespace watcher reconnecting"); - tokio::select! { - () = tokio::time::sleep(retry_delay) => {} - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - } + if let Some(spec) = spec { + let pod_env = spec_pod_env(Some(spec)); + if let Some(template) = spec.template.as_ref() { + root.insert( + "podTemplate".to_string(), + sandbox_template_to_k8s_with_validated_config( + template, + driver_gpu_requirements(spec.resource_requirements.as_ref()), + &pod_env, + Some(spec), + &driver_config, + inject_workspace, + params, + ), + ); + if !template.agent_socket_path.is_empty() { + root.insert( + "agentSocket".to_string(), + serde_json::json!(template.agent_socket_path), + ); } - retry_attempt = retry_attempt.saturating_add(1); } - }); + } - info!( - label_selector = %label_selector, - "operator namespace label watcher spawned" - ); + if inject_workspace { + root.insert( + "volumeClaimTemplates".to_string(), + default_workspace_volume_claim_templates( + params.workspace_default_storage_size, + params.workspace_storage_class, + ), + ); + } + + // podTemplate is required by the Kubernetes CRD - ensure it's always present + if !root.contains_key("podTemplate") { + let pod_env = spec_pod_env(spec); + root.insert( + "podTemplate".to_string(), + sandbox_template_to_k8s_with_validated_config( + &SandboxTemplate::default(), + driver_gpu_requirements(spec.and_then(|s| s.resource_requirements.as_ref())), + &pod_env, + spec, + &driver_config, + inject_workspace, + params, + ), + ); + } + + Ok(serde_json::Value::Object( + std::iter::once(("spec".to_string(), serde_json::Value::Object(root))).collect(), + )) } -fn namespace_watcher_retry_delay(attempt: u32, jitter_seed: u64) -> Duration { - let base_secs = 2_u64.saturating_mul(1_u64 << attempt.min(4)).min(24); - let max_jitter_secs = base_secs / 4; - let mixed_seed = - jitter_seed.wrapping_add(u64::from(attempt).wrapping_mul(0x9e37_79b9_7f4a_7c15)); - let jitter_secs = mixed_seed % (max_jitter_secs + 1); - Duration::from_secs(base_secs + jitter_secs) +#[cfg(test)] +fn sandbox_template_to_k8s( + template: &SandboxTemplate, + gpu: bool, + spec_environment: &std::collections::HashMap, + inject_workspace: bool, + params: &SandboxPodParams<'_>, +) -> serde_json::Value { + let gpu_requirements = gpu.then_some(GpuResourceRequirements { count: None }); + let driver_config = KubernetesSandboxDriverConfig::from_template(template) + .expect("test Kubernetes driver_config should be valid"); + sandbox_template_to_k8s_with_validated_config( + template, + gpu_requirements.as_ref(), + spec_environment, + None, + &driver_config, + inject_workspace, + params, + ) } -fn load_namespace_file(path: &Path) -> Result, String> { - let contents = std::fs::read_to_string(path) - .map_err(|e| format!("failed to read {}: {e}", path.display()))?; - let names: Vec = serde_json::from_str(&contents) - .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; - Ok(names.into_iter().collect()) +#[cfg(test)] +fn sandbox_template_to_k8s_with_gpu_requirements( + template: &SandboxTemplate, + gpu_requirements: Option<&GpuResourceRequirements>, + spec_environment: &std::collections::HashMap, + inject_workspace: bool, + params: &SandboxPodParams<'_>, +) -> serde_json::Value { + let driver_config = KubernetesSandboxDriverConfig::from_template(template) + .expect("test Kubernetes driver_config should be valid"); + sandbox_template_to_k8s_with_validated_config( + template, + gpu_requirements, + spec_environment, + None, + &driver_config, + inject_workspace, + params, + ) } -fn spawn_namespace_file_watcher( - path: PathBuf, - allowlist: OperatorNamespaceAllowlist, - mut shutdown_rx: tokio::sync::watch::Receiver, -) { - match load_namespace_file(&path) { - Ok(names) => { - let count = names.len(); - allowlist.replace(names); - info!( - path = %path.display(), - total = count, - "operator namespace allowlist loaded from file" +fn sandbox_template_to_k8s_with_validated_config( + template: &SandboxTemplate, + gpu_requirements: Option<&GpuResourceRequirements>, + spec_environment: &std::collections::HashMap, + sandbox_spec: Option<&openshell_core::proto::compute::v1::DriverSandboxSpec>, + driver_config: &KubernetesSandboxDriverConfig, + inject_workspace: bool, + params: &SandboxPodParams<'_>, +) -> serde_json::Value { + let mut metadata = serde_json::Map::new(); + let mut pod_labels = template + .labels + .iter() + .map(|(key, value)| (key.clone(), serde_json::Value::String(value.clone()))) + .collect::>(); + let proxy_pod_topology = params.topology == SupervisorTopology::ProxyPod; + if params.provider_spiffe_enabled || proxy_pod_topology { + pod_labels.insert( + LABEL_MANAGED_BY.to_string(), + serde_json::Value::String(LABEL_MANAGED_BY_VALUE.to_string()), + ); + if !params.sandbox_id.is_empty() { + pod_labels.insert( + LABEL_SANDBOX_ID.to_string(), + serde_json::Value::String(params.sandbox_id.to_string()), ); } - Err(err) => { + } + if proxy_pod_topology { + pod_labels.insert( + LABEL_SANDBOX_ROLE.to_string(), + serde_json::Value::String(SANDBOX_ROLE_AGENT.to_string()), + ); + } + if !pod_labels.is_empty() { + metadata.insert("labels".to_string(), serde_json::Value::Object(pod_labels)); + } + // Carry the sandbox UUID as a pod annotation so the gateway can resolve + // a projected SA token claim (pod name + uid) back to a sandbox identity + // when the supervisor calls `IssueSandboxToken` at startup. The gateway + // also verifies the pod's controlling Sandbox ownerReference against the + // live CR before accepting this annotation. Its K8s Role does NOT grant + // `patch pods`, so this annotation is effectively immutable post-create. + let mut pod_annotations = platform_config_struct(template, "annotations") + .and_then(|v| match v { + serde_json::Value::Object(map) => Some(map), + _ => None, + }) + .unwrap_or_default(); + if !params.sandbox_id.is_empty() { + pod_annotations.insert( + LABEL_SANDBOX_ID.to_string(), + serde_json::Value::String(params.sandbox_id.to_string()), + ); + } + if !pod_annotations.is_empty() { + metadata.insert( + "annotations".to_string(), + serde_json::Value::Object(pod_annotations), + ); + } + + let mut spec = serde_json::Map::new(); + let runtime_class_name = platform_config_string(template, "runtime_class_name") + .or_else(|| { + (!driver_config.pod.runtime_class_name.is_empty()) + .then(|| driver_config.pod.runtime_class_name.clone()) + }) + .or_else(|| { + (!params.default_runtime_class_name.is_empty()) + .then(|| params.default_runtime_class_name.to_string()) + }); + if let Some(runtime_class) = runtime_class_name { + spec.insert( + "runtimeClassName".to_string(), + serde_json::json!(runtime_class), + ); + } + if let Some(node_selector) = platform_config_struct(template, "node_selector") { + spec.insert("nodeSelector".to_string(), node_selector); + } + if let Some(tolerations) = platform_config_struct(template, "tolerations") { + spec.insert("tolerations".to_string(), tolerations); + } + apply_pod_driver_config(&mut spec, &driver_config.pod); + + // Per-sandbox portable intent overrides the cluster-wide default. This + // driver owns the Kubernetes-specific `hostUsers` translation. Accept the + // former platform_config encoding during rolling upgrades from gateways + // that predate the typed field. + let use_user_namespaces = template + .user_namespaces + .or_else(|| platform_config_bool(template, "host_users").map(|host_users| !host_users)) + .unwrap_or(params.enable_user_namespaces); + + if use_user_namespaces { + spec.insert("hostUsers".to_string(), serde_json::json!(false)); + if gpu_requirements.is_some() { warn!( - error = %err, - "failed to load initial operator namespace file, allowlist empty" + "GPU sandbox with user namespaces enabled — \ + NVIDIA device plugin compatibility is unverified" ); } } - let watch_dir = path - .parent() - .unwrap_or_else(|| Path::new(".")) - .to_path_buf(); - let debounce = Duration::from_secs(1); + if !params.service_account_name.is_empty() { + spec.insert( + "serviceAccountName".to_string(), + serde_json::json!(params.service_account_name), + ); + } - tokio::spawn(async move { - let (tx, mut rx) = mpsc::unbounded_channel(); + let image_pull_secrets = image_pull_secret_refs(params.image_pull_secrets); + if !image_pull_secrets.is_empty() { + spec.insert( + "imagePullSecrets".to_string(), + serde_json::Value::Array(image_pull_secrets), + ); + } - let mut watcher = - match notify::recommended_watcher(move |res: Result| { - if let Ok(event) = res - && matches!( - event.kind, - notify::EventKind::Modify(_) | notify::EventKind::Create(_) - ) - { - let _ = tx.send(()); - } - }) { - Ok(w) => w, - Err(e) => { - warn!( - error = %e, - "failed to start operator namespace file watcher, hot-reload disabled" - ); - return; - } - }; + // Disable service account token auto-mounting for security hardening. + // Sandbox pods should not have access to the Kubernetes API by default. + spec.insert( + "automountServiceAccountToken".to_string(), + serde_json::json!(false), + ); + // Do not let kubelet replace the canonical main-process generation after + // the supervisor exits. The gateway records that exit as terminal Error. + spec.insert("restartPolicy".to_string(), serde_json::json!("Never")); - if let Err(e) = notify::Watcher::watch( - &mut watcher, - &watch_dir, - notify::RecursiveMode::NonRecursive, - ) { - warn!( - error = %e, - dir = %watch_dir.display(), - "failed to watch operator namespace file directory, hot-reload disabled" + let mut container = serde_json::Map::new(); + container.insert("name".to_string(), serde_json::json!("agent")); + // Use template image if provided, otherwise fall back to default + let image = if template.image.is_empty() { + params.default_image + } else { + &template.image + }; + if !image.is_empty() { + container.insert("image".to_string(), serde_json::json!(image)); + if !params.image_pull_policy.is_empty() { + container.insert( + "imagePullPolicy".to_string(), + serde_json::json!(params.image_pull_policy), ); - return; } + } - info!( - path = %path.display(), - "operator namespace file watcher started" - ); + // Build environment variables - start with OpenShell-required vars + let env = build_env_list( + None, + &template.environment, + spec_environment, + sandbox_spec, + params.sandbox_id, + params.sandbox_name, + params.grpc_endpoint, + params.ssh_socket_path, + !params.client_tls_secret_name.is_empty(), + provider_spiffe_socket_path(params), + ); - loop { - let got_event = tokio::select! { - event = rx.recv() => event.is_some(), - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - continue; - } - }; - if !got_event { - warn!("operator namespace file watcher disconnected"); - break; - } + container.insert("env".to_string(), serde_json::Value::Array(env)); - loop { - tokio::select! { - () = tokio::time::sleep(debounce) => { - match load_namespace_file(&path) { - Ok(names) => { - let count = names.len(); - allowlist.replace(names); - info!( - total = count, - "operator namespace allowlist reloaded from file" - ); - } - Err(err) => { - warn!( - error = %err, - "failed to reload operator namespace file, keeping existing allowlist" - ); - } - } - break; - } - r = rx.recv() => { - if r.is_some() { - continue; - } - warn!("operator namespace file watcher disconnected"); - return; - } - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return; - } - } - } - } + let mut capabilities: Vec<&str> = vec!["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"]; + if use_user_namespaces { + // In a user namespace the bounding set is reset. SETUID/SETGID are + // needed for the supervisor to drop privileges to the sandbox user. + // DAC_READ_SEARCH is needed for cross-UID /proc//fd/ access + // for process identity resolution in network policy enforcement. + capabilities.extend(["SETUID", "SETGID", "DAC_READ_SEARCH"]); + } + let mut security_context = serde_json::json!({ + "capabilities": { + "add": capabilities } }); -} - -#[cfg(test)] -mod tests { - use super::*; - use openshell_core::progress::{ - PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, - PROGRESS_COMPLETE_STEP_KEY, - }; - use openshell_core::proto::compute::v1::{GpuResourceRequirements, ResourceRequirements}; - use prost_types::{Struct, Value, value::Kind}; - use std::collections::BTreeSet; - - static ENV_LOCK: std::sync::LazyLock> = - std::sync::LazyLock::new(|| std::sync::Mutex::new(())); - - #[tokio::test] - async fn tracing_create_sandbox_failure_exports_a_kubernetes_operation_span() { - use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; - use tracing::instrument::WithSubscriber as _; - use tracing_subscriber::layer::SubscriberExt as _; - - let _tracing_lock = crate::otel_tracing::test_lock().await; - let exporter = InMemorySpanExporterBuilder::new().build(); - let provider = SdkTracerProvider::builder() - .with_simple_exporter(exporter.clone()) - .build(); - let subscriber = tracing_subscriber::registry().with(crate::otel_tracing::layer(&provider)); - let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); - - driver - .create_sandbox(&Sandbox::default()) - .with_subscriber(subscriber) - .await - .expect_err("missing sandbox name should fail"); - provider.force_flush().unwrap(); - - let spans = exporter.get_finished_spans().unwrap(); - let span = spans - .iter() - .find(|span| span.name == "kubernetes.create_sandbox") - .expect("create operation span"); - assert!(matches!( - span.status, - opentelemetry::trace::Status::Error { .. } - )); - provider.shutdown().unwrap(); + if let Some(profile) = params.app_armor_profile { + security_context["appArmorProfile"] = app_armor_profile_to_k8s(profile); } + container.insert("securityContext".to_string(), security_context); - #[tokio::test] - async fn sandbox_annotation_propagates_the_active_w3c_trace_context() { - use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; - use tracing_subscriber::layer::SubscriberExt as _; - - let _tracing_lock = crate::otel_tracing::test_lock().await; - let exporter = InMemorySpanExporterBuilder::new().build(); - let provider = SdkTracerProvider::builder() - .with_simple_exporter(exporter) - .build(); - let subscriber = tracing_subscriber::registry().with(crate::otel_tracing::layer(&provider)); - - let annotations = tracing::subscriber::with_default(subscriber, || { - let span = tracing::info_span!("kubernetes.create_sandbox"); - let _entered = span.enter(); - let mut annotations = BTreeMap::new(); - add_trace_context_annotation(&mut annotations); - annotations - }); - - let carrier: serde_json::Value = serde_json::from_str( - annotations - .get("opentelemetry.io/trace-context") - .expect("agent-sandbox trace-context annotation"), - ) - .expect("annotation should contain a JSON propagation carrier"); - let traceparent = carrier["traceparent"] - .as_str() - .expect("carrier should contain traceparent"); - assert!(traceparent.starts_with("00-")); - assert_eq!(traceparent.len(), 55); - - provider.shutdown().unwrap(); + // Mount client TLS secret for mTLS to the server. Gateway identity uses + // the projected ServiceAccount bootstrap token. Provider token grants may + // additionally mount the SPIFFE Workload API socket. + let mut volume_mounts: Vec = Vec::new(); + if !params.client_tls_secret_name.is_empty() { + volume_mounts.push(serde_json::json!({ + "name": CLIENT_TLS_VOLUME_NAME, + "mountPath": openshell_core::container_paths::CLIENT_TLS_DIR, + "readOnly": true + })); } - - fn json_struct(value: serde_json::Value) -> Struct { - let serde_json::Value::Object(object) = value else { - panic!("expected JSON object"); - }; - openshell_core::proto_struct::json_object_to_struct(object) - .expect("test JSON must convert to a protobuf Struct") + if params.provider_spiffe_enabled { + volume_mounts.push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), + "readOnly": true, + })); } + volume_mounts.push(serde_json::json!({ + "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, + "mountPath": SERVICE_ACCOUNT_TOKEN_MOUNT_PATH, + "readOnly": true, + })); + volume_mounts.extend( + driver_config + .containers + .agent + .volume_mounts + .iter() + .map(kubernetes_driver_volume_mount_to_k8s), + ); + container.insert( + "volumeMounts".to_string(), + serde_json::Value::Array(volume_mounts), + ); - fn sandbox_to_k8s_spec_for_test( - spec: Option<&SandboxSpec>, - params: &SandboxPodParams<'_>, - ) -> serde_json::Value { - sandbox_to_k8s_spec(spec, params).expect("test Kubernetes driver_config should be valid") + if let Some(resources) = container_resources(template, gpu_requirements) { + container.insert("resources".to_string(), resources); } - - fn kube_api_error(code: u16, message: &str) -> KubeError { - KubeError::Api(kube::core::ErrorResponse { - status: if code == 404 { - "404 Not Found".to_string() - } else { - "Failure".to_string() - }, - message: message.to_string(), - reason: "Failed to parse error data".to_string(), - code, - }) + apply_agent_driver_resources(&mut container, &driver_config.containers.agent.resources); + if params.topology == SupervisorTopology::ProxyPod { + let agent_config = &driver_config.containers.agent; + if !agent_config.command.is_empty() { + container.insert( + "command".to_string(), + serde_json::json!(agent_config.command), + ); + } + if !agent_config.args.is_empty() { + container.insert("args".to_string(), serde_json::json!(agent_config.args)); + } } + spec.insert( + "containers".to_string(), + serde_json::Value::Array(vec![serde_json::Value::Object(container)]), + ); - fn expired_watch_error() -> watcher::Error { - watcher::Error::WatchError(kube::core::ErrorResponse { - status: "Failure".to_string(), - message: "too old resource version".to_string(), - reason: "Expired".to_string(), - code: 410, - }) + // Add TLS secret volume. Combined mode uses mode 0400 because the + // supervisor starts as root and drops privileges before running workload + // children. Sidecar mode keeps the process supervisor non-root, so it uses + // pod fsGroup + 0440 to preserve gateway session and SSH control behavior. + let mut volumes: Vec = Vec::new(); + if !params.client_tls_secret_name.is_empty() { + let client_tls_default_mode = match params.topology { + SupervisorTopology::Combined => 0o400, + SupervisorTopology::Sidecar | SupervisorTopology::ProxyPod => 0o440, + }; + volumes.push(serde_json::json!({ + "name": CLIENT_TLS_VOLUME_NAME, + "secret": { + "secretName": params.client_tls_secret_name, + "defaultMode": client_tls_default_mode + } + })); } - - #[tokio::test] - async fn sandbox_watcher_error_does_not_hide_restarted_recovery_event() { - let recovered = DynamicObject { - types: None, - metadata: ObjectMeta { - name: Some("recovered-sandbox".to_string()), - ..Default::default() - }, - data: serde_json::json!({}), + if has_upstream_proxy_credentials(params) { + let secret_name = params + .proxy_auth_secret_name + .expect("complete proxy credential reference has a Secret name"); + let secret_key = params + .proxy_auth_secret_key + .expect("complete proxy credential reference has a Secret key"); + // The credential volume is mounted only into the container that runs + // network supervision. Sidecar mode uses the pod fsGroup already + // required for its non-root network supervisor. + let default_mode = match params.topology { + // `combined` and `proxy-pod` are rejected by + // `validate_upstream_proxy_config`; use the most restrictive mode. + SupervisorTopology::Combined | SupervisorTopology::ProxyPod => 0o400, + SupervisorTopology::Sidecar => 0o440, }; - let source = futures::stream::iter([ - Err(expired_watch_error()), - Ok(Event::Restarted(vec![recovered])), - ]); - let mut stream = continue_on_watcher_errors(source, "sandbox-resource"); + volumes.push(serde_json::json!({ + "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, + "secret": { + "secretName": secret_name, + "defaultMode": default_mode, + "items": [{ + "key": secret_key, + "path": upstream_proxy_auth_file_name(), + }] + } + })); + } + if params.provider_spiffe_enabled { + volumes.push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "csi": { + "driver": "csi.spiffe.io", + "readOnly": true + } + })); + } + // Projected ServiceAccountToken volume — kubelet writes a short-lived + // audience-bound JWT into /var/run/secrets/openshell/token and rotates + // it automatically. The supervisor exchanges this for a gateway-minted + // JWT via `IssueSandboxToken` once at startup. In sidecar topology both + // supervisor containers run with the sandbox GID and need group-read access. + let sa_token_default_mode = match params.topology { + SupervisorTopology::Combined => 0o400, + SupervisorTopology::Sidecar | SupervisorTopology::ProxyPod => 0o440, + }; + volumes.push(serde_json::json!({ + "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, + "projected": { + "sources": [{ + "serviceAccountToken": { + "audience": "openshell-gateway", + "expirationSeconds": params.sa_token_ttl_secs, + "path": "token" + } + }], + "defaultMode": sa_token_default_mode + } + })); + volumes.extend( + driver_config + .volumes + .iter() + .map(kubernetes_driver_volume_to_k8s), + ); + spec.insert("volumes".to_string(), serde_json::Value::Array(volumes)); - let event = stream - .next() - .await - .expect("410 Expired must not terminate the watcher stream"); - let Event::Restarted(objects) = event else { - panic!("expected kube-runtime recovery to emit Restarted"); - }; - assert_eq!(objects.len(), 1); - assert_eq!( - objects[0].metadata.name.as_deref(), - Some("recovered-sandbox") - ); - assert!( - stream.next().await.is_none(), - "source closure must be preserved" - ); + // Add hostAliases so sandbox pods can reach the Docker host. + apply_host_gateway_aliases(&mut spec, params.host_gateway_ip); + + let mut template_value = serde_json::Map::new(); + if !metadata.is_empty() { + template_value.insert("metadata".to_string(), serde_json::Value::Object(metadata)); } + template_value.insert("spec".to_string(), serde_json::Value::Object(spec)); - #[tokio::test(start_paused = true)] - async fn outward_watch_stream_survives_expired_error_and_backoff_recovery() { - let recovered = DynamicObject { - types: None, - metadata: ObjectMeta { - name: Some("recovered-sandbox".to_string()), - namespace: Some("recovered-namespace".to_string()), - labels: Some(BTreeMap::from([ - (LABEL_SANDBOX_ID.to_string(), "sandbox-id".to_string()), - (LABEL_SANDBOX_NAME.to_string(), "sandbox-name".to_string()), - (LABEL_SANDBOX_WORKSPACE.to_string(), "workspace".to_string()), - ( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ), - ])), - ..Default::default() - }, - data: serde_json::json!({}), - }; - let source = futures::stream::iter([ - Err(expired_watch_error()), - Ok(Event::Restarted(vec![recovered])), - ]) - .chain(futures::stream::pending()); - let sandbox_stream = recovering_watcher_stream(source, "sandbox-resource").boxed(); - let mut outward = cluster_wide_watch_stream(sandbox_stream, "default".to_string()); + let mut result = serde_json::Value::Object(template_value); - let event = outward - .next() - .await - .expect("outward stream must stay open through recovery") - .expect("recoverable watcher error must not reach the outward stream"); - let Some(watch_sandboxes_event::Payload::Sandbox(event)) = event.payload else { - panic!("expected recovered sandbox event"); - }; - let sandbox = event.sandbox.expect("sandbox payload must be populated"); - assert_eq!(sandbox.id, "sandbox-id"); - assert_eq!(sandbox.namespace, "recovered-namespace"); + match params.topology { + SupervisorTopology::Combined => { + apply_supervisor_sideload_with_params(&mut result, params); + } + SupervisorTopology::Sidecar => { + apply_supervisor_sidecar_topology( + &mut result, + &template.environment, + spec_environment, + params, + ); + } + SupervisorTopology::ProxyPod => { + apply_supervisor_proxy_pod_topology(&mut result, params); + } + } - let next = outward.next(); - futures::pin_mut!(next); - assert!( - futures::poll!(next).is_pending(), - "outward stream must remain open after the recovered event" + // Inject workspace persistence (init container + PVC volume mount) so + // that /sandbox data survives pod rescheduling. Skipped when the user + // provides custom storage through driver_config. + if inject_workspace { + apply_workspace_persistence( + &mut result, + image, + params.image_pull_policy, + params.sandbox_uid, + params.sandbox_gid, + params.topology, ); } - #[tokio::test] - async fn kubernetes_event_watcher_error_does_not_hide_restarted_recovery_event() { - let source = futures::stream::iter([ - Err(expired_watch_error()), - Ok(Event::Restarted(vec![KubeEventObj::default()])), - ]); - let mut stream = continue_on_watcher_errors(source, "kubernetes-event"); + result +} - let event = stream - .next() - .await - .expect("410 Expired must not terminate the watcher stream"); - let Event::Restarted(events) = event else { - panic!("expected kube-runtime recovery to emit Restarted"); - }; - assert_eq!(events.len(), 1); - assert!( - stream.next().await.is_none(), - "source closure must be preserved" - ); +fn apply_pod_driver_config( + spec: &mut serde_json::Map, + config: &KubernetesPodDriverConfig, +) { + if !config.node_selector.is_empty() { + let node_selector = spec + .entry("nodeSelector".to_string()) + .or_insert_with(|| serde_json::json!({})); + merge_string_map(node_selector, &config.node_selector); } - fn authenticated_token_review(username: &str) -> TokenReviewStatus { - TokenReviewStatus { - authenticated: Some(true), - audiences: Some(vec![SANDBOX_TOKEN_AUDIENCE.to_string()]), - user: Some(UserInfo { - username: Some(username.to_string()), - extra: Some(BTreeMap::from([ - (POD_NAME_EXTRA.to_string(), vec!["sandbox-pod".to_string()]), - (POD_UID_EXTRA.to_string(), vec!["pod-uid".to_string()]), - ])), - ..Default::default() - }), - ..Default::default() - } - } - - #[test] - fn token_review_uses_configured_service_account_and_pod_binding() { - let status = authenticated_token_review("system:serviceaccount:workspaces:sandbox-sa"); - let identity = token_review_identity(&status, "sandbox-sa") - .unwrap() - .expect("authenticated identity"); - assert_eq!(identity.namespace, "workspaces"); - assert_eq!(identity.pod_name, "sandbox-pod"); - assert_eq!(identity.pod_uid, "pod-uid"); + if !config.priority_class_name.is_empty() { + spec.entry("priorityClassName".to_string()) + .or_insert_with(|| serde_json::json!(config.priority_class_name)); } - #[test] - fn token_review_rejects_a_different_service_account() { - let status = authenticated_token_review("system:serviceaccount:workspaces:other"); - let error = token_review_identity(&status, "sandbox-sa").unwrap_err(); - assert_eq!(error.code(), tonic::Code::PermissionDenied); + if !config.tolerations.is_empty() { + let tolerations = spec + .entry("tolerations".to_string()) + .or_insert_with(|| serde_json::json!([])); + if let Some(existing) = tolerations.as_array_mut() { + existing.extend(config.tolerations.iter().cloned()); + } else { + *tolerations = serde_json::Value::Array(config.tolerations.clone()); + } } +} - #[test] - fn token_review_rejects_wrong_audience_and_missing_pod_binding() { - let mut wrong_audience = - authenticated_token_review("system:serviceaccount:workspaces:sandbox-sa"); - wrong_audience.audiences = Some(vec!["kubernetes.default.svc".to_string()]); - let error = token_review_identity(&wrong_audience, "sandbox-sa").unwrap_err(); - assert_eq!(error.code(), tonic::Code::Unauthenticated); - - let mut missing_binding = - authenticated_token_review("system:serviceaccount:workspaces:sandbox-sa"); - missing_binding.user.as_mut().unwrap().extra = None; - let error = token_review_identity(&missing_binding, "sandbox-sa").unwrap_err(); - assert_eq!(error.code(), tonic::Code::PermissionDenied); +fn apply_agent_driver_resources( + container: &mut serde_json::Map, + resources: &KubernetesContainerResourceConfig, +) { + if resources.requests.is_empty() && resources.limits.is_empty() { + return; } - #[test] - fn token_review_returns_none_when_not_authenticated() { - let status = TokenReviewStatus { - authenticated: Some(false), - error: Some("token rejected".to_string()), - ..Default::default() - }; + let target = container + .entry("resources".to_string()) + .or_insert_with(|| serde_json::json!({})); + apply_resource_quantity_map(target, "requests", &resources.requests); + apply_resource_quantity_map(target, "limits", &resources.limits); +} - assert!( - token_review_identity(&status, "sandbox-sa") - .unwrap() - .is_none() - ); +fn merge_string_map(target: &mut serde_json::Value, values: &BTreeMap) { + if !target.is_object() { + *target = serde_json::json!({}); } - - #[test] - fn authentication_namespace_validation_covers_each_workspace_mode() { - let mut config = KubernetesComputeConfig { - namespace: "openshell".to_string(), - ..Default::default() - }; - assert!(accepts_auth_namespace(&config, None, "openshell")); - assert!(!accepts_auth_namespace(&config, None, "other")); - - config.workspace_mode = WorkspaceMode::Managed; - config.gateway_id = "gateway-a".to_string(); - assert!(accepts_auth_namespace( - &config, - None, - "openshell-gateway-a-workspace-a" - )); - assert!(!accepts_auth_namespace( - &config, - None, - "openshell-gateway-b-workspace-a" - )); - - config.workspace_mode = WorkspaceMode::Operator; - let allowlist = OperatorNamespaceAllowlist::from_set(BTreeSet::from([ - "team-a".to_string(), - "team-b".to_string(), - ])); - assert!(accepts_auth_namespace(&config, Some(&allowlist), "team-a")); - assert!(!accepts_auth_namespace(&config, Some(&allowlist), "team-c")); - assert!(!accepts_auth_namespace(&config, None, "team-a")); + let target = target + .as_object_mut() + .expect("target was converted to object"); + for (key, value) in values { + target + .entry(key.clone()) + .or_insert_with(|| serde_json::json!(value)); } +} - fn sandbox_owner_for_test(name: &str, uid: &str) -> OwnerReference { - OwnerReference { - api_version: "agents.x-k8s.io/v1beta1".to_string(), - block_owner_deletion: None, - controller: Some(true), - kind: SANDBOX_KIND.to_string(), - name: name.to_string(), - uid: uid.to_string(), - } +fn apply_resource_quantity_map( + target: &mut serde_json::Value, + section: &str, + values: &BTreeMap, +) { + if values.is_empty() { + return; } - - fn sandbox_object_for_test(uid: &str, sandbox_id: &str) -> DynamicObject { - let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( - SANDBOX_GROUP, - SANDBOX_VERSION_V1BETA1, - SANDBOX_KIND, - )); - let mut sandbox = DynamicObject::new("sandbox-a", &resource); - sandbox.metadata.uid = Some(uid.to_string()); - sandbox.metadata.labels = Some(BTreeMap::from([( - LABEL_SANDBOX_ID.to_string(), - sandbox_id.to_string(), - )])); - sandbox + if !target.is_object() { + *target = serde_json::json!({}); } + let target = target + .as_object_mut() + .expect("target was converted to object"); + let section_value = target + .entry(section.to_string()) + .or_insert_with(|| serde_json::json!({})); + merge_string_map(section_value, values); +} - #[test] - fn pod_identity_requires_matching_uid_annotation_and_controlling_owner() { - let owner = sandbox_owner_for_test("sandbox-a", "sandbox-uid-a"); - let pod = Pod { - metadata: ObjectMeta { - uid: Some("pod-uid-a".to_string()), - annotations: Some(BTreeMap::from([( - LABEL_SANDBOX_ID.to_string(), - "sandbox-id-a".to_string(), - )])), - owner_references: Some(vec![owner.clone()]), - ..Default::default() - }, - ..Default::default() - }; - - validate_pod_uid(&pod, "pod-uid-a").expect("matching pod UID"); - assert_eq!(pod_sandbox_id(&pod).unwrap(), "sandbox-id-a"); - assert_eq!(sandbox_owner_reference(&pod).unwrap(), &owner); - - let error = validate_pod_uid(&pod, "other-pod-uid").unwrap_err(); - assert_eq!(error.code(), tonic::Code::PermissionDenied); - - let mut missing_annotation = pod.clone(); - missing_annotation.metadata.annotations = None; - let error = pod_sandbox_id(&missing_annotation).unwrap_err(); - assert_eq!(error.code(), tonic::Code::PermissionDenied); +fn image_pull_secret_refs(secrets: &[String]) -> Vec { + secrets + .iter() + .map(|secret| secret.trim()) + .filter(|secret| !secret.is_empty()) + .map(|secret| serde_json::json!({ "name": secret })) + .collect() +} - let mut non_controlling = pod; - non_controlling.metadata.owner_references.as_mut().unwrap()[0].controller = Some(false); - let error = sandbox_owner_reference(&non_controlling).unwrap_err(); - assert_eq!(error.code(), tonic::Code::PermissionDenied); - } +fn k8s_object(value: serde_json::Value) -> T +where + T: DeserializeOwned, +{ + serde_json::from_value(value).expect("driver rendered an invalid Kubernetes object") +} - #[test] - fn sandbox_owner_identity_requires_matching_uid_and_sandbox_id() { - let owner = sandbox_owner_for_test("sandbox-a", "sandbox-uid-a"); - let sandbox = sandbox_object_for_test("sandbox-uid-a", "sandbox-id-a"); - validate_sandbox_owner_identity(&owner, "sandbox-id-a", &sandbox) - .expect("matching owner identity"); +fn generate_proxy_pod_ca() -> Result<(String, String), KubernetesDriverError> { + let ca_key = KeyPair::generate().map_err(|err| { + KubernetesDriverError::Message(format!("failed to generate CA key: {err}")) + })?; - let mismatched_owner = sandbox_object_for_test("sandbox-uid-b", "sandbox-id-a"); - let error = - validate_sandbox_owner_identity(&owner, "sandbox-id-a", &mismatched_owner).unwrap_err(); - assert_eq!(error.code(), tonic::Code::PermissionDenied); + let mut params = CertificateParams::default(); + params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + params + .distinguished_name + .push(DnType::CommonName, "OpenShell Proxy Pod Sandbox CA"); + params + .distinguished_name + .push(DnType::OrganizationName, "OpenShell"); + params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; - let mismatched_annotation = sandbox_object_for_test("sandbox-uid-a", "sandbox-id-b"); - let error = validate_sandbox_owner_identity(&owner, "sandbox-id-a", &mismatched_annotation) - .unwrap_err(); - assert_eq!(error.code(), tonic::Code::PermissionDenied); - } + let ca_cert = params.self_signed(&ca_key).map_err(|err| { + KubernetesDriverError::Message(format!("failed to generate CA certificate: {err}")) + })?; + Ok((ca_cert.pem(), ca_key.serialize_pem())) +} - #[test] - fn sandbox_api_version_probe_retries_on_structured_and_raw_404() { - let structured = kube_api_error(404, "could not find the requested resource"); - assert!(should_try_next_sandbox_api_version(&structured)); +fn proxy_pod_owner_reference( + sandbox_cr: &DynamicObject, + api_version: &str, + controller: bool, +) -> Result { + let name = + sandbox_cr.metadata.name.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("created Sandbox is missing name".into()) + })?; + let uid = + sandbox_cr.metadata.uid.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("created Sandbox is missing uid".into()) + })?; + Ok(serde_json::json!({ + "apiVersion": sandbox_cr + .types + .as_ref() + .map_or(api_version, |types| types.api_version.as_str()), + "kind": SANDBOX_KIND, + "name": name, + "uid": uid, + "controller": controller, + "blockOwnerDeletion": false, + })) +} - let raw = kube_api_error(404, "404 page not found\n"); - assert!(should_try_next_sandbox_api_version(&raw)); +fn proxy_pod_labels(sandbox_id: &str, role: &str, gateway_id: &str) -> serde_json::Value { + let mut labels = serde_json::Map::new(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + serde_json::json!(LABEL_MANAGED_BY_VALUE), + ); + labels.insert(LABEL_SANDBOX_ID.to_string(), serde_json::json!(sandbox_id)); + labels.insert(LABEL_SANDBOX_ROLE.to_string(), serde_json::json!(role)); + // Gateway ownership so reconciliation can scope list/reap to this gateway. + if !gateway_id.is_empty() { + labels.insert(LABEL_GATEWAY_ID.to_string(), serde_json::json!(gateway_id)); } + serde_json::Value::Object(labels) +} - #[test] - fn lifecycle_patch_uses_version_specific_operating_state() { - let beta_stop = sandbox_operating_state_patch(SANDBOX_VERSION_V1BETA1, "42", false); - assert_eq!(beta_stop["metadata"]["resourceVersion"], "42"); - assert_eq!(beta_stop["spec"]["operatingMode"], "Suspended"); - assert!(beta_stop["spec"].get("replicas").is_none()); - - let alpha_start = sandbox_operating_state_patch(SANDBOX_VERSION_V1ALPHA1, "43", true); - assert_eq!(alpha_start["metadata"]["resourceVersion"], "43"); - assert_eq!(alpha_start["spec"]["replicas"], 1); - assert!(alpha_start["spec"].get("operatingMode").is_none()); - } +fn proxy_pod_match_labels(sandbox_id: &str, role: &str) -> serde_json::Value { + let mut labels = serde_json::Map::new(); + labels.insert(LABEL_SANDBOX_ID.to_string(), serde_json::json!(sandbox_id)); + labels.insert(LABEL_SANDBOX_ROLE.to_string(), serde_json::json!(role)); + serde_json::Value::Object(labels) +} - #[test] - fn stop_timeout_includes_pod_grace_period_and_reconcile_headroom() { - let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( - SANDBOX_GROUP, - SANDBOX_VERSION_V1BETA1, - SANDBOX_KIND, - )); - let mut sandbox = DynamicObject::new("sandbox", &resource); +fn proxy_pod_object_meta( + name: &str, + namespace: &str, + sandbox_id: &str, + role: &str, + gateway_id: &str, + owner_ref: serde_json::Value, +) -> serde_json::Value { + serde_json::json!({ + "name": name, + "namespace": namespace, + "labels": proxy_pod_labels(sandbox_id, role, gateway_id), + "annotations": { + "openshell.ai/sandbox-id": sandbox_id + }, + "ownerReferences": [owner_ref] + }) +} - assert_eq!( - kubernetes_sandbox_stop_timeout(&sandbox), - Duration::from_secs(60), - "an omitted grace period uses the Kubernetes 30-second default" +fn proxy_pod_supervisor_env( + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) -> Vec { + let mut env = Vec::new(); + apply_required_env( + &mut env, + params.sandbox_id, + params.sandbox_name, + params.grpc_endpoint, + "", + false, + provider_spiffe_socket_path(params), + ); + if !params.client_tls_secret_name.is_empty() { + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CA, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/ca.crt"), ); - - sandbox.data = serde_json::json!({ - "spec": { - "podTemplate": { - "spec": {"terminationGracePeriodSeconds": 45} - } - } - }); - assert_eq!( - kubernetes_sandbox_stop_timeout(&sandbox), - Duration::from_secs(75) + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CERT, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.crt"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_KEY, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.key"), ); } + copy_log_level_env(&mut env, template_environment, spec_environment); + upsert_env( + &mut env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "proxy-pod", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + PROXY_POD_NETWORK_ENFORCEMENT_MODE, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + "relaxed", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_BIND_ADDR, + &format!("0.0.0.0:{PROXY_POD_PROXY_PORT}"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_CA_CERT_PATH, + &format!("{PROXY_POD_CA_SECRET_MOUNT_PATH}/{PROXY_POD_CA_CERT_FILE}"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_CA_KEY_PATH, + &format!("{PROXY_POD_CA_SECRET_MOUNT_PATH}/{PROXY_POD_CA_KEY_FILE}"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SANDBOX_UID, + ¶ms.sandbox_uid.to_string(), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SANDBOX_GID, + ¶ms.sandbox_gid.to_string(), + ); + env +} - #[test] - fn stop_poll_interval_backs_off_to_cap() { - let mut interval = STOP_INITIAL_POLL_INTERVAL; - let expected = [ - Duration::from_millis(500), - Duration::from_secs(1), - Duration::from_secs(2), - Duration::from_secs(2), - ]; +fn proxy_pod_ca_secret( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, + owner_ref: serde_json::Value, + cert_pem: &str, + key_pem: &str, +) -> Secret { + let mut string_data = serde_json::Map::new(); + string_data.insert( + PROXY_POD_CA_CERT_FILE.to_string(), + serde_json::json!(cert_pem), + ); + string_data.insert( + PROXY_POD_CA_KEY_FILE.to_string(), + serde_json::json!(key_pem), + ); + k8s_object(serde_json::json!({ + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": names.proxy_ca_secret, + "namespace": params.namespace, + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR, params.gateway_id), + "ownerReferences": [owner_ref], + }, + "type": "Opaque", + "stringData": serde_json::Value::Object(string_data) + })) +} - for expected_interval in expected { - interval = next_stop_poll_interval(interval); - assert_eq!(interval, expected_interval); +fn proxy_pod_supervisor_service( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, + owner_ref: serde_json::Value, +) -> Service { + k8s_object(serde_json::json!({ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": names.service, + "namespace": params.namespace, + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR, params.gateway_id), + "ownerReferences": [owner_ref], + }, + "spec": { + "clusterIP": "None", + "publishNotReadyAddresses": true, + "selector": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR), + "ports": [ + { + "name": "http-proxy", + "port": PROXY_POD_PROXY_PORT, + "targetPort": PROXY_POD_PROXY_PORT, + "protocol": "TCP" + } + ] } - } - - #[test] - fn stopped_status_requires_published_condition() { - let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( - SANDBOX_GROUP, - SANDBOX_VERSION_V1ALPHA1, - SANDBOX_KIND, - )); - let mut sandbox = DynamicObject::new("sandbox", &resource); - sandbox.data = serde_json::json!({"status": {"replicas": 0}}); + })) +} - assert!( - !kubernetes_sandbox_has_stopped_condition(&sandbox), - "v1alpha1 omits a zero status replica count on the wire; it is not a usable completion signal" - ); +/// The set of Kubernetes objects that back one proxy-pod sandbox alongside its +/// Sandbox CR. All are owner-referenced to the CR for garbage collection. +struct ProxyPodCompanions { + secret: Secret, + service: Service, + agent_egress: NetworkPolicy, + supervisor_ingress: NetworkPolicy, + supervisor_deployment: Deployment, +} - sandbox.data = serde_json::json!({ - "status": { - "conditions": [{"type": "Suspended", "status": "True"}] +/// Render the full companion set from already-resolved inputs. Shared by the +/// create path (inputs from the sandbox spec) and the reconciliation path +/// (inputs reconstructed from the CR) so both produce identical objects. +#[allow(clippy::too_many_arguments)] +fn build_proxy_pod_companions( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + pod_driver_config: &KubernetesPodDriverConfig, + placement: &ProxyPodPlacement, + supervisor_replicas: u32, + deployment_owner_ref: serde_json::Value, + dependent_owner_ref: serde_json::Value, + ca_cert_pem: &str, + ca_key_pem: &str, +) -> ProxyPodCompanions { + ProxyPodCompanions { + secret: proxy_pod_ca_secret( + names, + params, + dependent_owner_ref.clone(), + ca_cert_pem, + ca_key_pem, + ), + service: proxy_pod_supervisor_service(names, params, dependent_owner_ref.clone()), + // No owner reference: the gateway manages this fence's lifecycle so it + // outlives the workload pod on deletion. + agent_egress: proxy_pod_agent_egress_network_policy(names, params), + supervisor_ingress: proxy_pod_supervisor_ingress_network_policy( + names, + params, + dependent_owner_ref, + ), + supervisor_deployment: proxy_pod_supervisor_deployment( + names, + template_environment, + spec_environment, + params, + pod_driver_config, + placement, + supervisor_replicas, + deployment_owner_ref, + ), + } +} + +/// Create a companion object, treating an `AlreadyExists` (409) conflict as +/// success. This makes companion provisioning idempotent so it is safe to run +/// repeatedly from the reconciliation path without clobbering existing objects. +/// +/// `verify_ownership` controls whether a 409 triggers an ownership-verifying +/// GET. It is `false` for the CA Secret because the gateway deliberately holds +/// no Secret read permission (least privilege); the companion name is keyed on +/// the immutable sandbox UUID, so a 409 already implies the object is this +/// sandbox's own. Non-secret companions verify via a metadata GET. +/// Create the agent egress fence, or validate an existing same-name policy. +/// +/// The fence carries no owner reference, so ownership verification cannot vouch +/// for it. On an `AlreadyExists` conflict, fetch the existing policy and confirm +/// its enforcement fields (`spec`) and `sandbox-id` label match what we intended +/// to create; fail closed on any mismatch so a stale or altered policy is never +/// treated as a valid fence. If the conflicting policy has vanished by the time +/// we read it (409 then 404), the fence is *absent* — never treat that as +/// provisioned; retry the create so the workload is never left at default-allow. +async fn create_or_validate_egress_fence( + api: &Api, + expected: &NetworkPolicy, +) -> Result<(), KubernetesDriverError> { + const DESC: &str = "proxy-pod agent egress NetworkPolicy"; + const MAX_ATTEMPTS: usize = 4; + for _ in 0..MAX_ATTEMPTS { + match tokio::time::timeout( + KUBE_API_TIMEOUT, + api.create(&PostParams::default(), expected), + ) + .await + { + Ok(Ok(_)) => return Ok(()), + Ok(Err(KubeError::Api(err))) if err.code == 409 => { + let name = expected.metadata.name.clone().unwrap_or_default(); + let existing = match tokio::time::timeout(KUBE_API_TIMEOUT, api.get(&name)).await { + Ok(Ok(existing)) => existing, + // Conflicting policy vanished after the 409: the fence is now + // absent. Loop back and re-create it rather than reporting a + // non-existent boundary as provisioned. + Ok(Err(KubeError::Api(err))) if err.code == 404 => continue, + Ok(Err(err)) => return Err(KubernetesDriverError::from_kube(err)), + Err(_elapsed) => { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s validating {DESC} {name}", + KUBE_API_TIMEOUT.as_secs() + ))); + } + }; + let expected_sandbox_id = expected + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)); + let existing_sandbox_id = existing + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)); + if existing.spec == expected.spec && existing_sandbox_id == expected_sandbox_id { + return Ok(()); + } + return Err(KubernetesDriverError::Message(format!( + "{DESC} {name} already exists but its enforcement does not match the intended \ + fence (selector, egress rules, or sandbox-id differ); refusing to treat it \ + as provisioned" + ))); } - }); - assert!(kubernetes_sandbox_has_stopped_condition(&sandbox)); + Ok(Err(err)) => return Err(KubernetesDriverError::from_kube(err)), + Err(_elapsed) => { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s creating {DESC}", + KUBE_API_TIMEOUT.as_secs() + ))); + } + } } + // Exhausted retries always re-creating/re-reading a vanishing fence: fail + // closed rather than proceed without a boundary. + Err(KubernetesDriverError::Message(format!( + "{DESC} could not be provisioned after {MAX_ATTEMPTS} attempts (create/verify kept racing \ + a vanishing policy)" + ))) +} - #[test] - fn beta_stop_requires_suspended_condition_and_deleted_pod() { - let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( - SANDBOX_GROUP, - SANDBOX_VERSION_V1BETA1, - SANDBOX_KIND, - )); - let mut sandbox = DynamicObject::new("sandbox", &resource); - - assert!(!kubernetes_sandbox_stop_is_complete( - SANDBOX_VERSION_V1BETA1, - &sandbox, - true, - )); - - sandbox.data = serde_json::json!({ - "status": { - "conditions": [{"type": "Suspended", "status": "True"}] +async fn create_companion_if_absent( + api: &Api, + obj: &K, + description: &str, + verify_ownership: bool, +) -> Result<(), KubernetesDriverError> +where + K: kube::Resource + Clone + std::fmt::Debug + serde::Serialize + DeserializeOwned + Sync, + ::DynamicType: Default, +{ + match tokio::time::timeout(KUBE_API_TIMEOUT, api.create(&PostParams::default(), obj)).await { + Ok(Ok(_)) => Ok(()), + Ok(Err(KubeError::Api(err))) if err.code == 409 => { + if verify_ownership { + verify_companion_ownership(api, obj, description).await + } else { + Ok(()) } - }); - assert!(!kubernetes_sandbox_stop_is_complete( - SANDBOX_VERSION_V1BETA1, - &sandbox, - false, - )); - assert!(kubernetes_sandbox_stop_is_complete( - SANDBOX_VERSION_V1BETA1, - &sandbox, - true, - )); - assert!(kubernetes_sandbox_stop_is_complete( - SANDBOX_VERSION_V1ALPHA1, - &DynamicObject::new("sandbox", &resource), - true, - )); + } + Ok(Err(err)) => Err(KubernetesDriverError::from_kube(err)), + Err(_elapsed) => Err(KubernetesDriverError::Message(format!( + "timed out after {}s creating {description}", + KUBE_API_TIMEOUT.as_secs() + ))), + } +} + +/// On an `AlreadyExists` conflict, confirm the existing object belongs to the +/// same sandbox instance before treating the create as idempotent. A companion +/// is owner-referenced to its Sandbox CR, whose UID is per-instance, so an +/// object owned by a different CR UID is a stale leftover from a prior instance +/// (or an unrelated object). Adopting it would give the new sandbox a +/// mis-selecting egress policy or an unreachable supervisor, so we fail closed. +async fn verify_companion_ownership( + api: &Api, + obj: &K, + description: &str, +) -> Result<(), KubernetesDriverError> +where + K: kube::Resource + Clone + std::fmt::Debug + serde::Serialize + DeserializeOwned + Sync, + ::DynamicType: Default, +{ + let name = obj.meta().name.clone().unwrap_or_default(); + let expected_uids: HashSet<&str> = obj + .meta() + .owner_references + .iter() + .flatten() + .map(|owner| owner.uid.as_str()) + .collect(); + // Without an owner reference to compare against we cannot prove identity; + // there is nothing to verify, so accept (companions always carry one). + if expected_uids.is_empty() { + return Ok(()); } - - #[test] - fn stop_failure_only_rejects_terminal_suspension_condition() { - let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( - SANDBOX_GROUP, - SANDBOX_VERSION_V1BETA1, - SANDBOX_KIND, - )); - let mut sandbox = DynamicObject::new("sandbox", &resource); - sandbox.data = serde_json::json!({ - "status": { - "conditions": [{ - "type": "Suspended", - "status": "False", - "reason": "PodNotOwned", - "message": "Refused to delete pod because it is not owned by this sandbox" - }] + match tokio::time::timeout(KUBE_API_TIMEOUT, api.get(&name)).await { + Ok(Ok(existing)) => { + let same_instance = existing + .meta() + .owner_references + .iter() + .flatten() + .any(|owner| expected_uids.contains(owner.uid.as_str())); + if same_instance { + Ok(()) + } else { + Err(KubernetesDriverError::Message(format!( + "{description} {name} already exists but is owned by a different sandbox \ + instance; refusing to adopt a stale companion" + ))) } - }); - - assert_eq!( - kubernetes_sandbox_stop_failure(&sandbox).as_deref(), - Some( - "Kubernetes sandbox stop rejected: Refused to delete pod because it is not owned by this sandbox" - ) - ); - - sandbox.data["status"]["conditions"][0]["status"] = serde_json::json!("Unknown"); - sandbox.data["status"]["conditions"][0]["reason"] = serde_json::json!("PodStateUnknown"); - assert!( - kubernetes_sandbox_stop_failure(&sandbox).is_none(), - "an unknown pod state can recover on a later controller reconciliation" - ); + } + // Raced with garbage collection: the conflicting object is already gone, + // so a later reconcile pass will recreate it cleanly. + Ok(Err(KubeError::Api(err))) if err.code == 404 => Ok(()), + // Cannot read the object to verify (should not happen with the rendered + // RBAC, which grants get on non-secret companions). Accept rather than + // wedge reconciliation: the UUID-keyed name already implies it is ours. + Ok(Err(KubeError::Api(err))) if err.code == 403 => { + warn!( + companion = %description, + name = %name, + "Cannot verify companion ownership (forbidden); accepting existing object by UUID-keyed name" + ); + Ok(()) + } + Ok(Err(err)) => Err(KubernetesDriverError::from_kube(err)), + Err(_elapsed) => Err(KubernetesDriverError::Message(format!( + "timed out after {}s verifying ownership of {description} {name}", + KUBE_API_TIMEOUT.as_secs() + ))), } +} - #[test] - fn sandbox_api_version_probe_keeps_non_404_errors() { - let err = kube_api_error(403, "sandboxes.agents.x-k8s.io is forbidden"); - assert!(!should_try_next_sandbox_api_version(&err)); +/// Reconstruct the supervisor's node placement from a Sandbox CR's rendered +/// agent pod, so a reconciled supervisor lands where the workload can pair with +/// it. The agent pod already carries the merged placement, so reading it back is +/// both accurate and cluster-domain-agnostic. +fn proxy_pod_placement_from_cr(obj: &DynamicObject) -> ProxyPodPlacement { + let Some(pod_spec) = obj + .data + .get("spec") + .and_then(|spec| spec.get("podTemplate")) + .and_then(|template| template.get("spec")) + else { + return ProxyPodPlacement::default(); + }; + ProxyPodPlacement { + runtime_class_name: pod_spec + .get("runtimeClassName") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.is_empty()) + .map(str::to_string), + node_selector: pod_spec + .get("nodeSelector") + .filter(|value| value.as_object().is_some_and(|map| !map.is_empty())) + .cloned(), + tolerations: pod_spec + .get("tolerations") + .filter(|value| value.as_array().is_some_and(|list| !list.is_empty())) + .cloned(), + } +} + +/// Read the log-level env back from a Sandbox CR's rendered agent pod so a +/// reconciled supervisor keeps the same verbosity as the original. +fn proxy_pod_log_level_env_from_cr( + obj: &DynamicObject, +) -> std::collections::HashMap { + let mut env = std::collections::HashMap::new(); + let containers = obj + .data + .get("spec") + .and_then(|spec| spec.get("podTemplate")) + .and_then(|template| template.get("spec")) + .and_then(|spec| spec.get("containers")) + .and_then(serde_json::Value::as_array); + let Some(containers) = containers else { + return env; + }; + for container in containers { + let Some(entries) = container.get("env").and_then(serde_json::Value::as_array) else { + continue; + }; + for entry in entries { + if entry.get("name").and_then(serde_json::Value::as_str) + == Some(openshell_core::sandbox_env::LOG_LEVEL) + && let Some(value) = entry.get("value").and_then(serde_json::Value::as_str) + { + env.insert( + openshell_core::sandbox_env::LOG_LEVEL.to_string(), + value.to_string(), + ); + } + } } + env +} - fn rendered_env<'a>(container: &'a serde_json::Value, name: &str) -> Option<&'a str> { - container["env"] - .as_array()? - .iter() - .find(|item| item.get("name").and_then(|value| value.as_str()) == Some(name))? - .get("value")? - .as_str() +#[allow(clippy::too_many_arguments)] +fn proxy_pod_supervisor_deployment( + names: &ProxyPodResourceNames, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, + pod_config: &KubernetesPodDriverConfig, + placement: &ProxyPodPlacement, + replicas: u32, + owner_ref: serde_json::Value, +) -> Deployment { + let mut container = serde_json::json!({ + "name": SUPERVISOR_NETWORK_SIDECAR_NAME, + "image": params.supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network", + ], + "env": proxy_pod_supervisor_env(template_environment, spec_environment, params), + "ports": [ + {"name": "http-proxy", "containerPort": PROXY_POD_PROXY_PORT, "protocol": "TCP"} + ], + "readinessProbe": { + "tcpSocket": {"port": PROXY_POD_PROXY_PORT}, + "periodSeconds": 2, + "failureThreshold": 30 + }, + "securityContext": { + "runAsUser": params.proxy_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + } + }, + "volumeMounts": [ + { + "name": "openshell-sa-token", + "mountPath": "/var/run/secrets/openshell", + "readOnly": true + }, + { + "name": "openshell-proxy-pod-ca-source", + "mountPath": PROXY_POD_CA_SECRET_MOUNT_PATH, + "readOnly": true + }, + proxy_pod_ca_tls_volume_mount(false), + ] + }); + // Route egress through the operator's corporate upstream proxy, matching + // sidecar topology. Credentials are excluded because the proxy-pod + // supervisor does not mount the auth Secret. + container["command"] + .as_array_mut() + .expect("network supervisor command is an array") + .extend( + upstream_proxy_cli_args(params, false) + .into_iter() + .map(serde_json::Value::String), + ); + if !params.supervisor_image_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + } + if !params.client_tls_secret_name.is_empty() { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": "openshell-client-tls", + "mountPath": SIDECAR_CLIENT_TLS_MOUNT_PATH, + "readOnly": true + })); + } + if params.provider_spiffe_enabled { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), + "readOnly": true, + })); + } + if let Some(profile) = params.app_armor_profile { + container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); } - #[test] - fn driver_config_rejects_invalid_shape() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "pod": "not-an-object" - }))), - ..SandboxTemplate::default() - }; + let mut spec = serde_json::json!({ + "serviceAccountName": params.service_account_name, + "automountServiceAccountToken": false, + "securityContext": { + "fsGroup": params.sandbox_gid + }, + "containers": [container], + "volumes": [ + { + "name": "openshell-sa-token", + "projected": { + "sources": [{ + "serviceAccountToken": { + "audience": "openshell-gateway", + "expirationSeconds": params.sa_token_ttl_secs, + "path": "token" + } + }], + "defaultMode": 0o440 + } + }, + { + "name": "openshell-proxy-pod-ca-source", + "secret": { + "secretName": names.proxy_ca_secret, + "defaultMode": 0o440 + } + }, + { + "name": "openshell-proxy-pod-tls", + "emptyDir": {} + } + ] + }); + // Match the workload's runtime-class precedence: public platform_config, + // then driver_config.pod, then the cluster default. + let runtime_class_name = placement + .runtime_class_name + .clone() + .or_else(|| { + (!pod_config.runtime_class_name.is_empty()) + .then(|| pod_config.runtime_class_name.clone()) + }) + .or_else(|| { + (!params.default_runtime_class_name.is_empty()) + .then(|| params.default_runtime_class_name.to_string()) + }); + if let Some(runtime_class) = runtime_class_name { + spec["runtimeClassName"] = serde_json::json!(runtime_class); + } + if let Some(spec_obj) = spec.as_object_mut() { + apply_host_gateway_aliases(spec_obj, params.host_gateway_ip); + } + let image_pull_secrets = image_pull_secret_refs(params.image_pull_secrets); + if !image_pull_secrets.is_empty() { + spec["imagePullSecrets"] = serde_json::Value::Array(image_pull_secrets); + } + if !params.client_tls_secret_name.is_empty() { + spec["volumes"] + .as_array_mut() + .expect("volumes is an array") + .push(serde_json::json!({ + "name": "openshell-client-tls", + "secret": { + "secretName": params.client_tls_secret_name, + "defaultMode": 0o440 + } + })); + } + if params.provider_spiffe_enabled { + spec["volumes"] + .as_array_mut() + .expect("volumes is an array") + .push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "csi": { + "driver": "csi.spiffe.io", + "readOnly": true + } + })); + } + if let Some(spec_obj) = spec.as_object_mut() { + // Seed platform_config placement first so driver_config.pod merges on top + // with the same precedence the workload uses (per-key node-selector + // override, appended tolerations). + if let Some(node_selector) = placement.node_selector.clone() { + spec_obj.insert("nodeSelector".to_string(), node_selector); + } + if let Some(tolerations) = placement.tolerations.clone() { + spec_obj.insert("tolerations".to_string(), tolerations); + } + apply_pod_driver_config(spec_obj, pod_config); + } + + k8s_object(serde_json::json!({ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": proxy_pod_object_meta( + &names.supervisor_deployment, + params.namespace, + params.sandbox_id, + SANDBOX_ROLE_SUPERVISOR, + params.gateway_id, + owner_ref + ), + "spec": { + "replicas": replicas, + "selector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR) + }, + "template": { + "metadata": { + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR, params.gateway_id), + "annotations": { + "openshell.ai/sandbox-id": params.sandbox_id + } + }, + "spec": spec + } + } + })) +} - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); +/// Build the DNS egress rules for the agent pod. +/// +/// Emits one rule per configured peer, because peers may listen on different +/// ports and a `NetworkPolicy` rule applies its port list to every `to` entry +/// in that rule. +/// +/// `peer.port` is the destination **pod** port. Egress rules with a +/// `podSelector` peer are evaluated after `Service` address translation, so a +/// cluster whose DNS `Service` maps 53 onto a different container port needs +/// that container port configured. Upstream `CoreDNS` listens on 53; +/// `OpenShift`'s `dns-default` listens on 5353 and maps 53 to it. +/// +/// Returns an empty vector for an empty peer list. This is deliberately +/// fail-closed: a `NetworkPolicy` egress rule with an empty `to` array matches +/// *every* destination, so emitting one here would silently open DNS-port +/// egress to the whole cluster. Emitting no rule denies DNS instead, and +/// `validate_dns_peers` rejects an empty list at startup so a correctly +/// configured driver never reaches that state. +fn proxy_pod_dns_egress_rules(peers: &[ProxyPodDnsPeer]) -> Vec { + peers + .iter() + .map(|peer| { + let mut entry = serde_json::Map::new(); + if !peer.namespace_labels.is_empty() { + entry.insert( + "namespaceSelector".to_string(), + serde_json::json!({"matchLabels": peer.namespace_labels}), + ); + } + if !peer.pod_labels.is_empty() { + entry.insert( + "podSelector".to_string(), + serde_json::json!({"matchLabels": peer.pod_labels}), + ); + } + serde_json::json!({ + "to": [serde_json::Value::Object(entry)], + "ports": [ + {"protocol": "UDP", "port": peer.port}, + {"protocol": "TCP", "port": peer.port} + ] + }) + }) + .collect() +} - assert!(err.contains("invalid kubernetes driver_config")); - } +/// Egress rules permitting the in-pod process supervisor to reach the gateway +/// (TCP only). Same selector shape as DNS peers; the port is the gateway's. +fn proxy_pod_gateway_egress_rules(peers: &[ProxyPodDnsPeer]) -> Vec { + peers + .iter() + .map(|peer| { + let mut entry = serde_json::Map::new(); + if !peer.namespace_labels.is_empty() { + entry.insert( + "namespaceSelector".to_string(), + serde_json::json!({"matchLabels": peer.namespace_labels}), + ); + } + if !peer.pod_labels.is_empty() { + entry.insert( + "podSelector".to_string(), + serde_json::json!({"matchLabels": peer.pod_labels}), + ); + } + serde_json::json!({ + "to": [serde_json::Value::Object(entry)], + "ports": [{"protocol": "TCP", "port": peer.port}] + }) + }) + .collect() +} - #[test] - fn driver_config_rejects_unknown_fields() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "cdi_devices": ["nvidia.com/gpu=0"] - }))), - ..SandboxTemplate::default() - }; +fn proxy_pod_agent_egress_network_policy( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, +) -> NetworkPolicy { + let mut egress = vec![serde_json::json!({ + "to": [{ + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR) + } + }], + "ports": [ + {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT} + ] + })]; + egress.extend(proxy_pod_dns_egress_rules(params.proxy_pod_dns_peers)); + // Permit the in-pod process supervisor's own gateway session (relays, policy, + // log push, token bootstrap). The workload's children still reach only the + // proxy on 3128; this rule is for the supervisor, not the workload. + egress.extend(proxy_pod_gateway_egress_rules( + params.proxy_pod_gateway_peers, + )); - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + // Deliberately NO ownerReference: this egress policy is the workload's fence, + // and it must outlive the workload pod during deletion. Owner-reference + // garbage collection deletes it concurrently with the pod (siblings of the + // Sandbox CR), which would reopen direct egress for a pod that ignores + // SIGTERM during its termination grace period. The gateway instead deletes + // it explicitly after the pod is gone (delete_sandbox) and reaps orphans in + // reconciliation, so it is never collected while the workload can still run. + k8s_object(serde_json::json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": names.agent_egress_network_policy, + "namespace": params.namespace, + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_AGENT, params.gateway_id), + // Record the guarded workload pod's name so delete/reap can confirm + // it is gone by a scoped `get`, never a cluster-wide pod list. The + // agent pod is named after its Sandbox CR (== cr_name). + "annotations": { + ANNOTATION_AGENT_POD_NAME: params.cr_name, + }, + }, + "spec": { + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_AGENT) + }, + "policyTypes": ["Egress"], + "egress": egress + } + })) +} - assert!(err.contains("unknown field")); +fn proxy_pod_supervisor_ingress_network_policy( + names: &ProxyPodResourceNames, + params: &SandboxPodParams<'_>, + owner_ref: serde_json::Value, +) -> NetworkPolicy { + k8s_object(serde_json::json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": names.supervisor_ingress_network_policy, + "namespace": params.namespace, + "labels": proxy_pod_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR, params.gateway_id), + "ownerReferences": [owner_ref], + }, + "spec": { + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_SUPERVISOR) + }, + "policyTypes": ["Ingress"], + "ingress": [{ + "from": [{ + "podSelector": { + "matchLabels": proxy_pod_match_labels(params.sandbox_id, SANDBOX_ROLE_AGENT) + } + }], + "ports": [ + {"protocol": "TCP", "port": PROXY_POD_PROXY_PORT} + ] + }] + } + })) +} + +fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { + let mut value = serde_json::json!({ + "type": profile.to_k8s_type() + }); + if let Some(localhost_profile) = profile.localhost_profile() { + value["localhostProfile"] = serde_json::json!(localhost_profile); } + value +} - #[test] - fn driver_config_for_spec_rejects_unknown_fields() { - let sandbox = Sandbox { - id: "sandbox-123".to_string(), - spec: Some(SandboxSpec { - template: Some(SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "gpu_device_ids": ["0000:2d:00.0"] - }))), - ..Default::default() - }), - ..Default::default() - }), - ..Default::default() +fn container_resources( + template: &SandboxTemplate, + gpu_requirements: Option<&GpuResourceRequirements>, +) -> Option { + // Start from the raw resources passthrough in platform_config (preserves + // custom resource types like GPU limits that users set via the public API + // Struct), then overlay the typed DriverResourceRequirements on top. + let mut resources = + platform_config_struct(template, "resources_raw").unwrap_or_else(|| serde_json::json!({})); + + // Overlay typed CPU/memory from DriverResourceRequirements. + if let Some(ref req) = template.resources { + let obj = resources.as_object_mut().unwrap(); + let mut apply = |section: &str, key: &str, value: &str| { + if !value.is_empty() { + let sec = obj.entry(section).or_insert_with(|| serde_json::json!({})); + sec[key] = serde_json::json!(value); + } }; + apply("limits", "cpu", &req.cpu_limit); + apply("limits", "memory", &req.memory_limit); - let err = kubernetes_driver_config_for_spec(sandbox.spec.as_ref(), None).unwrap_err(); - assert!(err.contains("unknown field")); - assert!(err.contains("gpu_device_ids")); + let cpu_request = if req.cpu_request.is_empty() { + &req.cpu_limit + } else { + &req.cpu_request + }; + let memory_request = if req.memory_request.is_empty() { + &req.memory_limit + } else { + &req.memory_request + }; + apply("requests", "cpu", cpu_request); + apply("requests", "memory", memory_request); } - #[test] - fn driver_config_pvc_subpath_mounts_render_in_pod_template() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": { - "claim_name": "pvc-user-data-123", - "read_only": false - } - }], - "containers": { - "agent": { - "volume_mounts": [ - { - "name": "user-data", - "mount_path": "/sandbox/.openshell/workspace", - "sub_path": "workspace", - "read_only": false - }, - { - "name": "user-data", - "mount_path": "/sandbox/.openshell/memory", - "sub_path": "memory" - } - ] - } - } - }))), - ..SandboxTemplate::default() - }; - let spec = SandboxSpec { - template: Some(template), - ..SandboxSpec::default() - }; - - let cr = sandbox_to_k8s_spec_for_test(Some(&spec), &SandboxPodParams::default()); - let pod_template = &cr["spec"]["podTemplate"]; + if let Some(gpu) = gpu_requirements { + let quantity = gpu.count.unwrap_or(1).to_string(); + apply_gpu_limit(&mut resources, &quantity); + } + if resources.as_object().is_some_and(serde_json::Map::is_empty) { + None + } else { + Some(resources) + } +} - let volumes = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist"); - let user_volume = volumes - .iter() - .find(|volume| volume["name"] == "user-data") - .expect("user PVC volume should be rendered"); - assert_eq!( - user_volume["persistentVolumeClaim"]["claimName"], - "pvc-user-data-123" - ); - assert_eq!(user_volume["persistentVolumeClaim"]["readOnly"], false); +fn apply_gpu_limit(resources: &mut serde_json::Value, quantity: &str) { + let Some(resources_obj) = resources.as_object_mut() else { + *resources = serde_json::json!({}); + return apply_gpu_limit(resources, quantity); + }; - let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] - .as_array() - .expect("volumeMounts should exist"); - let workspace_mount = mounts - .iter() - .find(|mount| mount["mountPath"] == "/sandbox/.openshell/workspace") - .expect("workspace subPath mount should be rendered"); - assert_eq!(workspace_mount["name"], "user-data"); - assert_eq!(workspace_mount["subPath"], "workspace"); - assert_eq!(workspace_mount["readOnly"], false); + let limits = resources_obj + .entry("limits") + .or_insert_with(|| serde_json::json!({})); + let Some(limits_obj) = limits.as_object_mut() else { + *limits = serde_json::json!({}); + return apply_gpu_limit(resources, quantity); + }; - let memory_mount = mounts - .iter() - .find(|mount| mount["mountPath"] == "/sandbox/.openshell/memory") - .expect("memory subPath mount should be rendered"); - assert_eq!(memory_mount["name"], "user-data"); - assert_eq!(memory_mount["subPath"], "memory"); - assert_eq!(memory_mount["readOnly"], true); + limits_obj.insert(GPU_RESOURCE_NAME.to_string(), serde_json::json!(quantity)); +} - let spec_obj = cr["spec"].as_object().expect("spec should be an object"); - assert!( - !spec_obj.contains_key("volumeClaimTemplates"), - "explicit /sandbox driver_config mounts should skip the default workspace VCT" - ); - let has_workspace_init = pod_template["spec"]["initContainers"] - .as_array() - .is_some_and(|containers| { - containers - .iter() - .any(|container| container["name"] == WORKSPACE_INIT_CONTAINER_NAME) - }); - assert!( - !has_workspace_init, - "explicit /sandbox driver_config mounts should skip the default workspace init container" +#[allow(clippy::too_many_arguments)] +fn build_env_list( + existing_env: Option<&Vec>, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + sandbox_spec: Option<&openshell_core::proto::compute::v1::DriverSandboxSpec>, + sandbox_id: &str, + sandbox_name: &str, + grpc_endpoint: &str, + ssh_socket_path: &str, + tls_enabled: bool, + provider_spiffe_socket_path: Option<&str>, +) -> Vec { + let mut env = existing_env.cloned().unwrap_or_default(); + apply_env_map(&mut env, template_environment); + apply_env_map(&mut env, spec_environment); + let mut user_env = template_environment.clone(); + user_env.extend(spec_environment.clone()); + if !user_env.is_empty() + && let Ok(json) = serde_json::to_string(&user_env) + { + upsert_env( + &mut env, + openshell_core::sandbox_env::USER_ENVIRONMENT, + &json, ); } + let main_process = + openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox_spec) + .expect("main process config serialization cannot fail"); + upsert_env( + &mut env, + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + &main_process, + ); + apply_required_env( + &mut env, + sandbox_id, + sandbox_name, + grpc_endpoint, + ssh_socket_path, + tls_enabled, + provider_spiffe_socket_path, + ); + env +} - #[test] - fn driver_config_accepts_read_write_pvc_with_multiple_subpath_mounts() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": { - "claim_name": "pvc-user-data", - "read_only": false - } - }], - "containers": { - "agent": { - "volume_mounts": [ - { - "name": "user-data", - "mount_path": "/sandbox/.openshell/workspace", - "sub_path": "workspace", - "read_only": false - }, - { - "name": "user-data", - "mount_path": "/sandbox/.openshell/memory", - "sub_path": "memory", - "read_only": false - }, - { - "name": "user-data", - "mount_path": "/sandbox/.openshell/sessions", - "sub_path": "sessions", - "read_only": false - } - ] - } - } - }))), - ..SandboxTemplate::default() - }; - - let config = KubernetesSandboxDriverConfig::from_template(&template) - .expect("read-write PVC with multiple subPath mounts should validate"); +fn apply_env_map( + env: &mut Vec, + values: &std::collections::HashMap, +) { + for (key, value) in values { + upsert_env(env, key, value); + } +} - assert_eq!(config.volumes.len(), 1); - assert_eq!(config.volumes[0].name, "user-data"); - assert_eq!( - config.volumes[0].persistent_volume_claim.claim_name, - "pvc-user-data" +// Required env vars are passed individually for clarity at call sites; grouping into a struct +// would not improve readability for this internal helper. +fn apply_required_env( + env: &mut Vec, + sandbox_id: &str, + sandbox_name: &str, + grpc_endpoint: &str, + ssh_socket_path: &str, + tls_enabled: bool, + provider_spiffe_socket_path: Option<&str>, +) { + upsert_env(env, openshell_core::sandbox_env::SANDBOX_ID, sandbox_id); + upsert_env(env, openshell_core::sandbox_env::SANDBOX, sandbox_name); + upsert_env(env, openshell_core::sandbox_env::ENDPOINT, grpc_endpoint); + upsert_env( + env, + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ); + // Runtime capabilities are driver-owned. Kubernetes topologies do not yet + // provide the complete policy DNS and transparent TCP substrate. + upsert_env( + env, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + "", + ); + if !ssh_socket_path.is_empty() { + upsert_env( + env, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + ssh_socket_path, ); - assert!(!config.volumes[0].persistent_volume_claim.read_only); - assert_eq!(config.containers.agent.volume_mounts.len(), 3); - assert!( - config - .containers - .agent - .volume_mounts - .iter() - .all(|mount| !mount.read_only) + } + // TLS cert paths for sandbox-to-server mTLS. Only set when TLS is enabled + // and the client TLS secret is mounted into the sandbox pod. + if tls_enabled { + upsert_env( + env, + openshell_core::sandbox_env::TLS_CA, + "/etc/openshell-tls/client/ca.crt", + ); + upsert_env( + env, + openshell_core::sandbox_env::TLS_CERT, + "/etc/openshell-tls/client/tls.crt", + ); + upsert_env( + env, + openshell_core::sandbox_env::TLS_KEY, + "/etc/openshell-tls/client/tls.key", ); - assert!(config.has_explicit_sandbox_data_mount()); } - - #[test] - fn driver_config_rejects_duplicate_pvc_volume_names() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [ - { - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-a"} - }, - { - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-b"} - } - ] - }))), - ..SandboxTemplate::default() - }; - - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); - - assert!(err.contains("duplicate kubernetes driver_config volume")); + // Projected ServiceAccount token written by kubelet (see the volume + // definition in `sandbox_template_to_k8s`). The supervisor reads this + // and exchanges it for a gateway-minted JWT via `IssueSandboxToken`. + upsert_env( + env, + openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, + "/var/run/secrets/openshell/token", + ); + if let Some(socket_path) = provider_spiffe_socket_path { + upsert_env( + env, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + socket_path, + ); } +} - #[test] - fn driver_config_rejects_duplicate_pvc_volume_mount_targets() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [ - { - "name": "user-data", - "mount_path": "/sandbox/.openshell/workspace" - }, - { - "name": "user-data", - "mount_path": "/sandbox/.openshell/workspace" - } - ] - } - } - }))), - ..SandboxTemplate::default() - }; +fn provider_spiffe_socket_path<'a>(params: &'a SandboxPodParams<'a>) -> Option<&'a str> { + params + .provider_spiffe_enabled + .then_some(params.provider_spiffe_workload_api_socket_path) +} - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); +fn spiffe_socket_mount_path(socket_path: &str) -> String { + Path::new(socket_path) + .parent() + .and_then(Path::to_str) + .filter(|path| !path.is_empty() && *path != "/") + .expect("provider SPIFFE socket path should be validated before pod rendering") + .to_string() +} - assert!(err.contains("duplicate kubernetes driver_config mount target")); +fn upsert_env(env: &mut Vec, name: &str, value: &str) { + if let Some(existing) = env + .iter_mut() + .find(|item| item.get("name").and_then(|value| value.as_str()) == Some(name)) + { + *existing = serde_json::json!({"name": name, "value": value}); + return; } - #[test] - fn driver_config_accepts_dns1123_subdomain_pvc_claim_name() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc.user-data.123"} - }] - }))), - ..SandboxTemplate::default() - }; + env.push(serde_json::json!({"name": name, "value": value})); +} - let config = KubernetesSandboxDriverConfig::from_template(&template) - .expect("DNS-1123 subdomain PVC names should validate"); +fn apply_resolved_identity_env(env: &mut Vec, uid: u32, gid: u32) { + remove_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER); + remove_env(env, openshell_core::sandbox_env::SANDBOX_UID); + remove_env(env, openshell_core::sandbox_env::SANDBOX_GID); + upsert_env(env, openshell_core::sandbox_env::OCI_IMAGE_USER, ""); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_UID, + &uid.to_string(), + ); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_GID, + &gid.to_string(), + ); +} - assert_eq!( - config.volumes[0].persistent_volume_claim.claim_name, - "pvc.user-data.123" - ); - } +fn remove_env(env: &mut Vec, name: &str) { + env.retain(|item| item.get("name").and_then(|value| value.as_str()) != Some(name)); +} - #[test] - fn driver_config_rejects_invalid_volume_label_and_claim_name() { - for (field, config) in [ - ( - "volumes[].name", - serde_json::json!({ - "volumes": [{ - "name": "User_Data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }] - }), - ), - ( - "volumes[].persistent_volume_claim.claim_name", - serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "Pvc_User_Data"} - }] - }), - ), - ] { - let template = SandboxTemplate { - driver_config: Some(json_struct(config)), - ..SandboxTemplate::default() - }; +fn remove_volume_mount(volume_mounts: &mut Vec, name: &str) { + volume_mounts.retain(|mount| mount.get("name").and_then(|value| value.as_str()) != Some(name)); +} - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); - assert!( - err.contains(field) && err.contains("DNS-1123"), - "expected invalid {field} to fail DNS-1123 validation, got {err}" - ); +/// Node-placement overrides sourced from a sandbox template's public +/// `platform_config` (the typed/legacy path the workload pod honors). The +/// proxy-pod supervisor must apply the same overrides so it lands on a node the +/// workload can also use; otherwise same-node affinity can be unschedulable and +/// runtime-class mismatches (e.g. Kata vs default) split the pair across +/// incompatible runtimes. +#[derive(Default)] +struct ProxyPodPlacement { + runtime_class_name: Option, + node_selector: Option, + tolerations: Option, +} + +impl ProxyPodPlacement { + fn from_template(template: Option<&SandboxTemplate>) -> Self { + let Some(template) = template else { + return Self::default(); + }; + Self { + runtime_class_name: platform_config_string(template, "runtime_class_name"), + node_selector: platform_config_struct(template, "node_selector"), + tolerations: platform_config_struct(template, "tolerations"), } } +} - #[test] - fn driver_config_rejects_mounts_referencing_unknown_volumes() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "known-data", - "persistent_volume_claim": {"claim_name": "pvc-known"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "missing-data", - "mount_path": "/sandbox/.openshell/workspace", - "sub_path": "workspace" - }] - } - } - }))), - ..SandboxTemplate::default() - }; +/// Extract a string value from the template's `platform_config` Struct. +fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option { + let config = template.platform_config.as_ref()?; + let value = config.fields.get(key)?; + match value.kind.as_ref() { + Some(prost_types::value::Kind::StringValue(s)) if !s.is_empty() => Some(s.clone()), + _ => None, + } +} - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); +fn platform_config_bool(template: &SandboxTemplate, key: &str) -> Option { + let config = template.platform_config.as_ref()?; + let value = config.fields.get(key)?; + match value.kind.as_ref() { + Some(prost_types::value::Kind::BoolValue(value)) => Some(*value), + _ => None, + } +} - assert!(err.contains("unknown kubernetes driver_config volume 'missing-data'")); +/// Extract a nested Struct value from the template's `platform_config`, +/// converting it to `serde_json::Value`. +fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option { + let config = template.platform_config.as_ref()?; + let value = config.fields.get(key)?; + let json = value_to_json(value); + // Return None for null/empty objects so callers can distinguish + // "field absent" from "field present but empty". + match &json { + serde_json::Value::Null => None, + serde_json::Value::Object(m) if m.is_empty() => None, + _ => Some(json), } +} - #[test] - fn driver_config_rejects_shared_reserved_mount_targets() { - for mount_path in [ - "/", - "/sandbox", - "/etc/openshell", - "/etc/openshell-tls/client", - "/opt/openshell/bin", - ] { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": mount_path - }] - } - } - }))), - ..SandboxTemplate::default() - }; +/// Convert a `Sandbox` CR's status into the driver contract's status. +/// +/// `topology` decides the supervisor-session model reported to the gateway. +/// `proxy-pod` has no in-sandbox process supervisor, so no `ConnectSupervisor` +/// session will ever open; the gateway must derive readiness from the +/// conditions below instead of waiting forever. The agent pod's +/// `wait-for-proxy` init container is what makes that safe: the pod does not +/// become Ready until its paired supervisor is accepting connections. +/// Whether a proxy-pod sandbox's supervisor Deployment currently has no +/// available replica. A missing Deployment counts as unavailable; a transient +/// API error does not (returns `false`), so readiness never flaps on a blip. +/// Tri-state supervisor availability. `Unknown` (an API error or timeout on the +/// Deployment GET) is deliberately distinct from `Available`: callers must not +/// treat "could not determine" as "up", or a transient blip would republish a +/// dead-egress sandbox as `Ready`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SupervisorAvailability { + Available, + Unavailable, + Unknown, +} + +/// Supervisor availability derived from a supervisor `Deployment` object already +/// in hand (e.g. from a watch event), so no GET is needed and the result is +/// never `Unknown`. +fn supervisor_availability_from_deployment(deployment: &Deployment) -> SupervisorAvailability { + let available = deployment + .status + .as_ref() + .and_then(|status| status.available_replicas) + .unwrap_or(0) + >= 1; + if available { + SupervisorAvailability::Available + } else { + SupervisorAvailability::Unavailable + } +} - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); - assert!( - err.contains("mount path") || err.contains("mount target"), - "expected protected mount target {mount_path:?} to be rejected, got {err}" +async fn proxy_pod_supervisor_availability( + client: &Client, + namespace: &str, + deployment_name: &str, +) -> SupervisorAvailability { + let deployments: Api = Api::namespaced(client.clone(), namespace); + match tokio::time::timeout(KUBE_API_TIMEOUT, deployments.get_opt(deployment_name)).await { + Ok(Ok(Some(deployment))) => supervisor_availability_from_deployment(&deployment), + // A missing Deployment is a definite absence, not an error. + Ok(Ok(None)) => SupervisorAvailability::Unavailable, + Ok(Err(err)) => { + warn!( + deployment = %deployment_name, + error = %err, + "Could not determine proxy-pod supervisor availability; treating supervisor as not ready" ); + SupervisorAvailability::Unknown + } + Err(_elapsed) => { + warn!( + deployment = %deployment_name, + "Timed out checking proxy-pod supervisor availability; treating supervisor as not ready" + ); + SupervisorAvailability::Unknown } } +} - #[test] - fn driver_config_rejects_kubernetes_static_protected_mount_targets() { - let spec = SandboxSpec { - template: Some(SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": "/var/run/secrets/openshell" - }] - } - } - }))), - ..SandboxTemplate::default() - }), - ..SandboxSpec::default() - }; +/// Map a Sandbox CR to a `DriverSandbox`, folding proxy-pod supervisor +/// availability into readiness. Used by the watch paths so a CR event never +/// republishes a sandbox as `Ready` while its supervisor Deployment is down — +/// matching what the get/list paths already report. +async fn sandbox_from_object_with_supervisor_readiness( + client: &Client, + namespace: &str, + obj: DynamicObject, + fallback_topology: SupervisorTopology, +) -> Result<(String, Sandbox), String> { + let cr_topology = topology_from_object(&obj, fallback_topology); + let cr_namespace = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| namespace.to_string()); + let cr_name = obj.metadata.name.clone().unwrap_or_default(); + let sandbox_id = sandbox_id_from_object(&obj).unwrap_or_default(); + let (kube_name, mut sandbox) = sandbox_from_object(namespace, obj, fallback_topology)?; + if cr_topology == SupervisorTopology::ProxyPod && !sandbox_id.is_empty() { + let names = proxy_pod_resource_names(&cr_name, &sandbox_id); + // Fail closed: keep `Ready` only when the supervisor is confirmed + // available. `Unavailable` and `Unknown` (a GET error/timeout) both + // downgrade to `Provisioning`, so a watch event never republishes a + // sandbox as `Ready` while its policy-enforced egress path may be down. + if proxy_pod_supervisor_availability(client, &cr_namespace, &names.supervisor_deployment) + .await + != SupervisorAvailability::Available + { + mark_supervisor_unavailable(&mut sandbox); + } + } + Ok((kube_name, sandbox)) +} - let err = kubernetes_driver_config_for_spec(Some(&spec), None).unwrap_err(); +/// The `openshell.ai/sandbox-id` a supervisor Deployment belongs to, or `None` +/// when the label is absent or empty (not an OpenShell-managed supervisor). +fn supervisor_deployment_sandbox_id(deployment: &Deployment) -> Option { + deployment + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .filter(|id| !id.is_empty()) + .cloned() +} - assert!(err.contains("/var/run/secrets/openshell")); +/// Push a refreshed sandbox status in response to a supervisor Deployment +/// change so proxy-pod readiness reflects supervisor availability within seconds +/// instead of waiting for the reconcile sweep. `availability` is taken from the +/// watch event object itself, so the CR re-read applies the exact observed state +/// rather than re-fetching the Deployment (which could fail open on a transient +/// error and republish a dead-egress sandbox as `Ready`). Returns `false` only +/// when the watch consumer has gone away, signalling the caller to stop. +async fn emit_supervisor_readiness_refresh( + driver: &KubernetesComputeDriver, + tx: &mpsc::Sender>, + deployment: &Deployment, + availability: SupervisorAvailability, +) -> bool { + let Some(sandbox_id) = supervisor_deployment_sandbox_id(deployment) else { + return true; + }; + match driver + .lookup_sandbox_with_readiness(&sandbox_id, Some(availability)) + .await + { + Ok(Some(sandbox)) => { + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + }; + tx.send(Ok(event)).await.is_ok() + } + // The CR is already gone; the sandbox watch emits its own Deleted event. + Ok(None) => true, + Err(err) => { + warn!( + sandbox_id = %sandbox_id, + error = %err, + "Failed to refresh sandbox status after supervisor Deployment change" + ); + true + } } +} - #[test] - fn driver_config_allows_spiffe_workload_path_without_provider_spiffe() { - let spec = SandboxSpec { - template: Some(SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": "/spiffe-workload-api" - }] - } - } - }))), - ..SandboxTemplate::default() - }), - ..SandboxSpec::default() - }; +/// Turn one supervisor Deployment watch event into sandbox status refreshes. +/// Returns `false` when the watch consumer has gone away. +async fn handle_supervisor_deployment_event( + driver: &KubernetesComputeDriver, + tx: &mpsc::Sender>, + event: Event, +) -> bool { + match event { + Event::Applied(deployment) => { + let availability = supervisor_availability_from_deployment(&deployment); + emit_supervisor_readiness_refresh(driver, tx, &deployment, availability).await + } + // A deleted supervisor Deployment is unambiguously unavailable; do not + // re-fetch and risk reading nothing (or a stale replica) back. + Event::Deleted(deployment) => { + emit_supervisor_readiness_refresh( + driver, + tx, + &deployment, + SupervisorAvailability::Unavailable, + ) + .await + } + Event::Restarted(deployments) => { + for deployment in deployments { + let availability = supervisor_availability_from_deployment(&deployment); + if !emit_supervisor_readiness_refresh(driver, tx, &deployment, availability).await { + return false; + } + } + true + } + } +} - kubernetes_driver_config_for_spec(Some(&spec), None) - .expect("SPIFFE workload path should only be protected when SPIFFE is enabled"); +/// Desired supervisor replica count from a Sandbox CR's operating state: `1` +/// while running, `0` while suspended. Defaults to `1` (a freshly created CR is +/// running) when no operating state is recorded. Lets reconciliation restore the +/// correct replica count after a crash between the operating-state patch and the +/// supervisor scale. +fn desired_supervisor_replicas(obj: &DynamicObject) -> u32 { + let spec = obj.data.get("spec"); + // v1beta1 encodes desired state as spec.operatingMode. + if let Some(mode) = spec + .and_then(|spec| spec.get("operatingMode")) + .and_then(serde_json::Value::as_str) + { + return u32::from(!mode.eq_ignore_ascii_case("Suspended")); + } + // v1alpha1 encodes it as spec.replicas (0 or 1). + if let Some(replicas) = spec + .and_then(|spec| spec.get("replicas")) + .and_then(serde_json::Value::as_u64) + { + return u32::from(replicas > 0); + } + 1 +} + +/// Force a proxy-pod sandbox's `Ready` condition to `False` because its +/// supervisor Deployment has no available replica. The agent pod's own Ready +/// condition (which the CR carries) cannot see the separate supervisor, so +/// without this the sandbox would report Ready while policy-enforced egress is +/// dead. +fn mark_supervisor_unavailable(sandbox: &mut Sandbox) { + // `DependenciesNotReady` is on the gateway's transient-reason allowlist, so + // the sandbox becomes `Provisioning` (recoverable) rather than terminal + // `Error`: readiness returns to `Ready` once the supervisor Deployment does. + const REASON: &str = "DependenciesNotReady"; + const MESSAGE: &str = "proxy-pod network supervisor Deployment has no available replica"; + let Some(status) = sandbox.status.as_mut() else { + return; + }; + if let Some(ready) = status + .conditions + .iter_mut() + .find(|condition| condition.r#type == "Ready") + { + ready.status = "False".to_string(); + ready.reason = REASON.to_string(); + ready.message = MESSAGE.to_string(); + } else { + status.conditions.push(SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: REASON.to_string(), + message: MESSAGE.to_string(), + last_transition_time: String::new(), + }); } +} - #[test] - fn driver_config_rejects_invalid_kubernetes_sub_paths() { - for sub_path in ["/workspace", "../workspace"] { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": "/sandbox/.openshell/workspace", - "sub_path": sub_path - }] - } - } - }))), - ..SandboxTemplate::default() - }; +fn status_from_object(obj: &DynamicObject, topology: SupervisorTopology) -> Option { + let status = obj.data.get("status")?; + let status_obj = status.as_object()?; - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); - assert!( - err.contains("mount subpath must be relative"), - "expected invalid sub_path {sub_path:?} to be rejected, got {err}" - ); - } + let conditions = status_obj + .get("conditions") + .and_then(|val| val.as_array()) + .map(|items| { + items + .iter() + .filter_map(condition_from_value) + .collect::>() + }) + .unwrap_or_default(); + + Some(SandboxStatus { + sandbox_name: status_obj + .get("sandboxName") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + instance_id: status_obj + .get("agentPod") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + agent_fd: status_obj + .get("agentFd") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + sandbox_fd: status_obj + .get("sandboxFd") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + conditions, + deleting: obj.metadata.deletion_timestamp.is_some(), + // Every topology now runs an in-sandbox process supervisor that owns a + // gateway session and serves relays, so all report `Required`. (The + // proxy-pod variant keeps only the network proxy out of the pod.) + supervisor_session_model: match topology { + SupervisorTopology::Combined + | SupervisorTopology::Sidecar + | SupervisorTopology::ProxyPod => SupervisorSessionModel::Required as i32, + }, + }) +} + +fn kubernetes_sandbox_has_stopped_condition(obj: &DynamicObject) -> bool { + obj.data + .get("status") + .and_then(|status| status.get("conditions")) + .and_then(serde_json::Value::as_array) + .is_some_and(|conditions| { + conditions.iter().any(|condition| { + condition.get("type").and_then(serde_json::Value::as_str) + == Some(SANDBOX_SUSPENDED_CONDITION) + && condition + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| status.eq_ignore_ascii_case("true")) + }) + }) +} + +fn kubernetes_sandbox_stop_is_complete( + api_version: &str, + obj: &DynamicObject, + pod_is_gone: bool, +) -> bool { + if api_version == SANDBOX_VERSION_V1ALPHA1 { + // v1alpha1 omits a usable stopped condition. + pod_is_gone + } else { + kubernetes_sandbox_has_stopped_condition(obj) && pod_is_gone } +} - #[test] - fn driver_config_defaults_pvc_mounts_to_read_only() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": "/sandbox/.openshell/workspace", - "sub_path": "workspace" - }] - } - } - }))), - ..SandboxTemplate::default() - }; +fn kubernetes_sandbox_stop_failure(obj: &DynamicObject) -> Option { + obj.data + .get("status")? + .get("conditions")? + .as_array()? + .iter() + .find_map(|condition| { + let is_terminal = condition.get("type").and_then(serde_json::Value::as_str) + == Some(SANDBOX_SUSPENDED_CONDITION) + && condition + .get("status") + .and_then(serde_json::Value::as_str) + .is_some_and(|status| status.eq_ignore_ascii_case("false")) + && condition.get("reason").and_then(serde_json::Value::as_str) + == Some(SANDBOX_SUSPENDED_POD_NOT_OWNED_REASON); + if !is_terminal { + return None; + } + + let message = condition + .get("message") + .and_then(serde_json::Value::as_str) + .filter(|message| !message.is_empty()) + .unwrap_or("backing pod is not owned by this sandbox"); + Some(format!("Kubernetes sandbox stop rejected: {message}")) + }) +} + +async fn kubernetes_sandbox_pod_is_gone( + pod_api: &Api, + pod_name: &str, + deadline: tokio::time::Instant, +) -> Result { + let request_timeout = + KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(tokio::time::Instant::now())); + if request_timeout.is_zero() { + return Ok(false); + } + + match tokio::time::timeout(request_timeout, pod_api.get(pod_name)).await { + Ok(Ok(_)) => Ok(false), + Ok(Err(KubeError::Api(err))) if err.code == 404 => Ok(true), + Ok(Err(err)) => Err(err.to_string()), + Err(_) => Err(format!( + "timed out after {}s waiting for Kubernetes API while checking sandbox pod termination", + request_timeout.as_secs() + )), + } +} + +fn kubernetes_sandbox_stop_timeout(obj: &DynamicObject) -> Duration { + let termination_grace_period = obj + .data + .get("spec") + .and_then(|spec| spec.get("podTemplate")) + .and_then(|template| template.get("spec")) + .and_then(|spec| spec.get("terminationGracePeriodSeconds")) + .and_then(serde_json::Value::as_u64) + .map_or(DEFAULT_POD_TERMINATION_GRACE_PERIOD, Duration::from_secs); + + // The controller must observe the desired state, wait for the pod grace + // period and kubelet teardown, then reconcile the deleted pod into the + // Sandbox status. Keep one API timeout of headroom around that grace. + termination_grace_period.saturating_add(KUBE_API_TIMEOUT) +} + +fn next_stop_poll_interval(current: Duration) -> Duration { + current.saturating_mul(2).min(STOP_MAX_POLL_INTERVAL) +} + +fn sandbox_operating_state_patch( + api_version: &str, + resource_version: &str, + running: bool, +) -> serde_json::Value { + if api_version == SANDBOX_VERSION_V1BETA1 { + serde_json::json!({ + "metadata": {"resourceVersion": resource_version}, + "spec": {"operatingMode": if running { "Running" } else { "Suspended" }} + }) + } else { + serde_json::json!({ + "metadata": {"resourceVersion": resource_version}, + "spec": {"replicas": i32::from(running)} + }) + } +} + +fn condition_from_value(value: &serde_json::Value) -> Option { + let obj = value.as_object()?; + Some(SandboxCondition { + r#type: obj.get("type")?.as_str()?.to_string(), + status: obj.get("status")?.as_str()?.to_string(), + reason: obj + .get("reason") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + message: obj + .get("message") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + last_transition_time: obj + .get("lastTransitionTime") + .and_then(|val| val.as_str()) + .unwrap_or_default() + .to_string(), + }) +} + +fn spawn_namespace_label_watcher( + client: Client, + label_selector: String, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + let ns_api: Api = Api::all(client); + let watcher_config = watcher::Config::default().labels(&label_selector); + let jitter_seed = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |duration| { + duration.as_secs() ^ u64::from(duration.subsec_nanos()) + }); + + tokio::spawn(async move { + let mut retry_attempt = 0; + loop { + let mut stream = watcher::watcher(ns_api.clone(), watcher_config.clone()).boxed(); + + loop { + let event = tokio::select! { + result = stream.try_next() => result, + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + match event { + Ok(Some(Event::Applied(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.insert(name.to_string()) + { + info!(namespace = name, "operator namespace added to allowlist"); + } + } + Ok(Some(Event::Deleted(ns))) => { + retry_attempt = 0; + if let Some(name) = ns.metadata.name.as_deref() + && allowlist.remove(name) + { + info!( + namespace = name, + "operator namespace removed from allowlist" + ); + } + } + Ok(Some(Event::Restarted(namespaces))) => { + retry_attempt = 0; + let names: std::collections::BTreeSet = namespaces + .into_iter() + .filter_map(|ns| ns.metadata.name) + .collect(); + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist replaced from full relist" + ); + } + Ok(None) => { + warn!("operator namespace watcher stream ended unexpectedly"); + break; + } + Err(err) => { + warn!(error = %err, "operator namespace watcher stream error"); + break; + } + } + } + + let retry_delay = namespace_watcher_retry_delay(retry_attempt, jitter_seed); + warn!(?retry_delay, "operator namespace watcher reconnecting"); + tokio::select! { + () = tokio::time::sleep(retry_delay) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + retry_attempt = retry_attempt.saturating_add(1); + } + }); + + info!( + label_selector = %label_selector, + "operator namespace label watcher spawned" + ); +} + +fn namespace_watcher_retry_delay(attempt: u32, jitter_seed: u64) -> Duration { + let base_secs = 2_u64.saturating_mul(1_u64 << attempt.min(4)).min(24); + let max_jitter_secs = base_secs / 4; + let mixed_seed = + jitter_seed.wrapping_add(u64::from(attempt).wrapping_mul(0x9e37_79b9_7f4a_7c15)); + let jitter_secs = mixed_seed % (max_jitter_secs + 1); + Duration::from_secs(base_secs + jitter_secs) +} + +fn load_namespace_file(path: &Path) -> Result, String> { + let contents = std::fs::read_to_string(path) + .map_err(|e| format!("failed to read {}: {e}", path.display()))?; + let names: Vec = serde_json::from_str(&contents) + .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; + Ok(names.into_iter().collect()) +} + +fn spawn_namespace_file_watcher( + path: PathBuf, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + path = %path.display(), + total = count, + "operator namespace allowlist loaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to load initial operator namespace file, allowlist empty" + ); + } + } + + let watch_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let debounce = Duration::from_secs(1); + + tokio::spawn(async move { + let (tx, mut rx) = mpsc::unbounded_channel(); + + let mut watcher = + match notify::recommended_watcher(move |res: Result| { + if let Ok(event) = res + && matches!( + event.kind, + notify::EventKind::Modify(_) | notify::EventKind::Create(_) + ) + { + let _ = tx.send(()); + } + }) { + Ok(w) => w, + Err(e) => { + warn!( + error = %e, + "failed to start operator namespace file watcher, hot-reload disabled" + ); + return; + } + }; + + if let Err(e) = notify::Watcher::watch( + &mut watcher, + &watch_dir, + notify::RecursiveMode::NonRecursive, + ) { + warn!( + error = %e, + dir = %watch_dir.display(), + "failed to watch operator namespace file directory, hot-reload disabled" + ); + return; + } + + info!( + path = %path.display(), + "operator namespace file watcher started" + ); + + loop { + let got_event = tokio::select! { + event = rx.recv() => event.is_some(), + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + if !got_event { + warn!("operator namespace file watcher disconnected"); + break; + } + + loop { + tokio::select! { + () = tokio::time::sleep(debounce) => { + match load_namespace_file(&path) { + Ok(names) => { + let count = names.len(); + allowlist.replace(names); + info!( + total = count, + "operator namespace allowlist reloaded from file" + ); + } + Err(err) => { + warn!( + error = %err, + "failed to reload operator namespace file, keeping existing allowlist" + ); + } + } + break; + } + r = rx.recv() => { + if r.is_some() { + continue; + } + warn!("operator namespace file watcher disconnected"); + return; + } + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::progress::{ + PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, + PROGRESS_COMPLETE_STEP_KEY, + }; + use openshell_core::proto::compute::v1::{GpuResourceRequirements, ResourceRequirements}; + use prost_types::{Struct, Value, value::Kind}; + use std::collections::BTreeSet; + + static ENV_LOCK: std::sync::LazyLock> = + std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + + #[tokio::test] + async fn tracing_create_sandbox_failure_exports_a_kubernetes_operation_span() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing::instrument::WithSubscriber as _; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = crate::otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(crate::otel_tracing::layer(&provider)); + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + + driver + .create_sandbox(&Sandbox::default()) + .with_subscriber(subscriber) + .await + .expect_err("missing sandbox name should fail"); + provider.force_flush().unwrap(); + + let spans = exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|span| span.name == "kubernetes.create_sandbox") + .expect("create operation span"); + assert!(matches!( + span.status, + opentelemetry::trace::Status::Error { .. } + )); + provider.shutdown().unwrap(); + } + + #[tokio::test] + async fn sandbox_annotation_propagates_the_active_w3c_trace_context() { + use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; + use tracing_subscriber::layer::SubscriberExt as _; + + let _tracing_lock = crate::otel_tracing::test_lock().await; + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter) + .build(); + let subscriber = tracing_subscriber::registry().with(crate::otel_tracing::layer(&provider)); + + let annotations = tracing::subscriber::with_default(subscriber, || { + let span = tracing::info_span!("kubernetes.create_sandbox"); + let _entered = span.enter(); + let mut annotations = BTreeMap::new(); + add_trace_context_annotation(&mut annotations); + annotations + }); + + let carrier: serde_json::Value = serde_json::from_str( + annotations + .get("opentelemetry.io/trace-context") + .expect("agent-sandbox trace-context annotation"), + ) + .expect("annotation should contain a JSON propagation carrier"); + let traceparent = carrier["traceparent"] + .as_str() + .expect("carrier should contain traceparent"); + assert!(traceparent.starts_with("00-")); + assert_eq!(traceparent.len(), 55); + + provider.shutdown().unwrap(); + } + + fn json_struct(value: serde_json::Value) -> Struct { + let serde_json::Value::Object(object) = value else { + panic!("expected JSON object"); + }; + openshell_core::proto_struct::json_object_to_struct(object) + .expect("test JSON must convert to a protobuf Struct") + } + + fn sandbox_to_k8s_spec_for_test( + spec: Option<&SandboxSpec>, + params: &SandboxPodParams<'_>, + ) -> serde_json::Value { + sandbox_to_k8s_spec(spec, params).expect("test Kubernetes driver_config should be valid") + } + + fn kube_api_error(code: u16, message: &str) -> KubeError { + KubeError::Api(kube::core::ErrorResponse { + status: if code == 404 { + "404 Not Found".to_string() + } else { + "Failure".to_string() + }, + message: message.to_string(), + reason: "Failed to parse error data".to_string(), + code, + }) + } + + fn expired_watch_error() -> watcher::Error { + watcher::Error::WatchError(kube::core::ErrorResponse { + status: "Failure".to_string(), + message: "too old resource version".to_string(), + reason: "Expired".to_string(), + code: 410, + }) + } + + #[tokio::test] + async fn sandbox_watcher_error_does_not_hide_restarted_recovery_event() { + let recovered = DynamicObject { + types: None, + metadata: ObjectMeta { + name: Some("recovered-sandbox".to_string()), + ..Default::default() + }, + data: serde_json::json!({}), + }; + let source = futures::stream::iter([ + Err(expired_watch_error()), + Ok(Event::Restarted(vec![recovered])), + ]); + let mut stream = continue_on_watcher_errors(source, "sandbox-resource"); + + let event = stream + .next() + .await + .expect("410 Expired must not terminate the watcher stream"); + let Event::Restarted(objects) = event else { + panic!("expected kube-runtime recovery to emit Restarted"); + }; + assert_eq!(objects.len(), 1); + assert_eq!( + objects[0].metadata.name.as_deref(), + Some("recovered-sandbox") + ); + assert!( + stream.next().await.is_none(), + "source closure must be preserved" + ); + } + + #[tokio::test(start_paused = true)] + async fn outward_watch_stream_survives_expired_error_and_backoff_recovery() { + let recovered = DynamicObject { + types: None, + metadata: ObjectMeta { + name: Some("recovered-sandbox".to_string()), + namespace: Some("recovered-namespace".to_string()), + labels: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "sandbox-id".to_string()), + (LABEL_SANDBOX_NAME.to_string(), "sandbox-name".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "workspace".to_string()), + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + ])), + ..Default::default() + }, + data: serde_json::json!({}), + }; + let source = futures::stream::iter([ + Err(expired_watch_error()), + Ok(Event::Restarted(vec![recovered])), + ]) + .chain(futures::stream::pending()); + let sandbox_stream = recovering_watcher_stream(source, "sandbox-resource").boxed(); + let (tx, rx) = mpsc::channel(256); + let mut outward = cluster_wide_watch_stream( + sandbox_stream, + "default".to_string(), + SupervisorTopology::Combined, + tx, + rx, + ); + + let event = outward + .next() + .await + .expect("outward stream must stay open through recovery") + .expect("recoverable watcher error must not reach the outward stream"); + let Some(watch_sandboxes_event::Payload::Sandbox(event)) = event.payload else { + panic!("expected recovered sandbox event"); + }; + let sandbox = event.sandbox.expect("sandbox payload must be populated"); + assert_eq!(sandbox.id, "sandbox-id"); + assert_eq!(sandbox.namespace, "recovered-namespace"); + + let next = outward.next(); + futures::pin_mut!(next); + assert!( + futures::poll!(next).is_pending(), + "outward stream must remain open after the recovered event" + ); + } + + #[tokio::test] + async fn kubernetes_event_watcher_error_does_not_hide_restarted_recovery_event() { + let source = futures::stream::iter([ + Err(expired_watch_error()), + Ok(Event::Restarted(vec![KubeEventObj::default()])), + ]); + let mut stream = continue_on_watcher_errors(source, "kubernetes-event"); + + let event = stream + .next() + .await + .expect("410 Expired must not terminate the watcher stream"); + let Event::Restarted(events) = event else { + panic!("expected kube-runtime recovery to emit Restarted"); + }; + assert_eq!(events.len(), 1); + assert!( + stream.next().await.is_none(), + "source closure must be preserved" + ); + } + + fn authenticated_token_review(username: &str) -> TokenReviewStatus { + TokenReviewStatus { + authenticated: Some(true), + audiences: Some(vec![SANDBOX_TOKEN_AUDIENCE.to_string()]), + user: Some(UserInfo { + username: Some(username.to_string()), + extra: Some(BTreeMap::from([ + (POD_NAME_EXTRA.to_string(), vec!["sandbox-pod".to_string()]), + (POD_UID_EXTRA.to_string(), vec!["pod-uid".to_string()]), + ])), + ..Default::default() + }), + ..Default::default() + } + } + + #[test] + fn token_review_uses_configured_service_account_and_pod_binding() { + let status = authenticated_token_review("system:serviceaccount:workspaces:sandbox-sa"); + let identity = token_review_identity(&status, "sandbox-sa") + .unwrap() + .expect("authenticated identity"); + assert_eq!(identity.namespace, "workspaces"); + assert_eq!(identity.pod_name, "sandbox-pod"); + assert_eq!(identity.pod_uid, "pod-uid"); + } + + #[test] + fn token_review_rejects_a_different_service_account() { + let status = authenticated_token_review("system:serviceaccount:workspaces:other"); + let error = token_review_identity(&status, "sandbox-sa").unwrap_err(); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + } + + #[test] + fn token_review_rejects_wrong_audience_and_missing_pod_binding() { + let mut wrong_audience = + authenticated_token_review("system:serviceaccount:workspaces:sandbox-sa"); + wrong_audience.audiences = Some(vec!["kubernetes.default.svc".to_string()]); + let error = token_review_identity(&wrong_audience, "sandbox-sa").unwrap_err(); + assert_eq!(error.code(), tonic::Code::Unauthenticated); + + let mut missing_binding = + authenticated_token_review("system:serviceaccount:workspaces:sandbox-sa"); + missing_binding.user.as_mut().unwrap().extra = None; + let error = token_review_identity(&missing_binding, "sandbox-sa").unwrap_err(); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + } + + #[test] + fn token_review_returns_none_when_not_authenticated() { + let status = TokenReviewStatus { + authenticated: Some(false), + error: Some("token rejected".to_string()), + ..Default::default() + }; + + assert!( + token_review_identity(&status, "sandbox-sa") + .unwrap() + .is_none() + ); + } + + #[test] + fn authentication_namespace_validation_covers_each_workspace_mode() { + let mut config = KubernetesComputeConfig { + namespace: "openshell".to_string(), + ..Default::default() + }; + assert!(accepts_auth_namespace(&config, None, "openshell")); + assert!(!accepts_auth_namespace(&config, None, "other")); + + config.workspace_mode = WorkspaceMode::Managed; + config.gateway_id = "gateway-a".to_string(); + assert!(accepts_auth_namespace( + &config, + None, + "openshell-gateway-a-workspace-a" + )); + assert!(!accepts_auth_namespace( + &config, + None, + "openshell-gateway-b-workspace-a" + )); + + config.workspace_mode = WorkspaceMode::Operator; + let allowlist = OperatorNamespaceAllowlist::from_set(BTreeSet::from([ + "team-a".to_string(), + "team-b".to_string(), + ])); + assert!(accepts_auth_namespace(&config, Some(&allowlist), "team-a")); + assert!(!accepts_auth_namespace(&config, Some(&allowlist), "team-c")); + assert!(!accepts_auth_namespace(&config, None, "team-a")); + } + + fn sandbox_owner_for_test(name: &str, uid: &str) -> OwnerReference { + OwnerReference { + api_version: "agents.x-k8s.io/v1beta1".to_string(), + block_owner_deletion: None, + controller: Some(true), + kind: SANDBOX_KIND.to_string(), + name: name.to_string(), + uid: uid.to_string(), + } + } + + fn sandbox_object_for_test(uid: &str, sandbox_id: &str) -> DynamicObject { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut sandbox = DynamicObject::new("sandbox-a", &resource); + sandbox.metadata.uid = Some(uid.to_string()); + sandbox.metadata.labels = Some(BTreeMap::from([( + LABEL_SANDBOX_ID.to_string(), + sandbox_id.to_string(), + )])); + sandbox + } + + #[test] + fn pod_identity_requires_matching_uid_annotation_and_controlling_owner() { + let owner = sandbox_owner_for_test("sandbox-a", "sandbox-uid-a"); + let pod = Pod { + metadata: ObjectMeta { + uid: Some("pod-uid-a".to_string()), + annotations: Some(BTreeMap::from([( + LABEL_SANDBOX_ID.to_string(), + "sandbox-id-a".to_string(), + )])), + owner_references: Some(vec![owner.clone()]), + ..Default::default() + }, + ..Default::default() + }; + + validate_pod_uid(&pod, "pod-uid-a").expect("matching pod UID"); + assert_eq!(pod_sandbox_id(&pod).unwrap(), "sandbox-id-a"); + assert_eq!(sandbox_owner_reference(&pod).unwrap(), &owner); + + let error = validate_pod_uid(&pod, "other-pod-uid").unwrap_err(); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + + let mut missing_annotation = pod.clone(); + missing_annotation.metadata.annotations = None; + let error = pod_sandbox_id(&missing_annotation).unwrap_err(); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + + let mut non_controlling = pod; + non_controlling.metadata.owner_references.as_mut().unwrap()[0].controller = Some(false); + let error = sandbox_owner_reference(&non_controlling).unwrap_err(); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + } + + #[test] + fn sandbox_owner_identity_requires_matching_uid_and_sandbox_id() { + let owner = sandbox_owner_for_test("sandbox-a", "sandbox-uid-a"); + let sandbox = sandbox_object_for_test("sandbox-uid-a", "sandbox-id-a"); + validate_sandbox_owner_identity(&owner, "sandbox-id-a", &sandbox) + .expect("matching owner identity"); + + let mismatched_owner = sandbox_object_for_test("sandbox-uid-b", "sandbox-id-a"); + let error = + validate_sandbox_owner_identity(&owner, "sandbox-id-a", &mismatched_owner).unwrap_err(); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + + let mismatched_annotation = sandbox_object_for_test("sandbox-uid-a", "sandbox-id-b"); + let error = validate_sandbox_owner_identity(&owner, "sandbox-id-a", &mismatched_annotation) + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + } + + #[test] + fn sandbox_api_version_probe_retries_on_structured_and_raw_404() { + let structured = kube_api_error(404, "could not find the requested resource"); + assert!(should_try_next_sandbox_api_version(&structured)); + + let raw = kube_api_error(404, "404 page not found\n"); + assert!(should_try_next_sandbox_api_version(&raw)); + } + + #[test] + fn lifecycle_patch_uses_version_specific_operating_state() { + let beta_stop = sandbox_operating_state_patch(SANDBOX_VERSION_V1BETA1, "42", false); + assert_eq!(beta_stop["metadata"]["resourceVersion"], "42"); + assert_eq!(beta_stop["spec"]["operatingMode"], "Suspended"); + assert!(beta_stop["spec"].get("replicas").is_none()); + + let alpha_start = sandbox_operating_state_patch(SANDBOX_VERSION_V1ALPHA1, "43", true); + assert_eq!(alpha_start["metadata"]["resourceVersion"], "43"); + assert_eq!(alpha_start["spec"]["replicas"], 1); + assert!(alpha_start["spec"].get("operatingMode").is_none()); + } + + #[test] + fn stop_timeout_includes_pod_grace_period_and_reconcile_headroom() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut sandbox = DynamicObject::new("sandbox", &resource); + + assert_eq!( + kubernetes_sandbox_stop_timeout(&sandbox), + Duration::from_secs(60), + "an omitted grace period uses the Kubernetes 30-second default" + ); + + sandbox.data = serde_json::json!({ + "spec": { + "podTemplate": { + "spec": {"terminationGracePeriodSeconds": 45} + } + } + }); + assert_eq!( + kubernetes_sandbox_stop_timeout(&sandbox), + Duration::from_secs(75) + ); + } + + #[test] + fn stop_poll_interval_backs_off_to_cap() { + let mut interval = STOP_INITIAL_POLL_INTERVAL; + let expected = [ + Duration::from_millis(500), + Duration::from_secs(1), + Duration::from_secs(2), + Duration::from_secs(2), + ]; + + for expected_interval in expected { + interval = next_stop_poll_interval(interval); + assert_eq!(interval, expected_interval); + } + } + + #[test] + fn stopped_status_requires_published_condition() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1ALPHA1, + SANDBOX_KIND, + )); + let mut sandbox = DynamicObject::new("sandbox", &resource); + sandbox.data = serde_json::json!({"status": {"replicas": 0}}); + + assert!( + !kubernetes_sandbox_has_stopped_condition(&sandbox), + "v1alpha1 omits a zero status replica count on the wire; it is not a usable completion signal" + ); + + sandbox.data = serde_json::json!({ + "status": { + "conditions": [{"type": "Suspended", "status": "True"}] + } + }); + assert!(kubernetes_sandbox_has_stopped_condition(&sandbox)); + } + + #[test] + fn beta_stop_requires_suspended_condition_and_deleted_pod() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut sandbox = DynamicObject::new("sandbox", &resource); + + assert!(!kubernetes_sandbox_stop_is_complete( + SANDBOX_VERSION_V1BETA1, + &sandbox, + true, + )); + + sandbox.data = serde_json::json!({ + "status": { + "conditions": [{"type": "Suspended", "status": "True"}] + } + }); + assert!(!kubernetes_sandbox_stop_is_complete( + SANDBOX_VERSION_V1BETA1, + &sandbox, + false, + )); + assert!(kubernetes_sandbox_stop_is_complete( + SANDBOX_VERSION_V1BETA1, + &sandbox, + true, + )); + assert!(kubernetes_sandbox_stop_is_complete( + SANDBOX_VERSION_V1ALPHA1, + &DynamicObject::new("sandbox", &resource), + true, + )); + } + + #[test] + fn stop_failure_only_rejects_terminal_suspension_condition() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut sandbox = DynamicObject::new("sandbox", &resource); + sandbox.data = serde_json::json!({ + "status": { + "conditions": [{ + "type": "Suspended", + "status": "False", + "reason": "PodNotOwned", + "message": "Refused to delete pod because it is not owned by this sandbox" + }] + } + }); + + assert_eq!( + kubernetes_sandbox_stop_failure(&sandbox).as_deref(), + Some( + "Kubernetes sandbox stop rejected: Refused to delete pod because it is not owned by this sandbox" + ) + ); + + sandbox.data["status"]["conditions"][0]["status"] = serde_json::json!("Unknown"); + sandbox.data["status"]["conditions"][0]["reason"] = serde_json::json!("PodStateUnknown"); + assert!( + kubernetes_sandbox_stop_failure(&sandbox).is_none(), + "an unknown pod state can recover on a later controller reconciliation" + ); + } + + #[test] + fn sandbox_api_version_probe_keeps_non_404_errors() { + let err = kube_api_error(403, "sandboxes.agents.x-k8s.io is forbidden"); + assert!(!should_try_next_sandbox_api_version(&err)); + } + + fn rendered_env<'a>(container: &'a serde_json::Value, name: &str) -> Option<&'a str> { + container["env"] + .as_array()? + .iter() + .find(|item| item.get("name").and_then(|value| value.as_str()) == Some(name))? + .get("value")? + .as_str() + } + + #[test] + fn driver_config_rejects_invalid_shape() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "pod": "not-an-object" + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("invalid kubernetes driver_config")); + } + + #[test] + fn driver_config_rejects_unknown_fields() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "cdi_devices": ["nvidia.com/gpu=0"] + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("unknown field")); + } + + #[test] + fn driver_config_for_spec_rejects_unknown_fields() { + let sandbox = Sandbox { + id: "sandbox-123".to_string(), + spec: Some(SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "gpu_device_ids": ["0000:2d:00.0"] + }))), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + + let err = kubernetes_driver_config_for_spec(sandbox.spec.as_ref(), None).unwrap_err(); + assert!(err.contains("unknown field")); + assert!(err.contains("gpu_device_ids")); + } + + #[test] + fn driver_config_pvc_subpath_mounts_render_in_pod_template() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": { + "claim_name": "pvc-user-data-123", + "read_only": false + } + }], + "containers": { + "agent": { + "volume_mounts": [ + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace", + "read_only": false + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/memory", + "sub_path": "memory" + } + ] + } + } + }))), + ..SandboxTemplate::default() + }; + let spec = SandboxSpec { + template: Some(template), + ..SandboxSpec::default() + }; + + let cr = sandbox_to_k8s_spec_for_test(Some(&spec), &SandboxPodParams::default()); + let pod_template = &cr["spec"]["podTemplate"]; + + let volumes = pod_template["spec"]["volumes"] + .as_array() + .expect("volumes should exist"); + let user_volume = volumes + .iter() + .find(|volume| volume["name"] == "user-data") + .expect("user PVC volume should be rendered"); + assert_eq!( + user_volume["persistentVolumeClaim"]["claimName"], + "pvc-user-data-123" + ); + assert_eq!(user_volume["persistentVolumeClaim"]["readOnly"], false); + + let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] + .as_array() + .expect("volumeMounts should exist"); + let workspace_mount = mounts + .iter() + .find(|mount| mount["mountPath"] == "/sandbox/.openshell/workspace") + .expect("workspace subPath mount should be rendered"); + assert_eq!(workspace_mount["name"], "user-data"); + assert_eq!(workspace_mount["subPath"], "workspace"); + assert_eq!(workspace_mount["readOnly"], false); + + let memory_mount = mounts + .iter() + .find(|mount| mount["mountPath"] == "/sandbox/.openshell/memory") + .expect("memory subPath mount should be rendered"); + assert_eq!(memory_mount["name"], "user-data"); + assert_eq!(memory_mount["subPath"], "memory"); + assert_eq!(memory_mount["readOnly"], true); + + let spec_obj = cr["spec"].as_object().expect("spec should be an object"); + assert!( + !spec_obj.contains_key("volumeClaimTemplates"), + "explicit /sandbox driver_config mounts should skip the default workspace VCT" + ); + let has_workspace_init = pod_template["spec"]["initContainers"] + .as_array() + .is_some_and(|containers| { + containers + .iter() + .any(|container| container["name"] == WORKSPACE_INIT_CONTAINER_NAME) + }); + assert!( + !has_workspace_init, + "explicit /sandbox driver_config mounts should skip the default workspace init container" + ); + } + + #[test] + fn driver_config_accepts_read_write_pvc_with_multiple_subpath_mounts() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": { + "claim_name": "pvc-user-data", + "read_only": false + } + }], + "containers": { + "agent": { + "volume_mounts": [ + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace", + "read_only": false + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/memory", + "sub_path": "memory", + "read_only": false + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/sessions", + "sub_path": "sessions", + "read_only": false + } + ] + } + } + }))), + ..SandboxTemplate::default() + }; + + let config = KubernetesSandboxDriverConfig::from_template(&template) + .expect("read-write PVC with multiple subPath mounts should validate"); + + assert_eq!(config.volumes.len(), 1); + assert_eq!(config.volumes[0].name, "user-data"); + assert_eq!( + config.volumes[0].persistent_volume_claim.claim_name, + "pvc-user-data" + ); + assert!(!config.volumes[0].persistent_volume_claim.read_only); + assert_eq!(config.containers.agent.volume_mounts.len(), 3); + assert!( + config + .containers + .agent + .volume_mounts + .iter() + .all(|mount| !mount.read_only) + ); + assert!(config.has_explicit_sandbox_data_mount()); + } + + #[test] + fn driver_config_rejects_duplicate_pvc_volume_names() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [ + { + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-a"} + }, + { + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-b"} + } + ] + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("duplicate kubernetes driver_config volume")); + } + + #[test] + fn driver_config_rejects_duplicate_pvc_volume_mount_targets() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [ + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace" + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace" + } + ] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("duplicate kubernetes driver_config mount target")); + } + + #[test] + fn driver_config_accepts_dns1123_subdomain_pvc_claim_name() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc.user-data.123"} + }] + }))), + ..SandboxTemplate::default() + }; + + let config = KubernetesSandboxDriverConfig::from_template(&template) + .expect("DNS-1123 subdomain PVC names should validate"); + + assert_eq!( + config.volumes[0].persistent_volume_claim.claim_name, + "pvc.user-data.123" + ); + } + + #[test] + fn driver_config_rejects_invalid_volume_label_and_claim_name() { + for (field, config) in [ + ( + "volumes[].name", + serde_json::json!({ + "volumes": [{ + "name": "User_Data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }] + }), + ), + ( + "volumes[].persistent_volume_claim.claim_name", + serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "Pvc_User_Data"} + }] + }), + ), + ] { + let template = SandboxTemplate { + driver_config: Some(json_struct(config)), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + assert!( + err.contains(field) && err.contains("DNS-1123"), + "expected invalid {field} to fail DNS-1123 validation, got {err}" + ); + } + } + + #[test] + fn driver_config_rejects_mounts_referencing_unknown_volumes() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "known-data", + "persistent_volume_claim": {"claim_name": "pvc-known"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "missing-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace" + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("unknown kubernetes driver_config volume 'missing-data'")); + } + + #[test] + fn driver_config_rejects_shared_reserved_mount_targets() { + for mount_path in [ + "/", + "/sandbox", + "/etc/openshell", + "/etc/openshell-tls/client", + "/opt/openshell/bin", + ] { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": mount_path + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + assert!( + err.contains("mount path") || err.contains("mount target"), + "expected protected mount target {mount_path:?} to be rejected, got {err}" + ); + } + } + + #[test] + fn driver_config_rejects_kubernetes_static_protected_mount_targets() { + let spec = SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/var/run/secrets/openshell" + }] + } + } + }))), + ..SandboxTemplate::default() + }), + ..SandboxSpec::default() + }; + + let err = kubernetes_driver_config_for_spec(Some(&spec), None).unwrap_err(); + + assert!(err.contains("/var/run/secrets/openshell")); + } + + #[test] + fn driver_config_allows_spiffe_workload_path_without_provider_spiffe() { + let spec = SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/spiffe-workload-api" + }] + } + } + }))), + ..SandboxTemplate::default() + }), + ..SandboxSpec::default() + }; + + kubernetes_driver_config_for_spec(Some(&spec), None) + .expect("SPIFFE workload path should only be protected when SPIFFE is enabled"); + } + + #[test] + fn driver_config_rejects_invalid_kubernetes_sub_paths() { + for sub_path in ["/workspace", "../workspace"] { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": sub_path + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + assert!( + err.contains("mount subpath must be relative"), + "expected invalid sub_path {sub_path:?} to be rejected, got {err}" + ); + } + } + + #[test] + fn driver_config_defaults_pvc_mounts_to_read_only() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace" + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let pod_template = sandbox_template_to_k8s( + &template, + false, + &std::collections::HashMap::new(), + false, + &SandboxPodParams::default(), + ); + + let volume = pod_template["spec"]["volumes"] + .as_array() + .expect("volumes should exist") + .iter() + .find(|volume| volume["name"] == "user-data") + .expect("user volume should exist"); + assert_eq!(volume["persistentVolumeClaim"]["readOnly"], true); + + let mount = pod_template["spec"]["containers"][0]["volumeMounts"] + .as_array() + .expect("volumeMounts should exist") + .iter() + .find(|mount| mount["mountPath"] == "/sandbox/.openshell/workspace") + .expect("user mount should exist"); + assert_eq!(mount["readOnly"], true); + } + + #[test] + fn driver_config_rejects_read_write_mount_for_read_only_pvc_volume() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": { + "claim_name": "pvc-user-data", + "read_only": true + } + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "read_only": false + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("cannot set read_only=false")); + } + + #[test] + fn driver_config_rejects_reserved_kubernetes_volume_names() { + for volume_name in [ + CLIENT_TLS_VOLUME_NAME, + SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, + SPIFFE_WORKLOAD_API_VOLUME_NAME, + SUPERVISOR_VOLUME_NAME, + WORKSPACE_VOLUME_NAME, + ] { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": volume_name, + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }] + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + assert!( + err.contains("reserved for OpenShell-managed volumes"), + "expected reserved volume name {volume_name:?} to be rejected, got {err}" + ); + } + } + + #[test] + fn reserved_kubernetes_volume_names_cover_managed_pod_volumes() { + let params = SandboxPodParams { + client_tls_secret_name: "openshell-client-tls-secret", + provider_spiffe_enabled: true, + provider_spiffe_workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock", + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate::default(), + false, + &std::collections::HashMap::new(), + true, + ¶ms, + ); + let volume_names = pod_template["spec"]["volumes"] + .as_array() + .expect("volumes should exist") + .iter() + .filter_map(|volume| volume["name"].as_str()) + .collect::>(); + + for volume_name in volume_names { + assert!( + KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES.contains(&volume_name), + "managed volume {volume_name:?} should be reserved" + ); + } + } + + #[test] + fn driver_config_rejects_runtime_provider_spiffe_mount_path() { + let spec = SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/custom-spiffe" + }] + } + } + }))), + ..SandboxTemplate::default() + }), + ..SandboxSpec::default() + }; + + let err = + kubernetes_driver_config_for_spec(Some(&spec), Some("/custom-spiffe/spire-agent.sock")) + .unwrap_err(); + + assert!(err.contains("/custom-spiffe")); + } + + #[test] + fn validate_rejects_zero_gpu_count() { + let sandbox = Sandbox { + spec: Some(SandboxSpec { + resource_requirements: Some(ResourceRequirements { + gpu: Some(GpuResourceRequirements { count: Some(0) }), + }), + ..SandboxSpec::default() + }), + ..Sandbox::default() + }; + + let gpu_requirements = sandbox + .spec + .as_ref() + .and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref())); + let err = validate_gpu_request(gpu_requirements).unwrap_err(); + assert_eq!(err.code(), tonic::Code::InvalidArgument); + assert!(err.message().contains("gpu count must be greater than 0")); + } + + #[test] + fn kube_pulling_event_adds_image_progress_metadata() { + let mut metadata = std::collections::HashMap::new(); + + attach_kube_progress_metadata( + &mut metadata, + "Pulling", + "Pulling image \"ghcr.io/acme/sandbox:latest\"", + ); + + assert_eq!( + metadata.get(PROGRESS_ACTIVE_STEP_KEY).map(String::as_str), + Some(PROGRESS_STEP_PULLING_IMAGE) + ); + assert_eq!( + metadata.get(PROGRESS_ACTIVE_DETAIL_KEY).map(String::as_str), + Some("ghcr.io/acme/sandbox:latest") + ); + } + + #[test] + fn kube_pulled_event_adds_completed_image_progress_metadata() { + let mut metadata = std::collections::HashMap::new(); + + attach_kube_progress_metadata( + &mut metadata, + "Pulled", + "Successfully pulled image \"ghcr.io/acme/sandbox:latest\". Image size: 44040192 bytes.", + ); + + assert_eq!( + metadata.get(PROGRESS_COMPLETE_STEP_KEY).map(String::as_str), + Some(PROGRESS_STEP_PULLING_IMAGE) + ); + assert_eq!( + metadata + .get(PROGRESS_COMPLETE_LABEL_KEY) + .map(String::as_str), + Some("Image pulled (42 MB)") + ); + assert_eq!( + metadata.get(PROGRESS_ACTIVE_STEP_KEY).map(String::as_str), + Some(PROGRESS_STEP_STARTING_SANDBOX) + ); + } + + #[test] + fn supervisor_sideload_injects_run_as_user_zero() { + let mut pod_template = serde_json::json!({ + "spec": { + "containers": [{ + "name": "agent", + "image": "custom-image:latest", + "securityContext": { + "capabilities": { + "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] + } + } + }] + } + }); + + apply_supervisor_sideload( + &mut pod_template, + "custom-image:latest", + "IfNotPresent", + SupervisorSideloadMethod::InitContainer, + 1500, // sandbox_uid + 1500, // sandbox_gid + ); + + let sc = &pod_template["spec"]["containers"][0]["securityContext"]; + assert_eq!(sc["runAsUser"], 0, "runAsUser must be 0 for supervisor"); + // Capabilities should be preserved + assert!( + sc["capabilities"]["add"] + .as_array() + .unwrap() + .contains(&serde_json::json!("SYS_ADMIN")) + ); + } + + #[test] + fn supervisor_sideload_replaces_spoofed_identity_environment() { + let mut pod_template = serde_json::json!({ + "spec": { + "containers": [{ + "name": "agent", + "image": "custom-image:latest", + "env": [ + {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "spoofed"}, + {"name": openshell_core::sandbox_env::SANDBOX_UID, "value": "9999"}, + {"name": openshell_core::sandbox_env::SANDBOX_GID, "value": "9999"}, + {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "duplicate"} + ] + }] + } + }); + + apply_supervisor_sideload( + &mut pod_template, + "supervisor-image:latest", + "IfNotPresent", + SupervisorSideloadMethod::InitContainer, + 1500, + 1600, + ); + + let agent = &pod_template["spec"]["containers"][0]; + let env = agent["env"].as_array().unwrap(); + for name in [ + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX_UID, + openshell_core::sandbox_env::SANDBOX_GID, + ] { + assert_eq!( + env.iter().filter(|item| item["name"] == name).count(), + 1, + "{name} must have one driver-owned value" + ); + } + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::OCI_IMAGE_USER), + Some("") + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), + Some("1500") + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SANDBOX_GID), + Some("1600") + ); + } + + #[test] + fn supervisor_sideload_adds_security_context_when_missing() { + let mut pod_template = serde_json::json!({ + "spec": { + "containers": [{ + "name": "agent", + "image": "custom-image:latest" + }] + } + }); + + apply_supervisor_sideload( + &mut pod_template, + "supervisor-image:latest", + "IfNotPresent", + SupervisorSideloadMethod::InitContainer, + 1000, // sandbox_uid + 1000, // sandbox_gid + ); + + let sc = &pod_template["spec"]["containers"][0]["securityContext"]; + assert_eq!( + sc["runAsUser"], 0, + "runAsUser must be 0 even when no prior securityContext" + ); + } + + #[test] + fn supervisor_sideload_injects_emptydir_volume_init_container_and_mount() { + let mut pod_template = serde_json::json!({ + "spec": { + "containers": [{ + "name": "agent", + "image": "custom-image:latest" + }] + } + }); + + apply_supervisor_sideload( + &mut pod_template, + "supervisor-image:latest", + "IfNotPresent", + SupervisorSideloadMethod::InitContainer, + 1000, // sandbox_uid + 1000, // sandbox_gid + ); + + // Volume should be an emptyDir + let volumes = pod_template["spec"]["volumes"] + .as_array() + .expect("volumes should exist"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0]["name"], SUPERVISOR_VOLUME_NAME); + assert!( + volumes[0]["emptyDir"].is_object(), + "volume should be emptyDir, not hostPath" + ); + + // Init container should use the supervisor image, not the sandbox image + let init_containers = pod_template["spec"]["initContainers"] + .as_array() + .expect("initContainers should exist"); + assert_eq!(init_containers.len(), 1); + assert_eq!(init_containers[0]["name"], SUPERVISOR_INIT_CONTAINER_NAME); + assert_eq!(init_containers[0]["image"], "supervisor-image:latest"); + assert_eq!(init_containers[0]["imagePullPolicy"], "IfNotPresent"); + + // The init container must invoke the binary directly with + // `copy-self ` rather than depending on shell utilities. + let init_command = init_containers[0]["command"] + .as_array() + .expect("init container command should be set"); + assert_eq!(init_command.len(), 3, "expected [binary, copy-self, dest]"); + assert_eq!(init_command[0], SUPERVISOR_IMAGE_BINARY_PATH); + assert_eq!(init_command[1], "copy-self"); + assert_eq!( + init_command[2].as_str().unwrap(), + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + ); + assert!( + !init_command.iter().any(|v| v == "sh"), + "init container must not depend on a shell" + ); + + // `--workdir` is optional for standalone supervisor invocations and + // has no implicit default, so Kubernetes must pass its fixed workspace. + let command = pod_template["spec"]["containers"][0]["command"] + .as_array() + .expect("command should be set"); + assert_eq!( + command[0].as_str().unwrap(), + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + ); + assert_eq!( + command, + serde_json::json!([ + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT + ]) + .as_array() + .unwrap() + ); + + // Agent volume mount should be read-only + let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] + .as_array() + .expect("volumeMounts should exist"); + assert_eq!(mounts.len(), 1); + assert_eq!(mounts[0]["name"], SUPERVISOR_VOLUME_NAME); + assert_eq!(mounts[0]["mountPath"], SUPERVISOR_MOUNT_PATH); + assert_eq!(mounts[0]["readOnly"], true); + } + + #[test] + fn supervisor_sideload_image_volume_injects_image_source_without_init_container() { + let mut pod_template = serde_json::json!({ + "spec": { + "containers": [{ + "name": "agent", + "image": "custom-image:latest" + }] + } + }); + + apply_supervisor_sideload( + &mut pod_template, + "supervisor-image:latest", + "IfNotPresent", + SupervisorSideloadMethod::ImageVolume, + 1000, // sandbox_uid + 1000, // sandbox_gid + ); + + let volumes = pod_template["spec"]["volumes"] + .as_array() + .expect("volumes should exist"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0]["name"], SUPERVISOR_VOLUME_NAME); + assert_eq!(volumes[0]["image"]["reference"], "supervisor-image:latest"); + assert_eq!(volumes[0]["image"]["pullPolicy"], "IfNotPresent"); + assert!( + volumes[0]["emptyDir"].is_null(), + "image volume method must not use emptyDir" + ); + + assert!( + pod_template["spec"]["initContainers"].is_null(), + "image volume method must not inject init containers" + ); + + let command = pod_template["spec"]["containers"][0]["command"] + .as_array() + .expect("command should be set"); + assert_eq!( + command[0].as_str().unwrap(), + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + ); + + let sc = &pod_template["spec"]["containers"][0]["securityContext"]; + assert_eq!(sc["runAsUser"], 0); + + let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] + .as_array() + .expect("volumeMounts should exist"); + assert_eq!(mounts[0]["name"], SUPERVISOR_VOLUME_NAME); + assert_eq!(mounts[0]["mountPath"], SUPERVISOR_MOUNT_PATH); + assert_eq!(mounts[0]["readOnly"], true); + } + + #[test] + fn supervisor_image_volume_omits_pull_policy_when_empty() { + let mut pod_template = serde_json::json!({ + "spec": { + "containers": [{ + "name": "agent", + "image": "custom-image:latest" + }] + } + }); + + apply_supervisor_sideload( + &mut pod_template, + "supervisor-image:latest", + "", + SupervisorSideloadMethod::ImageVolume, + 1000, // sandbox_uid + 1000, // sandbox_gid + ); + + let volume = &pod_template["spec"]["volumes"][0]; + assert_eq!(volume["image"]["reference"], "supervisor-image:latest"); + assert!( + volume["image"].get("pullPolicy").is_none(), + "pullPolicy should be omitted when empty" + ); + } + + #[test] + fn sidecar_topology_renders_process_agent_and_network_sidecar() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + supervisor_image_pull_policy: "IfNotPresent", + grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", + client_tls_secret_name: "openshell-client-tls", + proxy_uid: 2200, + namespace: "default", + sandbox_uid: 1500, + sandbox_gid: 1500, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + environment: std::collections::HashMap::from([ + ( + openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), + "spoofed".to_string(), + ), + ( + openshell_core::sandbox_env::SANDBOX_UID.to_string(), + "9999".to_string(), + ), + ( + openshell_core::sandbox_env::SANDBOX_GID.to_string(), + "9999".to_string(), + ), + ]), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + assert_eq!(pod_template["spec"]["shareProcessNamespace"], true); + assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + assert_eq!(containers.len(), 2); + + let agent = containers + .iter() + .find(|container| container["name"] == "agent") + .unwrap(); + assert_eq!( + agent["command"], + serde_json::json!([ + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), + "--mode=process", + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT + ]) + ); + assert_eq!(agent["securityContext"]["runAsUser"], 1500); + assert_eq!(agent["securityContext"]["runAsGroup"], 1500); + assert_eq!(agent["securityContext"]["runAsNonRoot"], true); + assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false); + assert_eq!( + agent["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"] + }) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::TLS_CA), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), + Some(SIDECAR_SSH_SOCKET_FILE) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), + Some(SIDECAR_CONTROL_SOCKET) + ); + assert_eq!(rendered_env(agent, "OPENSHELL_SUPERVISOR_READY_FILE"), None); + assert_eq!(rendered_env(agent, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); + assert_eq!( + rendered_env(agent, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env(agent, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::PROXY_TLS_DIR), + Some(SIDECAR_TLS_MOUNT_PATH) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), + Some("1500") + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::OCI_IMAGE_USER), + Some("") + ); + + let sidecar = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!(sidecar["image"], "supervisor-image:latest"); + assert_eq!(sidecar["imagePullPolicy"], "IfNotPresent"); + assert_eq!( + sidecar["command"], + serde_json::json!([SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network"]) + ); + assert_eq!(sidecar["securityContext"]["runAsUser"], 0); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"], + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] + }) + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::ENDPOINT), + Some("https://openshell-gateway.openshell.svc:8080") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::SSH_SOCKET_PATH), + Some(SIDECAR_SSH_SOCKET_FILE) + ); + assert!( + SIDECAR_SSH_SOCKET_FILE.starts_with('@'), + "sidecar SSH relay must use a Linux abstract socket" + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_UID), + Some("1500") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_GID), + Some("1500") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::OCI_IMAGE_USER), + Some("") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), + Some(SIDECAR_CONTROL_SOCKET) + ); + assert_eq!( + rendered_env(sidecar, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env(sidecar, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env( + sidecar, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY + ), + None + ); + assert_eq!(rendered_env(sidecar, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::PROXY_TLS_DIR), + Some(SIDECAR_TLS_MOUNT_PATH) + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::TLS_CA), + Some("/etc/openshell-tls/proxy/client/ca.crt") + ); + let sidecar_mounts = sidecar["volumeMounts"].as_array().unwrap(); + assert!( + !sidecar_mounts + .iter() + .any(|mount| mount["name"] == "openshell-client-tls"), + "runtime sidecar should use the init-copied TLS files, not the root-owned Secret mount" + ); + let agent_mounts = agent["volumeMounts"].as_array().unwrap(); + assert!( + !agent_mounts + .iter() + .any(|mount| mount["name"] == "openshell-sa-token"), + "agent container must not mount gateway bootstrap token in sidecar topology" + ); + assert!( + !agent_mounts + .iter() + .any(|mount| mount["name"] == "openshell-client-tls"), + "agent container must not mount gateway client TLS secret in sidecar topology" + ); + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + let sa_token = volumes + .iter() + .find(|volume| volume["name"] == "openshell-sa-token") + .unwrap(); + assert_eq!(sa_token["projected"]["defaultMode"], 0o440); + let client_tls = volumes + .iter() + .find(|volume| volume["name"] == "openshell-client-tls") + .unwrap(); + assert_eq!(client_tls["secret"]["defaultMode"], 0o440); + + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["image"], "supervisor-image:latest"); + assert_eq!(network_init["imagePullPolicy"], "IfNotPresent"); + assert_eq!( + network_init["command"], + serde_json::json!([ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network-init", + "--proxy-uid", + "0", + "--proxy-gid", + "1500", + "--sidecar-state-dir", + SIDECAR_STATE_MOUNT_PATH, + "--sidecar-tls-dir", + SIDECAR_TLS_MOUNT_PATH + ]) + ); + assert_eq!( + network_init["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"], + "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] + }) + ); + let network_init_mounts = network_init["volumeMounts"].as_array().unwrap(); + assert!(network_init_mounts.iter().any(|mount| { + mount["name"] == "openshell-client-tls" + && mount["mountPath"] == "/etc/openshell-tls/client" + })); + } + #[test] + fn sidecar_topology_can_relax_process_binary_aware_network_policy() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + process_binary_aware_network_policy: false, + ..SandboxPodParams::default() + }; let pod_template = sandbox_template_to_k8s( - &template, + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, false, &std::collections::HashMap::new(), false, - &SandboxPodParams::default(), + ¶ms, ); - let volume = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist") + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let sidecar = containers .iter() - .find(|volume| volume["name"] == "user-data") - .expect("user volume should exist"); - assert_eq!(volume["persistentVolumeClaim"]["readOnly"], true); - - let mount = pod_template["spec"]["containers"][0]["volumeMounts"] - .as_array() - .expect("volumeMounts should exist") + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], true); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"] + }) + ); + assert_eq!( + rendered_env( + sidecar, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY + ), + Some("relaxed") + ); + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers .iter() - .find(|mount| mount["mountPath"] == "/sandbox/.openshell/workspace") - .expect("user mount should exist"); - assert_eq!(mount["readOnly"], true); - } - - #[test] - fn driver_config_rejects_read_write_mount_for_read_only_pvc_volume() { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": { - "claim_name": "pvc-user-data", - "read_only": true - } - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": "/sandbox/.openshell/workspace", - "read_only": false - }] - } - } - }))), - ..SandboxTemplate::default() - }; - - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); - - assert!(err.contains("cannot set read_only=false")); - } - - #[test] - fn driver_config_rejects_reserved_kubernetes_volume_names() { - for volume_name in [ - CLIENT_TLS_VOLUME_NAME, - SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, - SPIFFE_WORKLOAD_API_VOLUME_NAME, - SUPERVISOR_VOLUME_NAME, - WORKSPACE_VOLUME_NAME, - ] { - let template = SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": volume_name, - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }] - }))), - ..SandboxTemplate::default() - }; - - let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); - assert!( - err.contains("reserved for OpenShell-managed volumes"), - "expected reserved volume name {volume_name:?} to be rejected, got {err}" - ); - } + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["command"][3], "2200"); } #[test] - fn reserved_kubernetes_volume_names_cover_managed_pod_volumes() { + fn sidecar_topology_adds_shared_state_and_tls_volumes() { let params = SandboxPodParams { - client_tls_secret_name: "openshell-client-tls-secret", - provider_spiffe_enabled: true, - provider_spiffe_workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock", + topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::ImageVolume, + supervisor_image: "supervisor-image:latest", + grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", ..SandboxPodParams::default() }; let pod_template = sandbox_template_to_k8s( &SandboxTemplate::default(), false, &std::collections::HashMap::new(), - true, + false, ¶ms, ); - let volume_names = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist") + + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + assert!( + volumes + .iter() + .any(|volume| volume["name"] == SIDECAR_STATE_VOLUME_NAME) + ); + assert!( + volumes + .iter() + .any(|volume| volume["name"] == SIDECAR_TLS_VOLUME_NAME) + ); + assert!(volumes.iter().any(|volume| { + volume["name"] == SUPERVISOR_VOLUME_NAME && volume["image"].is_object() + })); + + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let sidecar = containers .iter() - .filter_map(|volume| volume["name"].as_str()) - .collect::>(); + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"], + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] + }) + ); + assert_eq!(sidecar["securityContext"]["runAsUser"], 0); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1000); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); - for volume_name in volume_names { - assert!( - KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES.contains(&volume_name), - "managed volume {volume_name:?} should be reserved" - ); + for container_name in ["agent", SUPERVISOR_NETWORK_SIDECAR_NAME] { + let container = containers + .iter() + .find(|container| container["name"] == container_name) + .unwrap(); + let mounts = container["volumeMounts"].as_array().unwrap(); + assert!(mounts.iter().any(|mount| { + mount["name"] == SIDECAR_STATE_VOLUME_NAME + && mount["mountPath"] == SIDECAR_STATE_MOUNT_PATH + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SIDECAR_TLS_VOLUME_NAME + && mount["mountPath"] == SIDECAR_TLS_MOUNT_PATH + })); } + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["command"][3], "0"); } #[test] - fn driver_config_rejects_runtime_provider_spiffe_mount_path() { - let spec = SandboxSpec { - template: Some(SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": "/custom-spiffe" - }] - } - } - }))), - ..SandboxTemplate::default() - }), - ..SandboxSpec::default() + fn sidecar_topology_rejects_proxy_uid_matching_sandbox_uid() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + proxy_uid: 1500, + namespace: "default", + sandbox_uid: 1500, + ..SandboxPodParams::default() }; - let err = - kubernetes_driver_config_for_spec(Some(&spec), Some("/custom-spiffe/spire-agent.sock")) - .unwrap_err(); - - assert!(err.contains("/custom-spiffe")); + let err = validate_proxy_identity(¶ms).unwrap_err(); + assert!(matches!(err, KubernetesDriverError::Precondition(_))); + assert!(err.to_string().contains("proxy_uid")); } #[test] - fn validate_rejects_zero_gpu_count() { - let sandbox = Sandbox { - spec: Some(SandboxSpec { - resource_requirements: Some(ResourceRequirements { - gpu: Some(GpuResourceRequirements { count: Some(0) }), - }), - ..SandboxSpec::default() - }), - ..Sandbox::default() + fn proxy_pod_topology_runs_process_supervisor_with_gateway_creds() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + cr_name: "example-sandbox", + grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + host_gateway_ip: "172.17.0.1", + ..SandboxPodParams::default() }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); - let gpu_requirements = sandbox - .spec - .as_ref() - .and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref())); - let err = validate_gpu_request(gpu_requirements).unwrap_err(); - assert_eq!(err.code(), tonic::Code::InvalidArgument); - assert!(err.message().contains("gpu count must be greater than 0")); - } - - #[test] - fn kube_pulling_event_adds_image_progress_metadata() { - let mut metadata = std::collections::HashMap::new(); + let names = proxy_pod_resource_names(params.cr_name, params.sandbox_id); + let service_dns = proxy_pod_service_dns(&names.service, "agents"); + let agent = &pod_template["spec"]["containers"][0]; - attach_kube_progress_metadata( - &mut metadata, - "Pulling", - "Pulling image \"ghcr.io/acme/sandbox:latest\"", + assert_eq!( + pod_template["metadata"]["labels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_AGENT ); - + // The agent container runs the process supervisor, not the workload + // directly, and it launches the workload from MAIN_PROCESS_SPEC. assert_eq!( - metadata.get(PROGRESS_ACTIVE_STEP_KEY).map(String::as_str), - Some(PROGRESS_STEP_PULLING_IMAGE) + agent["command"], + serde_json::json!([ + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), + "--mode=process", + "--workdir", + driver_mounts::DEFAULT_WORKSPACE_ROOT + ]) ); + // Gateway credentials are RETAINED (the in-pod supervisor owns its own + // scoped session), unlike the superseded network-only design. assert_eq!( - metadata.get(PROGRESS_ACTIVE_DETAIL_KEY).map(String::as_str), - Some("ghcr.io/acme/sandbox:latest") + rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), + Some(params.grpc_endpoint) ); - } - - #[test] - fn kube_pulled_event_adds_completed_image_progress_metadata() { - let mut metadata = std::collections::HashMap::new(); - - attach_kube_progress_metadata( - &mut metadata, - "Pulled", - "Successfully pulled image \"ghcr.io/acme/sandbox:latest\". Image size: 44040192 bytes.", + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY), + Some("proxy-pod") ); - + // The non-root agent cannot create a filesystem SSH socket under `/run`, + // so the relay must use a netns-scoped abstract socket. assert_eq!( - metadata.get(PROGRESS_COMPLETE_STEP_KEY).map(String::as_str), - Some(PROGRESS_STEP_PULLING_IMAGE) + rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), + Some(PROXY_POD_SSH_SOCKET_FILE) ); + assert!(PROXY_POD_SSH_SOCKET_FILE.starts_with('@')); assert_eq!( - metadata - .get(PROGRESS_COMPLETE_LABEL_KEY) - .map(String::as_str), - Some("Image pulled (42 MB)") + rendered_env(agent, openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE), + Some("proxy-pod") ); + // The workload's children still egress through the remote proxy Service. + // Relay-spawned exec processes reconstruct their proxy variables from + // PROXY_URL after clearing the inherited supervisor environment. + let proxy_url = format!("http://{service_dns}:3128"); assert_eq!( - metadata.get(PROGRESS_ACTIVE_STEP_KEY).map(String::as_str), - Some(PROGRESS_STEP_STARTING_SANDBOX) + rendered_env(agent, openshell_core::sandbox_env::PROXY_URL), + Some(proxy_url.as_str()) ); - } - - #[test] - fn supervisor_sideload_injects_run_as_user_zero() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest", - "securityContext": { - "capabilities": { - "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] - } - } - }] - } - }); + assert_eq!(rendered_env(agent, "HTTP_PROXY"), Some(proxy_url.as_str())); + assert_eq!( + rendered_env(agent, "SSL_CERT_FILE"), + Some("/etc/openshell-tls/proxy/ca-bundle.pem") + ); + assert_eq!( + agent["securityContext"]["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); + let proxy_tls_mount = agent["volumeMounts"] + .as_array() + .unwrap() + .iter() + .find(|mount| mount["name"] == "openshell-proxy-pod-tls") + .unwrap(); + assert_eq!(proxy_tls_mount["readOnly"], true); - apply_supervisor_sideload( - &mut pod_template, - "custom-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::InitContainer, - 1500, // sandbox_uid - 1500, // sandbox_gid + // Single pod (workload runs inside it); the supervisor binary and the + // retained credential volumes are all present. + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + assert_eq!(containers.len(), 1); + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + assert!(volumes.iter().any(|volume| { + volume["name"] == "openshell-proxy-pod-ca-source" + && volume["secret"]["secretName"] == names.proxy_ca_secret + })); + assert!(volumes.iter().any(|volume| { + volume["name"] == "openshell-proxy-pod-tls" && volume["emptyDir"].is_object() + })); + assert!( + volumes + .iter() + .any(|volume| volume["name"] == SUPERVISOR_VOLUME_NAME), + "supervisor binary volume must be mounted" + ); + assert!( + volumes + .iter() + .any(|volume| volume["name"] == SERVICE_ACCOUNT_TOKEN_VOLUME_NAME), + "SA token volume must be retained" ); - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!(sc["runAsUser"], 0, "runAsUser must be 0 for supervisor"); - // Capabilities should be preserved + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let ca_init = init_containers + .iter() + .find(|container| container["name"] == "openshell-proxy-ca-install") + .unwrap(); + assert_eq!(ca_init["image"], "supervisor-image:latest"); + assert_eq!(ca_init["securityContext"]["runAsUser"], 1500); + assert_eq!(ca_init["securityContext"]["readOnlyRootFilesystem"], true); + // Supervisor binary sideload init present (InitContainer method). assert!( - sc["capabilities"]["add"] - .as_array() - .unwrap() - .contains(&serde_json::json!("SYS_ADMIN")) + init_containers + .iter() + .any(|container| container["name"] == SUPERVISOR_INIT_CONTAINER_NAME) ); + + assert!(pod_template["spec"].get("affinity").is_none()); } #[test] - fn supervisor_sideload_replaces_spoofed_identity_environment() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest", - "env": [ - {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "spoofed"}, - {"name": openshell_core::sandbox_env::SANDBOX_UID, "value": "9999"}, - {"name": openshell_core::sandbox_env::SANDBOX_GID, "value": "9999"}, - {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "duplicate"} - ] - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::InitContainer, - 1500, - 1600, + fn proxy_pod_agent_pod_has_no_root_containers() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + cr_name: "example-sandbox", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1600, + // Force the init-container sideload so the supervisor copy-self init + // is present; on OpenShift's nonroot SCC it must not run as root. + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + true, + ¶ms, ); - let agent = &pod_template["spec"]["containers"][0]; - let env = agent["env"].as_array().unwrap(); - for name in [ - openshell_core::sandbox_env::OCI_IMAGE_USER, - openshell_core::sandbox_env::SANDBOX_UID, - openshell_core::sandbox_env::SANDBOX_GID, - ] { + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + // Supervisor binary sideload, CA install, wait-for-proxy, workspace seed. + assert_eq!(init_containers.len(), 4); + for container in containers.iter().chain(init_containers) { + let security_context = &container["securityContext"]; + assert_ne!( + security_context["runAsUser"], 0, + "{} must not run as root", + container["name"] + ); + assert_eq!(security_context["runAsNonRoot"], true); + assert_eq!(security_context["allowPrivilegeEscalation"], false); assert_eq!( - env.iter().filter(|item| item["name"] == name).count(), - 1, - "{name} must have one driver-owned value" + security_context["capabilities"]["drop"], + serde_json::json!(["ALL"]) ); } + } + + #[test] + fn proxy_pod_topology_supports_preferred_affinity() { + let mut spec = serde_json::Map::new(); + apply_proxy_pod_affinity(&mut spec, "sandbox-123", ProxyPodAffinity::Preferred); + + let preferred = + &spec["affinity"]["podAffinity"]["preferredDuringSchedulingIgnoredDuringExecution"][0]; + assert_eq!(preferred["weight"], 100); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") + preferred["podAffinityTerm"]["labelSelector"]["matchLabels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR ); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_GID), - Some("1600") + preferred["podAffinityTerm"]["topologyKey"], + "kubernetes.io/hostname" ); } #[test] - fn supervisor_sideload_adds_security_context_when_missing() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] + fn proxy_pod_topology_supports_required_affinity_without_replacing_existing_terms() { + let mut spec = serde_json::json!({ + "affinity": { + "podAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": [{ + "topologyKey": "topology.kubernetes.io/zone" + }] + } } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::InitContainer, - 1000, // sandbox_uid - 1000, // sandbox_gid - ); + }) + .as_object() + .unwrap() + .clone(); + apply_proxy_pod_affinity(&mut spec, "sandbox-123", ProxyPodAffinity::Required); - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!( - sc["runAsUser"], 0, - "runAsUser must be 0 even when no prior securityContext" - ); + let required = + spec["affinity"]["podAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"] + .as_array() + .unwrap(); + assert_eq!(required.len(), 2); + assert_eq!(required[0]["topologyKey"], "topology.kubernetes.io/zone"); + assert_eq!(required[1]["topologyKey"], "kubernetes.io/hostname"); } #[test] - fn supervisor_sideload_injects_emptydir_volume_init_container_and_mount() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } + fn proxy_pod_companion_resources_bind_one_agent_to_one_supervisor() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + service_account_name: "openshell-sandbox", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + cr_name: "example-sandbox", + grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + host_gateway_ip: "172.17.0.1", + ..SandboxPodParams::default() + }; + let names = proxy_pod_resource_names(params.cr_name, params.sandbox_id); + let owner_ref = serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", + "kind": "Sandbox", + "name": params.sandbox_name, + "uid": "sandbox-cr-uid", + "controller": true, + "blockOwnerDeletion": false }); - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::InitContainer, - 1000, // sandbox_uid - 1000, // sandbox_gid + let supervisor = serde_json::to_value(proxy_pod_supervisor_deployment( + &names, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + ¶ms, + &KubernetesPodDriverConfig::default(), + &ProxyPodPlacement::default(), + 1, + owner_ref.clone(), + )) + .unwrap(); + assert_eq!( + supervisor["metadata"]["ownerReferences"][0]["controller"], + true ); - - // Volume should be an emptyDir - let volumes = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist"); - assert_eq!(volumes.len(), 1); - assert_eq!(volumes[0]["name"], SUPERVISOR_VOLUME_NAME); - assert!( - volumes[0]["emptyDir"].is_object(), - "volume should be emptyDir, not hostPath" + assert_eq!( + supervisor["metadata"]["annotations"]["openshell.ai/sandbox-id"], + "sandbox-123" ); - - // Init container should use the supervisor image, not the sandbox image - let init_containers = pod_template["spec"]["initContainers"] - .as_array() - .expect("initContainers should exist"); - assert_eq!(init_containers.len(), 1); - assert_eq!(init_containers[0]["name"], SUPERVISOR_INIT_CONTAINER_NAME); - assert_eq!(init_containers[0]["image"], "supervisor-image:latest"); - assert_eq!(init_containers[0]["imagePullPolicy"], "IfNotPresent"); - - // The init container must invoke the binary directly with - // `copy-self ` rather than depending on shell utilities. - let init_command = init_containers[0]["command"] + assert_eq!( + supervisor["metadata"]["labels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR + ); + assert_eq!(supervisor["kind"], "Deployment"); + assert_eq!(supervisor["spec"]["replicas"], 1); + assert_eq!( + supervisor["spec"]["selector"]["matchLabels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR + ); + assert_eq!( + supervisor["spec"]["template"]["metadata"]["labels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR + ); + assert_eq!( + supervisor["spec"]["template"]["spec"]["hostAliases"][0]["ip"], + params.host_gateway_ip + ); + let hostnames = supervisor["spec"]["template"]["spec"]["hostAliases"][0]["hostnames"] .as_array() - .expect("init container command should be set"); - assert_eq!(init_command.len(), 3, "expected [binary, copy-self, dest]"); - assert_eq!(init_command[0], SUPERVISOR_IMAGE_BINARY_PATH); - assert_eq!(init_command[1], "copy-self"); + .unwrap(); + assert!(hostnames.contains(&serde_json::json!("host.openshell.internal"))); + let container = &supervisor["spec"]["template"]["spec"]["containers"][0]; assert_eq!( - init_command[2].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + rendered_env(container, openshell_core::sandbox_env::PROXY_BIND_ADDR), + Some("0.0.0.0:3128") + ); + + let agent_egress = + serde_json::to_value(proxy_pod_agent_egress_network_policy(&names, ¶ms)).unwrap(); + assert_eq!( + agent_egress["spec"]["policyTypes"], + serde_json::json!(["Egress"]) ); + // The egress fence must carry NO owner reference: it is gateway-managed + // so it outlives the workload pod during deletion rather than being + // garbage-collected concurrently with it. assert!( - !init_command.iter().any(|v| v == "sh"), - "init container must not depend on a shell" + agent_egress["metadata"].get("ownerReferences").is_none(), + "agent egress NetworkPolicy must have no ownerReferences: {agent_egress}" + ); + assert_eq!( + agent_egress["spec"]["podSelector"]["matchLabels"][LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_AGENT + ); + assert_eq!( + agent_egress["spec"]["egress"][0]["to"][0]["podSelector"]["matchLabels"] + [LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_SUPERVISOR ); - // `--workdir` is optional for standalone supervisor invocations and - // has no implicit default, so Kubernetes must pass its fixed workspace. - let command = pod_template["spec"]["containers"][0]["command"] - .as_array() - .expect("command should be set"); + let supervisor_ingress = serde_json::to_value(proxy_pod_supervisor_ingress_network_policy( + &names, ¶ms, owner_ref, + )) + .unwrap(); assert_eq!( - command[0].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + supervisor_ingress["spec"]["policyTypes"], + serde_json::json!(["Ingress"]) ); assert_eq!( - command, - serde_json::json!([ - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT - ]) - .as_array() - .unwrap() + supervisor_ingress["spec"]["ingress"][0]["from"][0]["podSelector"]["matchLabels"] + [LABEL_SANDBOX_ROLE], + SANDBOX_ROLE_AGENT ); + } - // Agent volume mount should be read-only - let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] + #[test] + fn proxy_pod_topology_rejects_proxy_uid_matching_sandbox_uid() { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + proxy_uid: 1500, + namespace: "default", + sandbox_uid: 1500, + ..SandboxPodParams::default() + }; + + let err = validate_proxy_identity(¶ms).unwrap_err(); + assert!(matches!(err, KubernetesDriverError::Precondition(_))); + assert!(err.to_string().contains("proxy-pod")); + } + + /// Every egress rule except the supervisor rule, which is the one carrying + /// the proxy port. + fn dns_egress_rules(policy: &NetworkPolicy) -> Vec { + let policy = serde_json::to_value(policy).unwrap(); + policy["spec"]["egress"] .as_array() - .expect("volumeMounts should exist"); - assert_eq!(mounts.len(), 1); - assert_eq!(mounts[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(mounts[0]["mountPath"], SUPERVISOR_MOUNT_PATH); - assert_eq!(mounts[0]["readOnly"], true); + .unwrap() + .iter() + .filter(|rule| { + !rule["ports"].as_array().is_some_and(|ports| { + ports + .iter() + .any(|port| port["port"] == i64::from(PROXY_POD_PROXY_PORT)) + }) + }) + .cloned() + .collect() + } + + fn proxy_pod_egress_policy_with_dns_peers(peers: &[ProxyPodDnsPeer]) -> NetworkPolicy { + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + cr_name: "example-sandbox", + proxy_pod_dns_peers: peers, + proxy_pod_gateway_peers: &[], + ..SandboxPodParams::default() + }; + proxy_pod_agent_egress_network_policy( + &proxy_pod_resource_names(params.cr_name, params.sandbox_id), + ¶ms, + ) + } + + fn sandbox_object_with_conditions(conditions: &[(&str, &str)]) -> DynamicObject { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut obj = DynamicObject::new("sandbox", &resource); + let conditions: Vec<_> = conditions + .iter() + .map(|(kind, status)| serde_json::json!({"type": kind, "status": status})) + .collect(); + obj.data = serde_json::json!({"status": {"conditions": conditions}}); + obj } #[test] - fn supervisor_sideload_image_volume_injects_image_source_without_init_container() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); + fn agent_command_override_is_rejected_in_every_topology() { + let config = KubernetesSandboxDriverConfig { + containers: KubernetesDriverContainersConfig { + agent: KubernetesContainerDriverConfig { + command: vec!["sleep".to_string(), "infinity".to_string()], + ..KubernetesContainerDriverConfig::default() + }, + }, + ..KubernetesSandboxDriverConfig::default() + }; - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::ImageVolume, - 1000, // sandbox_uid - 1000, // sandbox_gid - ); + // proxy-pod now runs the process supervisor as the entrypoint too, so the + // override is ignored (and rejected) in every topology. + for topology in [ + SupervisorTopology::Combined, + SupervisorTopology::Sidecar, + SupervisorTopology::ProxyPod, + ] { + let err = validate_agent_command_for_topology(&config, topology).unwrap_err(); + assert!(err.contains("not supported"), "{topology}: {err}"); + } + } - let volumes = pod_template["spec"]["volumes"] + /// Per-sandbox proxy-pod resources are named from the sandbox name, not + /// the Sandbox CR name -- the CR is `--`. Deriving them + /// from the CR name silently targets objects that do not exist, which + /// owner-reference GC then masks on delete but not on stop/start. + #[test] + fn proxy_pod_supervisor_inherits_workload_node_placement() { + let names = proxy_pod_resource_names("ws--dev", "sandbox-1"); + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor:latest", + namespace: "agents", + sandbox_id: "sandbox-1", + sandbox_name: "dev", + proxy_uid: 2000, + sandbox_uid: 1500, + sandbox_gid: 1500, + ..SandboxPodParams::default() + }; + let pod_config = KubernetesPodDriverConfig { + node_selector: std::iter::once(("pool".to_string(), "gpu".to_string())).collect(), + tolerations: vec![serde_json::json!({"key": "gpu", "operator": "Exists"})], + ..KubernetesPodDriverConfig::default() + }; + let dep = serde_json::to_value(proxy_pod_supervisor_deployment( + &names, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + ¶ms, + &pod_config, + &ProxyPodPlacement::default(), + 1, + serde_json::json!({}), + )) + .unwrap(); + let pod_spec = &dep["spec"]["template"]["spec"]; + assert_eq!(pod_spec["nodeSelector"]["pool"], "gpu"); + assert_eq!(pod_spec["tolerations"][0]["key"], "gpu"); + } + + /// The proxy-pod supervisor must route egress through the operator's + /// corporate upstream proxy, matching sidecar topology, or all permitted + /// traffic fails (or bypasses the required monitoring route). + #[test] + fn proxy_pod_supervisor_forwards_to_upstream_proxy() { + let names = proxy_pod_resource_names("ws--dev", "sandbox-1"); + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor:latest", + namespace: "agents", + sandbox_id: "sandbox-1", + sandbox_name: "dev", + proxy_uid: 2000, + sandbox_uid: 1500, + sandbox_gid: 1500, + https_proxy: Some("http://corp-proxy.example.com:3128"), + no_proxy: Some("10.0.0.0/8,.svc"), + ..SandboxPodParams::default() + }; + let dep = serde_json::to_value(proxy_pod_supervisor_deployment( + &names, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + ¶ms, + &KubernetesPodDriverConfig::default(), + &ProxyPodPlacement::default(), + 1, + serde_json::json!({}), + )) + .unwrap(); + let command = dep["spec"]["template"]["spec"]["containers"][0]["command"] .as_array() - .expect("volumes should exist"); - assert_eq!(volumes.len(), 1); - assert_eq!(volumes[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(volumes[0]["image"]["reference"], "supervisor-image:latest"); - assert_eq!(volumes[0]["image"]["pullPolicy"], "IfNotPresent"); + .unwrap() + .iter() + .map(|v| v.as_str().unwrap().to_string()) + .collect::>(); + let joined = command.join(" "); assert!( - volumes[0]["emptyDir"].is_null(), - "image volume method must not use emptyDir" + joined.contains("--upstream-proxy http://corp-proxy.example.com:3128"), + "supervisor command must forward to the upstream proxy: {joined}" ); + assert!( + joined.contains("--upstream-no-proxy 10.0.0.0/8,.svc"), + "supervisor command must carry no_proxy: {joined}" + ); + // Credentials are not mounted into the supervisor pod, so no auth-file arg. + assert!( + !joined.contains("--upstream-proxy-auth-file"), + "proxy-pod must not reference an unmounted auth file: {joined}" + ); + } + + /// The provider SPIFFE Workload API socket must live only in the supervisor + /// (proxy) pod, which mints provider credentials. The agent pod holds a + /// scoped process-kind credential that cannot access provider secrets, so it + /// must not carry an SVID: no CSI volume, no container mount, no env. + #[test] + fn proxy_pod_strips_provider_spiffe_from_agent_but_keeps_it_on_supervisor() { + let names = proxy_pod_resource_names("example-sandbox", "sandbox-123"); + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor-image:latest", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + cr_name: "example-sandbox", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + provider_spiffe_enabled: true, + provider_spiffe_workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock", + ..SandboxPodParams::default() + }; + // Agent pod: SPIFFE stripped everywhere. + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + let agent = &pod_template["spec"]["containers"][0]; assert!( - pod_template["spec"]["initContainers"].is_null(), - "image volume method must not inject init containers" + rendered_env( + agent, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET + ) + .is_none(), + "agent must not advertise the provider SPIFFE socket" + ); + let agent_mounts = agent["volumeMounts"] + .as_array() + .expect("agent volumeMounts"); + assert!( + !agent_mounts + .iter() + .any(|m| m["name"] == SPIFFE_WORKLOAD_API_VOLUME_NAME), + "agent container must not mount the SPIFFE workload API socket" + ); + let pod_volumes = pod_template["spec"]["volumes"] + .as_array() + .expect("pod volumes"); + assert!( + !pod_volumes + .iter() + .any(|v| v["name"] == SPIFFE_WORKLOAD_API_VOLUME_NAME), + "agent pod must not carry the SPIFFE CSI volume" ); - let command = pod_template["spec"]["containers"][0]["command"] + // Supervisor (proxy) pod: SPIFFE retained (it mints provider credentials). + let dep = serde_json::to_value(proxy_pod_supervisor_deployment( + &names, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + ¶ms, + &KubernetesPodDriverConfig::default(), + &ProxyPodPlacement::default(), + 1, + serde_json::json!({}), + )) + .unwrap(); + let sup_spec = &dep["spec"]["template"]["spec"]; + let sup_mounts = sup_spec["containers"][0]["volumeMounts"] .as_array() - .expect("command should be set"); - assert_eq!( - command[0].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + .expect("supervisor volumeMounts"); + assert!( + sup_mounts + .iter() + .any(|m| m["name"] == SPIFFE_WORKLOAD_API_VOLUME_NAME), + "supervisor container must mount the SPIFFE workload API socket" + ); + let sup_volumes = sup_spec["volumes"].as_array().expect("supervisor volumes"); + assert!( + sup_volumes + .iter() + .any(|v| v["name"] == SPIFFE_WORKLOAD_API_VOLUME_NAME + && v["csi"]["driver"] == "csi.spiffe.io"), + "supervisor pod must carry the SPIFFE CSI volume" ); + } - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!(sc["runAsUser"], 0); + /// The supervisor must also honor the public `platform_config` placement the + /// workload pod reads (runtime class, node selector, tolerations). Otherwise + /// the workload can land under Kata (or a required node) while the supervisor + /// takes the cluster default, breaking same-node pairing. + #[test] + fn proxy_pod_supervisor_inherits_platform_config_placement() { + let toleration = Struct { + fields: std::iter::once(( + "key".to_string(), + Value { + kind: Some(Kind::StringValue("dedicated".to_string())), + }, + )) + .collect(), + }; + let template = SandboxTemplate { + platform_config: Some(Struct { + fields: [ + ( + "runtime_class_name".to_string(), + Value { + kind: Some(Kind::StringValue("kata-containers".to_string())), + }, + ), + ( + "node_selector".to_string(), + Value { + kind: Some(Kind::StructValue(Struct { + fields: std::iter::once(( + "disktype".to_string(), + Value { + kind: Some(Kind::StringValue("ssd".to_string())), + }, + )) + .collect(), + })), + }, + ), + ( + "tolerations".to_string(), + Value { + kind: Some(Kind::ListValue(prost_types::ListValue { + values: vec![Value { + kind: Some(Kind::StructValue(toleration)), + }], + })), + }, + ), + ] + .into_iter() + .collect(), + }), + ..SandboxTemplate::default() + }; + let placement = ProxyPodPlacement::from_template(Some(&template)); - let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] - .as_array() - .expect("volumeMounts should exist"); - assert_eq!(mounts[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(mounts[0]["mountPath"], SUPERVISOR_MOUNT_PATH); - assert_eq!(mounts[0]["readOnly"], true); + let names = proxy_pod_resource_names("ws--dev", "sandbox-1"); + let params = SandboxPodParams { + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor:latest", + namespace: "agents", + sandbox_id: "sandbox-1", + sandbox_name: "dev", + proxy_uid: 2000, + sandbox_uid: 1500, + sandbox_gid: 1500, + // Cluster default must lose to the platform_config runtime class. + default_runtime_class_name: "gvisor", + ..SandboxPodParams::default() + }; + let dep = serde_json::to_value(proxy_pod_supervisor_deployment( + &names, + &std::collections::HashMap::new(), + &std::collections::HashMap::new(), + ¶ms, + &KubernetesPodDriverConfig::default(), + &placement, + 1, + serde_json::json!({}), + )) + .unwrap(); + let pod_spec = &dep["spec"]["template"]["spec"]; + assert_eq!(pod_spec["runtimeClassName"], "kata-containers"); + assert_eq!(pod_spec["nodeSelector"]["disktype"], "ssd"); + assert_eq!(pod_spec["tolerations"][0]["key"], "dedicated"); } + /// Companion reconciliation rebuilds a repaired supervisor's placement and + /// log level from the Sandbox CR's rendered agent pod, so it lands where the + /// workload can pair with it even after a crash lost the original spec. #[test] - fn supervisor_image_volume_omits_pull_policy_when_empty() { - let mut pod_template = serde_json::json!({ + fn proxy_pod_placement_and_log_level_recovered_from_cr() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut obj = DynamicObject::new("ws--dev", &resource); + obj.data = serde_json::json!({ "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] + "podTemplate": { + "spec": { + "runtimeClassName": "kata-containers", + "nodeSelector": {"disktype": "ssd"}, + "tolerations": [{"key": "dedicated", "operator": "Exists"}], + "containers": [{ + "name": "agent", + "env": [{ + "name": openshell_core::sandbox_env::LOG_LEVEL, + "value": "debug" + }] + }] + } + } } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "", - SupervisorSideloadMethod::ImageVolume, - 1000, // sandbox_uid - 1000, // sandbox_gid + }); + + let placement = proxy_pod_placement_from_cr(&obj); + assert_eq!( + placement.runtime_class_name.as_deref(), + Some("kata-containers") ); + assert_eq!(placement.node_selector.unwrap()["disktype"], "ssd"); + assert_eq!(placement.tolerations.unwrap()[0]["key"], "dedicated"); - let volume = &pod_template["spec"]["volumes"][0]; - assert_eq!(volume["image"]["reference"], "supervisor-image:latest"); - assert!( - volume["image"].get("pullPolicy").is_none(), - "pullPolicy should be omitted when empty" + let env = proxy_pod_log_level_env_from_cr(&obj); + assert_eq!( + env.get(openshell_core::sandbox_env::LOG_LEVEL) + .map(String::as_str), + Some("debug") ); } + /// An empty/absent pod template must not fabricate placement or env. #[test] - fn sidecar_topology_renders_process_agent_and_network_sidecar() { + fn proxy_pod_reconstruction_tolerates_missing_pod_template() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let obj = DynamicObject::new("ws--dev", &resource); + let placement = proxy_pod_placement_from_cr(&obj); + assert!(placement.runtime_class_name.is_none()); + assert!(placement.node_selector.is_none()); + assert!(placement.tolerations.is_none()); + assert!(proxy_pod_log_level_env_from_cr(&obj).is_empty()); + } + + #[test] + fn proxy_pod_pod_template_references_companions_by_cr_name() { + // Shared mode: CR name is `--`, distinct from the bare + // sandbox name. The workload pod's CA secret mount and proxy URL must + // use the CR-name-derived companion names, or the pod mounts a secret + // that does not exist (and never becomes Ready). let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - supervisor_image: "supervisor-image:latest", - supervisor_image_pull_policy: "IfNotPresent", - grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", - client_tls_secret_name: "openshell-client-tls", - proxy_uid: 2200, + topology: SupervisorTopology::ProxyPod, + supervisor_image: "supervisor:latest", + namespace: "agents", + sandbox_id: "sandbox-1", + sandbox_name: "dev", + cr_name: "team-a--dev", + proxy_uid: 2000, sandbox_uid: 1500, sandbox_gid: 1500, ..SandboxPodParams::default() }; let pod_template = sandbox_template_to_k8s( &SandboxTemplate { - image: "agent-image:latest".to_string(), - environment: std::collections::HashMap::from([ - ( - openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), - "spoofed".to_string(), - ), - ( - openshell_core::sandbox_env::SANDBOX_UID.to_string(), - "9999".to_string(), - ), - ( - openshell_core::sandbox_env::SANDBOX_GID.to_string(), - "9999".to_string(), - ), - ]), + image: "agent:latest".to_string(), ..SandboxTemplate::default() }, false, @@ -6506,236 +10772,247 @@ mod tests { false, ¶ms, ); - - assert_eq!(pod_template["spec"]["shareProcessNamespace"], true); - assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - assert_eq!(containers.len(), 2); - - let agent = containers - .iter() - .find(|container| container["name"] == "agent") - .unwrap(); - assert_eq!( - agent["command"], - serde_json::json!([ - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), - "--mode=process", - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT - ]) - ); - assert_eq!(agent["securityContext"]["runAsUser"], 1500); - assert_eq!(agent["securityContext"]["runAsGroup"], 1500); - assert_eq!(agent["securityContext"]["runAsNonRoot"], true); - assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false); - assert_eq!( - agent["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"] - }) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::TLS_CA), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), - Some(SIDECAR_SSH_SOCKET_FILE) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), - Some(SIDECAR_CONTROL_SOCKET) - ); - assert_eq!(rendered_env(agent, "OPENSHELL_SUPERVISOR_READY_FILE"), None); - assert_eq!(rendered_env(agent, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); + let names = proxy_pod_resource_names(params.cr_name, params.sandbox_id); + let service_dns = proxy_pod_service_dns(&names.service, "agents"); + let agent = &pod_template["spec"]["containers"][0]; assert_eq!( - rendered_env(agent, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), - None + rendered_env(agent, "HTTP_PROXY"), + Some(format!("http://{service_dns}:3128").as_str()) ); - assert_eq!( - rendered_env(agent, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), - None + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + assert!( + volumes.iter().any(|v| { + v["name"] == "openshell-proxy-pod-ca-source" + && v["secret"]["secretName"] == serde_json::json!(names.proxy_ca_secret) + }), + "workload CA volume must reference the CR-name-derived secret {}", + names.proxy_ca_secret ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::PROXY_TLS_DIR), - Some(SIDECAR_TLS_MOUNT_PATH) + } + + #[test] + fn proxy_pod_resource_names_disambiguate_by_sandbox_id() { + // Distinct sandbox instances get distinct companion names even when + // their CR names are identical, because uniqueness is keyed on the + // immutable per-instance UUID rather than the CR name. + let a = + proxy_pod_resource_names("workspace-a--dev", "11111111-1111-1111-1111-111111111111"); + let b = + proxy_pod_resource_names("workspace-a--dev", "22222222-2222-2222-2222-222222222222"); + assert_ne!(a.supervisor_deployment, b.supervisor_deployment); + assert_ne!(a.service, b.service); + assert_ne!(a.proxy_ca_secret, b.proxy_ca_secret); + assert_ne!(a.agent_egress_network_policy, b.agent_egress_network_policy); + assert_ne!( + a.supervisor_ingress_network_policy, + b.supervisor_ingress_network_policy ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") + } + + #[test] + fn proxy_pod_resource_names_are_stable_for_an_instance() { + // The same sandbox id yields the same names across calls (create, + // reconcile, scale all recompute them independently). + let first = proxy_pod_resource_names("ws--dev", "abc-123"); + let second = proxy_pod_resource_names("ws--dev", "abc-123"); + assert_eq!(first.supervisor_deployment, second.supervisor_deployment); + } + + #[test] + fn proxy_pod_resource_names_avoid_truncation_collision() { + // Two CR names that collided under the old 32-bit name hash — a shared + // 48-char prefix that truncates identically — now differ because the + // suffix is a 64-bit hash of the distinct sandbox UUIDs. + let long = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + let a = proxy_pod_resource_names( + &format!("{long}-955pct6t1ohlwg"), + "11111111-1111-1111-1111-111111111111", ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") + let b = proxy_pod_resource_names( + &format!("{long}-uw6jys21qzazvy"), + "22222222-2222-2222-2222-222222222222", ); + assert_ne!(a.supervisor_deployment, b.supervisor_deployment); + assert!(a.supervisor_deployment.len() <= MAX_KUBE_NAME_LEN); + assert!(b.supervisor_deployment.len() <= MAX_KUBE_NAME_LEN); + } - let sidecar = containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + #[test] + fn supervisor_topologies_require_a_supervisor_session() { + let obj = sandbox_object_with_conditions(&[("Ready", "True")]); + // proxy-pod now runs an in-pod process supervisor, so it too requires a + // session (relays are available). + for topology in [ + SupervisorTopology::Combined, + SupervisorTopology::Sidecar, + SupervisorTopology::ProxyPod, + ] { + let status = status_from_object(&obj, topology).unwrap(); + assert_eq!( + status.supervisor_session_model, + SupervisorSessionModel::Required as i32, + "{topology}" + ); + } + } + + #[test] + fn mark_supervisor_unavailable_forces_not_ready() { + let mut sandbox = Sandbox { + id: "id".to_string(), + name: "dev".to_string(), + namespace: "agents".to_string(), + spec: None, + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: String::new(), + message: String::new(), + last_transition_time: String::new(), + }], + ..SandboxStatus::default() + }), + workspace: "default".to_string(), + }; + mark_supervisor_unavailable(&mut sandbox); + let ready = sandbox + .status + .unwrap() + .conditions + .into_iter() + .find(|condition| condition.r#type == "Ready") .unwrap(); - assert_eq!(sidecar["image"], "supervisor-image:latest"); - assert_eq!(sidecar["imagePullPolicy"], "IfNotPresent"); - assert_eq!( - sidecar["command"], - serde_json::json!([SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network"]) - ); - assert_eq!(sidecar["securityContext"]["runAsUser"], 0); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); - assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false - ); - assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::ENDPOINT), - Some("https://openshell-gateway.openshell.svc:8080") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SSH_SOCKET_PATH), - Some(SIDECAR_SSH_SOCKET_FILE) - ); - assert!( - SIDECAR_SSH_SOCKET_FILE.starts_with('@'), - "sidecar SSH relay must use a Linux abstract socket" - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_GID), - Some("1500") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), - Some(SIDECAR_CONTROL_SOCKET) - ); - assert_eq!( - rendered_env(sidecar, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), - None - ); - assert_eq!( - rendered_env(sidecar, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), - None - ); - assert_eq!( - rendered_env( - sidecar, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY - ), - None - ); - assert_eq!(rendered_env(sidecar, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::PROXY_TLS_DIR), - Some(SIDECAR_TLS_MOUNT_PATH) - ); + assert_eq!(ready.status, "False"); + // Must be a transient reason so the gateway maps it to a recoverable + // Provisioning phase rather than terminal Error. + assert_eq!(ready.reason, "DependenciesNotReady"); + } + + #[test] + fn mark_supervisor_unavailable_adds_condition_when_absent() { + let mut sandbox = Sandbox { + id: "id".to_string(), + name: "dev".to_string(), + namespace: "agents".to_string(), + spec: None, + status: Some(SandboxStatus::default()), + workspace: "default".to_string(), + }; + mark_supervisor_unavailable(&mut sandbox); + let conditions = sandbox.status.unwrap().conditions; + assert_eq!(conditions.len(), 1); + assert_eq!(conditions[0].r#type, "Ready"); + assert_eq!(conditions[0].status, "False"); + } + + #[test] + fn topology_from_object_prefers_annotation_over_fallback() { + let mut obj = sandbox_object_with_conditions(&[("Ready", "True")]); + obj.metadata.annotations = Some(BTreeMap::from([( + ANNOTATION_SUPERVISOR_TOPOLOGY.to_string(), + "proxy-pod".to_string(), + )])); + // Even if the gateway is now configured for `combined`, a CR created + // under `proxy-pod` must be interpreted as `proxy-pod`. assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::TLS_CA), - Some("/etc/openshell-tls/proxy/client/ca.crt") + topology_from_object(&obj, SupervisorTopology::Combined), + SupervisorTopology::ProxyPod ); - let sidecar_mounts = sidecar["volumeMounts"].as_array().unwrap(); - assert!( - !sidecar_mounts - .iter() - .any(|mount| mount["name"] == "openshell-client-tls"), - "runtime sidecar should use the init-copied TLS files, not the root-owned Secret mount" - ); - let agent_mounts = agent["volumeMounts"].as_array().unwrap(); - assert!( - !agent_mounts - .iter() - .any(|mount| mount["name"] == "openshell-sa-token"), - "agent container must not mount gateway bootstrap token in sidecar topology" - ); - assert!( - !agent_mounts - .iter() - .any(|mount| mount["name"] == "openshell-client-tls"), - "agent container must not mount gateway client TLS secret in sidecar topology" - ); - let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); - let sa_token = volumes - .iter() - .find(|volume| volume["name"] == "openshell-sa-token") - .unwrap(); - assert_eq!(sa_token["projected"]["defaultMode"], 0o440); - let client_tls = volumes - .iter() - .find(|volume| volume["name"] == "openshell-client-tls") - .unwrap(); - assert_eq!(client_tls["secret"]["defaultMode"], 0o440); + } - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert_eq!(network_init["image"], "supervisor-image:latest"); - assert_eq!(network_init["imagePullPolicy"], "IfNotPresent"); + #[test] + fn topology_from_object_falls_back_without_annotation() { + let obj = sandbox_object_with_conditions(&[("Ready", "True")]); + // A CR predating the annotation keeps a non-proxy-pod fallback as-is. assert_eq!( - network_init["command"], - serde_json::json!([ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network-init", - "--proxy-uid", - "0", - "--proxy-gid", - "1500", - "--sidecar-state-dir", - SIDECAR_STATE_MOUNT_PATH, - "--sidecar-tls-dir", - SIDECAR_TLS_MOUNT_PATH - ]) + topology_from_object(&obj, SupervisorTopology::Sidecar), + SupervisorTopology::Sidecar + ); + } + + #[test] + fn desired_supervisor_replicas_follows_operating_state() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut running = DynamicObject::new("s", &resource); + running.data = serde_json::json!({"spec": {"operatingMode": "Running"}}); + assert_eq!(desired_supervisor_replicas(&running), 1); + + let mut suspended = DynamicObject::new("s", &resource); + suspended.data = serde_json::json!({"spec": {"operatingMode": "Suspended"}}); + assert_eq!(desired_supervisor_replicas(&suspended), 0); + + // v1alpha1 encodes it as spec.replicas. + let mut alpha_stopped = DynamicObject::new("s", &resource); + alpha_stopped.data = serde_json::json!({"spec": {"replicas": 0}}); + assert_eq!(desired_supervisor_replicas(&alpha_stopped), 0); + + // No operating state recorded defaults to running. + let bare = DynamicObject::new("s", &resource); + assert_eq!(desired_supervisor_replicas(&bare), 1); + } + + #[test] + fn topology_from_object_never_falls_back_to_proxy_pod() { + // An un-annotated CR was created before this branch, so it was never + // proxy-pod. Even when the gateway is now configured proxy-pod, it must + // not be misclassified as such (which would report it sessionless and + // hunt for companions it never had). + let obj = sandbox_object_with_conditions(&[("Ready", "True")]); + assert_eq!( + topology_from_object(&obj, SupervisorTopology::ProxyPod), + SupervisorTopology::Combined ); + } + + #[test] + fn sandbox_from_object_derives_session_model_from_persisted_topology() { + let mut obj = sandbox_object_with_conditions(&[("Ready", "True")]); + obj.metadata.name = Some("alpha--work".to_string()); + obj.metadata.annotations = Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), "uuid-123".to_string()), + (LABEL_SANDBOX_NAME.to_string(), "work".to_string()), + (LABEL_SANDBOX_WORKSPACE.to_string(), "alpha".to_string()), + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + ( + ANNOTATION_SUPERVISOR_TOPOLOGY.to_string(), + "proxy-pod".to_string(), + ), + ])); + + // The persisted `proxy-pod` annotation is honored over the `combined` + // fallback; every topology now requires a supervisor session. + let (_, sandbox) = + sandbox_from_object("default", obj, SupervisorTopology::Combined).unwrap(); assert_eq!( - network_init["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] - }) + sandbox.status.unwrap().supervisor_session_model, + SupervisorSessionModel::Required as i32 ); - let network_init_mounts = network_init["volumeMounts"].as_array().unwrap(); - assert!(network_init_mounts.iter().any(|mount| { - mount["name"] == "openshell-client-tls" - && mount["mountPath"] == "/etc/openshell-tls/client" - })); } + /// The agent pod must not report Ready before its paired supervisor is + /// serving: the gateway derives readiness from the pod for this topology, + /// so a pod that is up without a proxy would advertise egress it does not + /// have. #[test] - fn sidecar_topology_can_relax_process_binary_aware_network_policy() { + fn proxy_pod_agent_waits_for_its_paired_supervisor() { let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + topology: SupervisorTopology::ProxyPod, supervisor_image: "supervisor-image:latest", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + cr_name: "example-sandbox", proxy_uid: 2200, sandbox_uid: 1500, sandbox_gid: 1500, - process_binary_aware_network_policy: false, ..SandboxPodParams::default() }; let pod_template = sandbox_template_to_k8s( @@ -6748,127 +11025,157 @@ mod tests { false, ¶ms, ); - - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - let sidecar = containers + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let wait = init_containers .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) - .unwrap(); - assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], true); - assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false - ); - assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"] - }) - ); + .find(|c| c["name"] == PROXY_POD_WAIT_INIT_CONTAINER_NAME) + .expect("proxy-pod agent pod waits for its supervisor"); + + let command = wait["command"].as_array().unwrap(); + assert_eq!(command[1], "wait-for-tcp"); + let names = proxy_pod_resource_names(params.cr_name, params.sandbox_id); + let service_dns = proxy_pod_service_dns(&names.service, "agents"); + assert_eq!(command[2], format!("{service_dns}:3128")); + assert_eq!(wait["securityContext"]["runAsNonRoot"], true); assert_eq!( - rendered_env( - sidecar, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY - ), - Some("relaxed") + wait["securityContext"]["capabilities"]["drop"], + serde_json::json!(["ALL"]) ); - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert_eq!(network_init["command"][3], "2200"); } #[test] - fn sidecar_topology_adds_shared_state_and_tls_volumes() { + fn other_topologies_have_no_wait_for_proxy_init_container() { let params = SandboxPodParams { topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::ImageVolume, supervisor_image: "supervisor-image:latest", - grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", + namespace: "agents", + sandbox_id: "sandbox-123", + sandbox_name: "example-sandbox", + cr_name: "example-sandbox", ..SandboxPodParams::default() }; let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, false, &std::collections::HashMap::new(), false, ¶ms, ); - - let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); - assert!( - volumes - .iter() - .any(|volume| volume["name"] == SIDECAR_STATE_VOLUME_NAME) - ); + let init_containers = pod_template["spec"]["initContainers"] + .as_array() + .cloned() + .unwrap_or_default(); assert!( - volumes + !init_containers .iter() - .any(|volume| volume["name"] == SIDECAR_TLS_VOLUME_NAME) + .any(|c| c["name"] == PROXY_POD_WAIT_INIT_CONTAINER_NAME) ); - assert!(volumes.iter().any(|volume| { - volume["name"] == SUPERVISOR_VOLUME_NAME && volume["image"].is_object() - })); + } - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - let sidecar = containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + #[test] + fn proxy_pod_dns_peers_default_to_upstream_kube_system_conventions() { + let peers = crate::config::KubernetesProxyPodConfig::default().dns_peers; + let rules = dns_egress_rules(&proxy_pod_egress_policy_with_dns_peers(&peers)); + + assert_eq!(rules.len(), 2); + let mut apps = Vec::new(); + for rule in &rules { + let to = rule["to"].as_array().unwrap(); + assert_eq!(to.len(), 1); + assert_eq!( + to[0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + "kube-system" + ); + for port in rule["ports"].as_array().unwrap() { + assert_eq!(port["port"], 53); + } + apps.push(to[0]["podSelector"]["matchLabels"]["k8s-app"].clone()); + } + assert!(apps.contains(&serde_json::json!("kube-dns"))); + assert!(apps.contains(&serde_json::json!("coredns"))); + } + + /// `OpenShift` hosts cluster DNS in `openshift-dns`, not `kube-system`, and + /// labels the pods with `dns.operator.openshift.io/daemonset-dns=default`. + /// The upstream default matches nothing there, leaving the agent pod unable + /// to resolve even its own paired supervisor Service. + #[test] + fn proxy_pod_dns_peers_render_openshift_selectors() { + let peers = vec![ProxyPodDnsPeer { + namespace_labels: std::iter::once(( + "kubernetes.io/metadata.name".to_string(), + "openshift-dns".to_string(), + )) + .collect(), + pod_labels: std::iter::once(( + "dns.operator.openshift.io/daemonset-dns".to_string(), + "default".to_string(), + )) + .collect(), + port: 5353, + }]; + let rule = dns_egress_rules(&proxy_pod_egress_policy_with_dns_peers(&peers)) + .pop() .unwrap(); + let to = rule["to"].as_array().unwrap(); + + assert_eq!(to.len(), 1); assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) + to[0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], + "openshift-dns" ); - assert_eq!(sidecar["securityContext"]["runAsUser"], 0); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1000); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false - ); + to[0]["podSelector"]["matchLabels"]["dns.operator.openshift.io/daemonset-dns"], + "default" + ); + // OpenShift's dns-default Service maps 53 onto container port 5353. + // Egress rules match the destination pod port, so the rule must carry + // 5353 rather than the Service port. + for port in rule["ports"].as_array().unwrap() { + assert_eq!(port["port"], 5353); + } + } - for container_name in ["agent", SUPERVISOR_NETWORK_SIDECAR_NAME] { - let container = containers + /// A `NetworkPolicy` egress rule with an empty `to` array matches every + /// destination. Emitting one for an empty peer list would open DNS-port + /// egress cluster-wide, so the rule is omitted entirely instead. + #[test] + fn proxy_pod_empty_dns_peers_omit_the_rule_rather_than_allowing_all() { + let policy = proxy_pod_egress_policy_with_dns_peers(&[]); + assert!(dns_egress_rules(&policy).is_empty()); + + let policy = serde_json::to_value(&policy).unwrap(); + let egress = policy["spec"]["egress"].as_array().unwrap(); + assert_eq!(egress.len(), 1); + assert!( + !egress .iter() - .find(|container| container["name"] == container_name) - .unwrap(); - let mounts = container["volumeMounts"].as_array().unwrap(); - assert!(mounts.iter().any(|mount| { - mount["name"] == SIDECAR_STATE_VOLUME_NAME - && mount["mountPath"] == SIDECAR_STATE_MOUNT_PATH - })); - assert!(mounts.iter().any(|mount| { - mount["name"] == SIDECAR_TLS_VOLUME_NAME - && mount["mountPath"] == SIDECAR_TLS_MOUNT_PATH - })); - } - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert_eq!(network_init["command"][3], "0"); + .any(|rule| rule["to"].as_array().is_some_and(Vec::is_empty)) + ); } #[test] - fn sidecar_topology_rejects_proxy_uid_matching_sandbox_uid() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - proxy_uid: 1500, - sandbox_uid: 1500, - ..SandboxPodParams::default() - }; + fn proxy_pod_dns_peers_allow_a_namespace_only_peer() { + let peers = vec![ProxyPodDnsPeer { + namespace_labels: std::iter::once(( + "kubernetes.io/metadata.name".to_string(), + "openshift-dns".to_string(), + )) + .collect(), + pod_labels: BTreeMap::new(), + port: 5353, + }]; + let rule = dns_egress_rules(&proxy_pod_egress_policy_with_dns_peers(&peers)) + .pop() + .unwrap(); + let to = rule["to"].as_array().unwrap(); - let err = validate_sidecar_proxy_identity(¶ms).unwrap_err(); - assert!(matches!(err, KubernetesDriverError::Precondition(_))); - assert!(err.to_string().contains("proxy_uid")); + assert_eq!(to.len(), 1); + assert!(to[0].get("namespaceSelector").is_some()); + assert!(to[0].get("podSelector").is_none()); } /// Regression test: TLS mount path must match env var paths. @@ -7353,7 +11660,9 @@ mod tests { &mut pod_template, "openshell/sandbox:latest", "IfNotPresent", + 1000, // sandbox_uid 1000, // sandbox_gid + SupervisorTopology::Combined, ); // Init container @@ -7413,6 +11722,8 @@ mod tests { "my-custom-image:v2", "IfNotPresent", 1000, + 1000, + SupervisorTopology::Combined, ); let init_image = pod_template["spec"]["initContainers"][0]["image"] @@ -7435,7 +11746,14 @@ mod tests { } }); - apply_workspace_persistence(&mut pod_template, "img:latest", "Always", 1000); + apply_workspace_persistence( + &mut pod_template, + "img:latest", + "Always", + 1000, + 1000, + SupervisorTopology::Combined, + ); let cmd = pod_template["spec"]["initContainers"][0]["command"] .as_array() @@ -7461,6 +11779,37 @@ mod tests { ); } + #[test] + fn workspace_persistence_uses_non_root_init_container_for_proxy_pod() { + let mut pod_template = serde_json::json!({ + "spec": { + "containers": [{ + "name": "agent", + "image": "img:latest" + }] + } + }); + + apply_workspace_persistence( + &mut pod_template, + "img:latest", + "IfNotPresent", + 1500, + 1600, + SupervisorTopology::ProxyPod, + ); + + let security_context = &pod_template["spec"]["initContainers"][0]["securityContext"]; + assert_eq!(security_context["runAsUser"], 1500); + assert_eq!(security_context["runAsGroup"], 1600); + assert_eq!(security_context["runAsNonRoot"], true); + assert_eq!(security_context["allowPrivilegeEscalation"], false); + assert_eq!( + security_context["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); + } + #[test] fn workspace_persistence_skipped_when_inject_workspace_false() { let params = SandboxPodParams { @@ -8196,7 +12545,8 @@ mod tests { data: serde_json::json!({}), }; - let (kube_name, sandbox) = sandbox_from_object("default", obj).unwrap(); + let (kube_name, sandbox) = + sandbox_from_object("default", obj, SupervisorTopology::Combined).unwrap(); assert_eq!(kube_name, "alpha--work"); assert_eq!(sandbox.name, "work"); assert_eq!(sandbox.workspace, "alpha"); @@ -8225,7 +12575,8 @@ mod tests { data: serde_json::json!({}), }; - let (_, sandbox) = sandbox_from_object("default", obj).unwrap(); + let (_, sandbox) = + sandbox_from_object("default", obj, SupervisorTopology::Combined).unwrap(); assert_eq!(sandbox.name, "work"); assert_eq!(sandbox.workspace, "alpha"); assert_eq!(sandbox.id, "uuid-456"); @@ -8247,7 +12598,7 @@ mod tests { data: serde_json::json!({}), }; - let result = sandbox_from_object("default", obj); + let result = sandbox_from_object("default", obj, SupervisorTopology::Combined); assert!(result.is_err()); assert!(result.unwrap_err().contains("not managed by openshell")); } @@ -8278,7 +12629,8 @@ mod tests { data: serde_json::json!({}), }; - let (_, sandbox) = sandbox_from_object("openshell", obj).unwrap(); + let (_, sandbox) = + sandbox_from_object("openshell", obj, SupervisorTopology::Combined).unwrap(); assert_eq!(sandbox.namespace, "openshell-gw1-team-a"); assert_eq!(sandbox.workspace, "team-a"); } @@ -8303,7 +12655,7 @@ mod tests { data: serde_json::json!({}), }; - let result = sandbox_from_object("default", obj); + let result = sandbox_from_object("default", obj, SupervisorTopology::Combined); assert!(result.is_err()); assert!(result.unwrap_err().contains("missing sandbox workspace")); } @@ -8531,6 +12883,76 @@ mod tests { ); } + #[test] + fn supervisor_availability_from_deployment_reads_available_replicas() { + use k8s_openapi::api::apps::v1::DeploymentStatus; + let ready = Deployment { + status: Some(DeploymentStatus { + available_replicas: Some(1), + ..Default::default() + }), + ..Default::default() + }; + assert_eq!( + supervisor_availability_from_deployment(&ready), + SupervisorAvailability::Available + ); + + // Zero available replicas, and a status with no replica counts at all, + // both mean unavailable — never Unknown (the object is in hand). + let zero = Deployment { + status: Some(DeploymentStatus { + available_replicas: Some(0), + ..Default::default() + }), + ..Default::default() + }; + assert_eq!( + supervisor_availability_from_deployment(&zero), + SupervisorAvailability::Unavailable + ); + assert_eq!( + supervisor_availability_from_deployment(&Deployment::default()), + SupervisorAvailability::Unavailable + ); + } + + #[test] + fn supervisor_deployment_sandbox_id_reads_label() { + let deployment = Deployment { + metadata: ObjectMeta { + labels: Some(BTreeMap::from([( + LABEL_SANDBOX_ID.to_string(), + "sb-77".to_string(), + )])), + ..Default::default() + }, + ..Default::default() + }; + assert_eq!( + supervisor_deployment_sandbox_id(&deployment), + Some("sb-77".to_string()) + ); + } + + #[test] + fn supervisor_deployment_sandbox_id_none_when_missing_or_empty() { + let no_labels = Deployment::default(); + assert_eq!(supervisor_deployment_sandbox_id(&no_labels), None); + + let empty = Deployment { + metadata: ObjectMeta { + labels: Some(BTreeMap::from([( + LABEL_SANDBOX_ID.to_string(), + String::new(), + )])), + ..Default::default() + }, + ..Default::default() + }; + assert_eq!(supervisor_deployment_sandbox_id(&empty), None); + } + #[test] fn gateway_id_backfill_adopts_unlabelled_sandbox() { let labels = BTreeMap::from([( diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 027c350a26..22cd919667 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -131,8 +131,12 @@ impl ComputeDriver for ComputeDriverService { if credential.is_empty() { return Err(Status::invalid_argument("credential is required")); } - let sandbox_id = self.driver.authenticate_sandbox(&credential).await?; - Ok(Response::new(AuthenticateSandboxResponse { sandbox_id })) + let (sandbox_id, scoped_process_caller) = + self.driver.authenticate_sandbox(&credential).await?; + Ok(Response::new(AuthenticateSandboxResponse { + sandbox_id, + scoped_process_caller, + })) } async fn get_capabilities( @@ -224,17 +228,23 @@ impl ComputeDriver for ComputeDriverService { &self, request: Request, ) -> Result, Status> { - self.trace_rpc("driver.create_sandbox", "create_sandbox", async { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.driver - .create_sandbox(&sandbox) - .await - .map_err(|e| Status::from(openshell_core::ComputeDriverError::from(e)))?; - Ok(Response::new(CreateSandboxResponse {})) - }) + // Boxed: the create path awaits large sandbox spec/status types, so the + // future exceeds the `large_futures` threshold when kept inline. + self.trace_rpc( + "driver.create_sandbox", + "create_sandbox", + Box::pin(async { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + self.driver + .create_sandbox(&sandbox) + .await + .map_err(|e| Status::from(openshell_core::ComputeDriverError::from(e)))?; + Ok(Response::new(CreateSandboxResponse {})) + }), + ) .await } diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 28d3c77a7d..a72c72f477 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -8,9 +8,9 @@ pub mod otel_tracing; pub use config::{ AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, - managed_namespace_prefix, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesProxyPodConfig, + KubernetesSidecarConfig, ManagedSshIngressConfig, ProxyPodAffinity, ProxyPodDnsPeer, + SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 7690ccee85..d5b491777f 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -16,8 +16,8 @@ use openshell_driver_kubernetes::otel_tracing::compute_driver_rpc_layer; use openshell_driver_kubernetes::{ AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, - KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, - WorkspaceMode, + KubernetesProxyPodConfig, KubernetesSidecarConfig, ManagedSshIngressConfig, ProxyPodAffinity, + ProxyPodDnsPeer, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, }; #[derive(Parser, Debug)] @@ -169,6 +169,51 @@ struct Args { #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY_CONNECT_BY_HOSTNAME", action = ArgAction::SetTrue)] proxy_connect_by_hostname: bool, + /// UID for the proxy container in `proxy-pod` topology. + #[arg( + long = "proxy-pod-proxy-uid", + env = "OPENSHELL_K8S_PROXY_POD_PROXY_UID", + default_value_t = DEFAULT_PROXY_UID + )] + proxy_pod_proxy_uid: u32, + + #[arg( + long = "proxy-pod-affinity", + env = "OPENSHELL_K8S_PROXY_POD_AFFINITY", + default_value = "disabled" + )] + proxy_pod_affinity: ProxyPodAffinity, + + /// Cluster DNS peers for the proxy-pod agent egress `NetworkPolicy`, as a + /// JSON array of `{"namespace_labels": {..}, "pod_labels": {..}}` objects. + /// Defaults to the upstream kube-system conventions, which do not match + /// `OpenShift` or `NodeLocal` `DNSCache` deployments. + #[arg( + long = "proxy-pod-dns-peers", + env = "OPENSHELL_K8S_PROXY_POD_DNS_PEERS" + )] + proxy_pod_dns_peers: Option, + + /// Gateway peers for the proxy-pod agent egress `NetworkPolicy`, as a JSON + /// array of `{"namespace_labels": {..}, "pod_labels": {..}, "port": N}` + /// objects. Required for proxy-pod: the in-pod process supervisor needs + /// egress to the gateway. + #[arg( + long = "proxy-pod-gateway-peers", + env = "OPENSHELL_K8S_PROXY_POD_GATEWAY_PEERS" + )] + proxy_pod_gateway_peers: Option, + + /// Keep managing existing proxy-pod sandboxes (periodic reconcile and the + /// shared-mode supervisor Deployment readiness watch) after the configured + /// topology has been switched away from proxy-pod. Set during a + /// `retainCompanionRbac` migration. + #[arg( + long = "proxy-pod-retain-companion-management", + env = "OPENSHELL_K8S_PROXY_POD_RETAIN_COMPANION_MANAGEMENT" + )] + proxy_pod_retain_companion_management: bool, + #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -250,6 +295,22 @@ async fn main() -> Result<()> { }) .collect::>>()?; + let proxy_pod_dns_peers = match args.proxy_pod_dns_peers.as_deref() { + Some(raw) => serde_json::from_str::>(raw) + .into_diagnostic() + .map_err(|err| miette::miette!("--proxy-pod-dns-peers must be a JSON array: {err}"))?, + None => KubernetesProxyPodConfig::default().dns_peers, + }; + + let proxy_pod_gateway_peers = match args.proxy_pod_gateway_peers.as_deref() { + Some(raw) => serde_json::from_str::>(raw) + .into_diagnostic() + .map_err(|err| { + miette::miette!("--proxy-pod-gateway-peers must be a JSON array: {err}") + })?, + None => KubernetesProxyPodConfig::default().gateway_peers, + }; + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { @@ -278,6 +339,13 @@ async fn main() -> Result<()> { process_binary_aware_network_policy: args .sidecar_process_binary_aware_network_policy, }, + proxy_pod: KubernetesProxyPodConfig { + proxy_uid: args.proxy_pod_proxy_uid, + affinity: args.proxy_pod_affinity, + dns_peers: proxy_pod_dns_peers, + gateway_peers: proxy_pod_gateway_peers, + retain_companion_management: args.proxy_pod_retain_companion_management, + }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, proxy_auth_secret_name: args.proxy_auth_secret_name, diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index c3903a0067..5a1f7f6713 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -13,8 +13,9 @@ use crate::container::{ use futures::Stream; use openshell_core::ComputeDriverError; use openshell_core::proto::compute::v1::{ - DriverCondition, DriverSandbox, DriverSandboxStatus, WatchSandboxesDeletedEvent, - WatchSandboxesEvent, WatchSandboxesSandboxEvent, watch_sandboxes_event, + DriverCondition, DriverSandbox, DriverSandboxStatus, SupervisorSessionModel, + WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesSandboxEvent, + watch_sandboxes_event, }; use std::collections::HashMap; use std::pin::Pin; @@ -346,6 +347,7 @@ fn build_driver_sandbox( namespace: String::new(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: instance_name, instance_id, agent_fd: String::new(), @@ -688,6 +690,7 @@ mod tests { namespace: String::new(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: String::new(), instance_id: short_id("container-id-full"), agent_fd: String::new(), diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 46133f38fe..80bb053fd8 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -46,7 +46,7 @@ use openshell_core::proto::compute::v1::{ GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + StopSandboxRequest, StopSandboxResponse, SupervisorSessionModel, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, watch_sandboxes_event, @@ -5964,6 +5964,7 @@ fn sandbox_snapshot(sandbox: &Sandbox, condition: SandboxCondition, deleting: bo namespace: sandbox.namespace.clone(), workspace: sandbox.workspace.clone(), status: Some(SandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: sandbox.name.clone(), instance_id: String::new(), agent_fd: String::new(), @@ -5981,6 +5982,7 @@ fn status_with_condition( deleting: bool, ) -> SandboxStatus { SandboxStatus { + supervisor_session_model: SupervisorSessionModel::Unspecified as i32, sandbox_name: snapshot.name.clone(), instance_id: String::new(), agent_fd: String::new(), diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 41c6ca3c94..66547a9a39 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -82,6 +82,7 @@ use tokio::sync::mpsc::UnboundedSender; use tokio::time::timeout; const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; +const PROXY_POD_NETWORK_ENFORCEMENT_MODE: &str = "proxy-pod"; const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; @@ -155,7 +156,9 @@ pub async fn run_sandbox( } } + let external_network_enforcement = external_network_enforcement_enabled(); let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); + let proxy_pod_network_enforcement = proxy_pod_network_enforcement_enabled(); let process_enforcement_mode = process_enforcement_mode(); let process_uses_sidecar_control = process_enabled && !network_enabled && sidecar_network_enforcement; @@ -177,7 +180,6 @@ pub async fn run_sandbox( } else { None }; - // Extension credentials are owned by this supervisor and shared by every // gateway connection it opens, so the middleware registry's bearer slots // and the policy poll loop that rotates them stay the same objects. @@ -401,7 +403,7 @@ pub async fn run_sandbox( // it via setns(). The RAII handle lives in this frame for the duration // of the sandbox. #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { + let netns = if network_enabled && !external_network_enforcement { openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? } else { None @@ -566,7 +568,7 @@ pub async fn run_sandbox( let sidecar_control_server = if network_enabled && sidecar_network_enforcement { if !matches!(policy.network.mode, NetworkMode::Proxy) { return Err(miette::miette!( - "sidecar network enforcement requires proxy network mode" + "external network enforcement requires proxy network mode" )); } let socket = sidecar_control_socket().ok_or_else(|| { @@ -635,9 +637,9 @@ pub async fn run_sandbox( } #[cfg(not(target_os = "linux"))] - if network_enabled && sidecar_network_enforcement { + if network_enabled && external_network_enforcement { return Err(miette::miette!( - "sidecar network enforcement is only supported on Linux" + "external network enforcement is only supported on Linux" )); } @@ -858,6 +860,8 @@ pub async fn run_sandbox( sidecar_bootstrap_ca_file_paths .clone() .or_else(sidecar_ca_file_paths) + } else if proxy_pod_network_enforcement { + sidecar_ca_file_paths() } else { None } @@ -1181,12 +1185,26 @@ fn sidecar_network_enforcement_enabled() -> bool { .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) } +fn proxy_pod_network_enforcement_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) + .is_ok_and(|value| value == PROXY_POD_NETWORK_ENFORCEMENT_MODE) +} + +fn external_network_enforcement_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE).is_ok_and(|value| { + matches!( + value.as_str(), + SIDECAR_NETWORK_ENFORCEMENT_MODE | PROXY_POD_NETWORK_ENFORCEMENT_MODE + ) + }) +} + fn process_enforcement_mode() -> ProcessEnforcementMode { match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) .ok() .as_deref() { - Some("sidecar") => ProcessEnforcementMode::NetworkOnly, + Some("sidecar" | "proxy-pod") => ProcessEnforcementMode::NetworkOnly, _ => ProcessEnforcementMode::Full, } } diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 7108378654..2e87172701 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::sync::Arc; use std::sync::atomic::AtomicBool; +use std::time::Duration; use clap::Parser; use miette::{IntoDiagnostic, Result}; @@ -34,6 +35,14 @@ const COPY_SELF_SUBCOMMAND: &str = "copy-self"; const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; +/// Subcommand that blocks until a TCP endpoint accepts a connection. +/// +/// Used by the `proxy-pod` agent pod's init container to hold the workload +/// until its paired network supervisor is serving. Without it the workload +/// can start before the proxy exists, its early egress fails, and the +/// Kubernetes `Sandbox` reports Ready while no egress path is available. +const WAIT_FOR_TCP_SUBCOMMAND: &str = "wait-for-tcp"; + /// Default `--mode` value: run both supervisor leaves in a single binary. const DEFAULT_MODE: &str = "network,process"; const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; @@ -186,8 +195,9 @@ struct Args { #[arg(long, default_value = DEFAULT_MODE)] mode: Mode, - /// UID that the long-running Kubernetes network sidecar will run as. - /// `--mode=network-init` installs nftables rules that exempt this UID. + /// UID that the long-running Kubernetes network proxy will run as. + /// In sidecar topology, `--mode=network-init` installs nftables rules + /// that exempt this UID. #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] proxy_uid: u32, @@ -511,6 +521,58 @@ fn run_network_init( )) } +/// Block until `addr` accepts a TCP connection, or the timeout elapses. +/// +/// Deliberately dependency-free: this runs in an init container built from +/// the supervisor image, which has no shell networking tools. +fn wait_for_tcp(args: &[String]) -> Result<()> { + let addr = args.first().ok_or_else(|| { + miette::miette!( + "usage: openshell-sandbox {WAIT_FOR_TCP_SUBCOMMAND} [TIMEOUT_SECS]" + ) + })?; + let timeout_secs: u64 = match args.get(1) { + Some(raw) => raw + .parse() + .map_err(|_| miette::miette!("timeout must be a positive integer: {raw}"))?, + None => 180, + }; + + let deadline = std::time::Instant::now() + Duration::from_secs(timeout_secs); + let mut last_error = String::new(); + loop { + // Re-resolve every attempt: the paired supervisor Service may not have + // endpoints yet when the init container first runs. + match std::net::ToSocketAddrs::to_socket_addrs(&addr.as_str()) { + Ok(mut resolved) => { + let mut connected = false; + for socket_addr in &mut resolved { + match std::net::TcpStream::connect_timeout(&socket_addr, Duration::from_secs(5)) + { + Ok(_) => { + connected = true; + break; + } + Err(err) => last_error = err.to_string(), + } + } + if connected { + println!("network supervisor endpoint {addr} is accepting connections"); + return Ok(()); + } + } + Err(err) => last_error = err.to_string(), + } + + if std::time::Instant::now() >= deadline { + return Err(miette::miette!( + "timed out after {timeout_secs}s waiting for network supervisor at {addr}: {last_error}" + )); + } + std::thread::sleep(Duration::from_millis(500)); + } +} + fn main() -> Result<()> { // Handle `copy-self ` before clap so it works without any of the // sandbox flags. Kubernetes init containers invoke this path to seed an @@ -540,13 +602,19 @@ fn main() -> Result<()> { return validate_workspace(&raw_args[2..]); } + // Handle `wait-for-tcp [TIMEOUT_SECS]` before clap. Runs in the + // agent pod's init container, which has none of the supervisor's config. + if raw_args.get(1).map(String::as_str) == Some(WAIT_FOR_TCP_SUBCOMMAND) { + return wait_for_tcp(&raw_args[2..]); + } + let args = Args::parse(); if args.mode.network_init { - let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); + let proxy_group_id = args.proxy_gid.unwrap_or(args.proxy_uid); return run_network_init( args.proxy_uid, - proxy_gid, + proxy_group_id, &args.sidecar_state_dir, &args.sidecar_tls_dir, ); diff --git a/crates/openshell-server/src/auth/compute_driver.rs b/crates/openshell-server/src/auth/compute_driver.rs index 04caee61b1..0c95147c5a 100644 --- a/crates/openshell-server/src/auth/compute_driver.rs +++ b/crates/openshell-server/src/auth/compute_driver.rs @@ -42,17 +42,28 @@ impl Authenticator for ComputeDriverAuthenticator { return Ok(None); }; - let sandbox_id = self.compute.authenticate_sandbox(credential).await?; + let (sandbox_id, scoped_process_caller) = + self.compute.authenticate_sandbox(credential).await?; if sandbox_id.is_empty() { return Err(Status::permission_denied( "compute driver returned an empty sandbox identity", )); } + // A driver that authenticated a process-scoped caller (e.g. a proxy-pod + // agent pod) narrows the minted token to `Process`, which cannot read + // provider secrets or inference routing. + let caller_kind = if scoped_process_caller { + crate::auth::principal::SandboxCallerKind::Process + } else { + crate::auth::principal::SandboxCallerKind::Full + }; + Ok(Some(Principal::Sandbox(SandboxPrincipal { sandbox_id, source: SandboxIdentitySource::ComputeDriver { driver_name: self.compute.configured_driver_name().to_string(), + caller_kind, }, trust_domain: Some("openshell".to_string()), }))) @@ -103,7 +114,7 @@ mod tests { assert_eq!(principal.sandbox_id, "sandbox-a"); assert!(matches!( principal.source, - SandboxIdentitySource::ComputeDriver { ref driver_name } + SandboxIdentitySource::ComputeDriver { ref driver_name, .. } if driver_name == "external-kubernetes" )); } diff --git a/crates/openshell-server/src/auth/guard.rs b/crates/openshell-server/src/auth/guard.rs index edcd6bc013..99ebbc1202 100644 --- a/crates/openshell-server/src/auth/guard.rs +++ b/crates/openshell-server/src/auth/guard.rs @@ -116,6 +116,7 @@ mod tests { sandbox_id: id.to_string(), source: SandboxIdentitySource::BootstrapJwt { issuer: "openshell-gateway:test".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), }) diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs index 9567cc62d2..c5ead31157 100644 --- a/crates/openshell-server/src/auth/principal.rs +++ b/crates/openshell-server/src/auth/principal.rs @@ -15,6 +15,29 @@ //! to prevent cross-sandbox access (see issue #1354). use super::identity::Identity; +use serde::{Deserialize, Serialize}; + +/// The authority a gateway-minted sandbox credential carries. +/// +/// The credential is always bound to exactly one sandbox; this narrows *which +/// of that sandbox's* RPCs it may call. It is serialized into the sandbox JWT +/// (`caller_kind` claim) and read back onto [`SandboxPrincipal`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SandboxCallerKind { + /// Full supervisor authority (network + process halves in one place, as in + /// `combined`/`sidecar`). The default so tokens minted before this claim + /// existed keep working unchanged. + #[default] + Full, + /// Process-supervisor-only authority, for a topology where the network proxy + /// runs in a separate pod (proxy-pod). Denied the network-supervisor RPCs + /// that read provider secrets or mint upstream credentials + /// (`GetSandboxProviderEnvironment`, `ExchangeProviderSubjectToken`, + /// `GetInferenceBundle`); still permitted its own relays, logs, config, and + /// token refresh. + Process, +} /// Who is calling. /// @@ -57,6 +80,18 @@ pub struct SandboxPrincipal { pub trust_domain: Option, } +impl SandboxPrincipal { + /// The authority this credential carries. Only a gateway-minted JWT can be + /// scoped; every other source (bootstrap SA token, client cert) is `Full`. + #[must_use] + pub fn caller_kind(&self) -> SandboxCallerKind { + match self.source { + SandboxIdentitySource::BootstrapJwt { caller_kind, .. } => caller_kind, + _ => SandboxCallerKind::Full, + } + } +} + /// How a [`SandboxPrincipal`] was authenticated. /// /// Variant fields are populated by the producing authenticator and consumed @@ -66,12 +101,21 @@ pub struct SandboxPrincipal { pub enum SandboxIdentitySource { /// Gateway-minted JWT validated against the gateway's signing key. /// Produced by [`super::sandbox_jwt::SandboxJwtAuthenticator`]. - BootstrapJwt { issuer: String }, + BootstrapJwt { + issuer: String, + /// Authority carried by the token's `caller_kind` claim. + caller_kind: SandboxCallerKind, + }, /// Per-sandbox client certificate. Reserved for channel-bound sandbox /// identity. BootstrapCert { fingerprint: String }, /// Driver-native credential used to bootstrap a gateway-minted JWT via /// `IssueSandboxToken`. The named compute driver authenticated only the - /// sandbox identity; the gateway still authorizes the exchange. - ComputeDriver { driver_name: String }, + /// sandbox identity; the gateway still authorizes the exchange. `caller_kind` + /// narrows the minted token when the driver reports a process-scoped caller + /// (e.g. a proxy-pod agent pod), otherwise `Full`. + ComputeDriver { + driver_name: String, + caller_kind: SandboxCallerKind, + }, } diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 9dc10b8401..801d9d57bf 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -16,7 +16,7 @@ //! prevent algorithm-confusion attacks. use super::authenticator::Authenticator; -use super::principal::{Principal, SandboxIdentitySource, SandboxPrincipal}; +use super::principal::{Principal, SandboxCallerKind, SandboxIdentitySource, SandboxPrincipal}; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use jsonwebtoken::{ @@ -76,6 +76,10 @@ pub struct SandboxJwtClaims { /// Canonical sandbox UUID, denormalized from `sub` for cheap parsing /// without a SPIFFE library. pub sandbox_id: String, + /// Authority this token carries. `#[serde(default)]` = `Full`, so tokens + /// minted before this claim existed keep full authority. + #[serde(default)] + pub caller_kind: SandboxCallerKind, } /// Mints fresh sandbox JWTs. @@ -126,9 +130,21 @@ impl SandboxJwtIssuer { }) } - /// Mint a fresh token for `sandbox_id`. + /// Mint a fresh full-authority token for `sandbox_id`. #[allow(clippy::result_large_err)] // `tonic::Status` is the natural error here pub fn mint(&self, sandbox_id: &str) -> Result { + self.mint_with_caller_kind(sandbox_id, SandboxCallerKind::Full) + } + + /// Mint a fresh token for `sandbox_id` carrying `caller_kind`. The + /// `proxy-pod` topology mints a `Process`-kind token for the agent pod so it + /// cannot call the network-supervisor RPCs (provider secrets, inference). + #[allow(clippy::result_large_err)] // `tonic::Status` is the natural error here + pub fn mint_with_caller_kind( + &self, + sandbox_id: &str, + caller_kind: SandboxCallerKind, + ) -> Result { crate::install_jsonwebtoken_crypto_provider(); let now = now_secs(); @@ -144,6 +160,7 @@ impl SandboxJwtIssuer { iat: now, exp, sandbox_id: sandbox_id.to_string(), + caller_kind, }; let mut header = Header::new(Algorithm::EdDSA); header.kid = Some(self.kid.clone()); @@ -325,7 +342,10 @@ impl SandboxJwtAuthenticator { validate_exp(claims.exp)?; Ok(Some(Principal::Sandbox(SandboxPrincipal { sandbox_id: claims.sandbox_id, - source: SandboxIdentitySource::BootstrapJwt { issuer: claims.iss }, + source: SandboxIdentitySource::BootstrapJwt { + issuer: claims.iss, + caller_kind: claims.caller_kind, + }, trust_domain: Some("openshell".to_string()), }))) } @@ -455,7 +475,7 @@ mod tests { Principal::Sandbox(p) => { assert_eq!(p.sandbox_id, "sandbox-a"); match p.source { - SandboxIdentitySource::BootstrapJwt { issuer: iss } => { + SandboxIdentitySource::BootstrapJwt { issuer: iss, .. } => { assert_eq!(iss, "openshell-gateway:test-gateway"); } other => panic!("unexpected source: {other:?}"), @@ -465,6 +485,46 @@ mod tests { } } + #[tokio::test] + async fn caller_kind_round_trips_full_and_process() { + let (issuer, auth) = pair(); + for (kind, sandbox) in [ + (SandboxCallerKind::Full, "sandbox-full"), + (SandboxCallerKind::Process, "sandbox-proc"), + ] { + let minted = issuer.mint_with_caller_kind(sandbox, kind).unwrap(); + let principal = auth + .authenticate(&header_map_with_bearer(&minted.token), "/anything") + .await + .unwrap() + .expect("expected principal"); + match principal { + Principal::Sandbox(p) => assert_eq!(p.caller_kind(), kind, "{sandbox}"), + _ => panic!("expected Sandbox principal"), + } + } + // Plain `mint` is full authority. + let minted = issuer.mint("sandbox-default").unwrap(); + let principal = auth + .authenticate(&header_map_with_bearer(&minted.token), "/anything") + .await + .unwrap() + .expect("expected principal"); + match principal { + Principal::Sandbox(p) => assert_eq!(p.caller_kind(), SandboxCallerKind::Full), + _ => panic!("expected Sandbox principal"), + } + } + + #[test] + fn claims_without_caller_kind_default_to_full() { + // A token minted before the claim existed has no `caller_kind`; it must + // deserialize as full authority, not a scoped credential. + let json = r#"{"sub":"spiffe://openshell/sandbox/s","iss":"g","aud":"g","iat":0,"exp":0,"sandbox_id":"s"}"#; + let claims: SandboxJwtClaims = serde_json::from_str(json).unwrap(); + assert_eq!(claims.caller_kind, SandboxCallerKind::Full); + } + #[tokio::test] async fn extension_token_cannot_authenticate_as_a_sandbox() { let mat = generate_jwt_key().expect("jwt key"); @@ -595,6 +655,7 @@ mod tests { iat: now_secs() - 7200, exp: now_secs() - 3600, sandbox_id: "sandbox-c".to_string(), + caller_kind: SandboxCallerKind::Full, }; let mut header = Header::new(Algorithm::EdDSA); header.kid = Some(mat.kid); diff --git a/crates/openshell-server/src/auth/sandbox_methods.rs b/crates/openshell-server/src/auth/sandbox_methods.rs index 89f34d1253..42cf8747ae 100644 --- a/crates/openshell-server/src/auth/sandbox_methods.rs +++ b/crates/openshell-server/src/auth/sandbox_methods.rs @@ -16,10 +16,54 @@ pub fn is_sandbox_callable(path: &str) -> bool { super::method_authz::is_sandbox_callable(path) } +/// Sandbox-callable methods a `Process`-kind credential must NOT call. +/// +/// These are the network-supervisor RPCs that read provider secrets or mint +/// upstream credentials. In the `proxy-pod` topology they run only in the +/// separate proxy pod (a `Full`-kind credential); the in-pod process supervisor +/// holds a `Process`-kind credential and is denied them, so a compromised agent +/// pod cannot reach provider secrets even though it holds a gateway credential. +const PROCESS_CALLER_DENIED_METHODS: &[&str] = &[ + "/openshell.v1.OpenShell/GetSandboxProviderEnvironment", + "/openshell.v1.OpenShell/ExchangeProviderSubjectToken", + "/openshell.inference.v1.Inference/GetInferenceBundle", +]; + +/// Whether `path` is denied to a `Process`-kind sandbox credential. Callers +/// apply this only after [`is_sandbox_callable`] has already passed. +#[must_use] +pub fn is_process_caller_denied(path: &str) -> bool { + PROCESS_CALLER_DENIED_METHODS.contains(&path) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn process_caller_is_denied_network_supervisor_rpcs() { + for path in PROCESS_CALLER_DENIED_METHODS { + assert!(is_process_caller_denied(path), "{path}"); + // Everything denied to a process caller must still be sandbox-callable + // at all (a full-authority credential may call it). + assert!(is_sandbox_callable(path), "{path}"); + } + } + + #[test] + fn process_caller_keeps_control_plane_rpcs() { + for path in [ + "/openshell.v1.OpenShell/ConnectSupervisor", + "/openshell.v1.OpenShell/RelayStream", + "/openshell.v1.OpenShell/PushSandboxLogs", + "/openshell.v1.OpenShell/ReportMainProcessExit", + "/openshell.v1.OpenShell/RefreshSandboxToken", + "/openshell.v1.OpenShell/GetSandboxConfig", + ] { + assert!(!is_process_caller_denied(path), "{path}"); + } + } + #[test] fn supervisor_callbacks_are_allowed() { assert!(is_sandbox_callable( diff --git a/crates/openshell-server/src/auth/workspace_authz.rs b/crates/openshell-server/src/auth/workspace_authz.rs index e23d2287a3..71f4b3ace2 100644 --- a/crates/openshell-server/src/auth/workspace_authz.rs +++ b/crates/openshell-server/src/auth/workspace_authz.rs @@ -224,6 +224,7 @@ mod tests { sandbox_id: "sandbox-a".to_string(), source: SandboxIdentitySource::BootstrapJwt { issuer: "openshell-gateway:test".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), }) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 6aa1f70a0c..3bbbcf9c3d 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -28,8 +28,8 @@ use openshell_core::proto::compute::v1::{ GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, GetSandboxRequest, GpuResourceRequirements as DriverGpuResourceRequirements, ListSandboxesRequest, ResourceRequirements as DriverSandboxResourceRequirements, - StartSandboxRequest, StopSandboxRequest, ValidateSandboxCreateRequest, WatchSandboxesEvent, - WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, + StartSandboxRequest, StopSandboxRequest, SupervisorSessionModel, ValidateSandboxCreateRequest, + WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, watch_sandboxes_event, }; @@ -753,7 +753,13 @@ impl ComputeRuntime { self.driver_info.supports_sandbox_authentication } - pub(crate) async fn authenticate_sandbox(&self, credential: &str) -> Result { + /// Authenticate a driver-native sandbox credential. Returns the + /// authenticated sandbox ID and whether the driver reported a process-scoped + /// caller (e.g. a proxy-pod agent pod), which narrows the minted token. + pub(crate) async fn authenticate_sandbox( + &self, + credential: &str, + ) -> Result<(String, bool), Status> { if !self.supports_sandbox_authentication() { return Err(Status::unimplemented( "selected compute driver does not authenticate sandbox credentials", @@ -767,7 +773,10 @@ impl ComputeRuntime { driver.authenticate_sandbox(Request::new(request)).await }) .await - .map(|response| response.into_inner().sandbox_id) + .map(|response| { + let response = response.into_inner(); + (response.sandbox_id, response.scoped_process_caller) + }) } #[must_use] @@ -1320,6 +1329,7 @@ impl ComputeRuntime { let sandbox_id = transition.object_id().to_string(); let expected_resource_version = sandbox_resource_version(transition); let session_connected = self.supervisor_sessions.has_session(&sandbox_id); + self.record_supervisor_session_model(&sandbox_id, snapshot); match self .store .update_message_cas::(&sandbox_id, expected_resource_version, |sandbox| { @@ -2872,6 +2882,7 @@ impl ComputeRuntime { existing_phase: SandboxPhase, ) -> Result<(), String> { let session_connected = self.supervisor_sessions.has_session(&incoming.id); + self.record_supervisor_session_model(&incoming.id, &incoming); let sandbox = self .store .update_message_cas::( @@ -2908,6 +2919,17 @@ impl ComputeRuntime { Ok(()) } + /// Track whether this sandbox's topology can ever open a supervisor + /// session, so relay-backed RPCs fail fast with an explanation instead of + /// waiting out a timeout that cannot succeed. + fn record_supervisor_session_model(&self, sandbox_id: &str, snapshot: &DriverSandbox) { + let Some(status) = snapshot.status.as_ref() else { + return; + }; + self.supervisor_sessions + .set_sessionless(sandbox_id, sandbox_has_no_supervisor_session(status)); + } + pub async fn supervisor_session_connected( &self, sandbox_id: &str, @@ -3437,6 +3459,10 @@ impl ComputeRuntime { self.tracing_log_bus.remove(sandbox_id); self.tracing_log_bus.platform_event_bus.remove(sandbox_id); self.sandbox_watch_bus.remove(sandbox_id); + // Drop the sessionless marker on permanent removal only. It is a + // topology property that must survive stop/start, so it is not cleared + // in cleanup_stopped_sandbox_sessions. + self.supervisor_sessions.forget_sessionless(sandbox_id); } async fn reconcile_snapshot_sandbox( @@ -3999,6 +4025,7 @@ fn build_platform_resources_config( fn driver_status_from_public(status: &SandboxStatus) -> DriverSandboxStatus { DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: status.sandbox_name.clone(), instance_id: status.agent_pod.clone(), agent_fd: status.agent_fd.clone(), @@ -4269,16 +4296,32 @@ fn ensure_supervisor_ready_status(status: &mut Option, sandbox_na ); } +/// Whether the driver reports that this sandbox has no in-sandbox process +/// supervisor, and therefore no `ConnectSupervisor` session. +/// +/// Unset and `Required` both preserve the default contract, so a driver that +/// never sets the field behaves exactly as before. +fn sandbox_has_no_supervisor_session(status: &DriverSandboxStatus) -> bool { + status.supervisor_session_model() == SupervisorSessionModel::None +} + /// Compose the public `SandboxPhase` from backend driver state and supervisor session presence. /// /// The readiness decision is a gateway-owned safety invariant: `SandboxPhase::Ready` means /// "usable through this gateway." The driver contract is the extension point for custom backend /// readiness semantics. RFC-0010 lifecycle hooks observe this decision via `post_commit`; they /// do not modify it. +/// +/// Topologies with no in-sandbox process supervisor use that extension point. +/// They report `SupervisorSessionModel::None`, and the gateway then trusts the +/// backend `Ready` condition, because no session will ever arrive. Such a +/// sandbox is usable for policy-enforced network egress but cannot serve a +/// relay, so relay-backed RPCs are rejected rather than left to time out. struct ComposedPhase { phase: SandboxPhase, session_connected: bool, backend_ready_without_session: bool, + sessionless: bool, } impl ComposedPhase { @@ -4288,6 +4331,7 @@ impl ComposedPhase { driver_reports_runtime_readiness: bool, ) -> Self { let backend_phase = derive_phase(Some(incoming_status)); + let sessionless = sandbox_has_no_supervisor_session(incoming_status); // A live supervisor session is a stronger readiness signal than the backend phase. // set_supervisor_session_state may have already promoted the store record to Ready // before this driver snapshot arrived. Keep Ready rather than letting a lagging @@ -4296,6 +4340,10 @@ impl ComposedPhase { SandboxPhase::Error | SandboxPhase::Deleting | SandboxPhase::Stopped => backend_phase, SandboxPhase::Ready if driver_reports_runtime_readiness => SandboxPhase::Ready, _ if session_connected => SandboxPhase::Ready, + // No session will ever arrive for this topology. The driver is + // responsible for withholding its `Ready` condition until the + // out-of-sandbox supervisor is actually serving. + SandboxPhase::Ready if sessionless => SandboxPhase::Ready, _ => SandboxPhase::Provisioning, }; Self { @@ -4303,7 +4351,9 @@ impl ComposedPhase { session_connected, backend_ready_without_session: !driver_reports_runtime_readiness && backend_phase == SandboxPhase::Ready - && !session_connected, + && !session_connected + && !sessionless, + sessionless, } } @@ -4314,6 +4364,9 @@ impl ComposedPhase { spec: Option<&SandboxSpec>, ) { rewrite_user_facing_conditions(status, spec); + if self.sessionless { + ensure_no_supervisor_session_status(status, sandbox_name); + } if self.backend_ready_without_session { ensure_supervisor_not_connected_status(status, sandbox_name); } else if self.session_connected && self.phase == SandboxPhase::Ready { @@ -4350,6 +4403,74 @@ fn ensure_supervisor_not_ready_status(status: &mut Option, sandbo ); } +/// Condition type advertising whether a sandbox can serve relay-backed +/// operations. +/// +/// Carried in the public status so clients can tell that SSH, `exec`, port +/// forwarding, and file transfer are unavailable *before* attempting one, +/// rather than discovering it from a failed connection. Using a condition +/// avoids adding a field to the public `Sandbox` message. +pub const SUPERVISOR_SESSION_CONDITION: &str = "SupervisorSession"; + +/// Reason marking a `SupervisorSession=False` condition as a permanent property +/// of the topology (no in-sandbox supervisor) rather than a transient +/// disconnect. Only this reason is treated as durably sessionless. +pub const SUPERVISOR_SESSION_NOT_APPLICABLE_REASON: &str = "NotApplicable"; + +/// Whether a stored sandbox status marks the sandbox as having no supervisor +/// session (via the durable `SupervisorSession=False` condition with reason +/// `NotApplicable`). Lets any gateway replica reject relay-backed RPCs from +/// durable state, not only the reconciler lease holder that populates the +/// in-memory sessionless set. The reason is required so a future driver that +/// reports `SupervisorSession=False` for a *transient* disconnect is not given +/// a terminal relay rejection. +pub fn sandbox_status_is_sessionless(status: &SandboxStatus) -> bool { + status.conditions.iter().any(|condition| { + condition.r#type == SUPERVISOR_SESSION_CONDITION + && condition.status.eq_ignore_ascii_case("false") + && condition + .reason + .eq_ignore_ascii_case(SUPERVISOR_SESSION_NOT_APPLICABLE_REASON) + }) +} + +fn upsert_condition( + status: &mut Option, + sandbox_name: &str, + condition: SandboxCondition, +) { + let status = status.get_or_insert_with(|| SandboxStatus { + sandbox_name: sandbox_name.to_string(), + ..Default::default() + }); + + let condition_type = condition.r#type.clone(); + if let Some(existing) = status + .conditions + .iter_mut() + .find(|existing| existing.r#type == condition_type) + { + *existing = condition; + } else { + status.conditions.push(condition); + } +} + +/// Record that this sandbox's topology never opens a supervisor session. +fn ensure_no_supervisor_session_status(status: &mut Option, sandbox_name: &str) { + upsert_condition( + status, + sandbox_name, + SandboxCondition { + r#type: SUPERVISOR_SESSION_CONDITION.to_string(), + status: "False".to_string(), + reason: SUPERVISOR_SESSION_NOT_APPLICABLE_REASON.to_string(), + message: openshell_core::error::no_supervisor_session_message(), + last_transition_time: String::new(), + }, + ); +} + fn upsert_ready_condition( status: &mut Option, sandbox_name: &str, @@ -4563,6 +4684,7 @@ impl ComputeDriver for NoopTestDriver { Some(Ok(sandbox_id)) => Ok(tonic::Response::new( openshell_core::proto::compute::v1::AuthenticateSandboxResponse { sandbox_id: sandbox_id.clone(), + scoped_process_caller: false, }, )), Some(Err((code, message))) => Err(Status::new(*code, message.clone())), @@ -6032,6 +6154,7 @@ mod tests { fn make_driver_status(condition: DriverCondition) -> DriverSandboxStatus { DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "test".to_string(), instance_id: "test-pod".to_string(), agent_fd: String::new(), @@ -6049,6 +6172,7 @@ mod tests { workspace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: name.to_string(), instance_id: format!("{name}-pod"), agent_fd: String::new(), @@ -6212,6 +6336,108 @@ mod tests { assert_eq!(derive_phase(Some(&status)), SandboxPhase::Deleting); } + fn ready_driver_status() -> DriverSandboxStatus { + let mut condition = make_driver_condition("DependenciesReady", "Pod is Ready"); + condition.status = "True".to_string(); + make_driver_status(condition) + } + + #[test] + fn composed_phase_requires_a_session_by_default() { + let status = ready_driver_status(); + assert_eq!(derive_phase(Some(&status)), SandboxPhase::Ready); + + // Unset session model keeps the historical contract: backend Ready is + // not enough, the gateway waits for a supervisor session. + let composed = ComposedPhase::new(&status, false, false); + assert_eq!(composed.phase, SandboxPhase::Provisioning); + assert!(composed.backend_ready_without_session); + + let composed = ComposedPhase::new(&status, true, false); + assert_eq!(composed.phase, SandboxPhase::Ready); + } + + #[test] + fn composed_phase_trusts_the_backend_when_no_session_will_ever_arrive() { + let mut status = ready_driver_status(); + status.supervisor_session_model = SupervisorSessionModel::None as i32; + + let composed = ComposedPhase::new(&status, false, false); + assert_eq!(composed.phase, SandboxPhase::Ready); + // Not "waiting for a supervisor session" -- none is coming, so the + // sandbox must not advertise that it is still settling. + assert!(!composed.backend_ready_without_session); + } + + #[test] + fn sessionless_sandboxes_are_not_ready_until_the_backend_says_so() { + let mut status = make_driver_status(make_driver_condition( + "DependenciesNotReady", + "Pod exists with phase: Pending", + )); + status.supervisor_session_model = SupervisorSessionModel::None as i32; + + // The driver withholds its Ready condition until the paired supervisor + // is serving, so the gateway must not promote this to Ready. + assert_eq!( + ComposedPhase::new(&status, false, false).phase, + SandboxPhase::Provisioning + ); + } + + #[test] + fn sessionless_model_does_not_override_terminal_backend_phases() { + for (reason, expected) in [ + ("Failed", SandboxPhase::Error), + ("Suspended", SandboxPhase::Stopped), + ] { + let mut status = if reason == "Suspended" { + let mut status = make_driver_status(make_driver_condition("Suspended", "stopped")); + status.conditions[0].r#type = "Suspended".to_string(); + status.conditions[0].status = "True".to_string(); + status + } else { + let mut status = make_driver_status(make_driver_condition(reason, "failed")); + status.conditions[0].status = "False".to_string(); + status + }; + status.supervisor_session_model = SupervisorSessionModel::None as i32; + assert_eq!( + ComposedPhase::new(&status, false, false).phase, + expected, + "{reason}" + ); + } + } + + #[test] + fn sessionless_requires_not_applicable_reason() { + let sessionless = SandboxStatus { + conditions: vec![SandboxCondition { + r#type: SUPERVISOR_SESSION_CONDITION.to_string(), + status: "False".to_string(), + reason: SUPERVISOR_SESSION_NOT_APPLICABLE_REASON.to_string(), + ..Default::default() + }], + ..Default::default() + }; + assert!(sandbox_status_is_sessionless(&sessionless)); + + // SupervisorSession=False for a *transient* disconnect (any other reason) + // must NOT be treated as durably sessionless, or a future driver would + // get a terminal relay rejection during a temporary outage. + let transient = SandboxStatus { + conditions: vec![SandboxCondition { + r#type: SUPERVISOR_SESSION_CONDITION.to_string(), + status: "False".to_string(), + reason: "Disconnected".to_string(), + ..Default::default() + }], + ..Default::default() + }; + assert!(!sandbox_status_is_sessionless(&transient)); + } + #[test] fn derive_phase_returns_provisioning_for_transient_conditions() { let transient_conditions = [ @@ -7585,6 +7811,7 @@ mod tests { namespace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "sandbox-a".to_string(), instance_id: "agent-pod".to_string(), agent_fd: String::new(), @@ -9191,6 +9418,7 @@ mod tests { fn make_ready_driver_status() -> DriverSandboxStatus { DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "test".to_string(), instance_id: "test-pod".to_string(), agent_fd: String::new(), @@ -9208,6 +9436,7 @@ mod tests { fn make_deleting_driver_status() -> DriverSandboxStatus { DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "test".to_string(), instance_id: "test-pod".to_string(), agent_fd: String::new(), @@ -9485,6 +9714,7 @@ mod tests { namespace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "sandbox-a".to_string(), instance_id: "agent-pod".to_string(), agent_fd: String::new(), @@ -9506,6 +9736,7 @@ mod tests { namespace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "sandbox-a".to_string(), instance_id: "agent-pod".to_string(), agent_fd: String::new(), @@ -9719,6 +9950,7 @@ mod tests { namespace: "default".to_string(), spec: None, status: Some(DriverSandboxStatus { + supervisor_session_model: 0, sandbox_name: "sandbox-a".to_string(), instance_id: "agent-pod".to_string(), agent_fd: String::new(), diff --git a/crates/openshell-server/src/grpc/auth_rpc.rs b/crates/openshell-server/src/grpc/auth_rpc.rs index 104d639584..63dab3bcb1 100644 --- a/crates/openshell-server/src/grpc/auth_rpc.rs +++ b/crates/openshell-server/src/grpc/auth_rpc.rs @@ -93,9 +93,18 @@ pub async fn handle_issue_sandbox_token( ensure_sandbox_exists(state, &sandbox.sandbox_id).await?; - let minted = issuer.mint(&sandbox.sandbox_id)?; + // A proxy-pod agent pod (role=agent) gets a scoped `Process`-kind token that + // cannot read provider secrets or mint upstream credentials; every other pod + // (combined/sidecar, or a proxy-pod supervisor pod) gets full authority. The + // compute driver classifies the caller during `AuthenticateSandbox`. + let caller_kind = match &sandbox.source { + SandboxIdentitySource::ComputeDriver { caller_kind, .. } => *caller_kind, + _ => crate::auth::principal::SandboxCallerKind::Full, + }; + let minted = issuer.mint_with_caller_kind(&sandbox.sandbox_id, caller_kind)?; info!( sandbox_id = %sandbox.sandbox_id, + ?caller_kind, "issued gateway sandbox JWT" ); Ok(Response::new(IssueSandboxTokenResponse { @@ -144,7 +153,9 @@ pub async fn handle_refresh_sandbox_token( ensure_sandbox_exists(state, &sandbox.sandbox_id).await?; - let minted = issuer.mint(&sandbox.sandbox_id)?; + // Preserve the caller's authority across refresh: a process-kind token must + // not be upgraded to full authority by refreshing. + let minted = issuer.mint_with_caller_kind(&sandbox.sandbox_id, sandbox.caller_kind())?; let extension_credentials = if requested_extension_services.is_empty() { Vec::new() } else if !state @@ -360,6 +371,7 @@ mod tests { sandbox_id: sandbox_id.to_string(), source: SandboxIdentitySource::BootstrapJwt { issuer: "openshell-gateway:test-gateway".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), }) @@ -516,6 +528,7 @@ mod tests { sandbox_id: "sandbox-a".to_string(), source: SandboxIdentitySource::ComputeDriver { driver_name: "kubernetes".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), })); @@ -538,6 +551,7 @@ mod tests { sandbox_id: "sandbox-deleted".to_string(), source: SandboxIdentitySource::ComputeDriver { driver_name: "kubernetes".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), })); @@ -584,6 +598,7 @@ mod tests { sandbox_id: "sandbox-a".to_string(), source: SandboxIdentitySource::ComputeDriver { driver_name: "kubernetes".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), })); diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 0e4c74af95..3c5f11e6f9 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -6924,6 +6924,7 @@ mod tests { sandbox_id: sandbox_id.to_string(), source: SandboxIdentitySource::BootstrapJwt { issuer: "openshell-gateway:test".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), })); @@ -7285,6 +7286,7 @@ mod tests { sandbox_id: "test-sandbox".to_string(), source: SandboxIdentitySource::BootstrapJwt { issuer: "openshell-gateway:test".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: None, })); diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index d4d08da251..a93f0a4fe0 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -120,6 +120,23 @@ impl Drop for WatchSandboxStream { /// Fetch a sandbox by ID and authorize the caller in one step, returning /// `NOT_FOUND` for both missing and unauthorized sandboxes so that callers /// cannot distinguish the two cases (CWE-203). +/// Reject a relay-backed RPC when the sandbox's topology has no in-sandbox +/// supervisor session. Reads the durable `SupervisorSession=NotApplicable` +/// condition from the stored status so this holds on every gateway replica — +/// not only the reconciler lease holder that populates the in-memory set — and +/// returns the terminal `FailedPrecondition` the CLI must not retry, instead of +/// making the caller wait out a session that will never arrive. +fn reject_if_sessionless(sandbox: &Sandbox) -> Result<(), Status> { + if let Some(status) = sandbox.status.as_ref() + && crate::compute::sandbox_status_is_sessionless(status) + { + return Err(Status::failed_precondition( + openshell_core::error::no_supervisor_session_message(), + )); + } + Ok(()) +} + pub(super) async fn fetch_and_authorize_sandbox( state: &Arc, principal: &crate::auth::principal::Principal, @@ -1695,6 +1712,8 @@ pub(super) async fn handle_exec_sandbox( return Err(Status::failed_precondition("sandbox is not ready")); } + reject_if_sessionless(&sandbox)?; + // Open a relay channel through the supervisor session. Use a 15s // session-wait timeout, enough to cover a transient supervisor reconnect // while still failing quickly during normal operation. @@ -1702,7 +1721,15 @@ pub(super) async fn handle_exec_sandbox( .supervisor_sessions .open_relay(sandbox.object_id(), std::time::Duration::from_secs(15)) .await - .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; + .map_err(|e| { + // Preserve the original code: a sessionless topology returns + // FailedPrecondition (terminal), which the CLI must not retry as it + // would a transient Unavailable. + Status::new( + e.code(), + format!("supervisor relay failed: {}", e.message()), + ) + })?; let command_str = build_remote_exec_command(&req) .map_err(|e| Status::invalid_argument(format!("command construction failed: {e}")))?; @@ -1813,6 +1840,7 @@ pub(super) async fn handle_forward_tcp( if !sandbox_relay_reachable(state, &sandbox) { return Err(Status::failed_precondition("sandbox is not ready")); } + reject_if_sessionless(&sandbox)?; let connection_guard = acquire_forward_connection_guard(state, &init, &sandbox).await?; let (channel_id, relay_rx) = state @@ -1824,7 +1852,15 @@ pub(super) async fn handle_forward_tcp( std::time::Duration::from_secs(15), ) .await - .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; + .map_err(|e| { + // Preserve the original code: a sessionless topology returns + // FailedPrecondition (terminal), which the CLI must not retry as it + // would a transient Unavailable. + Status::new( + e.code(), + format!("supervisor relay failed: {}", e.message()), + ) + })?; let sandbox_id = sandbox.object_id().to_string(); let (tx, rx) = mpsc::channel::>(256); @@ -2136,11 +2172,25 @@ pub(super) async fn handle_exec_sandbox_interactive( return Err(Status::failed_precondition("sandbox is not ready")); } + // A sessionless topology (e.g. proxy-pod) has no in-sandbox supervisor to + // relay to. Reject from the durable status on every replica, so a follower + // returns the terminal FailedPrecondition immediately instead of waiting out + // the relay-open timeout and returning retryable Unavailable. + reject_if_sessionless(&sandbox)?; + let (channel_id, relay_rx) = state .supervisor_sessions .open_relay(sandbox.object_id(), std::time::Duration::from_secs(15)) .await - .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; + .map_err(|e| { + // Preserve the original code: a sessionless topology returns + // FailedPrecondition (terminal), which the CLI must not retry as it + // would a transient Unavailable. + Status::new( + e.code(), + format!("supervisor relay failed: {}", e.message()), + ) + })?; let command_str = build_remote_exec_command(&req) .map_err(|e| Status::invalid_argument(format!("command construction failed: {e}")))?; diff --git a/crates/openshell-server/src/inference.rs b/crates/openshell-server/src/inference.rs index b83fd6be4f..7ac8ad0855 100644 --- a/crates/openshell-server/src/inference.rs +++ b/crates/openshell-server/src/inference.rs @@ -1240,6 +1240,7 @@ mod tests { sandbox_id: "sandbox-a".to_string(), source: SandboxIdentitySource::BootstrapJwt { issuer: "openshell-gateway:test".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), }) diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 3ef774e4cb..9d8ae9e83e 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -1156,12 +1156,24 @@ where return Ok(status_response(status)); } } - Principal::Sandbox(_) => { + Principal::Sandbox(ref sandbox) => { if !crate::auth::sandbox_methods::is_sandbox_callable(&path) { return Ok(status_response(tonic::Status::permission_denied( "sandbox principals may not call this method", ))); } + // A process-kind credential (proxy-pod agent pod) is denied + // the network-supervisor RPCs that read provider secrets or + // mint upstream credentials; those run only in the proxy pod. + if sandbox.caller_kind() == crate::auth::principal::SandboxCallerKind::Process + && crate::auth::sandbox_methods::is_process_caller_denied(&path) + { + return Ok(status_response(tonic::Status::permission_denied( + "this method requires a full-authority sandbox credential; \ + the process-supervisor credential cannot access provider \ + secrets or inference routing", + ))); + } } Principal::Anonymous => { return Ok(status_response(tonic::Status::unauthenticated( @@ -2738,6 +2750,7 @@ mod tests { sandbox_id: "sandbox-a".to_string(), source: SandboxIdentitySource::BootstrapJwt { issuer: "openshell-gateway:test".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), }) diff --git a/crates/openshell-server/src/service_routing.rs b/crates/openshell-server/src/service_routing.rs index 3e80bc26f5..eb996976bb 100644 --- a/crates/openshell-server/src/service_routing.rs +++ b/crates/openshell-server/src/service_routing.rs @@ -193,6 +193,18 @@ impl ServiceRouteError { ) } + /// The sandbox's topology has no in-sandbox supervisor session to relay + /// through (e.g. proxy-pod), so HTTP service routing is not available. A + /// terminal condition, mirroring the `FailedPrecondition` the exec/forward + /// gRPC paths return for the same topologies. + const fn no_supervisor_session() -> Self { + Self::new( + StatusCode::PRECONDITION_FAILED, + "Sandbox topology has no in-sandbox supervisor; HTTP service routing is unavailable", + "no supervisor session", + ) + } + const fn invalid_request() -> Self { Self::new( StatusCode::BAD_REQUEST, @@ -297,6 +309,27 @@ async fn proxy_to_endpoint( ); return Err(err); } + // A sessionless topology (e.g. proxy-pod) has no in-sandbox supervisor to + // relay through. Reject from the durable status here — on every replica, not + // just the reconciler leader that populates the in-memory sessionless set — + // so an HA follower returns immediately instead of waiting out the relay-open + // timeout. Mirrors the guard the exec/interactive/forward gRPC paths apply. + if sandbox + .status + .as_ref() + .is_some_and(crate::compute::sandbox_status_is_sessionless) + { + let err = ServiceRouteError::no_supervisor_session(); + emit_service_http_failure( + &state, + &req, + &sandbox_name, + &service_name, + Some(&endpoint), + &err, + ); + return Err(err); + } let Ok(target_port) = u16::try_from(endpoint.target_port) else { let err = ServiceRouteError::endpoint_unavailable(); emit_service_http_failure( @@ -1070,6 +1103,14 @@ mod tests { ); } + #[test] + fn sessionless_route_error_is_terminal_precondition_failed() { + // A sessionless topology must return a terminal 4xx (like the exec/forward + // FailedPrecondition), not a retryable 5xx that a client would keep hitting. + let response = ServiceRouteError::no_supervisor_session().into_response(); + assert_eq!(response.status(), StatusCode::PRECONDITION_FAILED); + } + #[test] fn service_endpoint_config_event_includes_endpoint_metadata() { let event = diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index c8491dc1eb..ac55708dee 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::pin::Pin; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex}; @@ -76,6 +76,10 @@ pub struct SupervisorSessionRegistry { sessions: Mutex>, /// `channel_id` -> oneshot sender for the reverse CONNECT stream. pending_relays: Mutex>, + /// Sandboxes whose topology has no in-sandbox process supervisor, and so + /// will never register a session. Waiting for one is pointless, and the + /// caller deserves to know why rather than watching a timeout elapse. + sessionless: Mutex>, } struct PendingRelay { @@ -189,6 +193,15 @@ impl SupervisorSessionRegistry { sandbox_id: &str, timeout: Duration, ) -> Result, Status> { + // Topologies without an in-sandbox process supervisor never register a + // session. Fail immediately with an actionable message instead of + // burning the caller's timeout on a wait that cannot succeed. + if self.is_sessionless(sandbox_id) { + return Err(Status::failed_precondition( + openshell_core::error::no_supervisor_session_message(), + )); + } + let deadline = Instant::now() + timeout; let mut backoff = SESSION_WAIT_INITIAL_BACKOFF; @@ -233,6 +246,28 @@ impl SupervisorSessionRegistry { true } + /// Record whether a sandbox's topology can ever open a supervisor session. + /// + /// Driven by the compute driver's reported `SupervisorSessionModel`, so it + /// re-establishes itself from the next driver snapshot after a gateway + /// restart. + pub fn set_sessionless(&self, sandbox_id: &str, sessionless: bool) { + let mut set = self.sessionless.lock().unwrap(); + if sessionless { + set.insert(sandbox_id.to_string()); + } else { + set.remove(sandbox_id); + } + } + + pub fn is_sessionless(&self, sandbox_id: &str) -> bool { + self.sessionless.lock().unwrap().contains(sandbox_id) + } + + pub fn forget_sessionless(&self, sandbox_id: &str) { + self.sessionless.lock().unwrap().remove(sandbox_id); + } + pub fn is_current_session(&self, sandbox_id: &str, session_id: &str) -> bool { self.sessions .lock() @@ -1087,6 +1122,7 @@ mod tests { sandbox_id: sandbox_id.to_string(), source: SandboxIdentitySource::BootstrapJwt { issuer: "openshell-gateway:test".to_string(), + caller_kind: crate::auth::principal::SandboxCallerKind::Full, }, trust_domain: Some("openshell".to_string()), }) diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index 2275a60d34..0de47e545a 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -63,6 +63,28 @@ impl SandboxCa { }) } + /// Load an existing CA certificate and private key from PEM. + pub fn from_pem(ca_cert_pem: &str, ca_key_pem: &str) -> Result { + let ca_key = KeyPair::from_pem(ca_key_pem).into_diagnostic()?; + let ca_cert = CertificateParams::from_ca_cert_pem(ca_cert_pem) + .into_diagnostic()? + .self_signed(&ca_key) + .into_diagnostic()?; + + Ok(Self { + ca_cert, + ca_key, + ca_cert_pem: ca_cert_pem.to_string(), + }) + } + + /// Load an existing CA certificate and private key from files. + pub fn from_files(cert_path: &Path, key_path: &Path) -> Result { + let ca_cert_pem = std::fs::read_to_string(cert_path).into_diagnostic()?; + let ca_key_pem = std::fs::read_to_string(key_path).into_diagnostic()?; + Self::from_pem(&ca_cert_pem, &ca_key_pem) + } + /// Returns the CA certificate in PEM format. pub fn cert_pem(&self) -> &str { &self.ca_cert_pem @@ -559,4 +581,18 @@ mod tests { "bundle should contain at least one cert", ); } + + #[test] + fn sandbox_ca_loads_from_pem() { + let ca = SandboxCa::generate().unwrap(); + let key_pem = ca.ca_key.serialize_pem(); + let loaded = SandboxCa::from_pem(ca.cert_pem(), &key_pem).unwrap(); + + assert_eq!(loaded.cert_pem(), ca.cert_pem()); + assert!( + CertCache::new(loaded) + .get_or_generate("example.com") + .is_ok() + ); + } } diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 2a71702b4b..7067aa22f3 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -161,6 +161,62 @@ pub struct Networking { _transparent_tcp: Option, } +/// Resolve the L7 proxy CA. +/// +/// `trust_launch_env` gates the environment-provided CA file paths. They are a +/// proxy-pod/sidecar launch contract: those supervisors run standalone in a +/// separate container built from the trusted supervisor image, so their process +/// environment is set only by the driver. A combined-topology supervisor shares +/// the workload's container and inherits the workload image's baked-in +/// environment, which is untrusted — so it must ignore these paths and always +/// generate an ephemeral CA rather than load attacker-supplied key material. +fn sandbox_ca_for_proxy(trust_launch_env: bool) -> Result { + if !trust_launch_env { + return SandboxCa::generate(); + } + let cert_path = std::env::var(openshell_core::sandbox_env::PROXY_CA_CERT_PATH).ok(); + let key_path = std::env::var(openshell_core::sandbox_env::PROXY_CA_KEY_PATH).ok(); + match (cert_path, key_path) { + (Some(cert_path), Some(key_path)) => SandboxCa::from_files( + std::path::Path::new(&cert_path), + std::path::Path::new(&key_path), + ), + (None, None) => SandboxCa::generate(), + _ => Err(miette::miette!( + "{} and {} must be set together", + openshell_core::sandbox_env::PROXY_CA_CERT_PATH, + openshell_core::sandbox_env::PROXY_CA_KEY_PATH + )), + } +} + +/// Resolve an explicit proxy bind address from the environment. +/// +/// `trust_launch_env` gates this the same way as [`sandbox_ca_for_proxy`]: only +/// a standalone network supervisor (proxy-pod/sidecar) may take its bind address +/// from the environment. A combined-topology supervisor must never honor an +/// image-baked `PROXY_BIND_ADDR`; a value like `0.0.0.0:3128` would publish the +/// credential-bearing policy proxy on the pod network and let another workload +/// use the sandbox as a confused deputy. It binds to the namespace-scoped veth +/// IP instead (the caller's `proxy_bind_ip`). +fn explicit_proxy_bind_addr(trust_launch_env: bool) -> Result> { + if !trust_launch_env { + return Ok(None); + } + let Some(value) = std::env::var(openshell_core::sandbox_env::PROXY_BIND_ADDR) + .ok() + .filter(|value| !value.trim().is_empty()) + else { + return Ok(None); + }; + value.parse::().map(Some).map_err(|err| { + miette::miette!( + "invalid {} value {value:?}: {err}", + openshell_core::sandbox_env::PROXY_BIND_ADDR + ) + }) +} + /// Set up the networking stack: ephemeral CA + TLS state, proxy server, /// and the SSH-side proxy URL / netns FD. /// @@ -313,10 +369,15 @@ pub async fn run_networking( // the proxy, so it's owned here. let identity_cache = opa_engine.map(|_| Arc::new(BinaryIdentityCache::new())); - // Generate ephemeral CA and TLS state for HTTPS L7 inspection. - // The CA cert is written to disk so sandbox processes can trust it. + // Generate or load a CA and TLS state for HTTPS L7 inspection. The CA cert + // is written to disk so sandbox processes can trust it. + // A standalone network supervisor (proxy-pod/sidecar; `!process_enabled`) + // runs in a trusted separate container, so its launch environment (CA paths, + // bind address) is driver-controlled. A combined supervisor shares the + // workload's container and must not trust image-baked launch variables. + let trust_launch_env = !process_enabled; let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - match SandboxCa::generate() { + match sandbox_ca_for_proxy(trust_launch_env) { Ok(ca) => { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| openshell_core::container_paths::TLS_ROOT.to_string()); @@ -356,7 +417,7 @@ pub async fn run_networking( .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "enabled") - .message("TLS termination enabled: ephemeral CA generated") + .message("TLS termination enabled") .build() ); (Some(state), Some(paths)) @@ -391,7 +452,7 @@ pub async fn run_networking( .status(StatusId::Failure) .state(StateId::Disabled, "disabled") .message(format!( - "Failed to generate ephemeral CA, TLS termination disabled: {e}" + "Failed to initialize proxy CA, TLS termination disabled: {e}" )) .build() ); @@ -420,9 +481,11 @@ pub async fn run_networking( // originating inside the namespace can reach the proxy. Otherwise the // proxy falls back to the policy-declared http_addr (loopback in // tests, etc.). - let bind_addr = proxy_bind_ip.map(|ip| { - let port = proxy_policy.http_addr.map_or(3128, |addr| addr.port()); - SocketAddr::new(ip, port) + let bind_addr = explicit_proxy_bind_addr(trust_launch_env)?.or_else(|| { + proxy_bind_ip.map(|ip| { + let port = proxy_policy.http_addr.map_or(3128, |addr| addr.port()); + SocketAddr::new(ip, port) + }) }); // Build inference context for local routing of intercepted inference calls. @@ -521,3 +584,47 @@ mod transparent_runtime_tests { assert!(error.to_string().contains("allocation epoch is invalid")); } } + +#[cfg(test)] +mod launch_env_trust_tests { + use super::*; + + // A combined-topology supervisor (`trust_launch_env = false`) shares the + // workload's container, so image-baked launch variables are untrusted and + // must be ignored; a standalone network supervisor (proxy-pod/sidecar) + // honors the driver-set launch environment. These vars are process-global, + // so both directions live in one test to avoid racing sibling tests. + #[test] + #[allow(unsafe_code)] // std::env::set_var/remove_var require unsafe in Rust 2024 + fn launch_env_is_ignored_in_combined_topology() { + let bogus_cert = "/nonexistent/openshell-attacker-ca.crt"; + let bogus_key = "/nonexistent/openshell-attacker-ca.key"; + unsafe { + std::env::set_var(openshell_core::sandbox_env::PROXY_BIND_ADDR, "0.0.0.0:3128"); + std::env::set_var(openshell_core::sandbox_env::PROXY_CA_CERT_PATH, bogus_cert); + std::env::set_var(openshell_core::sandbox_env::PROXY_CA_KEY_PATH, bogus_key); + } + + // Untrusted (combined): the image-baked bind address is ignored, so the + // proxy falls back to the namespace-scoped veth IP instead of 0.0.0.0. + assert_eq!(explicit_proxy_bind_addr(false).unwrap(), None); + // Untrusted (combined): attacker CA paths are ignored; a fresh CA is + // generated rather than loaded from the (bogus) files. + assert!(sandbox_ca_for_proxy(false).is_ok()); + + // Trusted (proxy-pod/sidecar): the driver-set launch environment is honored. + assert_eq!( + explicit_proxy_bind_addr(true).unwrap(), + Some("0.0.0.0:3128".parse().unwrap()) + ); + // Trusted path actually reads the configured files, so bogus paths error + // rather than silently generating — proving the value is honored. + assert!(sandbox_ca_for_proxy(true).is_err()); + + unsafe { + std::env::remove_var(openshell_core::sandbox_env::PROXY_BIND_ADDR); + std::env::remove_var(openshell_core::sandbox_env::PROXY_CA_CERT_PATH); + std::env::remove_var(openshell_core::sandbox_env::PROXY_CA_KEY_PATH); + } + } +} diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index 2e2120f1d0..3238770cac 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -45,6 +45,7 @@ socket2 = { workspace = true } tempfile = "3" [dev-dependencies] +temp-env = "0.3" tempfile = "3" [lints] diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 2b4ea554ed..6c6ddaa97c 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -641,7 +641,7 @@ pub fn create_netns_for_proxy( /// Install pod-network bypass enforcement for Kubernetes sidecar topology. /// /// This runs in the current network namespace, not in a per-workload netns. -/// The rules allow loopback and the sidecar proxy UID, then reject direct +/// The rules allow loopback and the proxy UID, then reject direct /// TCP/UDP egress from other UIDs so traffic must use the sidecar's local /// proxy. /// diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 52557493cd..77b3249434 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -170,6 +170,10 @@ const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::TLS_KEY, openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::PROXY_URL, + openshell_core::sandbox_env::PROXY_BIND_ADDR, + openshell_core::sandbox_env::PROXY_CA_CERT_PATH, + openshell_core::sandbox_env::PROXY_CA_KEY_PATH, ]; pub fn is_supervisor_only_env_var(key: &str) -> bool { @@ -252,6 +256,35 @@ fn configured_user_environment() -> HashMap { .unwrap_or_default() } +fn configured_proxy_url( + policy: &SandboxPolicy, + netns_proxy_enabled: bool, +) -> Result> { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Ok(None); + } + + if let Ok(proxy_url) = std::env::var(openshell_core::sandbox_env::PROXY_URL) { + let trimmed = proxy_url.trim(); + if !trimmed.is_empty() { + return Ok(Some(trimmed.to_string())); + } + } + + let proxy = policy.network.proxy.as_ref().ok_or_else(|| { + miette::miette!("Network mode is set to proxy but no proxy configuration was provided") + })?; + + if netns_proxy_enabled { + let port = proxy.http_addr.map_or(3128, |addr| addr.port()); + return Ok(Some(format!("http://10.200.0.1:{port}"))); + } + + Ok(proxy + .http_addr + .map(|http_addr| format!("http://{http_addr}"))) +} + #[cfg(unix)] pub fn harden_child_process() -> Result<()> { use rustix::process::{Resource, Rlimit, setrlimit}; @@ -795,27 +828,11 @@ impl ProcessHandle { cmd.current_dir(dir); } - if matches!(policy.network.mode, NetworkMode::Proxy) { - let proxy = policy.network.proxy.as_ref().ok_or_else(|| { - miette::miette!( - "Network mode is set to proxy but no proxy configuration was provided" - ) - })?; - // When using network namespace, set proxy URL to the veth host IP - if netns_fd.is_some() { - // The proxy is on 10.200.0.1:3128 (or configured port) - let port = proxy.http_addr.map_or(3128, |addr| addr.port()); - let proxy_url = format!("http://10.200.0.1:{port}"); - // Both uppercase and lowercase variants: curl/wget use uppercase, - // gRPC C-core (libgrpc) checks lowercase http_proxy/https_proxy. - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } - } else if let Some(http_addr) = proxy.http_addr { - let proxy_url = format!("http://{http_addr}"); - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } + if let Some(proxy_url) = configured_proxy_url(policy, netns_fd.is_some())? { + // Both uppercase and lowercase variants: curl/wget use uppercase, + // gRPC C-core (libgrpc) checks lowercase http_proxy/https_proxy. + for (key, value) in child_env::proxy_env_vars(&proxy_url) { + cmd.env(key, value); } } @@ -991,17 +1008,9 @@ impl ProcessHandle { cmd.current_dir(dir); } - if matches!(policy.network.mode, NetworkMode::Proxy) { - let proxy = policy.network.proxy.as_ref().ok_or_else(|| { - miette::miette!( - "Network mode is set to proxy but no proxy configuration was provided" - ) - })?; - if let Some(http_addr) = proxy.http_addr { - let proxy_url = format!("http://{http_addr}"); - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } + if let Some(proxy_url) = configured_proxy_url(policy, false)? { + for (key, value) in child_env::proxy_env_vars(&proxy_url) { + cmd.env(key, value); } } diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index 8c47e789ba..773acb5e83 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -700,6 +700,13 @@ fn ssh_proxy_url_for_policy( return None; } + if let Ok(proxy_url) = std::env::var(openshell_core::sandbox_env::PROXY_URL) { + let trimmed = proxy_url.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + let proxy = policy.network.proxy.as_ref()?; if let Some(host) = netns_proxy_host { let port = proxy.http_addr.map_or(3128, |addr| addr.port()); @@ -768,6 +775,8 @@ mod tests { FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, }; + static PROXY_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + fn policy(mode: NetworkMode, http_addr: Option) -> SandboxPolicy { SandboxPolicy { version: 1, @@ -783,31 +792,57 @@ mod tests { } } + fn with_proxy_url(proxy_url: Option<&str>, test: F) -> T + where + F: FnOnce() -> T, + { + let _guard = PROXY_ENV_LOCK.lock().expect("proxy env lock poisoned"); + temp_env::with_var(openshell_core::sandbox_env::PROXY_URL, proxy_url, test) + } + #[test] fn ssh_proxy_url_uses_policy_addr_without_netns() { - let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 3128).into())); + with_proxy_url(None, || { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 3128).into())); - assert_eq!( - ssh_proxy_url_for_policy(&policy, None).as_deref(), - Some("http://127.0.0.1:3128") - ); + assert_eq!( + ssh_proxy_url_for_policy(&policy, None).as_deref(), + Some("http://127.0.0.1:3128") + ); + }); } #[test] fn ssh_proxy_url_prefers_netns_host_with_policy_port() { - let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); + with_proxy_url(None, || { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); - assert_eq!( - ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), - Some("http://10.200.0.1:8080") - ); + assert_eq!( + ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), + Some("http://10.200.0.1:8080") + ); + }); } #[test] fn ssh_proxy_url_skips_non_proxy_mode() { - let policy = policy(NetworkMode::Allow, Some(([127, 0, 0, 1], 3128).into())); + with_proxy_url(None, || { + let policy = policy(NetworkMode::Allow, Some(([127, 0, 0, 1], 3128).into())); + + assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); + }); + } + + #[test] + fn ssh_proxy_url_prefers_env_override() { + with_proxy_url(Some("http://openshell-supervisor.default.svc:3128"), || { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); - assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); + assert_eq!( + ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), + Some("http://openshell-supervisor.default.svc:3128") + ); + }); } #[cfg(unix)] diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 7dbd7964d0..6db351ed73 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -40,13 +40,17 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null \ + --set supervisor.topology=proxy-pod \ + --set sandboxServiceAccount.openshift.nonrootSCC=true +``` + ## Available versions | Tag | Source | Notes | @@ -226,6 +245,7 @@ discovery endpoint or its TLS CA. | sandboxServiceAccount.annotations | object | `{}` | Annotations to add to the generated sandbox service account. | | sandboxServiceAccount.create | bool | `true` | Create a service account for sandbox pods. | | sandboxServiceAccount.name | string | `""` | Existing service account name for sandbox pods when sandboxServiceAccount.create is false. | +| sandboxServiceAccount.openshift.nonrootSCC | bool | `false` | Grant the built-in OpenShift `nonroot-v2` SCC to the sandbox ServiceAccount. Required on OpenShift for "proxy-pod" topology: the driver assigns explicit non-root UIDs, which `restricted-v2` rejects because it only admits UIDs inside the namespace's openshift.io/sa.scc.uid-range annotation. No custom SCC is created — `nonroot-v2` ships with OpenShift and already permits exactly what this topology needs, keeping drop-ALL capabilities, no privilege escalation, and no host namespaces. Creates a ClusterRole + ClusterRoleBinding. Supported only with server.drivers.kubernetes.workspaceMode=shared: the ClusterRoleBinding is scoped to the static sandbox namespace, so it does not reach the dynamically created workspace namespaces used by managed and operator modes. Enabling it with a non-shared mode fails the Helm render. | | securityContext.allowPrivilegeEscalation | bool | `false` | Whether the gateway container can gain additional privileges. | | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | @@ -300,10 +320,15 @@ discovery endpoint or its TLS CA. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | +| supervisor.proxyPod.affinity | string | `"disabled"` | Same-node scheduling relationship between the workload pod and its paired proxy supervisor: disabled, preferred, or required. | +| supervisor.proxyPod.dnsPeers | list | `[]` | Cluster DNS peers permitted by the proxy-pod agent egress NetworkPolicy. Each entry sets `namespaceLabels`, `podLabels`, or both. Empty uses the upstream kube-system/kube-dns and kube-system/coredns conventions, which do NOT match OpenShift (cluster DNS runs in `openshift-dns`) or NodeLocal DNSCache. An agent pod with no matching DNS peer cannot resolve its own paired supervisor Service. `port` is the DNS *pod* port, not the Service port: egress rules with a podSelector match after Service address translation. Upstream CoreDNS listens on 53; OpenShift's dns-default listens on 5353 and maps 53 to it. For OpenShift: dnsPeers: - namespaceLabels: kubernetes.io/metadata.name: openshift-dns podLabels: dns.operator.openshift.io/daemonset-dns: default port: 5353 | +| supervisor.proxyPod.gatewayPeers | list | `[]` | Gateway peers the proxy-pod agent egress NetworkPolicy permits, so the in-pod process supervisor can reach the gateway for its session. Same shape as dnsPeers (namespaceLabels/podLabels + port). Empty (default) auto-renders a peer matching THIS chart's own gateway pods on the service port; override only for an external or differently-labeled gateway. | +| supervisor.proxyPod.proxyUid | int | `1337` | UID for the network supervisor in proxy-pod topology. The configured UID must not match the sandbox UID. | +| supervisor.proxyPod.retainCompanionRbac | bool | `false` | Render the proxy-pod companion, fence, and pod-inspection RBAC even when supervisor.topology is not proxy-pod, and tell the gateway to keep running background upkeep (periodic companion reconciliation and the shared-mode supervisor Deployment readiness watch) for those sandboxes. Set this true as a migration mode when switching a gateway away from proxy-pod while proxy-pod sandboxes still exist: the driver keeps managing them by their persisted creation-time topology, and without this flag their RBAC and upkeep would stop, breaking readiness, stop/start, repair, and safe fence cleanup. Renders `proxy_pod.retain_companion_management` in gateway.toml. Leave it true until all proxy-pod sandboxes have been deleted, then remove it. | | supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | | supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | -| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | +| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. "proxy-pod" runs network enforcement in a separate supervisor Deployment and restricts the agent pod to that supervisor through NetworkPolicy. | | tolerations | list | `[]` | Tolerations for the gateway pod. | | upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | | upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index cf8677741e..e5e29fb7da 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -40,13 +40,17 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null \ + --set supervisor.topology=proxy-pod \ + --set sandboxServiceAccount.openshift.nonrootSCC=true +``` + ## Available versions | Tag | Source | Notes | diff --git a/deploy/helm/openshell/ci/values-proxy-pod.yaml b/deploy/helm/openshell/ci/values-proxy-pod.yaml new file mode 100644 index 0000000000..b7cb533fd7 --- /dev/null +++ b/deploy/helm/openshell/ci/values-proxy-pod.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI/dev overlay for exercising the Kubernetes proxy-pod topology. +# +# This topology relies on Kubernetes NetworkPolicy enforcement: the agent pod is +# isolated to its paired supervisor pod plus DNS. The local k3s/k3d workflow +# must therefore run with the k3s network policy controller enabled, or with a +# custom policy-enforcing CNI installed before deploying this profile. +# +# Merge after values.yaml and ci/values-skaffold.yaml: +# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-proxy-pod.yaml +# +# Or set: +# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-proxy-pod.yaml +# before running `mise run e2e:kubernetes`. +supervisor: + topology: proxy-pod diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index ce32c72132..153961dfcb 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -121,6 +121,11 @@ deploy: #- ci/values-spire.yaml # To exercise the Kubernetes supervisor sidecar topology: #- ci/values-sidecar.yaml + # To exercise proxy-pod topology, use the proxy-pod Skaffold profile + # against a cluster with NetworkPolicy enforcement enabled. Stock k3s + # includes its embedded network policy controller; if you replace the + # CNI, install a policy-enforcing CNI before deploying this profile. + #- ci/values-proxy-pod.yaml # To test multi-replica external PostgreSQL behavior: #- ci/values-high-availability.yaml setValueTemplates: @@ -153,3 +158,8 @@ profiles: - op: add path: /deploy/helm/releases/0/valuesFiles/- value: ci/values-credential-driver-vault.yaml + - name: proxy-pod + patches: + - op: add + path: /deploy/helm/releases/0/valuesFiles/- + value: ci/values-proxy-pod.yaml diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index eb1ed8e1d0..948d47d79e 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -147,3 +147,63 @@ rules: - update {{- end }} {{- end }} + {{- if and (ne $workspaceMode "shared") (or (eq (.Values.supervisor.topology | default "combined") "proxy-pod") .Values.supervisor.proxyPod.retainCompanionRbac) }} + # Proxy-pod topology creates a supervisor Deployment, Service, CA Secret, and + # NetworkPolicy pair per sandbox in the sandbox's namespace. In managed and + # operator modes that namespace is per-workspace, so these permissions must be + # cluster-scoped; shared mode grants the same access through the namespaced + # Role instead. Companions are owner-referenced to the Sandbox CR and removed + # by garbage collection when it is deleted, so no `delete` verbs are granted — + # a compromised gateway must not be able to delete cluster resources. `get` + # supports crash-recovery reconciliation and, on deployments, readiness and + # the ServiceAccount-bootstrap owner-chain check. `patch` on deployments scales + # the supervisor on stop/start. The gateway never reads Secret contents, so + # Secrets get `create` only (no cluster-wide Secret read). Note: `list`/`watch` + # on deployments are intentionally NOT granted cluster-wide — the supervisor + # Deployment readiness watch is enabled only in shared (single-namespace) mode + # via the namespaced Role; managed/operator modes fold supervisor readiness in + # through get/list and the periodic reconcile to avoid cluster-wide Deployment + # enumeration. + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - get + - patch + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - create + - get + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + # The agent egress NetworkPolicy (the workload's egress fence) has no owner + # reference so it can outlive the workload pod during deletion; the gateway + # manages its lifecycle directly. `delete` tears it down after the pod is gone, + # and `list` lets reconciliation reap fences orphaned by a gateway crash. + # Owner-referenced NetworkPolicies (supervisor ingress) are garbage-collected + # with the Sandbox CR. + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + {{- end }} diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 083748aee3..320e1e73a1 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -219,6 +219,47 @@ data: proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} + [openshell.drivers.kubernetes.proxy_pod] + proxy_uid = {{ .Values.supervisor.proxyPod.proxyUid | default 1337 }} + affinity = {{ .Values.supervisor.proxyPod.affinity | default "disabled" | quote }} + retain_companion_management = {{ .Values.supervisor.proxyPod.retainCompanionRbac | default false }} + {{- range .Values.supervisor.proxyPod.dnsPeers }} + + [[openshell.drivers.kubernetes.proxy_pod.dns_peers]] + {{- with .namespaceLabels }} + {{- $pairs := list }}{{ range $k, $v := . }}{{ $pairs = append $pairs (printf "%q = %q" $k $v) }}{{ end }} + namespace_labels = { {{ join ", " $pairs }} } + {{- end }} + {{- with .podLabels }} + {{- $pairs := list }}{{ range $k, $v := . }}{{ $pairs = append $pairs (printf "%q = %q" $k $v) }}{{ end }} + pod_labels = { {{ join ", " $pairs }} } + {{- end }} + port = {{ .port | default 53 }} + {{- end }} + {{- if .Values.supervisor.proxyPod.gatewayPeers }} + {{- range .Values.supervisor.proxyPod.gatewayPeers }} + + [[openshell.drivers.kubernetes.proxy_pod.gateway_peers]] + {{- with .namespaceLabels }} + {{- $pairs := list }}{{ range $k, $v := . }}{{ $pairs = append $pairs (printf "%q = %q" $k $v) }}{{ end }} + namespace_labels = { {{ join ", " $pairs }} } + {{- end }} + {{- with .podLabels }} + {{- $pairs := list }}{{ range $k, $v := . }}{{ $pairs = append $pairs (printf "%q = %q" $k $v) }}{{ end }} + pod_labels = { {{ join ", " $pairs }} } + {{- end }} + port = {{ .port | default $.Values.service.port }} + {{- end }} + {{- else }} + + # Default: permit the in-pod process supervisor to reach this gateway's own + # pods on the service port. Override with supervisor.proxyPod.gatewayPeers. + [[openshell.drivers.kubernetes.proxy_pod.gateway_peers]] + namespace_labels = { "kubernetes.io/metadata.name" = {{ .Release.Namespace | quote }} } + pod_labels = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } + port = {{ .Values.service.port }} + {{- end }} + {{- if not $credentialDrivers }} [openshell.gateway.credential_storage] diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index dfd6423615..12d1082994 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -36,11 +36,71 @@ rules: # returned pod name and UID to the pod's `openshell.ai/sandbox-id` # annotation. patch is intentionally NOT granted — the annotation is set # once at pod create and must remain immutable for the lifetime of the - # sandbox. + # sandbox. create/delete/list/watch are intentionally not granted; the Agent + # Sandbox controller creates agent pods, and proxy-pod supervisors are + # managed through per-sandbox Deployments. - apiGroups: - "" resources: - pods verbs: - get + {{- if or (eq (.Values.supervisor.topology | default "combined") "proxy-pod") .Values.supervisor.proxyPod.retainCompanionRbac }} + # Proxy-pod topology creates one supervisor Deployment, one supervisor + # Service, and one CA Secret per sandbox, all owner-referenced to the Sandbox + # CR so Kubernetes garbage collection removes them when the CR is deleted — + # the gateway never deletes companions itself, so no `delete` verbs are + # granted. `get` supports crash-recovery reconciliation (verifying an existing + # companion's owner before adopting it) and, on deployments, the readiness and + # ServiceAccount-bootstrap owner-chain checks. `list` and `watch` on + # deployments back the supervisor Deployment watch that refreshes sandbox + # readiness when supervisor availability changes. `patch` on deployments scales + # the paired supervisor on stop/start. The gateway never reads Secret contents, + # so Secrets get `create` only. These permissions render only when the + # Kubernetes driver is configured for proxy-pod topology. + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - get + - list + - patch + - watch + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - create + - get + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + # The agent egress NetworkPolicy (the workload's egress fence) carries no + # owner reference so it can outlive the workload pod during deletion; the + # gateway therefore manages its lifecycle directly. `delete` tears it down + # after the pod is gone, and `list` lets reconciliation reap fences orphaned by + # a gateway crash. Owner-referenced NetworkPolicies (supervisor ingress) are + # still garbage-collected with the Sandbox CR. + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + {{- end }} {{- end }} diff --git a/deploy/helm/openshell/templates/sandbox-scc.yaml b/deploy/helm/openshell/templates/sandbox-scc.yaml new file mode 100644 index 0000000000..f408098032 --- /dev/null +++ b/deploy/helm/openshell/templates/sandbox-scc.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.sandboxServiceAccount.openshift.nonrootSCC }} +{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" }} +{{- if ne $workspaceMode "shared" }} +{{- fail (printf "sandboxServiceAccount.openshift.nonrootSCC is only supported with server.drivers.kubernetes.workspaceMode=shared (got %q). Managed and operator modes run sandboxes under ServiceAccounts in dynamically created workspace namespaces, which this single ClusterRoleBinding (scoped to the static sandbox namespace) does not cover, so nonroot-v2 would not be granted and non-root sandbox pods would be inadmissible. Grant the nonroot-v2 SCC per workspace namespace out-of-band, or use shared workspace mode." $workspaceMode) }} +{{- end }} +# Grant the built-in OpenShift `nonroot-v2` SCC to the sandbox ServiceAccount. +# +# No SecurityContextConstraints object is created: `nonroot-v2` ships with +# OpenShift and already permits exactly what proxy-pod topology needs. It is +# `restricted-v2` with `runAsUser: MustRunAsNonRoot` and `fsGroup: RunAsAny`, +# which admits the driver's explicit non-root UIDs (restricted-v2 rejects them +# because MustRunAsRange only allows UIDs inside the namespace's +# openshift.io/sa.scc.uid-range annotation). It keeps requiredDropCapabilities +# ALL, allowPrivilegeEscalation false, no privileged containers, no host +# namespaces, and seccomp runtime/default. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "openshell.fullname" . }}-sandbox-nonroot-scc + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: sandbox +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - nonroot-v2 + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "openshell.fullname" . }}-sandbox-nonroot-scc + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: sandbox +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "openshell.fullname" . }}-sandbox-nonroot-scc +subjects: + - kind: ServiceAccount + name: {{ include "openshell.sandboxServiceAccountName" . }} + namespace: {{ include "openshell.sandboxNamespace" . }} +{{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index afecada9b6..bb832deab2 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -122,3 +122,35 @@ tests: apiGroups: [""] resources: ["secrets"] any: true + + - it: grants proxy-pod Deployment RBAC without cluster-wide list/watch (managed) + set: + server.drivers.kubernetes.workspaceMode: managed + supervisor.topology: proxy-pod + asserts: + # create/get/patch only: readiness watch (list/watch) is shared-mode only, + # via the namespaced Role, to avoid cluster-wide Deployment enumeration. + - contains: + path: rules + content: + apiGroups: + - apps + resources: + - deployments + verbs: + - create + - get + - patch + - notContains: + path: rules + content: + apiGroups: + - apps + resources: + - deployments + verbs: + - create + - get + - list + - patch + - watch diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 1380cb18c5..e9e1942e1c 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -232,6 +232,18 @@ tests: path: data["gateway.toml"] pattern: 'supervisor[_]topology\s*=' + - it: renders proxy-pod supervisor topology under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + supervisor.topology: proxy-pod + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"proxy-pod"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'supervisor[_]topology\s*=' + - it: renders proxy uid under [openshell.drivers.kubernetes.sidecar] template: templates/gateway-config.yaml set: @@ -241,6 +253,40 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?proxy_uid\s*=\s*2200' + - it: renders proxy uid under [openshell.drivers.kubernetes.proxy_pod] + template: templates/gateway-config.yaml + set: + supervisor.proxyPod.proxyUid: 2300 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.proxy_pod\].*?proxy_uid\s*=\s*2300' + + - it: renders proxy pod affinity under [openshell.drivers.kubernetes.proxy_pod] + template: templates/gateway-config.yaml + set: + supervisor.proxyPod.affinity: preferred + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.proxy_pod\].*?affinity\s*=\s*"preferred"' + + - it: renders retain_companion_management from retainCompanionRbac + template: templates/gateway-config.yaml + set: + supervisor.proxyPod.retainCompanionRbac: true + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.proxy_pod\].*?retain_companion_management\s*=\s*true' + + - it: defaults retain_companion_management to false + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.proxy_pod\].*?retain_companion_management\s*=\s*false' + - it: renders process binary aware network policy under [openshell.drivers.kubernetes.sidecar] template: templates/gateway-config.yaml set: @@ -732,3 +778,59 @@ tests: asserts: - failedTemplate: errorMessage: "certManager.serverIssuerRef.name is set but certManager.enabled is false \u2014 the external server certificate, its Secret mount, and the gateway TLS configuration all require cert-manager to be enabled. Set certManager.enabled=true or remove certManager.serverIssuerRef.name." + + - it: omits proxy-pod dns_peers when none are configured, keeping driver defaults + template: templates/gateway-config.yaml + set: + supervisor.topology: proxy-pod + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'proxy_pod\.dns_peers' + + - it: renders OpenShift cluster DNS peers for proxy-pod topology + template: templates/gateway-config.yaml + set: + supervisor.topology: proxy-pod + supervisor.proxyPod.dnsPeers: + - namespaceLabels: + kubernetes.io/metadata.name: openshift-dns + podLabels: + dns.operator.openshift.io/daemonset-dns: default + port: 5353 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'port = 5353' + - matchRegex: + path: data["gateway.toml"] + pattern: '\[\[openshell\.drivers\.kubernetes\.proxy_pod\.dns_peers\]\]' + - matchRegex: + path: data["gateway.toml"] + pattern: 'namespace_labels = \{ "kubernetes\.io/metadata\.name" = "openshift-dns" \}' + - matchRegex: + path: data["gateway.toml"] + pattern: 'pod_labels = \{ "dns\.operator\.openshift\.io/daemonset-dns" = "default" \}' + + - it: renders multiple proxy-pod dns peers as repeated array-of-tables entries + template: templates/gateway-config.yaml + set: + supervisor.topology: proxy-pod + supervisor.proxyPod.dnsPeers: + - namespaceLabels: + kubernetes.io/metadata.name: openshift-dns + - namespaceLabels: + kubernetes.io/metadata.name: kube-system + podLabels: + k8s-app: node-local-dns + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'port = 53' + + - matchRegex: + path: data["gateway.toml"] + pattern: '(?s)dns_peers\]\].*dns_peers\]\]' + - matchRegex: + path: data["gateway.toml"] + pattern: 'pod_labels = \{ "k8s-app" = "node-local-dns" \}' diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 864e3a8512..4059b77cfb 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -57,6 +57,147 @@ tests: path: metadata.namespace value: other-ns + - it: grants only pod get for sandbox token bootstrap + template: templates/role.yaml + asserts: + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - pods + verbs: + - get + + - it: grants sandbox RBAC for proxy-pod supervisor Deployments + template: templates/role.yaml + set: + supervisor.topology: proxy-pod + asserts: + # No delete: companions are garbage-collected with the Sandbox CR. + # list + watch back the supervisor Deployment readiness watch. + - contains: + path: rules + content: + apiGroups: + - apps + resources: + - deployments + verbs: + - create + - get + - list + - patch + - watch + + - it: grants ReplicaSet get for proxy-pod supervisor token bootstrap + template: templates/role.yaml + set: + supervisor.topology: proxy-pod + asserts: + - contains: + path: rules + content: + apiGroups: + - apps + resources: + - replicasets + verbs: + - get + + - it: grants proxy-pod Service Secret and NetworkPolicy RBAC only in proxy-pod mode + template: templates/role.yaml + set: + supervisor.topology: proxy-pod + asserts: + # Services: create + get (get for 409-verify), no delete (GC-owned). + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - services + verbs: + - create + - get + # Secrets: create only — the gateway never reads or deletes Secret contents. + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - secrets + verbs: + - create + # NetworkPolicies: the gateway-managed egress fence needs delete + list + # for ordered teardown and orphan reaping. + - contains: + path: rules + content: + apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + + - it: omits proxy-pod RBAC in the default combined topology + template: templates/role.yaml + asserts: + - notContains: + path: rules + content: + apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - notContains: + path: rules + content: + apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - notContains: + path: rules + content: + apiGroups: + - "" + resources: + - services + - secrets + verbs: + - create + - delete + - get + - list + - watch + - notContains: + path: rules + content: + apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - watch + - it: uses explicit sandboxNamespace for sandbox RoleBinding template: templates/rolebinding.yaml set: diff --git a/deploy/helm/openshell/tests/sandbox_scc_test.yaml b/deploy/helm/openshell/tests/sandbox_scc_test.yaml new file mode 100644 index 0000000000..a6017e75e7 --- /dev/null +++ b/deploy/helm/openshell/tests/sandbox_scc_test.yaml @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: OpenShift sandbox SCC grant +templates: + - templates/sandbox-scc.yaml +tests: + - it: renders nothing by default so non-OpenShift installs never reference OpenShift APIs + asserts: + - hasDocuments: + count: 0 + + - it: grants use of the built-in nonroot-v2 SCC when enabled + set: + sandboxServiceAccount.openshift.nonrootSCC: true + documentIndex: 0 + asserts: + - isKind: + of: ClusterRole + - equal: + path: rules[0].resourceNames[0] + value: nonroot-v2 + - equal: + path: rules[0].verbs[0] + value: use + + - it: binds the SCC grant to the sandbox ServiceAccount + set: + sandboxServiceAccount.openshift.nonrootSCC: true + documentIndex: 1 + asserts: + - isKind: + of: ClusterRoleBinding + - equal: + path: subjects[0].kind + value: ServiceAccount + - equal: + path: subjects[0].name + value: RELEASE-NAME-openshell-sandbox + + - it: creates no SecurityContextConstraints object of its own + set: + sandboxServiceAccount.openshift.nonrootSCC: true + asserts: + - hasDocuments: + count: 2 diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 8f5b6fa51a..cf475170c4 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -48,6 +48,8 @@ supervisor: # "combined" runs the current single supervisor container in the agent pod. # "sidecar" runs network enforcement in a dedicated sidecar and the process # supervisor as a low-capability wrapper in the agent container. + # "proxy-pod" runs network enforcement in a separate supervisor Deployment and + # restricts the agent pod to that supervisor through NetworkPolicy. topology: "combined" sidecar: # -- UID for relaxed long-running network sidecars in sidecar topology. @@ -61,6 +63,48 @@ supervisor: # inspection capabilities, and enforces endpoint/L7 policy without matching # policy.binaries. processBinaryAwareNetworkPolicy: true + proxyPod: + # -- Render the proxy-pod companion, fence, and pod-inspection RBAC even when + # supervisor.topology is not proxy-pod, and tell the gateway to keep running + # background upkeep (periodic companion reconciliation and the shared-mode + # supervisor Deployment readiness watch) for those sandboxes. Set this true + # as a migration mode when switching a gateway away from proxy-pod while + # proxy-pod sandboxes still exist: the driver keeps managing them by their + # persisted creation-time topology, and without this flag their RBAC and + # upkeep would stop, breaking readiness, stop/start, repair, and safe fence + # cleanup. Renders `proxy_pod.retain_companion_management` in gateway.toml. + # Leave it true until all proxy-pod sandboxes have been deleted, then remove it. + retainCompanionRbac: false + # -- UID for the network supervisor in proxy-pod topology. The configured + # UID must not match the sandbox UID. + proxyUid: 1337 + # -- Same-node scheduling relationship between the workload pod and its + # paired proxy supervisor: disabled, preferred, or required. + affinity: disabled + # -- Cluster DNS peers permitted by the proxy-pod agent egress + # NetworkPolicy. Each entry sets `namespaceLabels`, `podLabels`, or both. + # Empty uses the upstream kube-system/kube-dns and kube-system/coredns + # conventions, which do NOT match OpenShift (cluster DNS runs in + # `openshift-dns`) or NodeLocal DNSCache. An agent pod with no matching DNS + # peer cannot resolve its own paired supervisor Service. + # + # `port` is the DNS *pod* port, not the Service port: egress rules with a + # podSelector match after Service address translation. Upstream CoreDNS + # listens on 53; OpenShift's dns-default listens on 5353 and maps 53 to it. + # For OpenShift: + # dnsPeers: + # - namespaceLabels: + # kubernetes.io/metadata.name: openshift-dns + # podLabels: + # dns.operator.openshift.io/daemonset-dns: default + # port: 5353 + dnsPeers: [] + # -- Gateway peers the proxy-pod agent egress NetworkPolicy permits, so the + # in-pod process supervisor can reach the gateway for its session. Same shape + # as dnsPeers (namespaceLabels/podLabels + port). Empty (default) auto-renders + # a peer matching THIS chart's own gateway pods on the service port; override + # only for an external or differently-labeled gateway. + gatewayPeers: [] # -- Operator-owned corporate forward proxy for policy-approved TLS egress # from Kubernetes sandboxes. The workload cannot select or override it. @@ -101,6 +145,20 @@ sandboxServiceAccount: annotations: {} # -- Existing service account name for sandbox pods when sandboxServiceAccount.create is false. name: "" + openshift: + # -- Grant the built-in OpenShift `nonroot-v2` SCC to the sandbox + # ServiceAccount. Required on OpenShift for "proxy-pod" topology: the + # driver assigns explicit non-root UIDs, which `restricted-v2` rejects + # because it only admits UIDs inside the namespace's + # openshift.io/sa.scc.uid-range annotation. No custom SCC is created — + # `nonroot-v2` ships with OpenShift and already permits exactly what this + # topology needs, keeping drop-ALL capabilities, no privilege escalation, + # and no host namespaces. Creates a ClusterRole + ClusterRoleBinding. + # Supported only with server.drivers.kubernetes.workspaceMode=shared: the + # ClusterRoleBinding is scoped to the static sandbox namespace, so it does + # not reach the dynamically created workspace namespaces used by managed and + # operator modes. Enabling it with a non-shared mode fails the Helm render. + nonrootSCC: false # Namespace-scoped resources needed to run sandboxes. Disable this when the # gateway and workspace prerequisites are managed as separate Helm releases diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index 43e7d0338b..8a61a28fed 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -9,7 +9,7 @@ position: 6 --- -The OpenShift install path is experimental. It currently requires running sandbox pods under the `privileged` SCC and installing the gateway with TLS disabled. Use only for evaluation on a private network. +The OpenShift install path is experimental. The default `combined` topology runs sandbox pods under the `privileged` SCC, and this guide installs the gateway with TLS disabled. Use only for evaluation on a private network. Only the `proxy-pod` topology runs sandbox pods under the built-in `nonroot-v2` SCC (set `sandboxServiceAccount.openshift.nonrootSCC=true`); the `sidecar` and `cni-sidecar` topologies still need a custom SCC (their network init container and root sidecar require added capabilities) — see [Topology](/kubernetes/topology). OpenShift's [Security Context Constraints](https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html) reject the chart's default pod security settings. Installing on OpenShift requires precreating the namespace, granting the `privileged` SCC to the sandbox service account, and overriding a few chart values so the cluster admission controller can assign UIDs and FS groups itself. @@ -39,12 +39,14 @@ oc create ns openshell ## Grant the privileged SCC to sandbox pods -Sandbox pods run under the `openshell-sandbox` service account in the `openshell` namespace and require the `privileged` SCC: +Sandbox pods run under the `openshell-sandbox` service account in the `openshell` namespace. The default `combined` topology requires the `privileged` SCC: ```shell oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell ``` +For the `proxy-pod` topology, skip this grant and instead set `sandboxServiceAccount.openshift.nonrootSCC=true` when installing the chart, which binds the built-in `nonroot-v2` SCC. The `sidecar` and `cni-sidecar` topologies are *not* covered by `nonroot-v2` — their UID-0 network init container and default root sidecar need added capabilities, so they require a custom SCC. See [Topology](/kubernetes/topology) for the per-topology privilege model. + ## Install the chart with OpenShift overrides ```shell @@ -61,6 +63,36 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ | `server.disableTls=true` | Runs the gateway over plaintext HTTP for simpler evaluation. | | `podSecurityContext.fsGroup=null` / `securityContext.runAsUser=null` | Clear the chart's hardcoded UID and fsGroup so OpenShift's SCC admission can assign them. | +The command above installs the default `combined` topology, which needs the +`privileged` SCC granted above. + +### Proxy-pod topology on OpenShift + +To run the least-privilege `proxy-pod` topology instead, skip the `privileged` +grant and install with the proxy-pod overrides. This binds the built-in +`nonroot-v2` SCC and declares OpenShift's cluster DNS peers, which the agent pod +needs to resolve its paired supervisor Service (OpenShift runs DNS in +`openshift-dns` on container port `5353`, not `kube-system`/`53`, so the default +peers do not match and are [required](/kubernetes/topology) to be set): + +```shell +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set server.disableTls=true \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null \ + --set supervisor.topology=proxy-pod \ + --set sandboxServiceAccount.openshift.nonrootSCC=true \ + --set 'supervisor.proxyPod.dnsPeers[0].namespaceLabels.kubernetes\.io/metadata\.name=openshift-dns' \ + --set 'supervisor.proxyPod.dnsPeers[0].podLabels.dns\.operator\.openshift\.io/daemonset-dns=default' \ + --set supervisor.proxyPod.dnsPeers[0].port=5353 +``` + +Proxy-pod also requires a cluster CNI that enforces Kubernetes NetworkPolicies +(OpenShift's default OVN-Kubernetes does). See [Topology](/kubernetes/topology) +for the full proxy-pod contract and tradeoffs. + ## Wait for the gateway to be ready ```shell diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index fb7881af2d..c16aaeac26 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -208,6 +208,8 @@ The most commonly changed values are: | `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). | | `supervisor.sidecar.proxyUid` | Non-root UID used when sidecar process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0 instead. The configured UID must not match the sandbox UID. | | `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | +| `supervisor.proxyPod.proxyUid` | Non-root UID used by the proxy-pod network supervisor. The UID must not match the sandbox UID. | +| `supervisor.proxyPod.affinity` | Same-node placement policy for workload and proxy pods: `disabled` (default), `preferred`, or `required`. | Use a values file for repeatable deployments: @@ -291,6 +293,10 @@ The namespaced Role covers sandbox lifecycle and identity: | `agents.x-k8s.io` | `sandboxes`, `sandboxes/status` | create, delete, get, list, patch, update, watch | | `""` | `events` | get, list, watch | | `""` | `pods` | get | +| `apps` | `deployments` | create, delete, get, list, watch | +| `apps` | `replicasets` | get | +| `""` | `services`, `secrets` | create, delete, get, list, watch | +| `networking.k8s.io` | `networkpolicies` | create, delete, get, list, watch | The ClusterRole grants node inspection and token validation: @@ -321,7 +327,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency ## Next Steps -- To choose between combined and sidecar sandbox pods, refer to [Topology](/kubernetes/topology). +- To choose between combined, sidecar, and proxy-pod sandbox topology, refer to [Topology](/kubernetes/topology). - To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress). - To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 869fc07f1b..003ada1088 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -3,14 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 title: "Kubernetes Sandbox Topology" sidebar-title: "Topology" -description: "Choose between combined and sidecar supervisor topology for Kubernetes sandbox pods." +description: "Choose between combined, sidecar, and proxy-pod topology for Kubernetes sandbox pods." keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Sidecar, Network Policy, RuntimeClass" position: 2 --- -Kubernetes sandbox pods can run the OpenShell supervisor in `combined` or -`sidecar` topology. Choose the topology based on which controls you need inside -the pod and how much privilege your cluster allows on the agent container. +Kubernetes sandbox pods can run the OpenShell supervisor in `combined`, +`sidecar`, or `proxy-pod` topology. Choose the topology based on which controls +you need inside the pod, how much privilege your cluster allows on the agent +container, and whether the cluster enforces Kubernetes NetworkPolicies. ## Choose a Topology @@ -22,6 +23,7 @@ lower-privilege agent container. |---|---|---| | `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | | `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Privilege-dropping and supervisor mount isolation do not run in the agent container. | +| `proxy-pod` | You need the network proxy (and its credentials) out of the agent pod with no in-pod network privilege, keep the full interactive feature set, and your cluster enforces Kubernetes NetworkPolicies. | Two pods per sandbox; requires a NetworkPolicy-enforcing CNI; the agent pod holds a scoped (process-kind) gateway credential. | ## Privilege Model @@ -33,6 +35,8 @@ The long-running container permissions differ by topology: | `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | | `sidecar` | Network supervisor sidecar, binary-aware mode (default) | `0:sandbox_gid` | `false` | Drops `ALL`; adds `SYS_PTRACE` and `DAC_READ_SEARCH` | Root sidecar inspects cross-UID workload `/proc` entries. The nftables fence exempts UID 0, so do not inject other root containers into these pods. | | `sidecar` | Network supervisor sidecar, endpoint/L7-only mode | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Non-root sidecar enforces endpoint and L7 policy without matching `policy.binaries`. | +| `proxy-pod` | Agent pod container, process supervisor + workload | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Runs the process supervisor (`--mode=process`); no added capabilities and no `NET_ADMIN` (network enforcement is external). | +| `proxy-pod` | Supervisor pod container, network proxy only | `proxyPod.proxyUid:sandbox_gid` | `false` | Drops `ALL` | Long-running proxy runs outside the agent pod without added capabilities. | Short-lived setup containers still have the permissions needed to prepare the pod: @@ -41,6 +45,9 @@ pod: |---|---|---|---|---|---| | `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. | | `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. | +| `combined` / `sidecar` | Workspace persistence init container | `0` | Not set | Not set | Seeds the default workspace PVC while preserving the existing topology behavior. | +| `proxy-pod` | Proxy CA install init container | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Copies proxy CA material into the agent pod TLS volume with a read-only root filesystem. | +| `proxy-pod` | Workspace persistence init container | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Seeds the default workspace PVC without granting UID 0 to the agent pod. | ## Combined Topology @@ -158,6 +165,105 @@ Sidecar pods use `shareProcessNamespace: true` so the network sidecar can resolve workload process and binary identity through `/proc/`. +## Proxy-Pod Topology + +Proxy-pod topology moves only the **network proxy** — L4/L7 enforcement, gateway +forwarding, and credential injection — into a separate per-sandbox supervisor +Deployment. The **process supervisor stays in the agent pod** alongside the +workload (running as `--mode=process`), so filesystem/process policy, SSH, +`exec`, upload/download, sync, and provider injection keep working. The agent +pod reaches the network proxy through a per-sandbox headless Service, and egress +is confined by Kubernetes NetworkPolicy rather than pod-local nftables — so the +agent pod needs no `NET_ADMIN`, no privileged init container, and no node +component. + +The in-pod process supervisor holds a **scoped, process-kind gateway +credential**: it can serve its sandbox's relays, push logs, read policy, and +refresh its token, but it cannot read provider secrets or mint upstream +credentials — those stay only in the separate proxy pod. Under a VM-based +RuntimeClass (Kata), the workload and the credential-bearing proxy are in +separate pods and therefore separate VMs/kernels. + +```mermaid +flowchart TB + Sandbox["agents.x-k8s.io Sandbox"] + + subgraph Namespace["Sandbox namespace"] + subgraph AgentPod["Agent pod"] + ProcSup["process supervisor
--mode=process"] + Workload["Sandbox workload"] + end + + SupervisorDeployment["Supervisor Deployment
1 replica"] + subgraph SupervisorPod["Supervisor pod"] + NetworkProxy["network supervisor proxy
proxyUid"] + end + + Service["Headless Service"] + ProxyCA["Proxy CA Secret"] + AgentEgressPolicy["NetworkPolicy
agent egress to supervisor + DNS"] + SupervisorIngressPolicy["NetworkPolicy
supervisor ingress from paired agent"] + end + + Gateway["OpenShell Gateway"] + External["External services"] + + Sandbox --> AgentPod + Sandbox --> SupervisorDeployment + SupervisorDeployment --> SupervisorPod + AgentPod -->|"egress allowed by NetworkPolicy"| Service + Service --> NetworkProxy + NetworkProxy -->|"gateway forwarding"| Gateway + NetworkProxy -->|"policy-enforced egress"| External + ProxyCA -. mounted .- AgentPod + ProxyCA -. mounted .- SupervisorPod + AgentEgressPolicy -. selects .- AgentPod + SupervisorIngressPolicy -. selects .- SupervisorPod +``` + +OpenShell creates these per-sandbox resources: + +- Agent pod labeled `openshell.ai/sandbox-role=agent`. +- Supervisor Deployment with one pod labeled `openshell.ai/sandbox-role=supervisor`. +- Headless Service for the supervisor pod. +- Proxy CA Secret shared through mounts. +- NetworkPolicy that limits agent egress to the supervisor pod and DNS. +- NetworkPolicy that accepts supervisor ingress only from the paired agent pod. + +The supervisor Deployment has a controlling `Sandbox` ownerReference so +Kubernetes garbage collection removes it when the sandbox is deleted. The +Deployment recreates the supervisor pod if the pod is deleted independently. + +Same-node scheduling is disabled by default. Set `proxy_pod.affinity` (or Helm +`supervisor.proxyPod.affinity`) to `preferred` for soft same-node placement or +`required` for hard same-node placement. Both modes match the paired supervisor +on `kubernetes.io/hostname` and preserve any affinity supplied by the workload. + +The agent container runs the OpenShell process supervisor, which launches the +workload the same way `combined` and `sidecar` do — from the sandbox spec, or +the command passed to `openshell sandbox create -- `. Because the +supervisor is the container entrypoint, `containers.agent.command`/`args` +overrides are rejected (as in the other topologies). + +The workload's children egress through the paired network proxy: the driver +injects standard proxy variables and the proxy CA trust into the agent +container, and the agent-egress NetworkPolicy allows the workload to reach only +that proxy (plus DNS). The in-pod process supervisor reaches the gateway +directly for its own session (relays, policy, log push) — the agent-egress +NetworkPolicy permits that via `proxy_pod.gateway_peers` (the Helm chart renders +it from the gateway's own pod labels by default). Network policy (L4/L7) is +enforced by the proxy pod; filesystem/process policy, SSH/`connect`, `exec`, +upload/download, sync, and provider injection are enforced by the in-pod process +supervisor. + + +Proxy-pod topology requires NetworkPolicy enforcement to work as OpenShell +expects. The target cluster must have a policy-enforcing CNI or equivalent +NetworkPolicy controller before deploying this topology. Without enforcement, +the agent pod is not forced through its paired supervisor proxy, so the +agent-to-supervisor isolation policy is only declarative. + + ## Credential Exposure Sidecar topology keeps gateway credentials in the network sidecar. The agent @@ -182,6 +288,18 @@ of the already-running workload entrypoint. Use `combined` topology when you need the full single-supervisor enforcement path; use additional runtime isolation when you need a stronger container boundary around sidecar workloads. +Proxy-pod topology keeps the process supervisor in the agent pod, so that pod +holds a gateway credential — but a **scoped, process-kind** one. The gateway +mints it (keyed on the pod's `sandbox-role=agent` label) so the agent pod can +serve its own relays, push logs, read policy, and refresh its token, but is +denied the network-supervisor RPCs that read provider secrets +(`GetSandboxProviderEnvironment`) or mint upstream credentials +(`ExchangeProviderSubjectToken`, `GetInferenceBundle`). Those run only in the +separate proxy pod, which holds the full-authority credential and all provider +secrets. A compromised agent pod is therefore bounded to its own sandbox's +control plane and never reaches provider secrets. Network egress is isolated by +the per-sandbox NetworkPolicies described above. + ## RuntimeClass Isolation Sidecar topology has been validated with Kata Containers. It does not currently @@ -195,6 +313,13 @@ mount-isolation controls that sidecar mode relaxes. Use them as an additional workload boundary, not as a replacement for the combined topology's full supervisor controls. +Proxy-pod topology has been tested with Kata Containers and gVisor and is +functional when the cluster enforces NetworkPolicies. Under Kata, the workload +(agent pod) and the credential-bearing network proxy (supervisor pod) run in +separate VMs with separate kernels, so a workload-VM kernel escape does not by +itself reach the proxy or its provider secrets — an isolation boundary unique to +proxy-pod among the topologies. + You can set a default runtime class in the Kubernetes driver configuration or override it per sandbox with driver config: @@ -204,9 +329,10 @@ openshell sandbox create \ -- claude ``` -## Enable Sidecar Mode +## Enable Alternate Topologies -For direct gateway TOML configuration, set the Kubernetes driver fields: +For direct gateway TOML configuration, set the Kubernetes driver fields for +sidecar mode: ```toml [openshell.drivers.kubernetes] @@ -222,7 +348,22 @@ runs the sidecar as UID 0 instead. The network init container exempts the effective sidecar UID from proxy redirection so the sidecar can reach the gateway. -When the Helm chart renders `gateway.toml`, set the equivalent chart values: +Set `topology="proxy-pod"` to use proxy-pod mode: + +```toml +[openshell.drivers.kubernetes] +topology = "proxy-pod" + +[openshell.drivers.kubernetes.proxy_pod] +proxy_uid = 1337 +affinity = "disabled" # disabled | preferred | required +``` + +`proxy_pod.proxy_uid` must be a non-root UID and must not match the sandbox UID. +It is used by the proxy supervisor pod created by the Deployment. + +When the Helm chart renders `gateway.toml`, set the equivalent chart values for +sidecar mode: ```yaml supervisor: @@ -232,6 +373,47 @@ supervisor: processBinaryAwareNetworkPolicy: true ``` +Set `supervisor.topology=proxy-pod` to use proxy-pod mode: + +```yaml +supervisor: + topology: proxy-pod + proxyPod: + proxyUid: 1337 + affinity: disabled +``` + +On OpenShift (and any cluster whose DNS is not the upstream +`kube-system`/`kube-dns` convention) you must also declare the cluster DNS peer +and grant the built-in `nonroot-v2` SCC — otherwise the agent pod cannot resolve +its supervisor Service and the supervisor pod is inadmissible: + +```yaml +supervisor: + topology: proxy-pod + proxyPod: + proxyUid: 1337 + affinity: disabled + # OpenShift runs cluster DNS in openshift-dns on container port 5353, not + # kube-system/53. Without this the agent egress NetworkPolicy denies DNS. + dnsPeers: + - namespaceLabels: + kubernetes.io/metadata.name: openshift-dns + podLabels: + dns.operator.openshift.io/daemonset-dns: default + port: 5353 +sandboxServiceAccount: + openshift: + # restricted-v2 rejects the driver's explicit non-root UIDs; nonroot-v2 + # accepts them. Supported only with shared workspace mode. + nonrootSCC: true +``` + +Changing `supervisor.topology` away from proxy-pod while proxy-pod sandboxes +still exist removes the RBAC those sandboxes need for lifecycle and cleanup. Set +`supervisor.proxyPod.retainCompanionRbac=true` during such a migration and leave +it set until every proxy-pod sandbox has been deleted. + Leave `topology` unset, or set it to `combined`, to keep the original single-container supervisor path. For Helm installs, leave `supervisor.topology` unset or set it to `combined`. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 7553e7deb4..01147fcd36 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -494,6 +494,8 @@ supervisor_sideload_method = "image-volume" # "combined" runs the existing single supervisor container with full process, # filesystem, and network enforcement in the agent container. "sidecar" moves # pod-level network enforcement and gateway session handling into a network sidecar. +# "proxy-pod" moves network enforcement and gateway forwarding into a separate +# supervisor Deployment and uses NetworkPolicy to force agent egress through it. topology = "combined" # Optional corporate HTTP forward proxy for policy-approved TLS egress. The # sandbox workload cannot select or override these settings. Only http:// proxy @@ -574,6 +576,36 @@ proxy_uid = 1337 # inspection capabilities, and enforce endpoint/L7 policy without matching # policy.binaries. process_binary_aware_network_policy = true + +[openshell.drivers.kubernetes.proxy_pod] +# UID used by the network supervisor pod. It must not match the sandbox UID. +proxy_uid = 1337 +# Same-node workload/supervisor placement: disabled, preferred, or required. +affinity = "disabled" +# Keep running background upkeep (periodic companion reconciliation and the +# shared-mode supervisor Deployment readiness watch) for existing proxy-pod +# sandboxes after `topology` is switched away from proxy-pod. Set true during a +# migration (Helm renders it from supervisor.proxyPod.retainCompanionRbac) and +# leave it set until every proxy-pod sandbox is deleted. Default false. +retain_companion_management = false +# Gateway peers the agent-egress NetworkPolicy permits, so the in-pod process +# supervisor can reach the gateway for its session. Same shape as dns_peers +# (selectors + the gateway's TCP port). Required for proxy-pod; the Helm chart +# renders it from the gateway's own pod labels by default. +[[openshell.drivers.kubernetes.proxy_pod.gateway_peers]] +port = 8080 +namespace_labels = { "kubernetes.io/metadata.name" = "openshell" } +pod_labels = { "app.kubernetes.io/name" = "openshell" } +# Cluster DNS peers the agent-egress NetworkPolicy allows. Empty (default) uses +# the upstream kube-system/kube-dns and kube-system/coredns conventions, which +# do NOT match OpenShift (cluster DNS runs in openshift-dns) or NodeLocal +# DNSCache. Each entry sets namespace_labels, pod_labels, or both, plus the DNS +# pod port (OpenShift's dns-default listens on 5353). An agent pod with no +# matching DNS peer cannot resolve its own paired supervisor Service. +[[openshell.drivers.kubernetes.proxy_pod.dns_peers]] +port = 5353 +namespace_labels = { "kubernetes.io/metadata.name" = "openshift-dns" } +pod_labels = { "dns.operator.openshift.io/daemonset-dns" = "default" } ``` In managed workspace mode, the Kubernetes driver copies each explicitly named diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 987e66b0d9..fc0f47327b 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -138,6 +138,20 @@ It overrides the gateway's configured default runtime class for that sandbox, while a typed `SandboxTemplate.runtime_class_name` value from the API still takes precedence. +In `proxy-pod` topology the sandbox image runs directly, with no supervisor to +launch a workload, so the container needs an entrypoint that stays running. +Set `containers.agent.command` and `containers.agent.args` when the image's own +entrypoint exits immediately: + +```shell +openshell sandbox create --name batch \ + --driver-config-json '{"kubernetes":{"containers":{"agent":{"command":["python","/app/agent.py"]}}}}' +``` + +These fields apply only to `proxy-pod`. The `combined` and `sidecar` topologies +run the OpenShell supervisor as the container entrypoint, so setting them there +is rejected rather than silently ignored. + Docker and Podman report the address through which their sandboxes can reach the gateway. If the primary listener covers that address, the gateway reuses it and sandbox JWT authentication restricts the supervisor to its callback RPC @@ -405,7 +419,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | -| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | +| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar, or `proxy-pod` to run network enforcement and gateway forwarding in a separate supervisor Deployment with NetworkPolicy isolation. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | | `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. | | `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. Requires `sidecar` topology. | @@ -413,7 +427,9 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret. | | `proxy_connect_by_hostname` | `upstreamProxy.connectByHostname` | Send hostnames rather than validated IPs in CONNECT requests. Use only when proxy ACLs require hostname targets. | | `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | Dedicated UID of at least `1000` used by the relaxed sidecar when process/binary-aware network policy is disabled. It must not match the workload UID. The default binary-aware sidecar runs as UID 0. The network init container exempts the effective sidecar UID from proxy redirection. | +| `proxy_pod.proxy_uid` | `supervisor.proxyPod.proxyUid` | Dedicated UID of at least `1000` used by the network supervisor in `proxy-pod` topology. It must not match the workload UID. | | `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. The default runs the sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. Set false to run as `proxy_uid`, drop both capabilities, and enforce endpoint/L7 policy without matching `policy.binaries`. | +| `proxy_pod.affinity` | `supervisor.proxyPod.affinity` | Configure same-node workload/supervisor placement as `disabled` (default), `preferred`, or `required`. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | @@ -451,6 +467,15 @@ identity mount isolation. Network policy still runs in the sidecar, and sidecar pods set `shareProcessNamespace: true` so the network sidecar can resolve process/binary identity through `/proc/`. +In `proxy-pod` topology, network enforcement runs in a separate non-root +supervisor Deployment with one pod, a headless Service, a proxy CA Secret, and +per-sandbox NetworkPolicies. The Deployment recreates the supervisor pod if it +is deleted. The sandbox container runs its image directly with proxy and CA +environment; it does not mount or execute the supervisor. Filesystem/process +policy, binary detection, SSH/exec, upload/download, sync, and provider +environment injection are therefore unavailable. Use `combined` or `sidecar` +when those process-supervisor features are required. + The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. Stop patches the existing resource rather than deleting it. For `v1beta1`, diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 18556d0f7b..a7e4bdaf20 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -32,6 +32,7 @@ e2e-kubernetes = ["e2e"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] +e2e-kubernetes-proxy-pod = ["e2e-kubernetes"] e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] e2e-oidc-pkce = [] @@ -128,6 +129,11 @@ name = "kubernetes_corporate_proxy" path = "tests/kubernetes_corporate_proxy.rs" required-features = ["e2e-kubernetes"] +[[test]] +name = "proxy_pod" +path = "tests/proxy_pod.rs" +required-features = ["e2e-kubernetes-proxy-pod"] + [[test]] name = "credential_drivers" path = "tests/credential_drivers.rs" diff --git a/e2e/rust/tests/live_policy_update.rs b/e2e/rust/tests/live_policy_update.rs index 7a1e12923a..555fa01e7d 100644 --- a/e2e/rust/tests/live_policy_update.rs +++ b/e2e/rust/tests/live_policy_update.rs @@ -517,8 +517,10 @@ async fn live_policy_update_from_empty_network_policies() { /// /// NOTE: This exercises the Docker-backed supervisor built from this branch. /// The exact `policy list` status wording ("Loaded"/"Superseded") may differ by -/// CLI version; the assertions below key on the effective version reaching 2 and -/// no revision remaining `Pending` once the acknowledgement lands. +/// CLI version; the assertions below key on the effective version reaching at +/// least 2 and no revision remaining `Pending` once the acknowledgement lands. +/// Multi-supervisor topologies may create a later revision while their network +/// and process leaves reconcile their runtime-specific policy views. #[tokio::test] async fn initial_sparse_policy_is_acknowledged_as_loaded() { // Repo-relative path to the sparse network-only policy fixture. @@ -543,7 +545,8 @@ async fn initial_sparse_policy_is_acknowledged_as_loaded() { // The enriched revision (2) is synced during startup; the acknowledgement // (LOADED) is delivered by the supervisor's poll loop shortly after Ready. - // Poll until the effective policy is version 2 and no revision is Pending. + // Poll until the effective policy is at least version 2 and no revision is + // Pending. A proxy-pod network supervisor may legitimately advance it again. let mut acknowledged = false; let mut last_list = String::new(); for _ in 0..30 { @@ -554,7 +557,7 @@ async fn initial_sparse_policy_is_acknowledged_as_loaded() { last_list = list.output.clone(); let pending = list.output.to_lowercase().contains("pending"); - if version == Some(2) && list.success && !pending { + if version.is_some_and(|version| version >= 2) && list.success && !pending { acknowledged = true; break; } @@ -563,7 +566,7 @@ async fn initial_sparse_policy_is_acknowledged_as_loaded() { assert!( acknowledged, - "enriched initial policy should reach revision 2 with no Pending revision.\n\ + "enriched initial policy should reach at least revision 2 with no Pending revision.\n\ last `policy list` output:\n{last_list}" ); diff --git a/e2e/rust/tests/proxy_pod.rs b/e2e/rust/tests/proxy_pod.rs new file mode 100644 index 0000000000..3e16c2d83d --- /dev/null +++ b/e2e/rust/tests/proxy_pod.rs @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-proxy-pod")] + +//! Capability coverage for the Kubernetes `proxy-pod` topology. +//! +//! `proxy-pod` keeps the process supervisor in the agent pod and moves only the +//! network proxy into a separate per-sandbox pod. Because the supervisor still +//! runs beside the workload, the relay-backed operations that a sandbox is +//! expected to offer — `exec` and file transfer — work exactly as they do in the +//! combined topology. This suite asserts that recovered behavior: +//! +//! - a sandbox reaches `Ready` with the process supervisor running non-root in +//! the agent pod (OpenShift `nonroot-v2` SCC), and +//! - `exec` and `upload`/`download` round-trip through the in-pod supervisor's +//! own gateway session. +//! +//! Two boundaries are covered elsewhere rather than here, because asserting them +//! reliably needs cluster policy enforcement that a generic e2e environment does +//! not guarantee: +//! +//! - the scoped process-kind credential (the agent pod's token is denied +//! provider/inference RPCs) is unit-tested in `openshell-server` and validated +//! on a policy-enforcing cluster via the gateway's `PERMISSION_DENIED` logs; +//! - the NetworkPolicy egress fence (workload egress reaches only the proxy pod) +//! is unit-tested in `openshell-driver-kubernetes` (generated policy shape) and +//! validated on a policy-enforcing cluster (direct egress blocked, proxied L7 +//! allow/deny enforced). + +use std::process::Stdio; +use std::time::Duration; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; + +/// Delete a sandbox by name, ignoring failures (best-effort cleanup). +async fn delete_sandbox(name: &str) { + let mut cmd = openshell_cmd(); + cmd.arg("sandbox") + .arg("delete") + .arg(name) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let _ = cmd.output().await; +} + +/// A `proxy-pod` sandbox reaches `Ready` and its in-pod supervisor serves the +/// relay-backed operations (`exec`, file transfer) that the network-only design +/// could not. +#[tokio::test] +async fn proxy_pod_runs_workload_and_serves_relays() { + let name = "e2e-proxy-pod"; + // Best-effort cleanup from a previous interrupted run. + delete_sandbox(name).await; + + // Plain create: no `containers.agent.command` override — the in-pod process + // supervisor launches the image's default workload and serves relays. Detach + // so the test drives readiness and relays explicitly. + let mut create = openshell_cmd(); + create + .arg("sandbox") + .arg("create") + .arg("--name") + .arg(name) + .arg("--detach") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let create_out = tokio::time::timeout(Duration::from_secs(300), create.output()) + .await + .expect("sandbox create timed out") + .expect("failed to spawn openshell"); + let create_text = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&create_out.stdout), + String::from_utf8_lossy(&create_out.stderr), + )); + assert!( + create_out.status.success(), + "proxy-pod create should succeed:\n{create_text}", + ); + + // The sandbox should be present and reach Ready. + let mut ready = false; + let mut last_list = String::new(); + for _ in 0..30 { + let mut list = openshell_cmd(); + list.arg("sandbox") + .arg("list") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let out = list.output().await.expect("failed to run sandbox list"); + last_list = strip_ansi(&String::from_utf8_lossy(&out.stdout)); + if last_list + .lines() + .any(|line| line.contains(name) && line.contains("Ready")) + { + ready = true; + break; + } + tokio::time::sleep(Duration::from_secs(2)).await; + } + assert!(ready, "proxy-pod sandbox never reached Ready:\n{last_list}"); + + // exec is relay-backed: it proves the in-pod supervisor accepts sessions and + // streams output. The network-only design rejected this. + let marker = "proxypod-exec-ok"; + let mut exec = openshell_cmd(); + exec.arg("sandbox") + .arg("exec") + .arg("--name") + .arg(name) + .arg("--") + .arg("echo") + .arg(marker) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let exec_out = tokio::time::timeout(Duration::from_secs(60), exec.output()) + .await + .expect("sandbox exec timed out") + .expect("failed to spawn openshell"); + let exec_text = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&exec_out.stdout), + String::from_utf8_lossy(&exec_out.stderr), + )); + assert!( + exec_out.status.success(), + "exec against proxy-pod should succeed:\n{exec_text}", + ); + assert!( + exec_text.contains(marker), + "exec output should contain the marker:\n{exec_text}", + ); + + // File transfer is also relay-backed. Upload a file, then read it back + // through exec to confirm the round-trip landed in the workspace. + let tmpdir = tempfile::tempdir().expect("create tmpdir"); + let local = tmpdir.path().join("proxypod-upload.txt"); + let content = "proxypod-sync-payload"; + std::fs::write(&local, content).expect("write local upload file"); + let local_str = local.to_str().expect("upload path is UTF-8"); + let remote = "/sandbox/proxypod-upload.txt"; + + let mut upload = openshell_cmd(); + upload + .arg("sandbox") + .arg("upload") + .arg(name) + .arg(local_str) + .arg(remote) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let upload_out = tokio::time::timeout(Duration::from_secs(60), upload.output()) + .await + .expect("sandbox upload timed out") + .expect("failed to spawn openshell"); + let upload_text = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&upload_out.stdout), + String::from_utf8_lossy(&upload_out.stderr), + )); + assert!( + upload_out.status.success(), + "upload to proxy-pod should succeed:\n{upload_text}", + ); + + let mut cat = openshell_cmd(); + cat.arg("sandbox") + .arg("exec") + .arg("--name") + .arg(name) + .arg("--") + .arg("cat") + .arg(remote) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let cat_out = tokio::time::timeout(Duration::from_secs(60), cat.output()) + .await + .expect("exec cat timed out") + .expect("failed to spawn openshell"); + let cat_text = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&cat_out.stdout), + String::from_utf8_lossy(&cat_out.stderr), + )); + assert!( + cat_text.contains(content), + "uploaded content should be readable in the sandbox:\n{cat_text}", + ); + + delete_sandbox(name).await; +} diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index bd7f246d24..c00917ab52 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -20,6 +20,12 @@ # files, relative to the repository root or absolute, to layer additional chart # configuration on top of ci/values-skaffold.yaml. # +# Proxy-pod topology: +# Use OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-proxy-pod.yaml +# or `mise run e2e:kubernetes:proxy-pod`. The target cluster must enforce +# Kubernetes NetworkPolicies; the ephemeral k3d/k3s path keeps k3s's embedded +# network policy controller enabled. +# # Image source: # - Ephemeral k3d mode builds local `openshell/{gateway,supervisor}:${IMAGE_TAG}` # images by default, imports them into k3d, then installs the chart. This @@ -94,6 +100,7 @@ VAULT_CHART_VERSION="${OPENSHELL_E2E_OPENBAO_CHART_VERSION:-0.28.3}" VAULT_DEV_ROOT_TOKEN="${OPENSHELL_E2E_VAULT_DEV_ROOT_TOKEN:-root}" CORPORATE_PROXY_FIXTURE_DEPLOYED=0 CORPORATE_PROXY_FIXTURE_SECRET="openshell-e2e-proxy-auth" +PROXY_POD_E2E=0 # Isolate CLI/SDK gateway metadata from the developer's real config. export XDG_CONFIG_HOME="${WORKDIR}/config" @@ -801,6 +808,9 @@ if [ -n "${OPENSHELL_E2E_KUBE_EXTRA_VALUES:-}" ]; then IFS=':' read -r -a extra_values_files <<< "${OPENSHELL_E2E_KUBE_EXTRA_VALUES}" for values_file in "${extra_values_files[@]}"; do [ -n "${values_file}" ] || continue + if [[ "${values_file}" == *"values-proxy-pod.yaml" ]]; then + PROXY_POD_E2E=1 + fi if [[ "${values_file}" != /* ]]; then values_file="${ROOT}/${values_file}" fi @@ -808,6 +818,11 @@ if [ -n "${OPENSHELL_E2E_KUBE_EXTRA_VALUES:-}" ]; then done fi +if [ "${PROXY_POD_E2E}" = "1" ]; then + echo "Proxy-pod e2e profile enabled; target cluster must enforce Kubernetes NetworkPolicies." + echo "Ephemeral k3d/k3s mode uses k3s's embedded NetworkPolicy controller unless the cluster is customized externally." +fi + if [ "${OPENSHELL_E2E_KUBE_DB_SCENARIOS:-0}" = "1" ]; then # --- Multi-scenario mode: test all database backends --- DB_PASSED=0 diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index b737e7de62..7c15a9c187 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -101,6 +101,11 @@ message AuthenticateSandboxRequest { message AuthenticateSandboxResponse { // Stable gateway-assigned sandbox ID authenticated by the driver. string sandbox_id = 1; + // When true, the driver authenticated a process-scoped caller (e.g. a + // proxy-pod agent pod). The gateway mints a narrowed `Process`-kind sandbox + // token that cannot read provider secrets or inference routing. Defaults to + // false (full-authority caller). + bool scoped_process_caller = 2; } message GetGatewayListenerRequirementsRequest {} @@ -266,6 +271,31 @@ message DriverSandboxStatus { repeated DriverCondition conditions = 5; // True when the compute platform has begun deleting this sandbox. bool deleting = 6; + // How the gateway should decide that this sandbox is Ready. + // + // Unset preserves the default contract: the gateway requires a live + // supervisor session before reporting Ready. Drivers whose topology has no + // in-sandbox process supervisor report SUPERVISOR_SESSION_MODEL_NONE so the + // gateway derives Ready from `conditions` alone. + SupervisorSessionModel supervisor_session_model = 7; +} + +// Whether a sandbox has an in-sandbox process supervisor that opens a +// `ConnectSupervisor` session with the gateway. +// +// The session is the transport for relays -- SSH, exec, port forwarding, and +// file transfer -- so this also tells the gateway which RPCs the sandbox can +// serve. A sandbox reporting NONE is reachable for policy-enforced network +// egress but cannot accept a relay. +enum SupervisorSessionModel { + // Default contract: a supervisor session is required for Ready and relays + // are expected to work. + SUPERVISOR_SESSION_MODEL_UNSPECIFIED = 0; + // A supervisor session is required before the sandbox is Ready. + SUPERVISOR_SESSION_MODEL_REQUIRED = 1; + // No supervisor session exists. Readiness comes from `conditions`, and + // relay-backed RPCs are unavailable for this sandbox. + SUPERVISOR_SESSION_MODEL_NONE = 2; } // Raw compute-platform condition. diff --git a/rfc/proxy-pod-topology-DRAFT.md b/rfc/proxy-pod-topology-DRAFT.md new file mode 100644 index 0000000000..ab5a0088fc --- /dev/null +++ b/rfc/proxy-pod-topology-DRAFT.md @@ -0,0 +1,1066 @@ +--- +authors: + - "@TaylorMutch" + - "@russellb" +state: draft +links: + - https://github.com/NVIDIA/OpenShell/pull/2077 - original proxy-pod topology PR from TaylorMutch + - https://github.com/NVIDIA/OpenShell/pull/2074 - kubernetes combined topology + - https://github.com/NVIDIA/OpenShell/pull/2076 - kubernetes sidecar topology + - https://github.com/NVIDIA/OpenShell/pull/2078 - cni-sidecar topology +--- + +# RFC NNNN - Proxy-Pod Supervisor Topology (and OpenShift Enablement) + + + +## Summary + +This RFC proposes `proxy-pod`, a Kubernetes supervisor topology that moves the +**network proxy** out of the sandbox pod into a paired, per-sandbox supervisor +`Deployment`, while **keeping the process supervisor in the sandbox pod**. The +sandbox pod runs the OpenShell process supervisor (`--mode=process`) alongside +the workload, so filesystem policy, process/binary identity, SSH, `connect`, +`exec`, upload/download, file sync, and provider injection all keep working. Only +the credential-bearing L4/L7 network proxy lives in the separate pod. Egress is +confined by two per-sandbox Kubernetes `NetworkPolicy` objects rather than by +pod-local nftables rules, so the sandbox pod needs **no `NET_ADMIN`/`SYS_ADMIN`, +no privileged init container, no shared node component** — and, via a +privilege-drop knob copied from `cni-sidecar`, can run entirely non-root under a +stock SCC. + +> **Revision note.** An earlier design of this RFC (retained below under +> [Superseded design](#superseded-design-network-only-no-in-pod-supervisor)) +> moved the *entire* supervisor into the separate pod, leaving the sandbox pod +> with no supervisor at all. That maximized isolation but gave up SSH/exec/sync, +> filesystem/process policy, and provider injection — too much for most users. +> This revision keeps those features by leaving the process supervisor in the +> sandbox pod and moving only the network proxy out. The OpenShift enablement +> (DNS peers, `nonroot-v2` SCC), the `NetworkPolicy` fence, the companion +> `Deployment`/`Service`/`Secret` set, and the readiness/lifecycle machinery are +> all carried forward unchanged from that work; the changes are confined to what +> now runs in the sandbox pod and to the credential it holds. + +Compared with the in-pod `sidecar`/`cni-sidecar` topologies, this delivers the +same interactive feature set with the network half in its own pod (its own +failure domain, its own Kata VM, and provider credentials never co-resident with +the workload) and confinement by `NetworkPolicy` instead of nftables — no +privileged init container and no node-level DaemonSet. The one new cost versus +the network-only design is that the sandbox pod again holds a gateway +credential; this RFC proposes **scoping that credential down** (a +process-supervisor `caller_kind` that cannot read provider secrets or mint +upstream credentials) so the sensitive capabilities stay only in the proxy pod. + +The OpenShift enablement is validated against a live OpenShift 4.x / +OVN-Kubernetes cluster: configurable DNS egress peers (the hardcoded upstream +`kube-system`/port-53 selectors do not hold on OpenShift), and a gated +`nonroot-v2` grant rather than a custom SCC. + +## Motivation + +OpenShell's `combined` topology runs the full supervisor inside the agent +container, which requires that container to carry `SYS_ADMIN`, `NET_ADMIN`, +`SYS_PTRACE`, and `SYSLOG`. The `sidecar` topology moves network enforcement to +a dedicated sidecar and drops the agent container to no added capabilities, but +still needs a **privileged network init container** in every sandbox pod to +install the nftables fence. [`cni-sidecar`](./cni-sidecar-topology-DRAFT.md) +removes that init container by pushing rule installation to a node-level CNI +plugin, but it moves the privilege rather than eliminating it: the CNI DaemonSet +runs `privileged` with host-path writes, and the binary-aware sidecar still runs +as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. + +All three share an assumption: OpenShell's enforcement point lives inside the +sandbox pod, so the pod must be granted whatever privilege that enforcement +requires. Some clusters will not accept that at any level. Multi-tenant +platforms, regulated environments, and clusters with strict admission policy +often permit only the baseline restricted profile for tenant workloads — no +added capabilities, no root containers, no privileged init containers, no +host-path DaemonSets installed on their behalf. On those clusters OpenShell is +currently not deployable at all. + +Such clusters do, however, almost always enforce `NetworkPolicy`, because that +is the tenant-isolation primitive their platform is already built on. If +OpenShell expresses its egress fence as `NetworkPolicy` instead of nftables, the +enforcement moves to machinery the cluster already runs and already trusts, and +the sandbox pod needs no privilege whatsoever. + +The features that depend on the supervisor sharing the workload's namespaces — +filesystem policy, binary identity, and the interactive session paths — do **not** +have to be given up to get there. They only require the *process* supervisor to +share those namespaces; they do not require the *network* proxy to. So this RFC +keeps the process supervisor in the sandbox pod (exactly where `sidecar` keeps +it) and moves only the network proxy out. The sandbox pod still needs the modest +privileges the process supervisor uses (and a knob to drop them), but never the +network-setup privileges (`NET_ADMIN`/`SYS_ADMIN`/nftables) — those move to the +proxy pod, and the fence becomes `NetworkPolicy`. The result runs on clusters +that permit no in-pod *network* privilege while still delivering the full +interactive contract. + +OpenShift is the concrete case driving this now. OpenShell's current OpenShift +guidance requires granting sandbox pods the `privileged` SCC and is documented +as experimental and evaluation-only. `cni-sidecar` improves on that but still +needs a custom SCC carrying `SYS_PTRACE` and `DAC_READ_SEARCH` plus +`runAsUser: RunAsAny`. `proxy-pod` needs neither: with the DNS fix proposed +below, it admits under the built-in, unmodified `nonroot-v2` SCC. That makes it +the first OpenShell topology that runs on OpenShift without a bespoke security +grant. + +## Non-goals + +- **Replacing `combined`, `sidecar`, or `cni-sidecar`.** All remain. `combined` + stays the default and the only topology providing the full supervisor + contract. `proxy-pod` is for clusters that cannot accept in-pod privilege. +- **Re-implementing the supervisor features.** Filesystem policy, process and + binary controls, SSH/`connect`, `exec`, upload/download, sync, and provider + injection are *preserved* by keeping the process supervisor in the sandbox pod + — they reuse the existing `--mode=process` / `sidecar` code paths unchanged. + This RFC does not build new implementations of them; it only relocates the + network proxy and the fence. +- **A zero-supervisor sandbox pod.** Moving the *entire* supervisor out (the + superseded design) is explicitly not the proposal; whether to retain it as a + separate maximal-isolation variant is an open question. +- **Working without `NetworkPolicy` enforcement.** The topology has no fallback + fence. On a cluster whose CNI ignores `NetworkPolicy`, the generated policies + are declarative only and the workload can bypass the proxy freely. This RFC + proposes failing loudly, not degrading quietly. +- **DNS-level exfiltration control.** The agent pod is permitted UDP/TCP 53 to + cluster DNS so name resolution works. DNS tunnelling is not addressed here. +- **Installing or configuring a CNI.** This RFC consumes whatever + `NetworkPolicy` implementation the cluster already runs. +- **Per-sandbox supervisor autoscaling or sharing.** The pairing is strictly + 1:1. A shared proxy serving many sandboxes is a different design. + +## Proposed design: in-pod process supervisor, out-of-pod network proxy + +This is the authoritative design. The [Superseded design](#superseded-design-network-only-no-in-pod-supervisor) +section that follows describes the earlier network-only variant; its +per-sandbox companion set, `NetworkPolicy` fence lifecycle, configurable DNS +peers, and OpenShift DNS analysis are all reused here and are not repeated. The +deltas below are what changes. + +### Overview + +```mermaid +flowchart TB + subgraph Namespace["Sandbox namespace"] + subgraph AgentPod["Agent pod — role=agent"] + Proc["openshell-sandbox --mode=process
process/binary policy, Landlock,
SSH + exec/forward/sync relays
runAsNonRoot (knob), no NET_ADMIN"] + Workload["Agent workload"] + end + Deployment["Supervisor Deployment (1 replica)"] + subgraph SupervisorPod["Supervisor pod — role=supervisor"] + Proxy["openshell-sandbox --mode=network
:3128 policy-enforced proxy
provider creds, inference, TLS intercept"] + end + Service["Headless Service"] + EgressNP["NetworkPolicy: agent egress
proxy:3128 + DNS + gateway"] + IngressNP["NetworkPolicy: supervisor ingress"] + end + Gateway["OpenShell Gateway"] + External["External services"] + + Proc --> Workload + Proc -->|"scoped gateway session:
relays, policy, logs"| Gateway + Workload -->|"HTTP(S)_PROXY"| Service --> Proxy + Proxy -->|"gateway session:
creds, inference, forwarding"| Gateway + Proxy -->|"policy-enforced egress"| External +``` + +The single `openshell-sandbox` binary already runs its network and process +halves independently by `--mode`; this is exactly the split `sidecar` uses, but +with the network half in a **separate pod** and the fence expressed as +`NetworkPolicy`. Crucially, **nothing crosses the pod boundary except workload +egress → proxy** (plain TCP + a trusted CA). The process supervisor stays whole +in the sandbox pod and owns its relays and gateway session **locally**, so none +of `sidecar`'s cross-container coupling (the peer-credentialed control socket, +the abstract SSH relay socket, the shared PID namespace, the loopback redirect) +is needed — those are the parts that would not survive a pod boundary, and this +design never reaches for them. + +### What runs where + +| | Agent pod (`role=agent`) | Supervisor pod (`role=supervisor`) | +|---|---|---| +| Process | `openshell-sandbox --mode=process` + workload | `openshell-sandbox --mode=network` | +| Enforces | filesystem/Landlock, process/binary identity, seccomp, privilege drop | L4/L7 network policy, TLS interception, credential injection, inference routing | +| Relays (SSH/exec/forward/sync) | **served locally** (has the workload's namespaces) | none | +| Gateway session | scoped process-supervisor session (see credential model) | full network session | +| Provider credentials | **never** | yes (isolated here) | +| Egress path | direct to gateway (session); workload children via `HTTP_PROXY` → proxy pod | policy-approved internet | +| Privilege | process caps (`SYS_PTRACE`,`DAC_READ_SEARCH`; `SETUID`/`SETGID` if root privilege-drop) — droppable via knob; **no `NET_ADMIN`/`SYS_ADMIN`** | non-root `proxy_uid`, drops `ALL` | + +### Cross-pod egress transport + +The workload's children reach the proxy the same way the network-only design +already does: injected `HTTP(S)_PROXY`/`NO_PROXY` env pointing at the paired +headless `Service` on `:3128`, plus the per-sandbox CA trust bundle. This is +advisory; the agent-egress `NetworkPolicy` is the real fence. A transparent +loopback redirect (as `sidecar` does with nftables) is deliberately **not** used, +because it would require `NET_ADMIN` in the sandbox pod — the privilege this +topology exists to avoid. The process supervisor therefore skips its own netns +creation (as it already does under `NETWORK_ENFORCEMENT_MODE`) and does not run +an in-pod proxy; a new enforcement-mode value selects "process supervisor + +remote proxy over the Service address," reusing the existing `PROXY_URL` / +`configured_proxy_url` plumbing. + +### Credential model: a scoped process-supervisor credential + +This is the one real regression versus the network-only design (where the +sandbox pod held no gateway identity at all), and the RFC proposes to bound it +tightly. The sandbox pod's process supervisor needs a gateway session for its +relays (`ConnectSupervisor`, `RelayStream`, `ReportMainProcessExit`), log push, +config read, and token refresh. The sandbox credential is already a +gateway-minted, Ed25519 **per-sandbox** JWT that can only act as its one sandbox +— no cross-sandbox, no cluster-wide, no admin RPC. But today its authority is a +fixed allowlist of all `sandbox`-callable RPCs, so it would also carry the two +capabilities the process supervisor does **not** need and that are the only +secret-bearing ones: `GetSandboxProviderEnvironment` (reads provider secrets) and +`ExchangeProviderSubjectToken` (mints upstream credentials), plus +`GetInferenceBundle` — all network-supervisor concerns. + +The RFC proposes adding a **`caller_kind` claim** to the sandbox JWT (mirroring +the existing `ExtensionJwtClaims.caller_kind`), minting the agent-pod token as a +`process`-kind credential, and rejecting those three RPCs for that kind at the +existing authorization chokepoint plus per-handler guards. The result: a +compromised agent pod can relay into / report on / renew **its own** sandbox, but +**cannot read provider secrets or mint upstream credentials** — those stay +exclusively in the proxy pod (a separate pod, and under Kata a separate VM). It +cannot be made literally read-only (it must push its own logs, report its own +exit, and refresh its own token), but "own-sandbox, control-plane-minimal, no +provider/inference" is achievable and is the proposed target. The full-authority +token is still minted for the proxy pod's network session. + +### NetworkPolicy delta + +The supervisor-ingress policy is unchanged. The agent-egress fence gains one +peer: the **gateway endpoint**, because the in-pod process supervisor now opens +its own gateway session (the network-only design's agent pod never talked to the +gateway, so its fence allowed only proxy:3128 + DNS). This is a deliberate, +documented widening of the agent's egress surface. Everything else — deny by +default, proxy:3128, DNS peers — carries over. + +### Privilege model and the drop knob + +The sandbox pod needs only the **process** supervisor's privileges, never the +network-setup ones. It reuses the `cni-sidecar`/`sidecar` privilege-drop pattern +verbatim: a boolean knob (secure-by-default) where the strict mode runs the +process supervisor with `SYS_PTRACE` + `DAC_READ_SEARCH` (for cross-UID `/proc` +binary attribution) and — if the operator wants supervisor-managed root→sandbox +privilege drop — as root with `SETUID`/`SETGID`; the relaxed mode drops those +caps, runs non-root as the resolved sandbox UID, and downgrades to endpoint/L7 +policy without `policy.binaries` matching (network enforcement is remote anyway). +`NET_ADMIN`/`NET_RAW`/`SYS_ADMIN` are never present in the sandbox pod under +either mode. The proxy pod is always non-root (`proxy_uid`, drops `ALL`). + +### Session model and readiness + +Unlike the network-only design, this topology **has** an in-sandbox supervisor +session, so it reports the `sidecar`/`combined` `SupervisorSessionModel` +(`REQUIRED`), relays are available, and readiness derives from the live process +supervisor session — not the `NONE` sessionless path. The separate proxy pod's +availability is still folded into readiness (a sandbox with a dead proxy is +`Provisioning`), reusing the network-only design's supervisor-`Deployment` +readiness watch + reconcile. The `wait-for-proxy` init container and the +companion lifecycle (create/reconcile/teardown, fence quiescence) carry over +unchanged. + +### Feature availability vs. the alternatives + +| Capability | `sidecar`/`cni-sidecar` | network-only `proxy-pod` (superseded) | **this design** | +|---|---|---|---| +| Network + L7 policy | yes | yes | yes | +| Filesystem / process / binary policy | yes | **no** | **yes** | +| SSH / `exec` / upload / sync | yes | **no** | **yes** | +| Provider injection | yes | **no** | **yes** | +| Workload output in `openshell logs` | yes | **no** | **yes** | +| Fence mechanism | nftables (in-pod) | `NetworkPolicy` | `NetworkPolicy` | +| Privileged init container / node DaemonSet | yes / (cni: yes) | no / no | **no / no** | +| Sandbox-pod network privilege (`NET_ADMIN`) | yes (init/sidecar) | none | **none** | +| Provider creds co-resident with workload | yes (same pod) | n/a | **no (separate pod/VM)** | +| Gateway credential in sandbox pod | yes (sidecar holds it) | **none** | scoped process-only | +| Pods per sandbox | 1 | 2 | 2 | + +The niche this fills: **`sidecar`'s feature set, confined by `NetworkPolicy` +instead of nftables (no privileged init, no node DaemonSet), with the network +half — and all provider credentials — isolated in its own pod/VM.** + +### OpenShift SCC implication (changed) + +Because the sandbox pod now runs the process supervisor, the "admits under stock +`nonroot-v2`" property depends on the drop knob. In **relaxed** mode the sandbox +pod is non-root with `drop: [ALL]` and admits under `nonroot-v2` (or stock +`restricted-v2` if UIDs are SCC-assigned). In **strict** (binary-aware) mode it +needs the same minimal custom SCC `cni-sidecar`/`sidecar` use (adds +`SYS_PTRACE` + `DAC_READ_SEARCH`, and `RunAsAny` if root privilege-drop is kept). +The proxy pod continues to admit under `nonroot-v2`. The OpenShift DNS-peer and +`nonroot-v2` analysis below still applies to the proxy pod verbatim. + +### Key tradeoffs to decide + +1. **Credential placement (accepted).** The sandbox pod holds a scoped + process-supervisor gateway credential. Recommended over the more complex + alternative of brokering all gateway access through the proxy pod (which would + re-introduce `sidecar`-style cross-pod relay bridging — the coupling this + design avoids). Recorded as an open question. +2. **Enum shape.** Whether this replaces the `proxy-pod` value outright, or ships + as a distinct value with the network-only design retained as a maximal + isolation variant. Recommended: this becomes `proxy-pod`; retain the + network-only path only if a concrete zero-in-pod-supervisor use case appears. + +## Superseded design (network-only, no in-pod supervisor) + +The sections below describe the earlier design that moved the entire supervisor +out of the sandbox pod. They are retained because most of their machinery — the +per-sandbox companion set, the `NetworkPolicy` fence and its lifecycle, the +configurable DNS peers, the OpenShift `nonroot-v2` analysis, and the +readiness/reconcile plumbing — is reused unchanged by the proposed design above. +Where a section describes the *sandbox pod running no supervisor* (privilege +model's agent row, credential isolation, "why relays cannot cross the pod +boundary," the network-only feature tables), it is superseded by the +corresponding subsection above. + +### Topology overview + +```mermaid +flowchart TB + Sandbox["agents.x-k8s.io Sandbox"] + + subgraph Namespace["Sandbox namespace"] + subgraph AgentPod["Agent pod — role=agent"] + Workload["Agent workload
sandbox image, run directly
runAsNonRoot, drops ALL"] + end + + Deployment["Supervisor Deployment
replicas: 1, owned by Sandbox CR"] + subgraph SupervisorPod["Supervisor pod — role=supervisor"] + Proxy["openshell-supervisor --mode=network
:3128 policy-enforced proxy"] + end + + Service["Headless Service
clusterIP: None"] + CA["Per-sandbox proxy CA Secret"] + EgressNP["NetworkPolicy: agent egress
supervisor ports + DNS only"] + IngressNP["NetworkPolicy: supervisor ingress
paired agent only"] + end + + Gateway["OpenShell Gateway"] + External["External services"] + + Sandbox --> AgentPod + Sandbox --> Deployment + Deployment --> SupervisorPod + AgentPod -->|"HTTP_PROXY / HTTPS_PROXY"| Service + Service --> Proxy + Proxy -->|"policy-enforced egress"| External + CA -. mounted .- AgentPod + CA -. mounted .- SupervisorPod + EgressNP -. selects .- AgentPod + IngressNP -. selects .- SupervisorPod +``` + +The key structural difference from every other topology: the supervisor is in a +**different pod, and therefore a different network namespace**. There is no +loopback to redirect to and no shared netns to install rules in, so the fence +cannot be nftables. It is `NetworkPolicy`, and that is the entire security +boundary. + +### Per-sandbox resources + +Creating one `proxy-pod` sandbox creates five OpenShell-managed objects +alongside the `Sandbox` CR, all in the sandbox namespace: + +| Object | Name pattern | Purpose | +|---|---|---| +| `Deployment` | `os-sup--` | Runs the network supervisor, 1 replica | +| `Service` | `os-svc--` | Headless; agent's proxy endpoint | +| `Secret` | `os-ca--` | Per-sandbox generated proxy CA cert + key | +| `NetworkPolicy` | `os-eg--` | Agent egress fence | +| `NetworkPolicy` | `os-ing--` | Supervisor ingress restriction | + +Names are `--` to stay within the 63-character +DNS label limit while remaining collision-resistant and human-recognizable. + +The `Deployment` carries a **controlling** `Sandbox` ownerReference; the +`Service`, CA `Secret`, and supervisor-ingress `NetworkPolicy` carry +non-controlling ones. Kubernetes garbage collection reclaims those four when the +sandbox is deleted. The `Deployment` recreates the supervisor pod if it is +deleted independently. + +The agent egress `NetworkPolicy` — the workload's egress fence — deliberately +carries **no** ownerReference. Owner-reference garbage collection does not order +sibling deletion, so a GC-owned fence would be removed concurrently with the +workload pod; a pod that ignores `SIGTERM` could then regain direct egress during +its termination grace period. Instead the gateway manages the fence's lifecycle +directly: it deletes the fence only after the workload pod is gone (the delete +path waits for the pod to disappear), and reconciliation reaps any fence orphaned +by a gateway crash (an `os-eg-*` policy whose Sandbox CR no longer exists). This +keeps the fence in place for exactly as long as the workload can still run. + +Because the supervisor pod is created by a `Deployment`, its owner chain is +`Pod → ReplicaSet → Deployment → Sandbox` rather than `Pod → Sandbox`. Gateway +ServiceAccount bootstrap must walk that chain to authenticate the supervisor, +validating each link's UID, which is why the topology needs `apps/replicasets: +get` and `apps/deployments: get` in the sandbox `Role`. In shared +(single-namespace) mode the topology also watches supervisor Deployments to keep +readiness current, so the namespaced `Role` additionally grants +`apps/deployments: list` and `watch`. Managed and operator modes omit those verbs +from the `ClusterRole` — a cluster-wide Deployment informer would be broad +enumeration a compromised gateway could abuse — and fold readiness in through +get/list and the periodic reconcile instead. + +### Privilege model + +| Component | UID | Priv. escalation | Capabilities | Notes | +|---|---|---|---|---| +| Agent workload container | `sandbox_uid:sandbox_gid` | false | drops `ALL` | Runs the sandbox image's own entrypoint. No supervisor. | +| Proxy CA init container | `sandbox_uid:sandbox_gid` | false | drops `ALL`, `readOnlyRootFilesystem` | Builds the CA bundle into an `emptyDir`. | +| Workspace init container | `sandbox_uid:sandbox_gid` | false | drops `ALL` | Seeds the workspace PVC. Non-root, unlike other topologies. | +| Supervisor container | `proxy_uid:sandbox_gid` | false | drops `ALL` | Separate pod. Holds all gateway credentials. | + +No container in either pod runs as root, requests a capability, or needs a +privileged init container, a shared process namespace, or a node-level +DaemonSet. This is the least-privileged configuration OpenShell produces. + +### Credential isolation + +The workload pod receives **no** gateway endpoint, bootstrap token, projected +ServiceAccount token, client TLS identity, or SPIFFE workload socket. It gets +only `HTTP_PROXY`/`HTTPS_PROXY` pointing at the paired Service, `NO_PROXY`, and +a CA trust bundle exposed through the environment variables the common runtimes +read (`SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `GIT_SSL_CAINFO`, +`NODE_EXTRA_CA_CERTS`, `DENO_CERT`). + +Credential isolation here is structural rather than procedural. The `sidecar` +topology keeps credentials out of the agent container but must defend a shared +control socket with peer-credential checks and one-shot listener semantics. +`proxy-pod` has no such socket: the credential simply is not in the pod, and the +two pods share no namespace, no filesystem, and no IPC. + +The workload also has no network path to the gateway. Only the supervisor +connects to the gateway (for policy, inference, log push, and token bootstrap); +the agent egress `NetworkPolicy` permits the workload to reach only the +supervisor's proxy port and cluster DNS. An earlier revision ran a raw TCP +forward from the supervisor to the gateway that the workload could reach; it was +removed because nothing on the workload consumed it and, under unauthenticated +gateway access, it was a policy-bypassing path to the gateway API. + +One consequence: because credentials are per-supervisor and the CA is generated +per sandbox, a `proxy-pod` sandbox cannot participate in the corporate +upstream-proxy credential feature, which mounts a `user:pass` Secret into the +container performing network supervision. Mounting it into the workload pod +would defeat the purpose. This RFC proposes rejecting that combination at +configuration validation rather than silently mounting it in the wrong place. + +Separate pods also raise the isolation ceiling under a VM-based `RuntimeClass`. +Kata Containers gives each *pod* its own lightweight VM and kernel; containers +within a pod share that VM. In every in-pod topology the workload and the +supervisor live in one pod, so a Kata VM escape — a kernel compromise inside +that shared VM — reaches the supervisor and its gateway credentials. Under +`proxy-pod` the workload and supervisor are separate pods and therefore separate +Kata VMs with separate kernels, so a kernel compromise in the workload VM does +not by itself reach the supervisor. This is unique to `proxy-pod`: it is the +only topology where the workload-to-supervisor boundary can be a hypervisor +boundary rather than a namespace boundary. + +### The NetworkPolicy contract + +Two policies define the fence: + +**Agent egress** (`policyTypes: [Egress]`, selecting `sandbox-role=agent`) permits +exactly two destinations: + +1. Pods labeled `sandbox-role=supervisor` for this sandbox ID, on TCP 3128 (the + policy-enforced HTTP CONNECT proxy). +2. Cluster DNS, on UDP 53 and TCP 53. + +Everything else is denied. **This is load-bearing.** `HTTP_PROXY` is only a +convention a workload may ignore; the egress policy is what makes ignoring it +useless. A cluster that does not enforce `NetworkPolicy` provides no fence at +all in this topology, which is why enforcement is a hard prerequisite and not a +recommendation. + +**Supervisor ingress** (`policyTypes: [Ingress]`, selecting +`sandbox-role=supervisor`) accepts only from the paired agent pod on those same +two ports. Supervisor egress is deliberately unrestricted: it must reach the +gateway and the policy-approved internet, and OpenShell policy — not +`NetworkPolicy` — governs where. + +The `sandbox-role` label selectors are scoped by sandbox ID, so two sandboxes in +one namespace cannot reach each other's supervisors. + +### Cluster DNS peers must be configurable + +The current implementation hardcodes the DNS peer as namespace +`kubernetes.io/metadata.name: kube-system` with pod labels `k8s-app: kube-dns` +or `k8s-app: coredns`. That encodes an upstream Kubernetes convention as if it +were a Kubernetes guarantee. It is not. + +On OpenShift 4.x, verified against a live 4.22.6 / OVN-Kubernetes cluster: +`kube-system` contains no DNS pods at all. Cluster DNS runs in namespace +`openshift-dns` as DaemonSet `dns-default`, with pods labeled +`dns.operator.openshift.io/daemonset-dns=default`. The hardcoded selector matches +nothing, so the agent pod's DNS egress falls through to the policy's implicit +deny and **no name resolution works** — including resolving the paired +supervisor's own Service name. The sandbox is inert. + +There is a second, subtler mismatch. A `NetworkPolicy` egress rule whose peer +is a `podSelector` is evaluated against the destination **pod** after `Service` +address translation, so its port list must name the DNS pods' *container* port. +Upstream `CoreDNS` listens on 53, so the Service port and container port +coincide and nobody notices. OpenShift's `dns-default` listens on **5353** and +maps 53 onto it, so a rule allowing port 53 matches nothing even with correct +selectors. This was confirmed empirically: with the right selectors but port +53, DNS failed both through the Service ClusterIP and directly against the DNS +pod IP; with 5353 it resolves. + +This RFC therefore proposes a configurable DNS peer list carrying both +selectors and a port: + +```toml +[openshell.drivers.kubernetes.proxy_pod] +proxy_uid = 1337 +affinity = "disabled" # disabled | preferred | required + +# Cluster DNS peers for the agent egress NetworkPolicy. Defaults to the +# upstream kube-system/kube-dns and kube-system/coredns conventions on port 53. +[[openshell.drivers.kubernetes.proxy_pod.dns_peers]] +namespace_labels = { "kubernetes.io/metadata.name" = "openshift-dns" } +pod_labels = { "dns.operator.openshift.io/daemonset-dns" = "default" } +port = 5353 +``` + +Each peer renders as its own egress rule, because a rule's port list applies to +every `to` entry in that rule and peers may listen on different ports. + +with the Helm equivalent under `supervisor.proxyPod.dnsPeers`. When unset, the +existing upstream defaults apply, so no behavior changes for current users. Each +entry becomes one `to` peer in the egress rule; multiple entries are additive. + +Configuration is the right shape rather than platform auto-detection: the driver +would otherwise need cluster-type inference and cluster-wide namespace or pod +read permissions it does not currently hold, and operators running NodeLocal +DNSCache or a non-default DNS deployment need the override regardless of +platform. + +### OpenShift SCC model + +OpenShift's `restricted-v2` SCC sets `runAsUser: MustRunAsRange` and +`fsGroup: MustRunAs`, admitting only UIDs inside the namespace's +`openshift.io/sa.scc.uid-range` annotation — on the verification cluster, +`1000000000/10000`. The driver assigns fixed UIDs (`sandbox_uid` default 1000, +`proxy_uid` default 1337), both far outside that range, so `restricted-v2` +rejects both pods. + +The built-in **`nonroot-v2`** SCC resolves this without a custom SCC. It is +`restricted-v2` with `runAsUser: MustRunAsNonRoot` and `fsGroup: RunAsAny`, +keeping `requiredDropCapabilities: [ALL]`, `allowPrivilegeEscalation: false`, +`allowPrivilegedContainer: false`, no host namespaces, and +`seccompProfiles: [runtime/default]`. Its `allowedCapabilities` is +`[NET_BIND_SERVICE]` only, which `proxy-pod` does not use. Its volume allowlist +covers every volume type the topology needs: `emptyDir`, `secret`, `projected`, +`persistentVolumeClaim`, `csi`, and `configMap`. + +`proxy-pod` therefore admits on OpenShift under an unmodified, Red Hat-shipped +SCC: + +```shell +oc adm policy add-scc-to-user nonroot-v2 -z openshell-sandbox -n openshell +``` + +Measured on the validation cluster, the two pods land on *different* SCCs, and +only one needs the grant: + +| Pod | Admitted under | UID | Why | +|---|---|---|---| +| Agent | `restricted-v2` | `1000810000` (SCC-assigned) | `sandbox_uid` is optional and was unset, so no explicit UID to reject | +| Supervisor | `nonroot-v2` | `1337` (explicit) | `proxy_pod.proxy_uid` always has a value, which `restricted-v2` rejects | + +Both ran with `capabilities.drop: ["ALL"]`, `allowPrivilegeEscalation: false`, +and `seccompProfile: RuntimeDefault`. + +This RFC proposes rendering that grant from the chart behind a gated value +(`sandboxServiceAccount.openshift.nonrootSCC`, default off, so non-OpenShift +installs never reference OpenShift-only APIs), mirroring how `cni-sidecar` +gates its SCC grants. + +The comparison across topologies is the strongest argument for `proxy-pod` on +OpenShift: + +| Topology | OpenShift SCC required | +|---|---| +| `combined` | `privileged` (current documented guidance, evaluation-only) | +| `sidecar` | custom SCC: `RunAsAny` + `SYS_PTRACE` + `DAC_READ_SEARCH` | +| `cni-sidecar` | custom sandbox SCC, plus `privileged` for the CNI DaemonSet | +| `proxy-pod` | built-in `nonroot-v2`, unmodified | + +An alternative worth recording: the driver could omit `runAsUser`/`runAsGroup`/ +`fsGroup` entirely on OpenShift and let SCC admission assign them from the +namespace range, which would admit under stock `restricted-v2` and require no +grant at all. The `proxy_uid != sandbox_uid` constraint exists to keep the +nftables fence from exempting the workload, and `proxy-pod` has no nftables +fence and no shared namespace, so the constraint is not security-relevant here. +This RFC does not propose it yet, because it interacts with workspace PVC +ownership and needs its own validation, but it is the natural follow-up and +would make `proxy-pod` zero-grant on OpenShift. The measurement above is direct +evidence that it would work: the agent pod already takes exactly this path. + +### Same-node placement + +`proxy_pod.affinity` controls pairing: `disabled` (default), `preferred`, or +`required`, matching the paired supervisor on `kubernetes.io/hostname` while +preserving any workload-supplied affinity terms. The default is off, which means +every workload byte crosses the pod network to another node. `preferred` is the +better operational default for latency-sensitive agents; `required` risks +unschedulable pairs under node pressure. The default is left at `disabled` in +this RFC but is a reasonable thing for reviewers to push back on. + +### Readiness without a supervisor session + +`SandboxPhase::Ready` was reachable only through a live `ConnectSupervisor` +session. That session is opened solely by `openshell-supervisor-process`, and +its `GatewayMessage` payload is relays — `RelayOpen`/`RelayClose` — plus session +control and heartbeats. So `Ready` has meant "the gateway can open relays into +this sandbox," which for `proxy-pod` will never be true and should not be. + +Left alone, this made the topology unusable: on OpenShift both pods ran and +policy-enforced egress worked end to end while the sandbox reported +`Provisioning` indefinitely, and every `Ready`-gated RPC — including `stop` and +`start` — was unreachable. + +This RFC proposes making the readiness contract explicit rather than implied. A +`SupervisorSessionModel` on `DriverSandboxStatus` lets a driver declare that a +sandbox has no in-sandbox process supervisor. `UNSPECIFIED` preserves the +existing behavior, so drivers that never set it are unaffected; the Kubernetes +driver reports `NONE` for `proxy-pod` and `REQUIRED` otherwise. The gateway then +derives readiness for such sandboxes from the backend conditions alone. + +Two consequences fall out of that and are part of the proposal: + +**Readiness must not become a lie.** With the session gate removed, `Ready` +follows the agent pod, which says nothing about whether the paired supervisor is +serving. A pod could be Ready with no egress path at all. The agent pod +therefore gains a `wait-for-proxy` init container that blocks until the paired +supervisor accepts connections on its proxy port, so pod readiness transitively +means egress works. This also closes a pre-existing ordering gap where the +workload could start before the proxy existed and its early requests simply +failed. + +**Relay-backed RPCs must fail honestly.** Once such sandboxes reach `Ready`, +`exec`, `connect`, port forwarding, and file transfer would pass their readiness +checks and then wait out a session timeout that cannot succeed. The same +declaration lets the gateway reject them immediately with an error naming the +topology. + +### Running a workload with no supervisor to launch it + +`proxy-pod` runs the sandbox image directly. Nothing supplies a command: the +initial command from `openshell sandbox create -- ` is delivered over the +supervisor session as an exec/SSH session after `Ready`, which this topology +does not have, and `DriverSandboxTemplate` has no `command`/`args` field. + +That is tolerable for images built to run a workload, but OpenShell's own +sandbox images use an interactive shell entrypoint. Under kubelet with no TTY it +reads EOF and exits 0, so the stock image produces a `CrashLoopBackOff` with +empty logs — verified on OpenShift, where only an image with a genuinely +long-running entrypoint stayed up. + +This RFC proposes accepting `containers.agent.command` and +`containers.agent.args` through the Kubernetes driver's existing `driver_config` +passthrough, alongside `resources` and `volume_mounts`. That needs no public API +change and reuses the documented escape hatch for driver-specific settings. The +fields are rejected in `combined` and `sidecar`, where the driver replaces the +container command with the supervisor binary and an override would be accepted +and then silently dropped. + +Adding `command`/`args` to the public `SandboxTemplate` remains the more +discoverable long-term answer, but it forces a semantic decision — the field is +genuinely inapplicable to topologies where the supervisor is the entrypoint — and +is deferred rather than resolved here. + +### Why relays cannot cross the pod boundary + +The relay protocol states the constraint directly: `RelayOpen`'s target is +"the target the supervisor should dial **inside the sandbox**." Every +relay-backed capability — SSH, `exec`, port forwarding, file transfer — is a +request to reach into the sandbox and connect to something. Three properties +make that impossible from a separate pod: + +- **The SSH server exists only in the process supervisor.** `russh` is a + dependency of `openshell-supervisor-process` and the gateway. + `openshell-supervisor-network`, the only supervisor `proxy-pod` runs, has no + SSH server at all. +- **Sessions must land in the workload's namespaces.** `ssh.rs` spawns PTY + shells and pipe-execs that need the workload's PID, mount, and user + namespaces, and for networking it calls `setns(fd, CLONE_NEWNET)` on a + dedicated thread to enter the sandbox network namespace — otherwise + connections reach the host loopback rather than the sandbox loopback where + services listen. A supervisor in another pod holds none of those namespaces. +- **The `sidecar` bridge does not generalize.** In `sidecar` the network + sidecar owns the gateway session but does not serve SSH itself; it bridges + relays to a Linux abstract socket owned by the process supervisor in the + agent container, verified by peer PID. That works only because both run in + one pod. + +SSHing into the supervisor pod would land a shell in the wrong container. + +One nuance is worth recording, because it narrows the gap. `RelayOpen` also +carries a `TcpRelayTarget`, used for port forwarding and service exposure, and +that is *not* structurally impossible here: the supervisor pod can dial the +agent pod's IP, since this design restricts agent **egress** and supervisor +**ingress** but leaves agent ingress open. The obstacle is practical rather +than architectural — `connect_in_netns` exists precisely because workloads +usually bind `127.0.0.1`, which is unreachable across pods, so it would work +for services bound to `0.0.0.0` and fail otherwise. The current implementation +rejects all relays uniformly, which is correct and safe; restoring TCP relays +alone is possible later and is the strongest argument for giving +`SupervisorSessionModel` a capability list rather than treating relays as +all-or-nothing. + +### Observability + +Network-layer observability survives intact; anything requiring visibility +inside the workload's namespaces does not. Log push to the gateway is gated on +the sandbox ID and gateway endpoint rather than on topology, and the proxy pod +has both, so `openshell logs ` carries `[sandbox]` lines as usual. +Confirmed on OpenShift: + +```text +[sandbox] [OCSF] NET:OPEN [MED] DENIED -(0) -> github.com:443 [engine:opa] [reason:network connections not allowed by policy] +[sandbox] [OCSF] CONFIG:LOADED [INFO] Acknowledged initial policy revision as loaded [version:1] +[sandbox] Flushed denial analysis to gateway proposals=2 summaries=2 +``` + +| Signal | `proxy-pod` | +|---|---| +| `NET:*` allow/deny with policy engine and reason | full | +| `CONFIG:*` policy and inference-route changes | full | +| Activity summaries and denial analysis for the policy advisor | full | +| Gateway-side logs | full | +| Workload stdout/stderr | **container log only** (`kubectl logs`), never `openshell logs` | +| Process and binary attribution on network events | **none** | +| `PROCESS:*`, `SSH:*`, Landlock/filesystem events | **none** | + +Two losses deserve emphasis. The workload's own output is no longer captured +by OpenShell at all: the workload is the container's PID 1 and no OpenShell +process shares that pod, so its output reaches only the container log. Anyone +driving OpenShell through the API rather than with cluster access cannot see it. + +And network events carry no actor: the denial above reads `-(0)`, an empty +process name and PID 0. Binary-aware attribution requires reading +`/proc/` across the workload's PID namespace, which a separate pod cannot +do. Operators can therefore answer what was denied but not which process +attempted it, which removes `policy.binaries` as both an enforcement and a +forensic tool. + +### Feature availability + +#### Enforcement + +| Capability | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | +|---|---|---|---|---| +| Network endpoint + L7 policy | yes | yes | yes | yes | +| Enforcement mechanism | in-pod nftables | in-pod nftables | node CNI rules | **`NetworkPolicy`** | +| Filesystem policy | yes | partial (Landlock) | partial (Landlock) | **no** | +| Process / binary identity | yes | yes | yes | **no** | +| `policy.binaries` matching | yes | yes | yes | **no** — no actor attribution | +| Dynamic provider env injection | yes | yes | yes | **no** | + +#### Session and file access + +All relay-backed, and all requiring the workload's namespaces: + +| Capability | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | +|---|---|---|---|---| +| SSH / `connect` | yes | yes | yes | **no** — structurally impossible | +| `exec` | yes | yes | yes | **no** — structurally impossible | +| Upload / download / sync | yes | yes | yes | **no** — structurally impossible | +| Port forwarding / service exposure | yes | yes | yes | **no today** — recoverable for `0.0.0.0` binds | +| Initial command from `sandbox create -- ` | yes | yes | yes | **no** — use `containers.agent.command` | + +#### Observability + +| Signal | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | +|---|---|---|---|---| +| `NET:*` allow/deny with reason | yes | yes | yes | yes | +| `CONFIG:*` policy and route changes | yes | yes | yes | yes | +| Denial analysis for the policy advisor | yes | yes | yes | yes | +| Workload stdout/stderr in `openshell logs` | yes | yes | yes | **no** — only in the `agent` container log via `kubectl logs ` | +| Actor process on network events | yes | yes | yes | **no** — renders as `-(0)` | +| `PROCESS:*` lifecycle events | yes | yes | yes | **no** | +| `SSH:*` events | yes | yes | yes | **no** | +| Landlock / filesystem events | yes | partial | partial | **no** | + +#### Operational posture + +| Property | `combined` | `sidecar` | `cni-sidecar` | `proxy-pod` | +|---|---|---|---|---| +| Privileged init container | no | **yes** | no | no | +| Added capabilities in sandbox pod | **yes** | no | no | no | +| Node-level privileged DaemonSet | no | no | **yes** | no | +| Requires `NetworkPolicy` enforcement | no | no | no | **yes** | +| Pods per sandbox | 1 | 1 | 1 | **2** | +| Workload/supervisor kernel isolation under Kata | no — one pod, one VM/kernel | no — one pod, one VM/kernel | no — one pod, one VM/kernel | **yes — separate pods, separate Kata VMs/kernels** | +| OpenShift SCC required | `privileged` | custom | custom + `privileged` CNI | **built-in `nonroot-v2`** | + +The dividing line is consistent: everything observable or enforceable at the +network boundary survives, and everything needing visibility inside the +workload's namespaces does not. `proxy-pod` suits batch and autonomous agent +workloads that need policy-enforced egress, ship their own long-running +entrypoint, and never need a human on the other end. Operators who want the +interactive workflow *and* low pod privilege should use `cni-sidecar`, which +keeps the full supervisor contract at the cost of a custom SCC and a +node-level DaemonSet. The two are complementary, not competing. + +## Implementation plan + +### In-pod process supervisor pivot (proposed direction) + +The network-only work below (Phases 1–5) landed the companion set, the +`NetworkPolicy` fence, OpenShift enablement, readiness/reconcile, and lifecycle — +all reused as-is. The pivot builds on that: + +- **P1 — Supervisor runtime.** A new `SUPERVISOR_TOPOLOGY`/`NETWORK_ENFORCEMENT_MODE` + value that runs `--mode=process` in the agent pod with `ProcessEnforcementMode` + selectable (Full vs relaxed), its own gateway session (policy/logs/relays), and + child egress pointed at the remote proxy `Service` via `HTTP(S)_PROXY` (reuse + `PROXY_URL`/`configured_proxy_url`); skip in-pod netns/nftables. +- **P2 — Scoped credential.** Add `caller_kind` to `SandboxJwtClaims`; mint the + agent-pod token as `process`-kind; reject `GetSandboxProviderEnvironment`, + `ExchangeProviderSubjectToken`, `GetInferenceBundle` for that kind at the + `multiplex` chokepoint + per-handler guards. Back-compat: absent `caller_kind` + = full authority. +- **P3 — Driver topology.** Render the agent pod as the `sidecar` `--mode=process` + container **retaining** gateway creds (SA token/client-TLS/SPIFFE/endpoint) + + proxy CA trust + `wait-for-proxy` init; render the proxy pod from the existing + `proxy_pod_supervisor_deployment`/companions; add the gateway-egress rule to the + agent-egress `NetworkPolicy`; thread the new topology through the ~30 match/gate + sites with **sidecar-like** session model and **proxy-pod-like** companion + lifecycle. +- **P4 — Privilege-drop knob + SCC.** Reuse the `cni-sidecar` pattern (config + field + effective-UID helper + capability branch + optional minimal SCC). +- **P5 — Docs, tests, e2e, cluster validation.** Topology/OpenShift/gateway-config + docs; unit + helm tests; extend the `proxy_pod` e2e suite to assert the + recovered features (exec/sync work) and the scoped credential; validate on the + OVN-Kubernetes cluster. + +### Prior work (network-only design) + +**Phase 1 — rebase and correctness (done).** Rebase PR #2077 onto current +`main`. Resolve the drift from multi-namespace gateway support (thread namespace +through the supervisor owner-chain walk and the cleanup path) and from the +corporate upstream-proxy feature (reject `proxy-pod` with proxy credential +Secrets at config validation, fail-closed). + +**Phase 2 — pre-OpenShift fixes.** Configurable `dns_peers` with upstream +defaults. Supervisor `Deployment` lifecycle on `stop_sandbox`, which currently +leaves the supervisor running and billable while the sandbox is stopped. Chart +plumbing and unit coverage for both. + +**Phase 3 — OpenShift enablement (validated).** Gated `nonroot-v2` grant in the +chart, then deployed to OpenShift 4.22.6 / OVN-Kubernetes. Measured results: + +| Check | Result | +|---|---| +| All five per-sandbox resources created | pass | +| Supervisor pod admitted and running | pass, under `nonroot-v2`, UID 1337 | +| Agent pod admitted and running | pass, under stock `restricted-v2`, SCC-assigned UID | +| DNS resolves from the agent pod | pass, only after the 5353 port fix | +| Agent resolves its paired supervisor `Service` | pass | +| Direct egress to the internet denied | pass | +| Direct egress to the gateway denied | pass | +| Egress to supervisor `:3128` allowed | pass | +| Policy-denied host through the proxy | pass, 403 at CONNECT | +| Policy-allowed host through the proxy | pass, HTTP 200 with the generated CA trusted | +| All resources reclaimed on delete | pass | +| Sandbox reaches `Ready` | pass, after the `SupervisorSessionModel` change | +| `wait-for-proxy` init container gates pod readiness | pass | +| Relay RPCs rejected with a topology error | pass, 43ms rather than a timeout | +| `sandbox stop` scales the supervisor to zero | pass | +| `sandbox start` scales it back and returns to service | pass | +| Stock sandbox image runs via `containers.agent.command` | pass, previously `CrashLoopBackOff` | + +Cluster testing also caught a bug the unit tests could not: the stop, start, +and delete paths derived per-sandbox resource names from the `Sandbox` CR name +rather than the sandbox name, which differ (`default--rdy` versus `rdy`). The +scale-down silently patched a Deployment that does not exist, and delete was +affected too but owner-reference garbage collection reclaimed the resources and +hid it. + +The remaining work is documenting the OpenShift path in +`docs/kubernetes/openshift.mdx`. + +**Phase 4 — test strategy.** The branch adds `mise run e2e:kubernetes:proxy-pod`, +but its `PROXY_POD_E2E` flag currently only prints warnings — it gates nothing. +The full Kubernetes e2e suite runs unchanged, and much of it drives sandboxes +through `exec`, SSH, upload, and sync, which this topology removes by design. A +run would fail broadly on absent capabilities and produce no signal about the +fence. `proxy-pod` needs a capability-scoped suite asserting what the topology +actually promises: egress denial, proxied egress, DNS, CA trust, and resource +GC. The capability-scoped `proxy_pod` suite now exists (`mise run +e2e:kubernetes:proxy-pod`) and runs in branch CI as `kubernetes-proxy-pod-e2e`. +Because CI's kind cluster uses a non-enforcing CNI, that job exercises the +control-plane contract — companion creation, readiness, and sessionless relay +rejection — but not the CNI-enforced egress isolation. The enforcement assertions +(egress denial and proxied egress) still need a policy-enforcing CNI in CI and +remain tracked as follow-up. + +**Phase 5 — graduation.** Ship experimental. Graduate once the scoped suite's +enforcement assertions run in CI on at least one policy-enforcing CNI, and the +OpenShift path is validated end to end. + +## Risks + +**Silent loss of enforcement on a non-enforcing CNI.** The highest-severity +risk. If `NetworkPolicy` is not enforced, the generated policies are inert, the +workload can route around the proxy, and everything still *looks* healthy — +pods run, the supervisor is ready, sandboxes report available. There is no +in-band signal. Mitigation should be active rather than documentary: a startup +probe that verifies a denied egress path is actually denied, failing the sandbox +if the fence is not real. Documentation alone is insufficient for a control +whose failure mode is invisible. This active negative-egress probe (and the +CI coverage on a policy-enforcing CNI that would exercise it) is still +outstanding and tracked as follow-up. + +**Supervisor liveness after startup.** A related but distinct failure: the +supervisor Deployment becoming unavailable *after* the sandbox reaches Ready. +The workload's `wait-for-proxy` init container only gates startup, and the agent +pod's own Ready condition cannot see the separate supervisor. This is now +mitigated: the driver folds supervisor Deployment availability into sandbox +status, so a sandbox whose supervisor has no available replica falls back to +`Provisioning` (Ready condition `False`, transient reason +`DependenciesNotReady`) rather than staying Ready with a dead egress path, and +recovers to `Ready` once the supervisor Deployment is available again. The driver +watches supervisor Deployments and pushes a refreshed status within seconds of an +availability change, so readiness does not lag behind the supervisor until the +next query or reconcile sweep; `get`/`list` queries and the periodic reconcile +fold in the same check as a backstop. + +**Confused-deputy via image-baked launch environment.** In `combined` topology +the supervisor shares the workload's container and inherits the workload image's +environment. Honoring image-baked `OPENSHELL_PROXY_BIND_ADDR` or +`OPENSHELL_PROXY_CA_*` there would let an untrusted image publish the +credential-bearing policy proxy on the pod network or substitute an attacker CA. +This is now mitigated: those launch variables are honored only by a standalone +network supervisor (`proxy-pod`/`sidecar`, which runs the trusted supervisor +image in a separate container); a combined supervisor ignores them, binding to +the namespace-scoped veth IP and generating an ephemeral CA. + +**Feature-set surprise.** An operator selecting `proxy-pod` for its security +properties may not anticipate that `openshell sandbox exec` and `connect` simply +stop working. The gateway should reject those RPCs for `proxy-pod` sandboxes +with an actionable error naming the topology, rather than failing obscurely. +This is now the behavior: relay-backed RPCs are rejected immediately with an +error naming the topology and pointing at `combined` or `sidecar`. + +**Resource multiplication.** Every sandbox becomes two pods plus three +supporting objects. At scale this doubles pod count, doubles scheduling +pressure, and adds five API objects per sandbox. Namespaces with pod quotas will +hit them at half the expected sandbox count. + +**Cross-node data path.** With affinity `disabled`, all workload egress crosses +the pod network. This adds latency to every request and makes the network path a +new failure mode that in-pod topologies do not have. + +**Per-sandbox CA key at rest.** Each sandbox generates a CA cert and private key +stored in a Kubernetes `Secret`. Anyone who can read Secrets in the sandbox +namespace can mint certificates that the workload will trust. The blast radius +is one sandbox, but it is a new key-at-rest surface that other topologies do not +create. + +**DNS as an open egress channel.** UDP/TCP 53 to cluster DNS is permitted and +unfiltered by OpenShell policy, leaving a DNS tunnelling path out of an +otherwise closed pod. + +**Supervisor restart decoupling.** The `Deployment` recreates the supervisor pod +independently of the agent pod. Unlike `sidecar`, where symmetric exit +guarantees a matched pair, an agent pod here can outlive its supervisor and +continue running with all egress denied until the replacement becomes ready. + +## Alternatives + +### Do nothing + +Clusters that permit no in-pod privilege remain unable to run OpenShell. On +OpenShift specifically, the documented path stays `privileged`-SCC and +evaluation-only. + +### Shared proxy for many sandboxes + +One supervisor `Deployment` per namespace instead of per sandbox would cut the +resource multiplication substantially. Rejected: policy is per sandbox, and a +shared proxy would need in-band sandbox attribution on every connection to +enforce the right policy, reintroducing a trust problem that 1:1 pairing avoids +structurally. + +### Sidecar container in the same pod, without the nftables fence + +Keeps one pod and removes the privileged init container, but without a fence the +workload reaches the network directly through the shared namespace and the proxy +becomes advisory. `NetworkPolicy` cannot help, because it cannot distinguish +containers within one pod. The separate pod is what makes the policy fence +expressible. + +### Rely on an admission webhook to inject proxy settings + +Moves configuration out of the driver but does not create a fence, and adds a +cluster-wide mutating webhook — often a harder sell than the workload permissions +it would replace. + +### Custom OpenShift SCC, as `cni-sidecar` uses + +Unnecessary here. `nonroot-v2` already grants exactly what `proxy-pod` needs. +Shipping a custom SCC when a built-in one suffices adds a cluster-scoped object +and an audit burden for no gain. + +### Auto-detect the DNS peers instead of configuring them + +Requires cluster-type inference plus cluster-wide namespace and pod read +permissions the driver does not hold, and still fails for NodeLocal DNSCache and +non-default DNS deployments. Configuration handles every case with no new RBAC. + +## Prior art + +- `combined`, `sidecar` (#2074, #2076) and `cni-sidecar` + ([RFC](./cni-sidecar-topology-DRAFT.md), #2078) — the in-pod topologies this + one departs from. +- Istio and Linkerd sidecar injection with `NetworkPolicy`-backed mesh + isolation: same reliance on the CNI enforcing policy, same + privilege-versus-enforcement tradeoff, and a comparable ambient/sidecar split. +- Kubernetes egress gateways (Cilium, Calico), which likewise centralize + policy-enforced egress outside the workload pod. + +## Open questions + +- **Credential placement.** The proposed design puts a scoped process-supervisor + credential in the sandbox pod. Is the `caller_kind` scoping (no + provider/inference) sufficient, or is it worth the extra complexity of brokering + all gateway access through the proxy pod so the sandbox pod holds no gateway + credential at all (at the cost of re-introducing cross-pod relay bridging)? +- **Enum shape.** Should the in-pod-process-supervisor design replace the + `proxy-pod` value, or ship as a distinct topology with the network-only + (zero-in-pod-supervisor) design retained as a separate maximal-isolation + variant? If distinct, what are they named? +- Should a startup fence-verification probe be a **requirement** for graduating + `proxy-pod` out of experimental, given that the failure mode of a + non-enforcing CNI is silent? +- Should `command`/`args` graduate from the Kubernetes `driver_config` + passthrough to the public `SandboxTemplate`, and if so what do they mean in + topologies where the supervisor is the container entrypoint? +- Should `openshell sandbox create -- ` be reinterpreted as the container + command in topologies with no session, rather than failing to deliver it? +- Should OpenShell publish a `proxy-pod`-suitable sandbox image with a + long-running entrypoint, so the default path works without `driver_config`? +- Should a future `SupervisorSessionModel` variant carry a capability list, so + the gateway can gate individual RPCs rather than treating relays as + all-or-nothing? +- Should `affinity` default to `preferred` rather than `disabled`, given that + the default sends all workload egress across nodes? +- Should the gateway reject `exec`/`connect`/`upload`/`sync` for `proxy-pod` + sandboxes at the RPC boundary with a topology-specific error? +- Should the driver drop explicit `runAsUser`/`runAsGroup`/`fsGroup` on + OpenShift so `proxy-pod` admits under stock `restricted-v2` with no SCC grant + at all, and what does that imply for workspace PVC ownership? +- Is per-sandbox CA generation the right model, or should the CA be issued by + the gateway and distributed, so the private key never rests in a namespace the + operator's tenants may be able to read? diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 8e86bc0643..c16b60a358 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -543,7 +543,71 @@ The shared state directory should preserve `sandbox_gid` inheritance `@openshell-sidecar-ssh`; the network sidecar verifies its peer PID before bridging gateway relay requests. No `ssh.sock` file should appear in the shared state directory. -Inspect all three when sandbox registration or egress enforcement fails: + +If `topology = "proxy-pod"` is rendered, each sandbox should have a +separate supervisor Deployment with one supervisor pod, a headless supervisor +Service, a proxy CA Secret, and two per-sandbox NetworkPolicies. The agent pod +should have `openshell.ai/sandbox-role=agent`; the supervisor pod should have +`openshell.ai/sandbox-role=supervisor`; both should share the same +`openshell.ai/sandbox-id`. The supervisor Deployment must have a controlling +`Sandbox` ownerReference. The Deployment pod template must carry the +`openshell.io/sandbox-id` annotation so the TokenReview bootstrap path can mint +a sandbox JWT. For supervisor pods, the gateway validates the +`Pod -> ReplicaSet -> Deployment -> Sandbox` owner chain, so missing +`apps/replicasets get` RBAC can also break bootstrap. Helm renders the +Deployment, ReplicaSet, Service, Secret, and NetworkPolicy RBAC when +`supervisor.topology=proxy-pod` (or when `supervisor.proxyPod.retainCompanionRbac=true` +during a migration away from proxy-pod), and scopes it by workspace mode: +`shared` grants it through the namespaced Role, while `managed` and `operator` +grant it through the ClusterRole (the sandbox namespace is per-workspace). +`list`/`watch` on `apps/deployments` are granted only through the namespaced Role +(shared mode): the supervisor-readiness Deployment watch runs only in shared +mode, and managed/operator modes deliberately avoid cluster-wide Deployment +enumeration, folding readiness in through get/list and the periodic reconcile +instead. If those resources fail with forbidden errors, confirm both the rendered +`gateway.toml` and Helm values use proxy-pod topology (or retainCompanionRbac) +and that the workspace mode's Role/ClusterRole was applied. +Companion cleanup is split: the owner-referenced Deployment, Service, CA Secret, +and supervisor-ingress NetworkPolicy are garbage-collected with the Sandbox CR +(the gateway holds no `delete` on them and no Secret read). The agent egress +NetworkPolicy — the workload's egress fence — carries no owner reference and is +deleted by the gateway only after the workload pod is gone, so a pod that ignores +SIGTERM cannot regain direct egress during its grace period; reconciliation reaps +any fence orphaned by a gateway crash (hence `delete`/`list` on networkpolicies). +Reconciliation runs at watch establishment and then periodically (~30s) while the +sandbox watch is up, so a transiently-failed stop-time supervisor scale-down or a +crash-orphaned fence is corrected without waiting for the watch to drop. If a +deleted sandbox leaves an `os-eg-...` NetworkPolicy behind, or a stopped +sandbox's `os-sup-...` Deployment keeps a replica, check that the gateway's +reconcile ran and that the workload pod actually terminated. When changing `supervisor.topology` away from proxy-pod while proxy-pod +sandboxes still exist, set `supervisor.proxyPod.retainCompanionRbac=true` and +leave it set until every such sandbox is deleted. The driver keeps managing them +by their persisted creation-time topology, so with the flag their companion RBAC, +periodic reconciliation, and readiness watch all keep working. Without it the RBAC +is removed and start/stop and crash-recovery for those sandboxes stop working +until the topology is restored. +If the agent cannot reach the gateway, check DNS to the headless Service, the +agent egress NetworkPolicy DNS exception for kube-dns/CoreDNS, and the +supervisor ingress NetworkPolicy allowing only that agent pod on port `3128`. + +A proxy-pod sandbox falls back to `Provisioning` (Ready condition `False`, +reason `DependenciesNotReady`) when its supervisor Deployment has no available +replica: the gateway folds supervisor Deployment availability into sandbox +status so a sandbox never stays Ready while its policy-enforced egress path is +down, and recovers to `Ready` once the supervisor does. In shared mode the +gateway also watches supervisor Deployments (hence `list`/`watch` on +`apps/deployments` in the namespaced Role) and pushes a refreshed status within +seconds of an availability change; managed/operator modes and every mode's +direct `get`/`list` queries and periodic reconcile fold in the same check, so +readiness is never wrong for long even without the watch. If a previously-Ready +sandbox drops to `Provisioning`, inspect the supervisor Deployment (`kubectl -n + get deploy `) and its pod. Companion resource names are keyed on the +immutable sandbox UUID, so the `os-sup-`/`os-svc-`/`os-ca-`/`os-eg-`/`os-ing-` +suffix is stable per sandbox instance and distinct across instances even when +sandbox names repeat. + +Inspect the relevant containers when sandbox registration or egress enforcement +fails: ```bash kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -E '^\[openshell\.drivers\.kubernetes\]|^topology\s*=' @@ -554,6 +618,15 @@ kubectl -n logs -c openshell-supervisor-networ kubectl -n logs -c agent --tail=200 ``` +In `proxy-pod` topology the network supervisor is NOT a container in the sandbox +pod — it runs in the separate per-sandbox supervisor `Deployment`. Get its logs +from that pod instead; the sandbox pod has only the workload `agent` container: + +```bash +kubectl -n logs deploy/ --tail=200 +kubectl -n logs -c agent --tail=200 +``` + #### Corporate upstream proxy When the deployment routes sandbox egress through a corporate HTTP forward diff --git a/skills/openshell-cli/SKILL.md b/skills/openshell-cli/SKILL.md index 48f84fa653..3412527848 100644 --- a/skills/openshell-cli/SKILL.md +++ b/skills/openshell-cli/SKILL.md @@ -229,6 +229,17 @@ remains usable only until its recorded expiry. ## Workflow 3: Sandbox Lifecycle +> **Proxy-pod topology is sessionless.** When the Kubernetes driver runs with +> `supervisor.topology=proxy-pod`, the sandbox has no in-pod supervisor session, +> so relay-backed operations — a trailing `-- `, `--upload`, +> `--forward`, `--editor`, `sandbox connect`, `sandbox exec`, `sandbox upload`, +> and `sandbox download` — are rejected with a topology-specific error. Run the +> workload as the container entrypoint via +> `--driver-config-json '{"kubernetes":{"containers":{"agent":{"command":[...]}}}}'` +> and create with `--detach`. Bake required files into the image instead of +> uploading. The rest of this workflow applies to `combined` and `sidecar` +> topologies. + ### Create with options ```bash diff --git a/tasks/helm.toml b/tasks/helm.toml index 525e9c2e51..1193a08e20 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -81,6 +81,11 @@ description = "Run skaffold dev with the Kubernetes supervisor sidecar topology dir = "deploy/helm/openshell" run = "skaffold dev -p sidecar-mtls" +["helm:skaffold:dev:proxy-pod"] +description = "Run skaffold dev with proxy-pod topology; requires NetworkPolicy enforcement in the target cluster" +dir = "deploy/helm/openshell" +run = "skaffold dev -p proxy-pod" + ["helm:skaffold:run"] description = "Run a one-shot Skaffold deploy and register its local plaintext gateway" dir = "deploy/helm/openshell" @@ -96,6 +101,11 @@ description = "Run skaffold run with the Kubernetes supervisor sidecar topology dir = "deploy/helm/openshell" run = "skaffold run -p sidecar-mtls" +["helm:skaffold:run:proxy-pod"] +description = "Run skaffold run with proxy-pod topology; requires NetworkPolicy enforcement in the target cluster" +dir = "deploy/helm/openshell" +run = "skaffold run -p proxy-pod" + ["helm:skaffold:delete"] description = "Run skaffold delete for deploy/helm/openshell" dir = "deploy/helm/openshell" @@ -111,6 +121,11 @@ description = "Run skaffold delete for the Kubernetes supervisor sidecar topolog dir = "deploy/helm/openshell" run = "skaffold delete -p sidecar-mtls" +["helm:skaffold:delete:proxy-pod"] +description = "Run skaffold delete for the Kubernetes proxy-pod topology" +dir = "deploy/helm/openshell" +run = "skaffold delete -p proxy-pod" + ["helm:skaffold:diagnose"] description = "Run skaffold diagnose for deploy/helm/openshell" dir = "deploy/helm/openshell" diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index dc8adb9bdf..eb1dcf7fe5 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -89,6 +89,10 @@ Environment: macOS uses k3d from mise (Docker required). Linux can use this flow only when k3d is installed explicitly; otherwise use kind or an existing cluster context. Pair with: mise run helm:skaffold:dev + +The proxy-pod Skaffold profile relies on Kubernetes NetworkPolicy enforcement. +This helper leaves k3s's embedded network policy controller enabled; if you +replace the CNI, install a policy-enforcing CNI before using that profile. EOF } diff --git a/tasks/test.toml b/tasks/test.toml index 2fc3c565e4..00ed578da2 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -186,6 +186,14 @@ env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidec depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:proxy-pod"] +description = "Run the capability-scoped proxy-pod Kubernetes e2e suite; requires NetworkPolicy enforcement in the target cluster" +# proxy-pod is network-only: it has no in-sandbox supervisor, so the generic +# suite's exec/session tests (e.g. smoke) cannot pass. Run only the +# proxy_pod suite, which exercises this topology's actual contract. +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e-kubernetes-proxy-pod", OPENSHELL_E2E_KUBE_TEST = "proxy_pod" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" }